1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28 /*
29 * This file and its contents are supplied under the terms of the
30 * Common Development and Distribution License ("CDDL"), version 1.0.
31 * You may only use this file in accordance with the terms of version
32 * 1.0 of the CDDL.
33 *
34 * A full copy of the text of the CDDL should have accompanied this
35 * source. A copy of the CDDL is also available via the Internet at
36 * http://www.illumos.org/license/CDDL.
37 *
38 * Copyright 2015 Pluribus Networks Inc.
39 * Copyright 2018 Joyent, Inc.
40 * Copyright 2022 Oxide Computer Company
41 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
42 */
43
44 #include <sys/cdefs.h>
45
46 #include <sys/types.h>
47 #ifndef WITHOUT_CAPSICUM
48 #include <sys/capsicum.h>
49 #endif
50 #include <sys/mman.h>
51 #include <sys/time.h>
52
53 #ifdef __FreeBSD__
54 #include <amd64/vmm/intel/vmcs.h>
55 #else
56 #include <sys/cpuset.h>
57 #include <intel/vmcs.h>
58 #endif
59
60 #include <machine/atomic.h>
61
62 #ifndef WITHOUT_CAPSICUM
63 #include <capsicum_helpers.h>
64 #endif
65 #include <stdio.h>
66 #include <stdlib.h>
67 #include <string.h>
68 #include <err.h>
69 #include <errno.h>
70 #include <libgen.h>
71 #include <unistd.h>
72 #include <assert.h>
73 #include <pthread.h>
74 #include <pthread_np.h>
75 #include <sysexits.h>
76 #include <stdbool.h>
77 #include <stdint.h>
78
79 #include <machine/vmm.h>
80 #ifndef WITHOUT_CAPSICUM
81 #include <machine/vmm_dev.h>
82 #endif
83 #ifdef __FreeBSD__
84 #include <machine/vmm_instruction_emul.h>
85 #endif
86 #include <vmmapi.h>
87
88 #include "bhyverun.h"
89 #include "acpi.h"
90 #include "atkbdc.h"
91 #include "bootrom.h"
92 #include "config.h"
93 #include "inout.h"
94 #include "debug.h"
95 #include "e820.h"
96 #include "fwctl.h"
97 #include "gdb.h"
98 #include "ioapic.h"
99 #include "kernemu_dev.h"
100 #include "mem.h"
101 #include "mevent.h"
102 #include "mptbl.h"
103 #include "pci_emul.h"
104 #include "pci_irq.h"
105 #include "pci_lpc.h"
106 #include "qemu_fwcfg.h"
107 #include "smbiostbl.h"
108 #include "tpm_device.h"
109 #include "xmsr.h"
110 #include "spinup_ap.h"
111 #include "rtc.h"
112 #include "vmgenc.h"
113 #ifndef __FreeBSD__
114 #include "privileges.h"
115 #endif
116
117 #define MB (1024UL * 1024)
118 #define GB (1024UL * MB)
119
120 static const char * const vmx_exit_reason_desc[] = {
121 [EXIT_REASON_EXCEPTION] = "Exception or non-maskable interrupt (NMI)",
122 [EXIT_REASON_EXT_INTR] = "External interrupt",
123 [EXIT_REASON_TRIPLE_FAULT] = "Triple fault",
124 [EXIT_REASON_INIT] = "INIT signal",
125 [EXIT_REASON_SIPI] = "Start-up IPI (SIPI)",
126 [EXIT_REASON_IO_SMI] = "I/O system-management interrupt (SMI)",
127 [EXIT_REASON_SMI] = "Other SMI",
128 [EXIT_REASON_INTR_WINDOW] = "Interrupt window",
129 [EXIT_REASON_NMI_WINDOW] = "NMI window",
130 [EXIT_REASON_TASK_SWITCH] = "Task switch",
131 [EXIT_REASON_CPUID] = "CPUID",
132 [EXIT_REASON_GETSEC] = "GETSEC",
133 [EXIT_REASON_HLT] = "HLT",
134 [EXIT_REASON_INVD] = "INVD",
135 [EXIT_REASON_INVLPG] = "INVLPG",
136 [EXIT_REASON_RDPMC] = "RDPMC",
137 [EXIT_REASON_RDTSC] = "RDTSC",
138 [EXIT_REASON_RSM] = "RSM",
139 [EXIT_REASON_VMCALL] = "VMCALL",
140 [EXIT_REASON_VMCLEAR] = "VMCLEAR",
141 [EXIT_REASON_VMLAUNCH] = "VMLAUNCH",
142 [EXIT_REASON_VMPTRLD] = "VMPTRLD",
143 [EXIT_REASON_VMPTRST] = "VMPTRST",
144 [EXIT_REASON_VMREAD] = "VMREAD",
145 [EXIT_REASON_VMRESUME] = "VMRESUME",
146 [EXIT_REASON_VMWRITE] = "VMWRITE",
147 [EXIT_REASON_VMXOFF] = "VMXOFF",
148 [EXIT_REASON_VMXON] = "VMXON",
149 [EXIT_REASON_CR_ACCESS] = "Control-register accesses",
150 [EXIT_REASON_DR_ACCESS] = "MOV DR",
151 [EXIT_REASON_INOUT] = "I/O instruction",
152 [EXIT_REASON_RDMSR] = "RDMSR",
153 [EXIT_REASON_WRMSR] = "WRMSR",
154 [EXIT_REASON_INVAL_VMCS] =
155 "VM-entry failure due to invalid guest state",
156 [EXIT_REASON_INVAL_MSR] = "VM-entry failure due to MSR loading",
157 [EXIT_REASON_MWAIT] = "MWAIT",
158 [EXIT_REASON_MTF] = "Monitor trap flag",
159 [EXIT_REASON_MONITOR] = "MONITOR",
160 [EXIT_REASON_PAUSE] = "PAUSE",
161 [EXIT_REASON_MCE_DURING_ENTRY] =
162 "VM-entry failure due to machine-check event",
163 [EXIT_REASON_TPR] = "TPR below threshold",
164 [EXIT_REASON_APIC_ACCESS] = "APIC access",
165 [EXIT_REASON_VIRTUALIZED_EOI] = "Virtualized EOI",
166 [EXIT_REASON_GDTR_IDTR] = "Access to GDTR or IDTR",
167 [EXIT_REASON_LDTR_TR] = "Access to LDTR or TR",
168 [EXIT_REASON_EPT_FAULT] = "EPT violation",
169 [EXIT_REASON_EPT_MISCONFIG] = "EPT misconfiguration",
170 [EXIT_REASON_INVEPT] = "INVEPT",
171 [EXIT_REASON_RDTSCP] = "RDTSCP",
172 [EXIT_REASON_VMX_PREEMPT] = "VMX-preemption timer expired",
173 [EXIT_REASON_INVVPID] = "INVVPID",
174 [EXIT_REASON_WBINVD] = "WBINVD",
175 [EXIT_REASON_XSETBV] = "XSETBV",
176 [EXIT_REASON_APIC_WRITE] = "APIC write",
177 [EXIT_REASON_RDRAND] = "RDRAND",
178 [EXIT_REASON_INVPCID] = "INVPCID",
179 [EXIT_REASON_VMFUNC] = "VMFUNC",
180 [EXIT_REASON_ENCLS] = "ENCLS",
181 [EXIT_REASON_RDSEED] = "RDSEED",
182 [EXIT_REASON_PM_LOG_FULL] = "Page-modification log full",
183 [EXIT_REASON_XSAVES] = "XSAVES",
184 [EXIT_REASON_XRSTORS] = "XRSTORS"
185 };
186
187 typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_exit *);
188
189 int guest_ncpus;
190 uint16_t cpu_cores, cpu_sockets, cpu_threads;
191
192 int raw_stdio = 0;
193
194 static char *progname;
195 static const int BSP = 0;
196
197 static cpuset_t cpumask;
198
199 static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu);
200
201 #ifndef __FreeBSD__
202 static struct vm_entry *vmentry;
203 #endif
204
205 static struct vcpu_info {
206 struct vmctx *ctx;
207 struct vcpu *vcpu;
208 int vcpuid;
209 } *vcpu_info;
210
211 #ifdef __FreeBSD__
212 static cpuset_t **vcpumap;
213 #endif
214
215 static void
usage(int code)216 usage(int code)
217 {
218
219 fprintf(stderr,
220 #ifdef __FreeBSD__
221 "Usage: %s [-AaCDeHhPSuWwxY]\n"
222 #else
223 "Usage: %s [-aCDdeHhPSuWwxY]\n"
224 #endif
225 " %*s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n"
226 #ifdef __FreeBSD__
227 " %*s [-G port] [-k config_file] [-l lpc] [-m mem] [-o var=value]\n"
228 " %*s [-p vcpu:hostcpu] [-r file] [-s pci] [-U uuid] vmname\n"
229
230 " -A: create ACPI tables\n"
231 #else
232 " %*s [-k <config_file>] [-l <lpc>] [-m mem] [-o <var>=<value>]\n"
233 " %*s [-s <pci>] [-U uuid] vmname\n"
234 #endif
235 " -a: local apic is in xAPIC mode (deprecated)\n"
236 #ifndef __FreeBSD__
237 " -B type,key=value,...: set SMBIOS information\n"
238 #endif
239 " -C: include guest memory in core file\n"
240 " -c: number of CPUs and/or topology specification\n"
241 " -D: destroy on power-off\n"
242 #ifndef __FreeBSD__
243 " -d: suspend cpu at boot\n"
244 #endif
245 " -e: exit on unhandled I/O access\n"
246 #ifdef __FreeBSD__
247 " -G: start a debug server\n"
248 #endif
249 " -H: vmexit from the guest on HLT\n"
250 " -h: help\n"
251 " -k: key=value flat config file\n"
252 " -K: PS2 keyboard layout\n"
253 " -l: LPC device configuration\n"
254 " -m: memory size\n"
255 " -o: set config 'var' to 'value'\n"
256 " -P: vmexit from the guest on pause\n"
257 #ifdef __FreeBSD__
258 " -p: pin 'vcpu' to 'hostcpu'\n"
259 #endif
260 " -S: guest memory cannot be swapped\n"
261 " -s: <slot,driver,configinfo> PCI slot config\n"
262 " -U: UUID\n"
263 " -u: RTC keeps UTC time\n"
264 " -W: force virtio to use single-vector MSI\n"
265 " -w: ignore unimplemented MSRs\n"
266 " -x: local APIC is in x2APIC mode\n"
267 " -Y: disable MPtable generation\n",
268 progname, (int)strlen(progname), "", (int)strlen(progname), "",
269 (int)strlen(progname), "");
270
271 exit(code);
272 }
273
274 /*
275 * XXX This parser is known to have the following issues:
276 * 1. It accepts null key=value tokens ",," as setting "cpus" to an
277 * empty string.
278 *
279 * The acceptance of a null specification ('-c ""') is by design to match the
280 * manual page syntax specification, this results in a topology of 1 vCPU.
281 */
282 static int
topology_parse(const char * opt)283 topology_parse(const char *opt)
284 {
285 char *cp, *str, *tofree;
286
287 if (*opt == '\0') {
288 set_config_value("sockets", "1");
289 set_config_value("cores", "1");
290 set_config_value("threads", "1");
291 set_config_value("cpus", "1");
292 return (0);
293 }
294
295 tofree = str = strdup(opt);
296 if (str == NULL)
297 errx(4, "Failed to allocate memory");
298
299 while ((cp = strsep(&str, ",")) != NULL) {
300 if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
301 set_config_value("cpus", cp + strlen("cpus="));
302 else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0)
303 set_config_value("sockets", cp + strlen("sockets="));
304 else if (strncmp(cp, "cores=", strlen("cores=")) == 0)
305 set_config_value("cores", cp + strlen("cores="));
306 else if (strncmp(cp, "threads=", strlen("threads=")) == 0)
307 set_config_value("threads", cp + strlen("threads="));
308 else if (strchr(cp, '=') != NULL)
309 goto out;
310 else
311 set_config_value("cpus", cp);
312 }
313 free(tofree);
314 return (0);
315
316 out:
317 free(tofree);
318 return (-1);
319 }
320
321 static int
parse_int_value(const char * key,const char * value,int minval,int maxval)322 parse_int_value(const char *key, const char *value, int minval, int maxval)
323 {
324 char *cp;
325 long lval;
326
327 errno = 0;
328 lval = strtol(value, &cp, 0);
329 if (errno != 0 || *cp != '\0' || cp == value || lval < minval ||
330 lval > maxval)
331 errx(4, "Invalid value for %s: '%s'", key, value);
332 return (lval);
333 }
334
335 /*
336 * Set the sockets, cores, threads, and guest_cpus variables based on
337 * the configured topology.
338 *
339 * The limits of UINT16_MAX are due to the types passed to
340 * vm_set_topology(). vmm.ko may enforce tighter limits.
341 */
342 static void
calc_topology(void)343 calc_topology(void)
344 {
345 const char *value;
346 bool explicit_cpus;
347 uint64_t ncpus;
348
349 value = get_config_value("cpus");
350 if (value != NULL) {
351 guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX);
352 explicit_cpus = true;
353 } else {
354 guest_ncpus = 1;
355 explicit_cpus = false;
356 }
357 value = get_config_value("cores");
358 if (value != NULL)
359 cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX);
360 else
361 cpu_cores = 1;
362 value = get_config_value("threads");
363 if (value != NULL)
364 cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX);
365 else
366 cpu_threads = 1;
367 value = get_config_value("sockets");
368 if (value != NULL)
369 cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX);
370 else
371 cpu_sockets = guest_ncpus;
372
373 /*
374 * Compute sockets * cores * threads avoiding overflow. The
375 * range check above insures these are 16 bit values.
376 */
377 ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads;
378 if (ncpus > UINT16_MAX)
379 errx(4, "Computed number of vCPUs too high: %ju",
380 (uintmax_t)ncpus);
381
382 if (explicit_cpus) {
383 if (guest_ncpus != (int)ncpus)
384 errx(4, "Topology (%d sockets, %d cores, %d threads) "
385 "does not match %d vCPUs",
386 cpu_sockets, cpu_cores, cpu_threads,
387 guest_ncpus);
388 } else
389 guest_ncpus = ncpus;
390 }
391
392 #ifdef __FreeBSD__
393 static int
pincpu_parse(const char * opt)394 pincpu_parse(const char *opt)
395 {
396 int vcpu, pcpu;
397 const char *value;
398 char *newval;
399 char key[16];
400
401 if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) {
402 fprintf(stderr, "invalid format: %s\n", opt);
403 return (-1);
404 }
405
406 if (vcpu < 0) {
407 fprintf(stderr, "invalid vcpu '%d'\n", vcpu);
408 return (-1);
409 }
410
411 if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
412 fprintf(stderr, "hostcpu '%d' outside valid range from "
413 "0 to %d\n", pcpu, CPU_SETSIZE - 1);
414 return (-1);
415 }
416
417 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
418 value = get_config_value(key);
419
420 if (asprintf(&newval, "%s%s%d", value != NULL ? value : "",
421 value != NULL ? "," : "", pcpu) == -1) {
422 perror("failed to build new cpuset string");
423 return (-1);
424 }
425
426 set_config_value(key, newval);
427 free(newval);
428 return (0);
429 }
430
431 static void
parse_cpuset(int vcpu,const char * list,cpuset_t * set)432 parse_cpuset(int vcpu, const char *list, cpuset_t *set)
433 {
434 char *cp, *token;
435 int pcpu, start;
436
437 CPU_ZERO(set);
438 start = -1;
439 token = __DECONST(char *, list);
440 for (;;) {
441 pcpu = strtoul(token, &cp, 0);
442 if (cp == token)
443 errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
444 if (pcpu < 0 || pcpu >= CPU_SETSIZE)
445 errx(4, "hostcpu '%d' outside valid range from 0 to %d",
446 pcpu, CPU_SETSIZE - 1);
447 switch (*cp) {
448 case ',':
449 case '\0':
450 if (start >= 0) {
451 if (start > pcpu)
452 errx(4, "Invalid hostcpu range %d-%d",
453 start, pcpu);
454 while (start < pcpu) {
455 CPU_SET(start, set);
456 start++;
457 }
458 start = -1;
459 }
460 CPU_SET(pcpu, set);
461 break;
462 case '-':
463 if (start >= 0)
464 errx(4, "invalid cpuset for vcpu %d: '%s'",
465 vcpu, list);
466 start = pcpu;
467 break;
468 default:
469 errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list);
470 }
471 if (*cp == '\0')
472 break;
473 token = cp + 1;
474 }
475 }
476
477 static void
build_vcpumaps(void)478 build_vcpumaps(void)
479 {
480 char key[16];
481 const char *value;
482 int vcpu;
483
484 vcpumap = calloc(guest_ncpus, sizeof(*vcpumap));
485 for (vcpu = 0; vcpu < guest_ncpus; vcpu++) {
486 snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu);
487 value = get_config_value(key);
488 if (value == NULL)
489 continue;
490 vcpumap[vcpu] = malloc(sizeof(cpuset_t));
491 if (vcpumap[vcpu] == NULL)
492 err(4, "Failed to allocate cpuset for vcpu %d", vcpu);
493 parse_cpuset(vcpu, value, vcpumap[vcpu]);
494 }
495 }
496
497 void
vm_inject_fault(struct vcpu * vcpu,int vector,int errcode_valid,int errcode)498 vm_inject_fault(struct vcpu *vcpu, int vector, int errcode_valid,
499 int errcode)
500 {
501 int error, restart_instruction;
502
503 restart_instruction = 1;
504
505 error = vm_inject_exception(vcpu, vector, errcode_valid, errcode,
506 restart_instruction);
507 assert(error == 0);
508 }
509 #endif /* __FreeBSD__ */
510
511 void *
paddr_guest2host(struct vmctx * ctx,uintptr_t gaddr,size_t len)512 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
513 {
514
515 return (vm_map_gpa(ctx, gaddr, len));
516 }
517
518 int
fbsdrun_virtio_msix(void)519 fbsdrun_virtio_msix(void)
520 {
521
522 return (get_config_bool_default("virtio_msix", true));
523 }
524
525 static void *
fbsdrun_start_thread(void * param)526 fbsdrun_start_thread(void *param)
527 {
528 char tname[MAXCOMLEN + 1];
529 struct vcpu_info *vi = param;
530 #ifdef __FreeBSD__
531 int error;
532 #endif
533
534 snprintf(tname, sizeof(tname), "vcpu %d", vi->vcpuid);
535 pthread_set_name_np(pthread_self(), tname);
536
537 #ifdef __FreeBSD__
538 if (vcpumap[vi->vcpuid] != NULL) {
539 error = pthread_setaffinity_np(pthread_self(),
540 sizeof(cpuset_t), vcpumap[vi->vcpuid]);
541 assert(error == 0);
542 }
543 #endif
544
545 gdb_cpu_add(vi->vcpu);
546
547 vm_loop(vi->ctx, vi->vcpu);
548
549 /* not reached */
550 exit(1);
551 return (NULL);
552 }
553
554 void
fbsdrun_addcpu(struct vcpu_info * vi,bool suspend)555 fbsdrun_addcpu(struct vcpu_info *vi, bool suspend)
556 {
557 pthread_t thr;
558 int error;
559
560 error = vm_activate_cpu(vi->vcpu);
561 if (error != 0)
562 err(EX_OSERR, "could not activate CPU %d", vi->vcpuid);
563
564 CPU_SET_ATOMIC(vi->vcpuid, &cpumask);
565
566 if (suspend)
567 (void) vm_suspend_cpu(vi->vcpu);
568
569 error = pthread_create(&thr, NULL, fbsdrun_start_thread, vi);
570 assert(error == 0);
571 }
572
573 static void
fbsdrun_deletecpu(int vcpu)574 fbsdrun_deletecpu(int vcpu)
575 {
576 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
577 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
578
579 pthread_mutex_lock(&resetcpu_mtx);
580 if (!CPU_ISSET(vcpu, &cpumask)) {
581 fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu);
582 exit(4);
583 }
584
585 CPU_CLR(vcpu, &cpumask);
586
587 if (vcpu != BSP) {
588 pthread_cond_signal(&resetcpu_cond);
589 pthread_mutex_unlock(&resetcpu_mtx);
590 pthread_exit(NULL);
591 /* NOTREACHED */
592 }
593
594 while (!CPU_EMPTY(&cpumask)) {
595 pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
596 }
597 pthread_mutex_unlock(&resetcpu_mtx);
598 }
599
600 #ifndef __FreeBSD__
601 static void
vmentry_mmio_read(struct vcpu * vcpu,uint64_t gpa,uint8_t bytes,uint64_t data)602 vmentry_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint8_t bytes, uint64_t data)
603 {
604 struct vm_entry *entry = &vmentry[vcpu_id(vcpu)];
605 struct vm_mmio *mmio = &entry->u.mmio;
606
607 assert(entry->cmd == VEC_DEFAULT);
608
609 entry->cmd = VEC_FULFILL_MMIO;
610 mmio->bytes = bytes;
611 mmio->read = 1;
612 mmio->gpa = gpa;
613 mmio->data = data;
614 }
615
616 static void
vmentry_mmio_write(struct vcpu * vcpu,uint64_t gpa,uint8_t bytes)617 vmentry_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint8_t bytes)
618 {
619 struct vm_entry *entry = &vmentry[vcpu_id(vcpu)];
620 struct vm_mmio *mmio = &entry->u.mmio;
621
622 assert(entry->cmd == VEC_DEFAULT);
623
624 entry->cmd = VEC_FULFILL_MMIO;
625 mmio->bytes = bytes;
626 mmio->read = 0;
627 mmio->gpa = gpa;
628 mmio->data = 0;
629 }
630
631 static void
vmentry_inout_read(struct vcpu * vcpu,uint16_t port,uint8_t bytes,uint32_t data)632 vmentry_inout_read(struct vcpu *vcpu, uint16_t port, uint8_t bytes,
633 uint32_t data)
634 {
635 struct vm_entry *entry = &vmentry[vcpu_id(vcpu)];
636 struct vm_inout *inout = &entry->u.inout;
637
638 assert(entry->cmd == VEC_DEFAULT);
639
640 entry->cmd = VEC_FULFILL_INOUT;
641 inout->bytes = bytes;
642 inout->flags = INOUT_IN;
643 inout->port = port;
644 inout->eax = data;
645 }
646
647 static void
vmentry_inout_write(struct vcpu * vcpu,uint16_t port,uint8_t bytes)648 vmentry_inout_write(struct vcpu *vcpu, uint16_t port, uint8_t bytes)
649 {
650 struct vm_entry *entry = &vmentry[vcpu_id(vcpu)];
651 struct vm_inout *inout = &entry->u.inout;
652
653 assert(entry->cmd == VEC_DEFAULT);
654
655 entry->cmd = VEC_FULFILL_INOUT;
656 inout->bytes = bytes;
657 inout->flags = 0;
658 inout->port = port;
659 inout->eax = 0;
660 }
661 #endif
662
663 static int
vmexit_inout(struct vmctx * ctx,struct vcpu * vcpu,struct vm_exit * vme)664 vmexit_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme)
665 {
666 int error;
667 struct vm_inout inout;
668 bool in;
669 uint8_t bytes;
670
671 inout = vme->u.inout;
672 in = (inout.flags & INOUT_IN) != 0;
673 bytes = inout.bytes;
674
675 error = emulate_inout(ctx, vcpu, &inout);
676 if (error) {
677 fprintf(stderr, "Unhandled %s%c 0x%04x at 0x%lx\n",
678 in ? "in" : "out",
679 bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'),
680 inout.port, vme->rip);
681 return (VMEXIT_ABORT);
682 } else {
683 /*
684 * Communicate the status of the inout operation back to the
685 * in-kernel instruction emulation.
686 */
687 if (in) {
688 vmentry_inout_read(vcpu, inout.port, bytes, inout.eax);
689 } else {
690 vmentry_inout_write(vcpu, inout.port, bytes);
691 }
692 return (VMEXIT_CONTINUE);
693 }
694 }
695
696 static int
vmexit_rdmsr(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)697 vmexit_rdmsr(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
698 {
699 uint64_t val;
700 uint32_t eax, edx;
701 int error;
702
703 val = 0;
704 error = emulate_rdmsr(vcpu, vme->u.msr.code, &val);
705 if (error != 0) {
706 fprintf(stderr, "rdmsr to register %#x on vcpu %d\n",
707 vme->u.msr.code, vcpu_id(vcpu));
708 if (get_config_bool("x86.strictmsr")) {
709 vm_inject_gp(vcpu);
710 return (VMEXIT_CONTINUE);
711 }
712 }
713
714 eax = val;
715 error = vm_set_register(vcpu, VM_REG_GUEST_RAX, eax);
716 assert(error == 0);
717
718 edx = val >> 32;
719 error = vm_set_register(vcpu, VM_REG_GUEST_RDX, edx);
720 assert(error == 0);
721
722 return (VMEXIT_CONTINUE);
723 }
724
725 static int
vmexit_wrmsr(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)726 vmexit_wrmsr(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
727 {
728 int error;
729
730 error = emulate_wrmsr(vcpu, vme->u.msr.code, vme->u.msr.wval);
731 if (error != 0) {
732 fprintf(stderr, "wrmsr to register %#x(%#lx) on vcpu %d\n",
733 vme->u.msr.code, vme->u.msr.wval, vcpu_id(vcpu));
734 if (get_config_bool("x86.strictmsr")) {
735 vm_inject_gp(vcpu);
736 return (VMEXIT_CONTINUE);
737 }
738 }
739 return (VMEXIT_CONTINUE);
740 }
741
742 #ifndef __FreeBSD__
743 static int
vmexit_run_state(struct vmctx * ctx __unused,struct vcpu * vcpu __unused,struct vm_exit * vme __unused)744 vmexit_run_state(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
745 struct vm_exit *vme __unused)
746 {
747 /*
748 * Run-state transitions (INIT, SIPI, etc) are handled in-kernel, so an
749 * exit to userspace with that code is not expected.
750 */
751 fprintf(stderr, "unexpected run-state VM exit");
752 return (VMEXIT_ABORT);
753 }
754
755 static int
vmexit_paging(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)756 vmexit_paging(struct vmctx *ctx __unused, struct vcpu *vcpu,
757 struct vm_exit *vme)
758 {
759 fprintf(stderr, "vm exit[%d]\n", vcpu_id(vcpu));
760 fprintf(stderr, "\treason\t\tPAGING\n");
761 fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
762 fprintf(stderr, "\tgpa\t\t0x%016lx\n", vme->u.paging.gpa);
763 fprintf(stderr, "\tfault_type\t\t%d\n", vme->u.paging.fault_type);
764
765 return (VMEXIT_ABORT);
766 }
767 #endif /* __FreeBSD__ */
768
769 #ifdef __FreeBSD__
770 #define DEBUG_EPT_MISCONFIG
771 #else
772 /* EPT misconfig debugging not possible now that raw VMCS access is gone */
773 #endif
774
775 #ifdef DEBUG_EPT_MISCONFIG
776 #define VMCS_GUEST_PHYSICAL_ADDRESS 0x00002400
777
778 static uint64_t ept_misconfig_gpa, ept_misconfig_pte[4];
779 static int ept_misconfig_ptenum;
780 #endif
781
782 static const char *
vmexit_vmx_desc(uint32_t exit_reason)783 vmexit_vmx_desc(uint32_t exit_reason)
784 {
785
786 if (exit_reason >= nitems(vmx_exit_reason_desc) ||
787 vmx_exit_reason_desc[exit_reason] == NULL)
788 return ("Unknown");
789 return (vmx_exit_reason_desc[exit_reason]);
790 }
791
792 static int
vmexit_vmx(struct vmctx * ctx,struct vcpu * vcpu,struct vm_exit * vme)793 vmexit_vmx(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme)
794 {
795
796 fprintf(stderr, "vm exit[%d]\n", vcpu_id(vcpu));
797 fprintf(stderr, "\treason\t\tVMX\n");
798 fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
799 fprintf(stderr, "\tinst_length\t%d\n", vme->inst_length);
800 fprintf(stderr, "\tstatus\t\t%d\n", vme->u.vmx.status);
801 fprintf(stderr, "\texit_reason\t%u (%s)\n", vme->u.vmx.exit_reason,
802 vmexit_vmx_desc(vme->u.vmx.exit_reason));
803 fprintf(stderr, "\tqualification\t0x%016lx\n",
804 vme->u.vmx.exit_qualification);
805 fprintf(stderr, "\tinst_type\t\t%d\n", vme->u.vmx.inst_type);
806 fprintf(stderr, "\tinst_error\t\t%d\n", vme->u.vmx.inst_error);
807 #ifdef DEBUG_EPT_MISCONFIG
808 if (vme->u.vmx.exit_reason == EXIT_REASON_EPT_MISCONFIG) {
809 vm_get_register(vcpu,
810 VMCS_IDENT(VMCS_GUEST_PHYSICAL_ADDRESS),
811 &ept_misconfig_gpa);
812 vm_get_gpa_pmap(ctx, ept_misconfig_gpa, ept_misconfig_pte,
813 &ept_misconfig_ptenum);
814 fprintf(stderr, "\tEPT misconfiguration:\n");
815 fprintf(stderr, "\t\tGPA: %#lx\n", ept_misconfig_gpa);
816 fprintf(stderr, "\t\tPTE(%d): %#lx %#lx %#lx %#lx\n",
817 ept_misconfig_ptenum, ept_misconfig_pte[0],
818 ept_misconfig_pte[1], ept_misconfig_pte[2],
819 ept_misconfig_pte[3]);
820 }
821 #endif /* DEBUG_EPT_MISCONFIG */
822 return (VMEXIT_ABORT);
823 }
824
825 static int
vmexit_svm(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)826 vmexit_svm(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
827 {
828
829 fprintf(stderr, "vm exit[%d]\n", vcpu_id(vcpu));
830 fprintf(stderr, "\treason\t\tSVM\n");
831 fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip);
832 fprintf(stderr, "\tinst_length\t%d\n", vme->inst_length);
833 fprintf(stderr, "\texitcode\t%#lx\n", vme->u.svm.exitcode);
834 fprintf(stderr, "\texitinfo1\t%#lx\n", vme->u.svm.exitinfo1);
835 fprintf(stderr, "\texitinfo2\t%#lx\n", vme->u.svm.exitinfo2);
836 return (VMEXIT_ABORT);
837 }
838
839 static int
vmexit_bogus(struct vmctx * ctx __unused,struct vcpu * vcpu __unused,struct vm_exit * vme)840 vmexit_bogus(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
841 struct vm_exit *vme)
842 {
843
844 assert(vme->inst_length == 0);
845
846 return (VMEXIT_CONTINUE);
847 }
848
849 static int
vmexit_hlt(struct vmctx * ctx __unused,struct vcpu * vcpu __unused,struct vm_exit * vme __unused)850 vmexit_hlt(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
851 struct vm_exit *vme __unused)
852 {
853
854 /*
855 * Just continue execution with the next instruction. We use
856 * the HLT VM exit as a way to be friendly with the host
857 * scheduler.
858 */
859 return (VMEXIT_CONTINUE);
860 }
861
862 static int
vmexit_pause(struct vmctx * ctx __unused,struct vcpu * vcpu __unused,struct vm_exit * vme __unused)863 vmexit_pause(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
864 struct vm_exit *vme __unused)
865 {
866 return (VMEXIT_CONTINUE);
867 }
868
869 static int
vmexit_mtrap(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)870 vmexit_mtrap(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
871 {
872
873 assert(vme->inst_length == 0);
874
875 gdb_cpu_mtrap(vcpu);
876
877 return (VMEXIT_CONTINUE);
878 }
879
880 static int
vmexit_inst_emul(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)881 vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu,
882 struct vm_exit *vme)
883 {
884 uint8_t i, valid;
885
886 fprintf(stderr, "Failed to emulate instruction sequence ");
887
888 valid = vme->u.inst_emul.num_valid;
889 if (valid != 0) {
890 assert(valid <= sizeof (vme->u.inst_emul.inst));
891 fprintf(stderr, "[");
892 for (i = 0; i < valid; i++) {
893 if (i == 0) {
894 fprintf(stderr, "%02x",
895 vme->u.inst_emul.inst[i]);
896 } else {
897 fprintf(stderr, ", %02x",
898 vme->u.inst_emul.inst[i]);
899 }
900 }
901 fprintf(stderr, "] ");
902 }
903 fprintf(stderr, "@ %rip = %x\n", vme->rip);
904
905 return (VMEXIT_ABORT);
906 }
907
908 #ifndef __FreeBSD__
909 static int
vmexit_mmio(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)910 vmexit_mmio(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme)
911 {
912 int err;
913 struct vm_mmio mmio;
914 bool is_read;
915
916 mmio = vme->u.mmio;
917 is_read = (mmio.read != 0);
918
919 err = emulate_mem(vcpu, &mmio);
920
921 if (err == ESRCH) {
922 fprintf(stderr, "Unhandled memory access to 0x%lx\n", mmio.gpa);
923
924 /*
925 * Access to non-existent physical addresses is not likely to
926 * result in fatal errors on hardware machines, but rather reads
927 * of all-ones or discarded-but-acknowledged writes.
928 */
929 mmio.data = ~0UL;
930 err = 0;
931 }
932
933 if (err == 0) {
934 if (is_read) {
935 vmentry_mmio_read(vcpu, mmio.gpa, mmio.bytes,
936 mmio.data);
937 } else {
938 vmentry_mmio_write(vcpu, mmio.gpa, mmio.bytes);
939 }
940 return (VMEXIT_CONTINUE);
941 }
942
943 fprintf(stderr, "Unhandled mmio error to 0x%lx: %d\n", mmio.gpa, err);
944 return (VMEXIT_ABORT);
945 }
946 #endif /* !__FreeBSD__ */
947
948 static int
vmexit_suspend(struct vmctx * ctx,struct vcpu * vcpu,struct vm_exit * vme)949 vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme)
950 {
951 enum vm_suspend_how how;
952 int vcpuid = vcpu_id(vcpu);
953
954 how = vme->u.suspended.how;
955
956 fbsdrun_deletecpu(vcpuid);
957
958 switch (how) {
959 case VM_SUSPEND_RESET:
960 exit(0);
961 case VM_SUSPEND_POWEROFF:
962 if (get_config_bool_default("destroy_on_poweroff", false))
963 vm_destroy(ctx);
964 exit(1);
965 case VM_SUSPEND_HALT:
966 exit(2);
967 case VM_SUSPEND_TRIPLEFAULT:
968 exit(3);
969 default:
970 fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
971 exit(100);
972 }
973 return (0); /* NOTREACHED */
974 }
975
976 static int
vmexit_debug(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme __unused)977 vmexit_debug(struct vmctx *ctx __unused, struct vcpu *vcpu,
978 struct vm_exit *vme __unused)
979 {
980
981 gdb_cpu_suspend(vcpu);
982 return (VMEXIT_CONTINUE);
983 }
984
985 static int
vmexit_breakpoint(struct vmctx * ctx __unused,struct vcpu * vcpu,struct vm_exit * vme)986 vmexit_breakpoint(struct vmctx *ctx __unused, struct vcpu *vcpu,
987 struct vm_exit *vme)
988 {
989
990 gdb_cpu_breakpoint(vcpu, vme);
991 return (VMEXIT_CONTINUE);
992 }
993
994 #ifdef __FreeBSD__
995 static int
vmexit_ipi(struct vmctx * ctx __unused,struct vcpu * vcpu __unused,struct vm_exit * vme)996 vmexit_ipi(struct vmctx *ctx __unused, struct vcpu *vcpu __unused,
997 struct vm_exit *vme)
998 {
999 int error = -1;
1000 int i;
1001 switch (vme->u.ipi.mode) {
1002 case APIC_DELMODE_INIT:
1003 CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
1004 error = vm_suspend_cpu(vcpu_info[i].vcpu);
1005 if (error) {
1006 warnx("%s: failed to suspend cpu %d\n",
1007 __func__, i);
1008 break;
1009 }
1010 }
1011 break;
1012 case APIC_DELMODE_STARTUP:
1013 CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) {
1014 spinup_ap(vcpu_info[i].vcpu,
1015 vme->u.ipi.vector << PAGE_SHIFT);
1016 }
1017 error = 0;
1018 break;
1019 default:
1020 break;
1021 }
1022
1023 return (error);
1024 }
1025 #endif
1026
1027 static const vmexit_handler_t handler[VM_EXITCODE_MAX] = {
1028 [VM_EXITCODE_INOUT] = vmexit_inout,
1029 #ifndef __FreeBSD__
1030 [VM_EXITCODE_MMIO] = vmexit_mmio,
1031 #endif
1032 [VM_EXITCODE_VMX] = vmexit_vmx,
1033 [VM_EXITCODE_SVM] = vmexit_svm,
1034 [VM_EXITCODE_BOGUS] = vmexit_bogus,
1035 [VM_EXITCODE_RDMSR] = vmexit_rdmsr,
1036 [VM_EXITCODE_WRMSR] = vmexit_wrmsr,
1037 [VM_EXITCODE_MTRAP] = vmexit_mtrap,
1038 [VM_EXITCODE_INST_EMUL] = vmexit_inst_emul,
1039 #ifndef __FreeBSD__
1040 [VM_EXITCODE_RUN_STATE] = vmexit_run_state,
1041 [VM_EXITCODE_PAGING] = vmexit_paging,
1042 #endif
1043 [VM_EXITCODE_SUSPENDED] = vmexit_suspend,
1044 [VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch,
1045 [VM_EXITCODE_DEBUG] = vmexit_debug,
1046 [VM_EXITCODE_BPT] = vmexit_breakpoint,
1047 #ifdef __FreeBSD__
1048 [VM_EXITCODE_IPI] = vmexit_ipi,
1049 #endif
1050 [VM_EXITCODE_HLT] = vmexit_hlt,
1051 [VM_EXITCODE_PAUSE] = vmexit_pause,
1052 };
1053
1054 static void
vm_loop(struct vmctx * ctx,struct vcpu * vcpu)1055 vm_loop(struct vmctx *ctx, struct vcpu *vcpu)
1056 {
1057 struct vm_exit vme;
1058 int error, rc;
1059 enum vm_exitcode exitcode;
1060 cpuset_t active_cpus;
1061 struct vm_entry *ventry;
1062
1063 error = vm_active_cpus(ctx, &active_cpus);
1064 assert(CPU_ISSET(vcpu_id(vcpu), &active_cpus));
1065
1066 ventry = &vmentry[vcpu_id(vcpu)];
1067
1068 while (1) {
1069 error = vm_run(vcpu, ventry, &vme);
1070 if (error != 0)
1071 break;
1072
1073 if (ventry->cmd != VEC_DEFAULT) {
1074 /*
1075 * Discard any lingering entry state after it has been
1076 * submitted via vm_run().
1077 */
1078 bzero(ventry, sizeof (*ventry));
1079 }
1080
1081 exitcode = vme.exitcode;
1082 if (exitcode >= VM_EXITCODE_MAX || handler[exitcode] == NULL) {
1083 fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n",
1084 exitcode);
1085 exit(4);
1086 }
1087
1088 rc = (*handler[exitcode])(ctx, vcpu, &vme);
1089
1090 switch (rc) {
1091 case VMEXIT_CONTINUE:
1092 break;
1093 case VMEXIT_ABORT:
1094 abort();
1095 default:
1096 exit(4);
1097 }
1098 }
1099 fprintf(stderr, "vm_run error %d, errno %d\n", error, errno);
1100 }
1101
1102 static int
num_vcpus_allowed(struct vmctx * ctx,struct vcpu * vcpu)1103 num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu)
1104 {
1105 uint16_t sockets, cores, threads, maxcpus;
1106 #ifdef __FreeBSD__
1107 int tmp, error;
1108
1109 /*
1110 * The guest is allowed to spinup more than one processor only if the
1111 * UNRESTRICTED_GUEST capability is available.
1112 */
1113 error = vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp);
1114 if (error != 0)
1115 return (1);
1116 #else
1117 int error;
1118 /* Unrestricted Guest is always enabled on illumos */
1119
1120 #endif /* __FreeBSD__ */
1121
1122 error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus);
1123 if (error == 0)
1124 return (maxcpus);
1125 else
1126 return (1);
1127 }
1128
1129 static void
fbsdrun_set_capabilities(struct vcpu * vcpu)1130 fbsdrun_set_capabilities(struct vcpu *vcpu)
1131 {
1132 int err, tmp;
1133
1134 #ifdef __FreeBSD__
1135 if (get_config_bool_default("x86.vmexit_on_hlt", false)) {
1136 err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp);
1137 if (err < 0) {
1138 fprintf(stderr, "VM exit on HLT not supported\n");
1139 exit(4);
1140 }
1141 vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1);
1142 }
1143 #else
1144 /*
1145 * We insist that vmexit-on-hlt is available on the host CPU, and enable
1146 * it by default. Configuration of that feature is done with both of
1147 * those facts in mind.
1148 */
1149 tmp = (int)get_config_bool_default("x86.vmexit_on_hlt", true);
1150 err = vm_set_capability(vcpu, VM_CAP_HALT_EXIT, tmp);
1151 if (err < 0) {
1152 fprintf(stderr, "VM exit on HLT not supported\n");
1153 exit(4);
1154 }
1155 #endif /* __FreeBSD__ */
1156
1157 if (get_config_bool_default("x86.vmexit_on_pause", false)) {
1158 /*
1159 * pause exit support required for this mode
1160 */
1161 err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp);
1162 if (err < 0) {
1163 fprintf(stderr,
1164 "SMP mux requested, no pause support\n");
1165 exit(4);
1166 }
1167 vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1);
1168 }
1169
1170 if (get_config_bool_default("x86.x2apic", false))
1171 err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED);
1172 else
1173 err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED);
1174
1175 if (err) {
1176 fprintf(stderr, "Unable to set x2apic state (%d)\n", err);
1177 exit(4);
1178 }
1179
1180 #ifdef __FreeBSD__
1181 vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1);
1182
1183 err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1);
1184 assert(err == 0);
1185 #endif
1186 }
1187
1188 static struct vmctx *
do_open(const char * vmname)1189 do_open(const char *vmname)
1190 {
1191 struct vmctx *ctx;
1192 int error;
1193 bool reinit, romboot;
1194
1195 reinit = romboot = false;
1196
1197 if (lpc_bootrom())
1198 romboot = true;
1199 #ifndef __FreeBSD__
1200 uint64_t create_flags = 0;
1201 if (get_config_bool_default("memory.use_reservoir", false)) {
1202 create_flags |= VCF_RESERVOIR_MEM;
1203 }
1204 error = vm_create(vmname, create_flags);
1205 #else
1206 error = vm_create(vmname);
1207 #endif /* __FreeBSD__ */
1208 if (error) {
1209 if (errno == EEXIST) {
1210 if (romboot) {
1211 reinit = true;
1212 } else {
1213 /*
1214 * The virtual machine has been setup by the
1215 * userspace bootloader.
1216 */
1217 }
1218 } else {
1219 perror("vm_create");
1220 exit(4);
1221 }
1222 } else {
1223 if (!romboot) {
1224 /*
1225 * If the virtual machine was just created then a
1226 * bootrom must be configured to boot it.
1227 */
1228 fprintf(stderr, "virtual machine cannot be booted\n");
1229 exit(4);
1230 }
1231 }
1232
1233 ctx = vm_open(vmname);
1234 if (ctx == NULL) {
1235 perror("vm_open");
1236 exit(4);
1237 }
1238
1239 #ifndef WITHOUT_CAPSICUM
1240 if (vm_limit_rights(ctx) != 0)
1241 err(EX_OSERR, "vm_limit_rights");
1242 #endif
1243
1244 if (reinit) {
1245 #ifndef __FreeBSD__
1246 error = vm_reinit(ctx, 0);
1247 #else
1248 error = vm_reinit(ctx);
1249 #endif
1250 if (error) {
1251 perror("vm_reinit");
1252 exit(4);
1253 }
1254 }
1255 error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads, 0);
1256 if (error)
1257 errx(EX_OSERR, "vm_set_topology");
1258 return (ctx);
1259 }
1260
1261 static void
spinup_vcpu(struct vcpu_info * vi,bool bsp,bool suspend)1262 spinup_vcpu(struct vcpu_info *vi, bool bsp, bool suspend)
1263 {
1264 int error;
1265
1266 if (!bsp) {
1267 #ifndef __FreeBSD__
1268 /*
1269 * On illumos, all APs are spun up halted and run-state
1270 * transitions (INIT, SIPI, etc) are handled in-kernel.
1271 */
1272 spinup_ap(vi->vcpu, 0);
1273 #endif
1274
1275 fbsdrun_set_capabilities(vi->vcpu);
1276
1277 #ifdef __FreeBSD__
1278 /*
1279 * Enable the 'unrestricted guest' mode for APs.
1280 *
1281 * APs startup in power-on 16-bit mode.
1282 */
1283 error = vm_set_capability(vi->vcpu, VM_CAP_UNRESTRICTED_GUEST, 1);
1284 assert(error == 0);
1285 #endif
1286 }
1287
1288 #ifndef __FreeBSD__
1289 /*
1290 * The value of 'suspend' for the BSP depends on whether the -d
1291 * (suspend_at_boot) flag was given to bhyve. Regardless of that
1292 * value we always want to set the BSP to VRS_RUN and all others to
1293 * VRS_HALT.
1294 */
1295 error = vm_set_run_state(vi->vcpu, bsp ? VRS_RUN : VRS_HALT, 0);
1296 assert(error == 0);
1297 #endif
1298
1299 fbsdrun_addcpu(vi, suspend);
1300 }
1301
1302 static bool
parse_config_option(const char * option)1303 parse_config_option(const char *option)
1304 {
1305 const char *value;
1306 char *path;
1307
1308 value = strchr(option, '=');
1309 if (value == NULL || value[1] == '\0')
1310 return (false);
1311 path = strndup(option, value - option);
1312 if (path == NULL)
1313 err(4, "Failed to allocate memory");
1314 set_config_value(path, value + 1);
1315 return (true);
1316 }
1317
1318 static void
parse_simple_config_file(const char * path)1319 parse_simple_config_file(const char *path)
1320 {
1321 FILE *fp;
1322 char *line, *cp;
1323 size_t linecap;
1324 unsigned int lineno;
1325
1326 fp = fopen(path, "r");
1327 if (fp == NULL)
1328 err(4, "Failed to open configuration file %s", path);
1329 line = NULL;
1330 linecap = 0;
1331 lineno = 1;
1332 for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) {
1333 if (*line == '#' || *line == '\n')
1334 continue;
1335 cp = strchr(line, '\n');
1336 if (cp != NULL)
1337 *cp = '\0';
1338 if (!parse_config_option(line))
1339 errx(4, "%s line %u: invalid config option '%s'", path,
1340 lineno, line);
1341 }
1342 free(line);
1343 fclose(fp);
1344 }
1345
1346 static void
parse_gdb_options(const char * opt)1347 parse_gdb_options(const char *opt)
1348 {
1349 const char *sport;
1350 char *colon;
1351
1352 if (opt[0] == 'w') {
1353 set_config_bool("gdb.wait", true);
1354 opt++;
1355 }
1356
1357 colon = strrchr(opt, ':');
1358 if (colon == NULL) {
1359 sport = opt;
1360 } else {
1361 *colon = '\0';
1362 colon++;
1363 sport = colon;
1364 set_config_value("gdb.address", opt);
1365 }
1366
1367 set_config_value("gdb.port", sport);
1368 }
1369
1370 static void
set_defaults(void)1371 set_defaults(void)
1372 {
1373
1374 set_config_bool("acpi_tables", false);
1375 set_config_bool("acpi_tables_in_memory", true);
1376 set_config_value("memory.size", "256M");
1377 set_config_bool("x86.strictmsr", true);
1378 set_config_value("lpc.fwcfg", "bhyve");
1379 }
1380
1381 int
main(int argc,char * argv[])1382 main(int argc, char *argv[])
1383 {
1384 int c, error;
1385 int max_vcpus, memflags;
1386 struct vcpu *bsp;
1387 struct vmctx *ctx;
1388 struct qemu_fwcfg_item *e820_fwcfg_item;
1389 size_t memsize;
1390 const char *optstr, *value, *vmname;
1391
1392 init_config();
1393 set_defaults();
1394 progname = basename(argv[0]);
1395
1396 #ifdef __FreeBSD__
1397 optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:l:K:U:";
1398 #else
1399 /* +d, +B, -p */
1400 optstr = "adehuwxACDHIPSWYk:f:o:G:c:s:m:l:B:K:U:";
1401 #endif
1402 while ((c = getopt(argc, argv, optstr)) != -1) {
1403 switch (c) {
1404 case 'a':
1405 set_config_bool("x86.x2apic", false);
1406 break;
1407 case 'A':
1408 set_config_bool("acpi_tables", true);
1409 break;
1410 case 'D':
1411 set_config_bool("destroy_on_poweroff", true);
1412 break;
1413 #ifndef __FreeBSD__
1414 case 'B':
1415 if (smbios_parse(optarg) != 0) {
1416 errx(EX_USAGE, "invalid SMBIOS "
1417 "configuration '%s'", optarg);
1418 }
1419 break;
1420 case 'd':
1421 set_config_bool("suspend_at_boot", true);
1422 break;
1423 #endif
1424 #ifdef __FreeBSD__
1425 case 'p':
1426 if (pincpu_parse(optarg) != 0) {
1427 errx(EX_USAGE, "invalid vcpu pinning "
1428 "configuration '%s'", optarg);
1429 }
1430 break;
1431 #endif
1432 case 'c':
1433 if (topology_parse(optarg) != 0) {
1434 errx(EX_USAGE, "invalid cpu topology "
1435 "'%s'", optarg);
1436 }
1437 break;
1438 case 'C':
1439 set_config_bool("memory.guest_in_core", true);
1440 break;
1441 case 'f':
1442 if (qemu_fwcfg_parse_cmdline_arg(optarg) != 0) {
1443 errx(EX_USAGE, "invalid fwcfg item '%s'", optarg);
1444 }
1445 break;
1446 case 'G':
1447 parse_gdb_options(optarg);
1448 break;
1449 case 'k':
1450 parse_simple_config_file(optarg);
1451 break;
1452 case 'K':
1453 set_config_value("keyboard.layout", optarg);
1454 break;
1455 case 'l':
1456 if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1457 lpc_print_supported_devices();
1458 exit(0);
1459 } else if (lpc_device_parse(optarg) != 0) {
1460 errx(EX_USAGE, "invalid lpc device "
1461 "configuration '%s'", optarg);
1462 }
1463 break;
1464 case 's':
1465 if (strncmp(optarg, "help", strlen(optarg)) == 0) {
1466 pci_print_supported_devices();
1467 exit(0);
1468 } else if (pci_parse_slot(optarg) != 0)
1469 exit(4);
1470 else
1471 break;
1472 case 'S':
1473 set_config_bool("memory.wired", true);
1474 break;
1475 case 'm':
1476 set_config_value("memory.size", optarg);
1477 break;
1478 case 'o':
1479 if (!parse_config_option(optarg))
1480 errx(EX_USAGE, "invalid configuration option '%s'", optarg);
1481 break;
1482 case 'H':
1483 set_config_bool("x86.vmexit_on_hlt", true);
1484 break;
1485 case 'I':
1486 /*
1487 * The "-I" option was used to add an ioapic to the
1488 * virtual machine.
1489 *
1490 * An ioapic is now provided unconditionally for each
1491 * virtual machine and this option is now deprecated.
1492 */
1493 break;
1494 case 'P':
1495 set_config_bool("x86.vmexit_on_pause", true);
1496 break;
1497 case 'e':
1498 set_config_bool("x86.strictio", true);
1499 break;
1500 case 'u':
1501 set_config_bool("rtc.use_localtime", false);
1502 break;
1503 case 'U':
1504 set_config_value("uuid", optarg);
1505 break;
1506 case 'w':
1507 set_config_bool("x86.strictmsr", false);
1508 break;
1509 case 'W':
1510 set_config_bool("virtio_msix", false);
1511 break;
1512 case 'x':
1513 set_config_bool("x86.x2apic", true);
1514 break;
1515 case 'Y':
1516 set_config_bool("x86.mptable", false);
1517 break;
1518 case 'h':
1519 usage(0);
1520 default:
1521 usage(1);
1522 }
1523 }
1524 argc -= optind;
1525 argv += optind;
1526
1527 if (argc > 1)
1528 usage(1);
1529
1530 if (argc == 1)
1531 set_config_value("name", argv[0]);
1532
1533 vmname = get_config_value("name");
1534 if (vmname == NULL)
1535 usage(1);
1536
1537 if (get_config_bool_default("config.dump", false)) {
1538 dump_config();
1539 exit(1);
1540 }
1541
1542 #ifndef __FreeBSD__
1543 illumos_priv_init();
1544 #endif
1545
1546 calc_topology();
1547
1548 #ifdef __FreeBSD__
1549 build_vcpumaps();
1550 #endif
1551
1552 value = get_config_value("memory.size");
1553 error = vm_parse_memsize(value, &memsize);
1554 if (error)
1555 errx(EX_USAGE, "invalid memsize '%s'", value);
1556
1557 ctx = do_open(vmname);
1558
1559 bsp = vm_vcpu_open(ctx, BSP);
1560 max_vcpus = num_vcpus_allowed(ctx, bsp);
1561 if (guest_ncpus > max_vcpus) {
1562 fprintf(stderr, "%d vCPUs requested but only %d available\n",
1563 guest_ncpus, max_vcpus);
1564 exit(4);
1565 }
1566
1567 fbsdrun_set_capabilities(bsp);
1568
1569 /* Allocate per-VCPU resources. */
1570 vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info));
1571 for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) {
1572 vcpu_info[vcpuid].ctx = ctx;
1573 vcpu_info[vcpuid].vcpuid = vcpuid;
1574 if (vcpuid == BSP)
1575 vcpu_info[vcpuid].vcpu = bsp;
1576 else
1577 vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid);
1578 }
1579
1580 memflags = 0;
1581 if (get_config_bool_default("memory.wired", false))
1582 memflags |= VM_MEM_F_WIRED;
1583 if (get_config_bool_default("memory.guest_in_core", false))
1584 memflags |= VM_MEM_F_INCORE;
1585 vm_set_memflags(ctx, memflags);
1586 #ifdef __FreeBSD__
1587 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1588 #else
1589 int _errno;
1590 do {
1591 errno = 0;
1592 error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
1593 _errno = errno;
1594 if (error != 0 && _errno == ENOMEM) {
1595 (void) fprintf(stderr, "Unable to allocate memory "
1596 "(%llu), retrying in 1 second\n", memsize);
1597 sleep(1);
1598 }
1599 } while (_errno == ENOMEM);
1600 #endif
1601 if (error) {
1602 fprintf(stderr, "Unable to set up memory (%d)\n", errno);
1603 exit(4);
1604 }
1605
1606 error = init_msr();
1607 if (error) {
1608 fprintf(stderr, "init_msr error %d", error);
1609 exit(4);
1610 }
1611
1612 init_mem(guest_ncpus);
1613 init_inout();
1614 #ifdef __FreeBSD__
1615 kernemu_dev_init();
1616 #endif
1617 init_bootrom(ctx);
1618 atkbdc_init(ctx);
1619 pci_irq_init(ctx);
1620 ioapic_init(ctx);
1621
1622 rtc_init(ctx);
1623 sci_init(ctx);
1624 #ifndef __FreeBSD__
1625 pmtmr_init(ctx);
1626 #endif
1627
1628 if (qemu_fwcfg_init(ctx) != 0) {
1629 fprintf(stderr, "qemu fwcfg initialization error");
1630 exit(4);
1631 }
1632
1633 if (qemu_fwcfg_add_file("opt/bhyve/hw.ncpu", sizeof(guest_ncpus),
1634 &guest_ncpus) != 0) {
1635 fprintf(stderr, "Could not add qemu fwcfg opt/bhyve/hw.ncpu");
1636 exit(4);
1637 }
1638
1639 if (e820_init(ctx) != 0) {
1640 fprintf(stderr, "Unable to setup E820");
1641 exit(4);
1642 }
1643
1644 #ifndef __FreeBSD__
1645 if (get_config_bool_default("e820.debug", false))
1646 e820_dump_table();
1647 #endif
1648
1649 /*
1650 * Exit if a device emulation finds an error in its initialization
1651 */
1652 if (init_pci(ctx) != 0) {
1653 perror("device emulation initialization error");
1654 exit(4);
1655 }
1656 if (init_tpm(ctx) != 0) {
1657 fprintf(stderr, "Failed to init TPM device");
1658 exit(4);
1659 }
1660
1661 /*
1662 * Initialize after PCI, to allow a bootrom file to reserve the high
1663 * region.
1664 */
1665 if (get_config_bool("acpi_tables"))
1666 vmgenc_init(ctx);
1667
1668 #ifdef __FreeBSD__
1669 init_gdb(ctx);
1670 #else
1671 if (value != NULL) {
1672 int port = atoi(value);
1673
1674 if (port < 0)
1675 init_mdb(ctx);
1676 else
1677 init_gdb(ctx);
1678 }
1679 #endif
1680
1681 if (lpc_bootrom()) {
1682 #ifdef __FreeBSD__
1683 if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) {
1684 fprintf(stderr, "ROM boot failed: unrestricted guest "
1685 "capability not available\n");
1686 exit(4);
1687 }
1688 #else
1689 /* Unrestricted Guest is always enabled on illumos */
1690 #endif
1691 error = vcpu_reset(bsp);
1692 assert(error == 0);
1693 }
1694
1695 /*
1696 * build the guest tables, MP etc.
1697 */
1698 if (get_config_bool_default("x86.mptable", true)) {
1699 error = mptable_build(ctx, guest_ncpus);
1700 if (error) {
1701 perror("error to build the guest tables");
1702 exit(4);
1703 }
1704 }
1705
1706 error = smbios_build(ctx);
1707 if (error != 0)
1708 exit(4);
1709
1710 if (get_config_bool("acpi_tables")) {
1711 error = acpi_build(ctx, guest_ncpus);
1712 assert(error == 0);
1713 }
1714
1715 e820_fwcfg_item = e820_get_fwcfg_item();
1716 if (e820_fwcfg_item == NULL) {
1717 fprintf(stderr, "invalid e820 table");
1718 exit(4);
1719 }
1720 if (qemu_fwcfg_add_file("etc/e820", e820_fwcfg_item->size,
1721 e820_fwcfg_item->data) != 0) {
1722 fprintf(stderr, "could not add qemu fwcfg etc/e820");
1723 exit(4);
1724 }
1725 free(e820_fwcfg_item);
1726
1727 if (lpc_bootrom() && strcmp(lpc_fwcfg(), "bhyve") == 0) {
1728 fwctl_init();
1729 }
1730
1731 /*
1732 * Change the proc title to include the VM name.
1733 */
1734 setproctitle("%s", vmname);
1735
1736 #ifndef WITHOUT_CAPSICUM
1737 caph_cache_catpages();
1738
1739 if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1)
1740 errx(EX_OSERR, "Unable to apply rights for sandbox");
1741
1742 if (caph_enter() == -1)
1743 errx(EX_OSERR, "cap_enter() failed");
1744 #endif
1745
1746 #ifndef __FreeBSD__
1747 illumos_priv_lock();
1748 #endif
1749
1750 #ifndef __FreeBSD__
1751 vmentry = calloc(guest_ncpus, sizeof(*vmentry));
1752 #endif
1753
1754 /*
1755 * Add all vCPUs.
1756 */
1757 for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) {
1758 #ifdef __FreeBSD__
1759 bool suspend = (vcpuid != BSP);
1760 #else
1761 bool suspend = vcpuid == BSP &&
1762 get_config_bool_default("suspend_at_boot", false);
1763 #endif
1764 spinup_vcpu(&vcpu_info[vcpuid], vcpuid == BSP, suspend);
1765 }
1766
1767 /*
1768 * Head off to the main event dispatch loop
1769 */
1770 mevent_dispatch();
1771
1772 exit(4);
1773 }
1774