1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 /* 31 * This file and its contents are supplied under the terms of the 32 * Common Development and Distribution License ("CDDL"), version 1.0. 33 * You may only use this file in accordance with the terms of version 34 * 1.0 of the CDDL. 35 * 36 * A full copy of the text of the CDDL should have accompanied this 37 * source. A copy of the CDDL is also available via the Internet at 38 * http://www.illumos.org/license/CDDL. 39 * 40 * Copyright 2015 Pluribus Networks Inc. 41 * Copyright 2019 Joyent, Inc. 42 * Copyright 2021 Oxide Computer Company 43 */ 44 45 #include <sys/cdefs.h> 46 __FBSDID("$FreeBSD$"); 47 48 #include <sys/param.h> 49 #include <sys/sysctl.h> 50 #include <sys/ioctl.h> 51 #ifdef __FreeBSD__ 52 #include <sys/linker.h> 53 #endif 54 #include <sys/mman.h> 55 #include <sys/module.h> 56 #include <sys/_iovec.h> 57 #include <sys/cpuset.h> 58 59 #include <x86/segments.h> 60 #include <machine/specialreg.h> 61 62 #include <errno.h> 63 #include <stdio.h> 64 #include <stdlib.h> 65 #include <assert.h> 66 #include <string.h> 67 #include <fcntl.h> 68 #include <unistd.h> 69 70 #include <libutil.h> 71 72 #include <machine/vmm.h> 73 #include <machine/vmm_dev.h> 74 75 #include "vmmapi.h" 76 77 #define MB (1024 * 1024UL) 78 #define GB (1024 * 1024 * 1024UL) 79 80 #ifndef __FreeBSD__ 81 /* shim to no-op for now */ 82 #define MAP_NOCORE 0 83 #define MAP_ALIGNED_SUPER 0 84 85 /* Rely on PROT_NONE for guard purposes */ 86 #define MAP_GUARD (MAP_PRIVATE | MAP_ANON | MAP_NORESERVE) 87 #endif 88 89 /* 90 * Size of the guard region before and after the virtual address space 91 * mapping the guest physical memory. This must be a multiple of the 92 * superpage size for performance reasons. 93 */ 94 #define VM_MMAP_GUARD_SIZE (4 * MB) 95 96 #define PROT_RW (PROT_READ | PROT_WRITE) 97 #define PROT_ALL (PROT_READ | PROT_WRITE | PROT_EXEC) 98 99 struct vmctx { 100 int fd; 101 uint32_t lowmem_limit; 102 int memflags; 103 size_t lowmem; 104 size_t highmem; 105 char *baseaddr; 106 char *name; 107 }; 108 109 #ifdef __FreeBSD__ 110 #define CREATE(x) sysctlbyname("hw.vmm.create", NULL, NULL, (x), strlen((x))) 111 #define DESTROY(x) sysctlbyname("hw.vmm.destroy", NULL, NULL, (x), strlen((x))) 112 113 int 114 vm_create(const char *name) 115 { 116 /* Try to load vmm(4) module before creating a guest. */ 117 if (modfind("vmm") < 0) 118 kldload("vmm"); 119 return (CREATE((char *)name)); 120 } 121 122 void 123 vm_destroy(struct vmctx *vm) 124 { 125 assert(vm != NULL); 126 127 if (vm->fd >= 0) 128 close(vm->fd); 129 DESTROY(vm->name); 130 131 free(vm); 132 } 133 134 #else 135 static int 136 vm_do_ctl(int cmd, void *req) 137 { 138 int ctl_fd; 139 140 ctl_fd = open(VMM_CTL_DEV, O_EXCL | O_RDWR); 141 if (ctl_fd < 0) { 142 return (-1); 143 } 144 145 if (ioctl(ctl_fd, cmd, req) == -1) { 146 int err = errno; 147 148 /* Do not lose ioctl errno through the close(2) */ 149 (void) close(ctl_fd); 150 errno = err; 151 return (-1); 152 } 153 (void) close(ctl_fd); 154 155 return (0); 156 } 157 158 int 159 vm_create(const char *name, uint64_t flags) 160 { 161 struct vm_create_req req; 162 163 (void) strncpy(req.name, name, VM_MAX_NAMELEN); 164 req.flags = flags; 165 166 return (vm_do_ctl(VMM_CREATE_VM, &req)); 167 } 168 169 void 170 vm_close(struct vmctx *vm) 171 { 172 assert(vm != NULL); 173 assert(vm->fd >= 0); 174 175 (void) close(vm->fd); 176 177 free(vm); 178 } 179 180 void 181 vm_destroy(struct vmctx *vm) 182 { 183 struct vm_destroy_req req; 184 185 assert(vm != NULL); 186 187 if (vm->fd >= 0) { 188 (void) close(vm->fd); 189 vm->fd = -1; 190 } 191 192 (void) strncpy(req.name, vm->name, VM_MAX_NAMELEN); 193 (void) vm_do_ctl(VMM_DESTROY_VM, &req); 194 195 free(vm); 196 } 197 #endif 198 199 static int 200 vm_device_open(const char *name) 201 { 202 int fd, len; 203 char *vmfile; 204 205 len = strlen("/dev/vmm/") + strlen(name) + 1; 206 vmfile = malloc(len); 207 assert(vmfile != NULL); 208 snprintf(vmfile, len, "/dev/vmm/%s", name); 209 210 /* Open the device file */ 211 fd = open(vmfile, O_RDWR, 0); 212 213 free(vmfile); 214 return (fd); 215 } 216 217 struct vmctx * 218 vm_open(const char *name) 219 { 220 struct vmctx *vm; 221 222 vm = malloc(sizeof(struct vmctx) + strlen(name) + 1); 223 assert(vm != NULL); 224 225 vm->fd = -1; 226 vm->memflags = 0; 227 vm->lowmem_limit = 3 * GB; 228 vm->name = (char *)(vm + 1); 229 strcpy(vm->name, name); 230 231 if ((vm->fd = vm_device_open(vm->name)) < 0) 232 goto err; 233 234 return (vm); 235 err: 236 free(vm); 237 return (NULL); 238 } 239 240 241 int 242 vm_parse_memsize(const char *optarg, size_t *ret_memsize) 243 { 244 char *endptr; 245 size_t optval; 246 int error; 247 248 optval = strtoul(optarg, &endptr, 0); 249 if (*optarg != '\0' && *endptr == '\0') { 250 /* 251 * For the sake of backward compatibility if the memory size 252 * specified on the command line is less than a megabyte then 253 * it is interpreted as being in units of MB. 254 */ 255 if (optval < MB) 256 optval *= MB; 257 *ret_memsize = optval; 258 error = 0; 259 } else 260 error = expand_number(optarg, ret_memsize); 261 262 return (error); 263 } 264 265 uint32_t 266 vm_get_lowmem_limit(struct vmctx *ctx) 267 { 268 269 return (ctx->lowmem_limit); 270 } 271 272 void 273 vm_set_lowmem_limit(struct vmctx *ctx, uint32_t limit) 274 { 275 276 ctx->lowmem_limit = limit; 277 } 278 279 void 280 vm_set_memflags(struct vmctx *ctx, int flags) 281 { 282 283 ctx->memflags = flags; 284 } 285 286 int 287 vm_get_memflags(struct vmctx *ctx) 288 { 289 290 return (ctx->memflags); 291 } 292 293 /* 294 * Map segment 'segid' starting at 'off' into guest address range [gpa,gpa+len). 295 */ 296 int 297 vm_mmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, int segid, vm_ooffset_t off, 298 size_t len, int prot) 299 { 300 struct vm_memmap memmap; 301 int error, flags; 302 303 memmap.gpa = gpa; 304 memmap.segid = segid; 305 memmap.segoff = off; 306 memmap.len = len; 307 memmap.prot = prot; 308 memmap.flags = 0; 309 310 if (ctx->memflags & VM_MEM_F_WIRED) 311 memmap.flags |= VM_MEMMAP_F_WIRED; 312 313 /* 314 * If this mapping already exists then don't create it again. This 315 * is the common case for SYSMEM mappings created by bhyveload(8). 316 */ 317 error = vm_mmap_getnext(ctx, &gpa, &segid, &off, &len, &prot, &flags); 318 if (error == 0 && gpa == memmap.gpa) { 319 if (segid != memmap.segid || off != memmap.segoff || 320 prot != memmap.prot || flags != memmap.flags) { 321 errno = EEXIST; 322 return (-1); 323 } else { 324 return (0); 325 } 326 } 327 328 error = ioctl(ctx->fd, VM_MMAP_MEMSEG, &memmap); 329 return (error); 330 } 331 332 int 333 vm_munmap_memseg(struct vmctx *ctx, vm_paddr_t gpa, size_t len) 334 { 335 struct vm_munmap munmap; 336 int error; 337 338 munmap.gpa = gpa; 339 munmap.len = len; 340 341 error = ioctl(ctx->fd, VM_MUNMAP_MEMSEG, &munmap); 342 return (error); 343 } 344 345 int 346 vm_mmap_getnext(struct vmctx *ctx, vm_paddr_t *gpa, int *segid, 347 vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) 348 { 349 struct vm_memmap memmap; 350 int error; 351 352 bzero(&memmap, sizeof(struct vm_memmap)); 353 memmap.gpa = *gpa; 354 error = ioctl(ctx->fd, VM_MMAP_GETNEXT, &memmap); 355 if (error == 0) { 356 *gpa = memmap.gpa; 357 *segid = memmap.segid; 358 *segoff = memmap.segoff; 359 *len = memmap.len; 360 *prot = memmap.prot; 361 *flags = memmap.flags; 362 } 363 return (error); 364 } 365 366 /* 367 * Return 0 if the segments are identical and non-zero otherwise. 368 * 369 * This is slightly complicated by the fact that only device memory segments 370 * are named. 371 */ 372 static int 373 cmpseg(size_t len, const char *str, size_t len2, const char *str2) 374 { 375 376 if (len == len2) { 377 if ((!str && !str2) || (str && str2 && !strcmp(str, str2))) 378 return (0); 379 } 380 return (-1); 381 } 382 383 static int 384 vm_alloc_memseg(struct vmctx *ctx, int segid, size_t len, const char *name) 385 { 386 struct vm_memseg memseg; 387 size_t n; 388 int error; 389 390 /* 391 * If the memory segment has already been created then just return. 392 * This is the usual case for the SYSMEM segment created by userspace 393 * loaders like bhyveload(8). 394 */ 395 error = vm_get_memseg(ctx, segid, &memseg.len, memseg.name, 396 sizeof(memseg.name)); 397 if (error) 398 return (error); 399 400 if (memseg.len != 0) { 401 if (cmpseg(len, name, memseg.len, VM_MEMSEG_NAME(&memseg))) { 402 errno = EINVAL; 403 return (-1); 404 } else { 405 return (0); 406 } 407 } 408 409 bzero(&memseg, sizeof(struct vm_memseg)); 410 memseg.segid = segid; 411 memseg.len = len; 412 if (name != NULL) { 413 n = strlcpy(memseg.name, name, sizeof(memseg.name)); 414 if (n >= sizeof(memseg.name)) { 415 errno = ENAMETOOLONG; 416 return (-1); 417 } 418 } 419 420 error = ioctl(ctx->fd, VM_ALLOC_MEMSEG, &memseg); 421 return (error); 422 } 423 424 int 425 vm_get_memseg(struct vmctx *ctx, int segid, size_t *lenp, char *namebuf, 426 size_t bufsize) 427 { 428 struct vm_memseg memseg; 429 size_t n; 430 int error; 431 432 memseg.segid = segid; 433 error = ioctl(ctx->fd, VM_GET_MEMSEG, &memseg); 434 if (error == 0) { 435 *lenp = memseg.len; 436 n = strlcpy(namebuf, memseg.name, bufsize); 437 if (n >= bufsize) { 438 errno = ENAMETOOLONG; 439 error = -1; 440 } 441 } 442 return (error); 443 } 444 445 static int 446 #ifdef __FreeBSD__ 447 setup_memory_segment(struct vmctx *ctx, vm_paddr_t gpa, size_t len, char *base) 448 #else 449 setup_memory_segment(struct vmctx *ctx, int segid, vm_paddr_t gpa, size_t len, 450 char *base) 451 #endif 452 { 453 char *ptr; 454 int error, flags; 455 456 /* Map 'len' bytes starting at 'gpa' in the guest address space */ 457 #ifdef __FreeBSD__ 458 error = vm_mmap_memseg(ctx, gpa, VM_SYSMEM, gpa, len, PROT_ALL); 459 #else 460 /* 461 * As we use two segments for lowmem/highmem the offset within the 462 * segment is 0 on illumos. 463 */ 464 error = vm_mmap_memseg(ctx, gpa, segid, 0, len, PROT_ALL); 465 #endif 466 if (error) 467 return (error); 468 469 flags = MAP_SHARED | MAP_FIXED; 470 if ((ctx->memflags & VM_MEM_F_INCORE) == 0) 471 flags |= MAP_NOCORE; 472 473 /* mmap into the process address space on the host */ 474 ptr = mmap(base + gpa, len, PROT_RW, flags, ctx->fd, gpa); 475 if (ptr == MAP_FAILED) 476 return (-1); 477 478 return (0); 479 } 480 481 int 482 vm_setup_memory(struct vmctx *ctx, size_t memsize, enum vm_mmap_style vms) 483 { 484 size_t objsize, len; 485 vm_paddr_t gpa; 486 char *baseaddr, *ptr; 487 int error; 488 489 assert(vms == VM_MMAP_ALL); 490 491 /* 492 * If 'memsize' cannot fit entirely in the 'lowmem' segment then 493 * create another 'highmem' segment above 4GB for the remainder. 494 */ 495 if (memsize > ctx->lowmem_limit) { 496 ctx->lowmem = ctx->lowmem_limit; 497 ctx->highmem = memsize - ctx->lowmem_limit; 498 objsize = 4*GB + ctx->highmem; 499 } else { 500 ctx->lowmem = memsize; 501 ctx->highmem = 0; 502 objsize = ctx->lowmem; 503 } 504 505 #ifdef __FreeBSD__ 506 error = vm_alloc_memseg(ctx, VM_SYSMEM, objsize, NULL); 507 if (error) 508 return (error); 509 #endif 510 511 /* 512 * Stake out a contiguous region covering the guest physical memory 513 * and the adjoining guard regions. 514 */ 515 len = VM_MMAP_GUARD_SIZE + objsize + VM_MMAP_GUARD_SIZE; 516 ptr = mmap(NULL, len, PROT_NONE, MAP_GUARD | MAP_ALIGNED_SUPER, -1, 0); 517 if (ptr == MAP_FAILED) 518 return (-1); 519 520 baseaddr = ptr + VM_MMAP_GUARD_SIZE; 521 522 #ifdef __FreeBSD__ 523 if (ctx->highmem > 0) { 524 gpa = 4*GB; 525 len = ctx->highmem; 526 error = setup_memory_segment(ctx, gpa, len, baseaddr); 527 if (error) 528 return (error); 529 } 530 531 if (ctx->lowmem > 0) { 532 gpa = 0; 533 len = ctx->lowmem; 534 error = setup_memory_segment(ctx, gpa, len, baseaddr); 535 if (error) 536 return (error); 537 } 538 #else 539 if (ctx->highmem > 0) { 540 error = vm_alloc_memseg(ctx, VM_HIGHMEM, ctx->highmem, NULL); 541 if (error) 542 return (error); 543 gpa = 4*GB; 544 len = ctx->highmem; 545 error = setup_memory_segment(ctx, VM_HIGHMEM, gpa, len, baseaddr); 546 if (error) 547 return (error); 548 } 549 550 if (ctx->lowmem > 0) { 551 error = vm_alloc_memseg(ctx, VM_LOWMEM, ctx->lowmem, NULL); 552 if (error) 553 return (error); 554 gpa = 0; 555 len = ctx->lowmem; 556 error = setup_memory_segment(ctx, VM_LOWMEM, gpa, len, baseaddr); 557 if (error) 558 return (error); 559 } 560 #endif 561 562 ctx->baseaddr = baseaddr; 563 564 return (0); 565 } 566 567 /* 568 * Returns a non-NULL pointer if [gaddr, gaddr+len) is entirely contained in 569 * the lowmem or highmem regions. 570 * 571 * In particular return NULL if [gaddr, gaddr+len) falls in guest MMIO region. 572 * The instruction emulation code depends on this behavior. 573 */ 574 void * 575 vm_map_gpa(struct vmctx *ctx, vm_paddr_t gaddr, size_t len) 576 { 577 578 if (ctx->lowmem > 0) { 579 if (gaddr < ctx->lowmem && len <= ctx->lowmem && 580 gaddr + len <= ctx->lowmem) 581 return (ctx->baseaddr + gaddr); 582 } 583 584 if (ctx->highmem > 0) { 585 if (gaddr >= 4*GB) { 586 if (gaddr < 4*GB + ctx->highmem && 587 len <= ctx->highmem && 588 gaddr + len <= 4*GB + ctx->highmem) 589 return (ctx->baseaddr + gaddr); 590 } 591 } 592 593 return (NULL); 594 } 595 596 size_t 597 vm_get_lowmem_size(struct vmctx *ctx) 598 { 599 600 return (ctx->lowmem); 601 } 602 603 size_t 604 vm_get_highmem_size(struct vmctx *ctx) 605 { 606 607 return (ctx->highmem); 608 } 609 610 #ifndef __FreeBSD__ 611 int 612 vm_get_devmem_offset(struct vmctx *ctx, int segid, off_t *mapoff) 613 { 614 struct vm_devmem_offset vdo; 615 int error; 616 617 vdo.segid = segid; 618 error = ioctl(ctx->fd, VM_DEVMEM_GETOFFSET, &vdo); 619 if (error == 0) 620 *mapoff = vdo.offset; 621 622 return (error); 623 } 624 #endif 625 626 void * 627 vm_create_devmem(struct vmctx *ctx, int segid, const char *name, size_t len) 628 { 629 #ifdef __FreeBSD__ 630 char pathname[MAXPATHLEN]; 631 #endif 632 size_t len2; 633 char *base, *ptr; 634 int fd, error, flags; 635 off_t mapoff; 636 637 fd = -1; 638 ptr = MAP_FAILED; 639 if (name == NULL || strlen(name) == 0) { 640 errno = EINVAL; 641 goto done; 642 } 643 644 error = vm_alloc_memseg(ctx, segid, len, name); 645 if (error) 646 goto done; 647 648 #ifdef __FreeBSD__ 649 strlcpy(pathname, "/dev/vmm.io/", sizeof(pathname)); 650 strlcat(pathname, ctx->name, sizeof(pathname)); 651 strlcat(pathname, ".", sizeof(pathname)); 652 strlcat(pathname, name, sizeof(pathname)); 653 654 fd = open(pathname, O_RDWR); 655 if (fd < 0) 656 goto done; 657 #else 658 if (vm_get_devmem_offset(ctx, segid, &mapoff) != 0) 659 goto done; 660 #endif 661 662 /* 663 * Stake out a contiguous region covering the device memory and the 664 * adjoining guard regions. 665 */ 666 len2 = VM_MMAP_GUARD_SIZE + len + VM_MMAP_GUARD_SIZE; 667 base = mmap(NULL, len2, PROT_NONE, MAP_GUARD | MAP_ALIGNED_SUPER, -1, 668 0); 669 if (base == MAP_FAILED) 670 goto done; 671 672 flags = MAP_SHARED | MAP_FIXED; 673 if ((ctx->memflags & VM_MEM_F_INCORE) == 0) 674 flags |= MAP_NOCORE; 675 676 #ifdef __FreeBSD__ 677 /* mmap the devmem region in the host address space */ 678 ptr = mmap(base + VM_MMAP_GUARD_SIZE, len, PROT_RW, flags, fd, 0); 679 #else 680 /* mmap the devmem region in the host address space */ 681 ptr = mmap(base + VM_MMAP_GUARD_SIZE, len, PROT_RW, flags, ctx->fd, 682 mapoff); 683 #endif 684 done: 685 if (fd >= 0) 686 close(fd); 687 return (ptr); 688 } 689 690 int 691 vm_set_desc(struct vmctx *ctx, int vcpu, int reg, 692 uint64_t base, uint32_t limit, uint32_t access) 693 { 694 int error; 695 struct vm_seg_desc vmsegdesc; 696 697 bzero(&vmsegdesc, sizeof(vmsegdesc)); 698 vmsegdesc.cpuid = vcpu; 699 vmsegdesc.regnum = reg; 700 vmsegdesc.desc.base = base; 701 vmsegdesc.desc.limit = limit; 702 vmsegdesc.desc.access = access; 703 704 error = ioctl(ctx->fd, VM_SET_SEGMENT_DESCRIPTOR, &vmsegdesc); 705 return (error); 706 } 707 708 int 709 vm_get_desc(struct vmctx *ctx, int vcpu, int reg, 710 uint64_t *base, uint32_t *limit, uint32_t *access) 711 { 712 int error; 713 struct vm_seg_desc vmsegdesc; 714 715 bzero(&vmsegdesc, sizeof(vmsegdesc)); 716 vmsegdesc.cpuid = vcpu; 717 vmsegdesc.regnum = reg; 718 719 error = ioctl(ctx->fd, VM_GET_SEGMENT_DESCRIPTOR, &vmsegdesc); 720 if (error == 0) { 721 *base = vmsegdesc.desc.base; 722 *limit = vmsegdesc.desc.limit; 723 *access = vmsegdesc.desc.access; 724 } 725 return (error); 726 } 727 728 int 729 vm_get_seg_desc(struct vmctx *ctx, int vcpu, int reg, struct seg_desc *seg_desc) 730 { 731 int error; 732 733 error = vm_get_desc(ctx, vcpu, reg, &seg_desc->base, &seg_desc->limit, 734 &seg_desc->access); 735 return (error); 736 } 737 738 int 739 vm_set_register(struct vmctx *ctx, int vcpu, int reg, uint64_t val) 740 { 741 int error; 742 struct vm_register vmreg; 743 744 bzero(&vmreg, sizeof(vmreg)); 745 vmreg.cpuid = vcpu; 746 vmreg.regnum = reg; 747 vmreg.regval = val; 748 749 error = ioctl(ctx->fd, VM_SET_REGISTER, &vmreg); 750 return (error); 751 } 752 753 int 754 vm_get_register(struct vmctx *ctx, int vcpu, int reg, uint64_t *ret_val) 755 { 756 int error; 757 struct vm_register vmreg; 758 759 bzero(&vmreg, sizeof(vmreg)); 760 vmreg.cpuid = vcpu; 761 vmreg.regnum = reg; 762 763 error = ioctl(ctx->fd, VM_GET_REGISTER, &vmreg); 764 *ret_val = vmreg.regval; 765 return (error); 766 } 767 768 int 769 vm_set_register_set(struct vmctx *ctx, int vcpu, unsigned int count, 770 const int *regnums, uint64_t *regvals) 771 { 772 int error; 773 struct vm_register_set vmregset; 774 775 bzero(&vmregset, sizeof(vmregset)); 776 vmregset.cpuid = vcpu; 777 vmregset.count = count; 778 vmregset.regnums = regnums; 779 vmregset.regvals = regvals; 780 781 error = ioctl(ctx->fd, VM_SET_REGISTER_SET, &vmregset); 782 return (error); 783 } 784 785 int 786 vm_get_register_set(struct vmctx *ctx, int vcpu, unsigned int count, 787 const int *regnums, uint64_t *regvals) 788 { 789 int error; 790 struct vm_register_set vmregset; 791 792 bzero(&vmregset, sizeof(vmregset)); 793 vmregset.cpuid = vcpu; 794 vmregset.count = count; 795 vmregset.regnums = regnums; 796 vmregset.regvals = regvals; 797 798 error = ioctl(ctx->fd, VM_GET_REGISTER_SET, &vmregset); 799 return (error); 800 } 801 802 int 803 vm_run(struct vmctx *ctx, int vcpu, const struct vm_entry *vm_entry, 804 struct vm_exit *vm_exit) 805 { 806 struct vm_entry entry; 807 808 bcopy(vm_entry, &entry, sizeof (entry)); 809 entry.cpuid = vcpu; 810 entry.exit_data = vm_exit; 811 812 return (ioctl(ctx->fd, VM_RUN, &entry)); 813 } 814 815 int 816 vm_suspend(struct vmctx *ctx, enum vm_suspend_how how) 817 { 818 struct vm_suspend vmsuspend; 819 820 bzero(&vmsuspend, sizeof(vmsuspend)); 821 vmsuspend.how = how; 822 return (ioctl(ctx->fd, VM_SUSPEND, &vmsuspend)); 823 } 824 825 #ifndef __FreeBSD__ 826 int 827 vm_reinit(struct vmctx *ctx, uint64_t flags) 828 { 829 struct vm_reinit reinit = { 830 .flags = flags 831 }; 832 833 return (ioctl(ctx->fd, VM_REINIT, &reinit)); 834 } 835 #else 836 int 837 vm_reinit(struct vmctx *ctx) 838 { 839 840 return (ioctl(ctx->fd, VM_REINIT, 0)); 841 } 842 #endif 843 844 int 845 vm_inject_exception(struct vmctx *ctx, int vcpu, int vector, int errcode_valid, 846 uint32_t errcode, int restart_instruction) 847 { 848 struct vm_exception exc; 849 850 exc.cpuid = vcpu; 851 exc.vector = vector; 852 exc.error_code = errcode; 853 exc.error_code_valid = errcode_valid; 854 exc.restart_instruction = restart_instruction; 855 856 return (ioctl(ctx->fd, VM_INJECT_EXCEPTION, &exc)); 857 } 858 859 #ifndef __FreeBSD__ 860 void 861 vm_inject_fault(struct vmctx *ctx, int vcpu, int vector, int errcode_valid, 862 int errcode) 863 { 864 int error; 865 struct vm_exception exc; 866 867 exc.cpuid = vcpu; 868 exc.vector = vector; 869 exc.error_code = errcode; 870 exc.error_code_valid = errcode_valid; 871 exc.restart_instruction = 1; 872 error = ioctl(ctx->fd, VM_INJECT_EXCEPTION, &exc); 873 874 assert(error == 0); 875 } 876 #endif /* __FreeBSD__ */ 877 878 int 879 vm_apicid2vcpu(struct vmctx *ctx, int apicid) 880 { 881 /* 882 * The apic id associated with the 'vcpu' has the same numerical value 883 * as the 'vcpu' itself. 884 */ 885 return (apicid); 886 } 887 888 int 889 vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector) 890 { 891 struct vm_lapic_irq vmirq; 892 893 bzero(&vmirq, sizeof(vmirq)); 894 vmirq.cpuid = vcpu; 895 vmirq.vector = vector; 896 897 return (ioctl(ctx->fd, VM_LAPIC_IRQ, &vmirq)); 898 } 899 900 int 901 vm_lapic_local_irq(struct vmctx *ctx, int vcpu, int vector) 902 { 903 struct vm_lapic_irq vmirq; 904 905 bzero(&vmirq, sizeof(vmirq)); 906 vmirq.cpuid = vcpu; 907 vmirq.vector = vector; 908 909 return (ioctl(ctx->fd, VM_LAPIC_LOCAL_IRQ, &vmirq)); 910 } 911 912 int 913 vm_lapic_msi(struct vmctx *ctx, uint64_t addr, uint64_t msg) 914 { 915 struct vm_lapic_msi vmmsi; 916 917 bzero(&vmmsi, sizeof(vmmsi)); 918 vmmsi.addr = addr; 919 vmmsi.msg = msg; 920 921 return (ioctl(ctx->fd, VM_LAPIC_MSI, &vmmsi)); 922 } 923 924 int 925 vm_ioapic_assert_irq(struct vmctx *ctx, int irq) 926 { 927 struct vm_ioapic_irq ioapic_irq; 928 929 bzero(&ioapic_irq, sizeof(struct vm_ioapic_irq)); 930 ioapic_irq.irq = irq; 931 932 return (ioctl(ctx->fd, VM_IOAPIC_ASSERT_IRQ, &ioapic_irq)); 933 } 934 935 int 936 vm_ioapic_deassert_irq(struct vmctx *ctx, int irq) 937 { 938 struct vm_ioapic_irq ioapic_irq; 939 940 bzero(&ioapic_irq, sizeof(struct vm_ioapic_irq)); 941 ioapic_irq.irq = irq; 942 943 return (ioctl(ctx->fd, VM_IOAPIC_DEASSERT_IRQ, &ioapic_irq)); 944 } 945 946 int 947 vm_ioapic_pulse_irq(struct vmctx *ctx, int irq) 948 { 949 struct vm_ioapic_irq ioapic_irq; 950 951 bzero(&ioapic_irq, sizeof(struct vm_ioapic_irq)); 952 ioapic_irq.irq = irq; 953 954 return (ioctl(ctx->fd, VM_IOAPIC_PULSE_IRQ, &ioapic_irq)); 955 } 956 957 int 958 vm_ioapic_pincount(struct vmctx *ctx, int *pincount) 959 { 960 961 return (ioctl(ctx->fd, VM_IOAPIC_PINCOUNT, pincount)); 962 } 963 964 int 965 vm_readwrite_kernemu_device(struct vmctx *ctx, int vcpu, vm_paddr_t gpa, 966 bool write, int size, uint64_t *value) 967 { 968 struct vm_readwrite_kernemu_device irp = { 969 .vcpuid = vcpu, 970 .access_width = fls(size) - 1, 971 .gpa = gpa, 972 .value = write ? *value : ~0ul, 973 }; 974 long cmd = (write ? VM_SET_KERNEMU_DEV : VM_GET_KERNEMU_DEV); 975 int rc; 976 977 rc = ioctl(ctx->fd, cmd, &irp); 978 if (rc == 0 && !write) 979 *value = irp.value; 980 return (rc); 981 } 982 983 int 984 vm_isa_assert_irq(struct vmctx *ctx, int atpic_irq, int ioapic_irq) 985 { 986 struct vm_isa_irq isa_irq; 987 988 bzero(&isa_irq, sizeof(struct vm_isa_irq)); 989 isa_irq.atpic_irq = atpic_irq; 990 isa_irq.ioapic_irq = ioapic_irq; 991 992 return (ioctl(ctx->fd, VM_ISA_ASSERT_IRQ, &isa_irq)); 993 } 994 995 int 996 vm_isa_deassert_irq(struct vmctx *ctx, int atpic_irq, int ioapic_irq) 997 { 998 struct vm_isa_irq isa_irq; 999 1000 bzero(&isa_irq, sizeof(struct vm_isa_irq)); 1001 isa_irq.atpic_irq = atpic_irq; 1002 isa_irq.ioapic_irq = ioapic_irq; 1003 1004 return (ioctl(ctx->fd, VM_ISA_DEASSERT_IRQ, &isa_irq)); 1005 } 1006 1007 int 1008 vm_isa_pulse_irq(struct vmctx *ctx, int atpic_irq, int ioapic_irq) 1009 { 1010 struct vm_isa_irq isa_irq; 1011 1012 bzero(&isa_irq, sizeof(struct vm_isa_irq)); 1013 isa_irq.atpic_irq = atpic_irq; 1014 isa_irq.ioapic_irq = ioapic_irq; 1015 1016 return (ioctl(ctx->fd, VM_ISA_PULSE_IRQ, &isa_irq)); 1017 } 1018 1019 int 1020 vm_isa_set_irq_trigger(struct vmctx *ctx, int atpic_irq, 1021 enum vm_intr_trigger trigger) 1022 { 1023 struct vm_isa_irq_trigger isa_irq_trigger; 1024 1025 bzero(&isa_irq_trigger, sizeof(struct vm_isa_irq_trigger)); 1026 isa_irq_trigger.atpic_irq = atpic_irq; 1027 isa_irq_trigger.trigger = trigger; 1028 1029 return (ioctl(ctx->fd, VM_ISA_SET_IRQ_TRIGGER, &isa_irq_trigger)); 1030 } 1031 1032 int 1033 vm_inject_nmi(struct vmctx *ctx, int vcpu) 1034 { 1035 struct vm_nmi vmnmi; 1036 1037 bzero(&vmnmi, sizeof(vmnmi)); 1038 vmnmi.cpuid = vcpu; 1039 1040 return (ioctl(ctx->fd, VM_INJECT_NMI, &vmnmi)); 1041 } 1042 1043 static const char *capstrmap[] = { 1044 [VM_CAP_HALT_EXIT] = "hlt_exit", 1045 [VM_CAP_MTRAP_EXIT] = "mtrap_exit", 1046 [VM_CAP_PAUSE_EXIT] = "pause_exit", 1047 #ifdef __FreeBSD__ 1048 [VM_CAP_UNRESTRICTED_GUEST] = "unrestricted_guest", 1049 #endif 1050 [VM_CAP_ENABLE_INVPCID] = "enable_invpcid", 1051 [VM_CAP_BPT_EXIT] = "bpt_exit", 1052 }; 1053 1054 int 1055 vm_capability_name2type(const char *capname) 1056 { 1057 int i; 1058 1059 for (i = 0; i < nitems(capstrmap); i++) { 1060 if (strcmp(capstrmap[i], capname) == 0) 1061 return (i); 1062 } 1063 1064 return (-1); 1065 } 1066 1067 const char * 1068 vm_capability_type2name(int type) 1069 { 1070 if (type >= 0 && type < nitems(capstrmap)) 1071 return (capstrmap[type]); 1072 1073 return (NULL); 1074 } 1075 1076 int 1077 vm_get_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap, 1078 int *retval) 1079 { 1080 int error; 1081 struct vm_capability vmcap; 1082 1083 bzero(&vmcap, sizeof(vmcap)); 1084 vmcap.cpuid = vcpu; 1085 vmcap.captype = cap; 1086 1087 error = ioctl(ctx->fd, VM_GET_CAPABILITY, &vmcap); 1088 *retval = vmcap.capval; 1089 return (error); 1090 } 1091 1092 int 1093 vm_set_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap, int val) 1094 { 1095 struct vm_capability vmcap; 1096 1097 bzero(&vmcap, sizeof(vmcap)); 1098 vmcap.cpuid = vcpu; 1099 vmcap.captype = cap; 1100 vmcap.capval = val; 1101 1102 return (ioctl(ctx->fd, VM_SET_CAPABILITY, &vmcap)); 1103 } 1104 1105 #ifdef __FreeBSD__ 1106 int 1107 vm_assign_pptdev(struct vmctx *ctx, int bus, int slot, int func) 1108 { 1109 struct vm_pptdev pptdev; 1110 1111 bzero(&pptdev, sizeof(pptdev)); 1112 pptdev.bus = bus; 1113 pptdev.slot = slot; 1114 pptdev.func = func; 1115 1116 return (ioctl(ctx->fd, VM_BIND_PPTDEV, &pptdev)); 1117 } 1118 1119 int 1120 vm_unassign_pptdev(struct vmctx *ctx, int bus, int slot, int func) 1121 { 1122 struct vm_pptdev pptdev; 1123 1124 bzero(&pptdev, sizeof(pptdev)); 1125 pptdev.bus = bus; 1126 pptdev.slot = slot; 1127 pptdev.func = func; 1128 1129 return (ioctl(ctx->fd, VM_UNBIND_PPTDEV, &pptdev)); 1130 } 1131 1132 int 1133 vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func, 1134 vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 1135 { 1136 struct vm_pptdev_mmio pptmmio; 1137 1138 bzero(&pptmmio, sizeof(pptmmio)); 1139 pptmmio.bus = bus; 1140 pptmmio.slot = slot; 1141 pptmmio.func = func; 1142 pptmmio.gpa = gpa; 1143 pptmmio.len = len; 1144 pptmmio.hpa = hpa; 1145 1146 return (ioctl(ctx->fd, VM_MAP_PPTDEV_MMIO, &pptmmio)); 1147 } 1148 1149 int 1150 vm_unmap_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func, 1151 vm_paddr_t gpa, size_t len) 1152 { 1153 struct vm_pptdev_mmio pptmmio; 1154 1155 bzero(&pptmmio, sizeof(pptmmio)); 1156 pptmmio.bus = bus; 1157 pptmmio.slot = slot; 1158 pptmmio.func = func; 1159 pptmmio.gpa = gpa; 1160 pptmmio.len = len; 1161 1162 return (ioctl(ctx->fd, VM_UNMAP_PPTDEV_MMIO, &pptmmio)); 1163 } 1164 1165 int 1166 vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func, 1167 uint64_t addr, uint64_t msg, int numvec) 1168 { 1169 struct vm_pptdev_msi pptmsi; 1170 1171 bzero(&pptmsi, sizeof(pptmsi)); 1172 pptmsi.vcpu = vcpu; 1173 pptmsi.bus = bus; 1174 pptmsi.slot = slot; 1175 pptmsi.func = func; 1176 pptmsi.msg = msg; 1177 pptmsi.addr = addr; 1178 pptmsi.numvec = numvec; 1179 1180 return (ioctl(ctx->fd, VM_PPTDEV_MSI, &pptmsi)); 1181 } 1182 1183 int 1184 vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func, 1185 int idx, uint64_t addr, uint64_t msg, uint32_t vector_control) 1186 { 1187 struct vm_pptdev_msix pptmsix; 1188 1189 bzero(&pptmsix, sizeof(pptmsix)); 1190 pptmsix.vcpu = vcpu; 1191 pptmsix.bus = bus; 1192 pptmsix.slot = slot; 1193 pptmsix.func = func; 1194 pptmsix.idx = idx; 1195 pptmsix.msg = msg; 1196 pptmsix.addr = addr; 1197 pptmsix.vector_control = vector_control; 1198 1199 return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix); 1200 } 1201 1202 int 1203 vm_get_pptdev_limits(struct vmctx *ctx, int bus, int slot, int func, 1204 int *msi_limit, int *msix_limit) 1205 { 1206 struct vm_pptdev_limits pptlimits; 1207 int error; 1208 1209 bzero(&pptlimits, sizeof (pptlimits)); 1210 pptlimits.bus = bus; 1211 pptlimits.slot = slot; 1212 pptlimits.func = func; 1213 1214 error = ioctl(ctx->fd, VM_GET_PPTDEV_LIMITS, &pptlimits); 1215 1216 *msi_limit = pptlimits.msi_limit; 1217 *msix_limit = pptlimits.msix_limit; 1218 1219 return (error); 1220 } 1221 1222 int 1223 vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func) 1224 { 1225 struct vm_pptdev ppt; 1226 1227 bzero(&ppt, sizeof(ppt)); 1228 ppt.bus = bus; 1229 ppt.slot = slot; 1230 ppt.func = func; 1231 1232 return ioctl(ctx->fd, VM_PPTDEV_DISABLE_MSIX, &ppt); 1233 } 1234 1235 #else /* __FreeBSD__ */ 1236 1237 int 1238 vm_assign_pptdev(struct vmctx *ctx, int pptfd) 1239 { 1240 struct vm_pptdev pptdev; 1241 1242 pptdev.pptfd = pptfd; 1243 return (ioctl(ctx->fd, VM_BIND_PPTDEV, &pptdev)); 1244 } 1245 1246 int 1247 vm_unassign_pptdev(struct vmctx *ctx, int pptfd) 1248 { 1249 struct vm_pptdev pptdev; 1250 1251 pptdev.pptfd = pptfd; 1252 return (ioctl(ctx->fd, VM_UNBIND_PPTDEV, &pptdev)); 1253 } 1254 1255 int 1256 vm_map_pptdev_mmio(struct vmctx *ctx, int pptfd, vm_paddr_t gpa, size_t len, 1257 vm_paddr_t hpa) 1258 { 1259 struct vm_pptdev_mmio pptmmio; 1260 1261 pptmmio.pptfd = pptfd; 1262 pptmmio.gpa = gpa; 1263 pptmmio.len = len; 1264 pptmmio.hpa = hpa; 1265 return (ioctl(ctx->fd, VM_MAP_PPTDEV_MMIO, &pptmmio)); 1266 } 1267 1268 int 1269 vm_unmap_pptdev_mmio(struct vmctx *ctx, int pptfd, vm_paddr_t gpa, size_t len) 1270 { 1271 struct vm_pptdev_mmio pptmmio; 1272 1273 bzero(&pptmmio, sizeof(pptmmio)); 1274 pptmmio.pptfd = pptfd; 1275 pptmmio.gpa = gpa; 1276 pptmmio.len = len; 1277 1278 return (ioctl(ctx->fd, VM_UNMAP_PPTDEV_MMIO, &pptmmio)); 1279 } 1280 1281 int 1282 vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, int pptfd, uint64_t addr, 1283 uint64_t msg, int numvec) 1284 { 1285 struct vm_pptdev_msi pptmsi; 1286 1287 pptmsi.vcpu = vcpu; 1288 pptmsi.pptfd = pptfd; 1289 pptmsi.msg = msg; 1290 pptmsi.addr = addr; 1291 pptmsi.numvec = numvec; 1292 return (ioctl(ctx->fd, VM_PPTDEV_MSI, &pptmsi)); 1293 } 1294 1295 int 1296 vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int pptfd, int idx, 1297 uint64_t addr, uint64_t msg, uint32_t vector_control) 1298 { 1299 struct vm_pptdev_msix pptmsix; 1300 1301 pptmsix.vcpu = vcpu; 1302 pptmsix.pptfd = pptfd; 1303 pptmsix.idx = idx; 1304 pptmsix.msg = msg; 1305 pptmsix.addr = addr; 1306 pptmsix.vector_control = vector_control; 1307 return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix); 1308 } 1309 1310 int 1311 vm_get_pptdev_limits(struct vmctx *ctx, int pptfd, int *msi_limit, 1312 int *msix_limit) 1313 { 1314 struct vm_pptdev_limits pptlimits; 1315 int error; 1316 1317 bzero(&pptlimits, sizeof (pptlimits)); 1318 pptlimits.pptfd = pptfd; 1319 error = ioctl(ctx->fd, VM_GET_PPTDEV_LIMITS, &pptlimits); 1320 1321 *msi_limit = pptlimits.msi_limit; 1322 *msix_limit = pptlimits.msix_limit; 1323 return (error); 1324 } 1325 1326 int 1327 vm_disable_pptdev_msix(struct vmctx *ctx, int pptfd) 1328 { 1329 struct vm_pptdev pptdev; 1330 1331 pptdev.pptfd = pptfd; 1332 return (ioctl(ctx->fd, VM_PPTDEV_DISABLE_MSIX, &pptdev)); 1333 } 1334 #endif /* __FreeBSD__ */ 1335 1336 uint64_t * 1337 vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv, 1338 int *ret_entries) 1339 { 1340 int error; 1341 1342 static struct vm_stats vmstats; 1343 1344 vmstats.cpuid = vcpu; 1345 1346 error = ioctl(ctx->fd, VM_STATS_IOC, &vmstats); 1347 if (error == 0) { 1348 if (ret_entries) 1349 *ret_entries = vmstats.num_entries; 1350 if (ret_tv) 1351 *ret_tv = vmstats.tv; 1352 return (vmstats.statbuf); 1353 } else 1354 return (NULL); 1355 } 1356 1357 const char * 1358 vm_get_stat_desc(struct vmctx *ctx, int index) 1359 { 1360 static struct vm_stat_desc statdesc; 1361 1362 statdesc.index = index; 1363 if (ioctl(ctx->fd, VM_STAT_DESC, &statdesc) == 0) 1364 return (statdesc.desc); 1365 else 1366 return (NULL); 1367 } 1368 1369 int 1370 vm_get_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state *state) 1371 { 1372 int error; 1373 struct vm_x2apic x2apic; 1374 1375 bzero(&x2apic, sizeof(x2apic)); 1376 x2apic.cpuid = vcpu; 1377 1378 error = ioctl(ctx->fd, VM_GET_X2APIC_STATE, &x2apic); 1379 *state = x2apic.state; 1380 return (error); 1381 } 1382 1383 int 1384 vm_set_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state state) 1385 { 1386 int error; 1387 struct vm_x2apic x2apic; 1388 1389 bzero(&x2apic, sizeof(x2apic)); 1390 x2apic.cpuid = vcpu; 1391 x2apic.state = state; 1392 1393 error = ioctl(ctx->fd, VM_SET_X2APIC_STATE, &x2apic); 1394 1395 return (error); 1396 } 1397 1398 #ifndef __FreeBSD__ 1399 int 1400 vcpu_reset(struct vmctx *vmctx, int vcpu) 1401 { 1402 struct vm_vcpu_reset vvr; 1403 1404 vvr.vcpuid = vcpu; 1405 vvr.kind = VRK_RESET; 1406 1407 return (ioctl(vmctx->fd, VM_RESET_CPU, &vvr)); 1408 } 1409 #else /* __FreeBSD__ */ 1410 /* 1411 * From Intel Vol 3a: 1412 * Table 9-1. IA-32 Processor States Following Power-up, Reset or INIT 1413 */ 1414 int 1415 vcpu_reset(struct vmctx *vmctx, int vcpu) 1416 { 1417 int error; 1418 uint64_t rflags, rip, cr0, cr4, zero, desc_base, rdx; 1419 uint32_t desc_access, desc_limit; 1420 uint16_t sel; 1421 1422 zero = 0; 1423 1424 rflags = 0x2; 1425 error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RFLAGS, rflags); 1426 if (error) 1427 goto done; 1428 1429 rip = 0xfff0; 1430 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RIP, rip)) != 0) 1431 goto done; 1432 1433 cr0 = CR0_NE; 1434 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR0, cr0)) != 0) 1435 goto done; 1436 1437 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR3, zero)) != 0) 1438 goto done; 1439 1440 cr4 = 0; 1441 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, cr4)) != 0) 1442 goto done; 1443 1444 /* 1445 * CS: present, r/w, accessed, 16-bit, byte granularity, usable 1446 */ 1447 desc_base = 0xffff0000; 1448 desc_limit = 0xffff; 1449 desc_access = 0x0093; 1450 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_CS, 1451 desc_base, desc_limit, desc_access); 1452 if (error) 1453 goto done; 1454 1455 sel = 0xf000; 1456 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CS, sel)) != 0) 1457 goto done; 1458 1459 /* 1460 * SS,DS,ES,FS,GS: present, r/w, accessed, 16-bit, byte granularity 1461 */ 1462 desc_base = 0; 1463 desc_limit = 0xffff; 1464 desc_access = 0x0093; 1465 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_SS, 1466 desc_base, desc_limit, desc_access); 1467 if (error) 1468 goto done; 1469 1470 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_DS, 1471 desc_base, desc_limit, desc_access); 1472 if (error) 1473 goto done; 1474 1475 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_ES, 1476 desc_base, desc_limit, desc_access); 1477 if (error) 1478 goto done; 1479 1480 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_FS, 1481 desc_base, desc_limit, desc_access); 1482 if (error) 1483 goto done; 1484 1485 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GS, 1486 desc_base, desc_limit, desc_access); 1487 if (error) 1488 goto done; 1489 1490 sel = 0; 1491 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_SS, sel)) != 0) 1492 goto done; 1493 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_DS, sel)) != 0) 1494 goto done; 1495 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_ES, sel)) != 0) 1496 goto done; 1497 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_FS, sel)) != 0) 1498 goto done; 1499 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_GS, sel)) != 0) 1500 goto done; 1501 1502 /* General purpose registers */ 1503 rdx = 0xf00; 1504 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RAX, zero)) != 0) 1505 goto done; 1506 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RBX, zero)) != 0) 1507 goto done; 1508 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RCX, zero)) != 0) 1509 goto done; 1510 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RDX, rdx)) != 0) 1511 goto done; 1512 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSI, zero)) != 0) 1513 goto done; 1514 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RDI, zero)) != 0) 1515 goto done; 1516 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RBP, zero)) != 0) 1517 goto done; 1518 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSP, zero)) != 0) 1519 goto done; 1520 1521 /* GDTR, IDTR */ 1522 desc_base = 0; 1523 desc_limit = 0xffff; 1524 desc_access = 0; 1525 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GDTR, 1526 desc_base, desc_limit, desc_access); 1527 if (error != 0) 1528 goto done; 1529 1530 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_IDTR, 1531 desc_base, desc_limit, desc_access); 1532 if (error != 0) 1533 goto done; 1534 1535 /* TR */ 1536 desc_base = 0; 1537 desc_limit = 0xffff; 1538 desc_access = 0x0000008b; 1539 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_TR, 0, 0, desc_access); 1540 if (error) 1541 goto done; 1542 1543 sel = 0; 1544 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_TR, sel)) != 0) 1545 goto done; 1546 1547 /* LDTR */ 1548 desc_base = 0; 1549 desc_limit = 0xffff; 1550 desc_access = 0x00000082; 1551 error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_LDTR, desc_base, 1552 desc_limit, desc_access); 1553 if (error) 1554 goto done; 1555 1556 sel = 0; 1557 if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_LDTR, 0)) != 0) 1558 goto done; 1559 1560 /* XXX cr2, debug registers */ 1561 1562 error = 0; 1563 done: 1564 return (error); 1565 } 1566 #endif /* __FreeBSD__ */ 1567 1568 int 1569 vm_get_gpa_pmap(struct vmctx *ctx, uint64_t gpa, uint64_t *pte, int *num) 1570 { 1571 int error, i; 1572 struct vm_gpa_pte gpapte; 1573 1574 bzero(&gpapte, sizeof(gpapte)); 1575 gpapte.gpa = gpa; 1576 1577 error = ioctl(ctx->fd, VM_GET_GPA_PMAP, &gpapte); 1578 1579 if (error == 0) { 1580 *num = gpapte.ptenum; 1581 for (i = 0; i < gpapte.ptenum; i++) 1582 pte[i] = gpapte.pte[i]; 1583 } 1584 1585 return (error); 1586 } 1587 1588 int 1589 vm_get_hpet_capabilities(struct vmctx *ctx, uint32_t *capabilities) 1590 { 1591 int error; 1592 struct vm_hpet_cap cap; 1593 1594 bzero(&cap, sizeof(struct vm_hpet_cap)); 1595 error = ioctl(ctx->fd, VM_GET_HPET_CAPABILITIES, &cap); 1596 if (capabilities != NULL) 1597 *capabilities = cap.capabilities; 1598 return (error); 1599 } 1600 1601 int 1602 vm_gla2gpa(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging, 1603 uint64_t gla, int prot, uint64_t *gpa, int *fault) 1604 { 1605 struct vm_gla2gpa gg; 1606 int error; 1607 1608 bzero(&gg, sizeof(struct vm_gla2gpa)); 1609 gg.vcpuid = vcpu; 1610 gg.prot = prot; 1611 gg.gla = gla; 1612 gg.paging = *paging; 1613 1614 error = ioctl(ctx->fd, VM_GLA2GPA, &gg); 1615 if (error == 0) { 1616 *fault = gg.fault; 1617 *gpa = gg.gpa; 1618 } 1619 return (error); 1620 } 1621 1622 int 1623 vm_gla2gpa_nofault(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging, 1624 uint64_t gla, int prot, uint64_t *gpa, int *fault) 1625 { 1626 struct vm_gla2gpa gg; 1627 int error; 1628 1629 bzero(&gg, sizeof(struct vm_gla2gpa)); 1630 gg.vcpuid = vcpu; 1631 gg.prot = prot; 1632 gg.gla = gla; 1633 gg.paging = *paging; 1634 1635 error = ioctl(ctx->fd, VM_GLA2GPA_NOFAULT, &gg); 1636 if (error == 0) { 1637 *fault = gg.fault; 1638 *gpa = gg.gpa; 1639 } 1640 return (error); 1641 } 1642 1643 #ifndef min 1644 #define min(a,b) (((a) < (b)) ? (a) : (b)) 1645 #endif 1646 1647 int 1648 vm_copy_setup(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging, 1649 uint64_t gla, size_t len, int prot, struct iovec *iov, int iovcnt, 1650 int *fault) 1651 { 1652 void *va; 1653 uint64_t gpa; 1654 int error, i, n, off; 1655 1656 for (i = 0; i < iovcnt; i++) { 1657 iov[i].iov_base = 0; 1658 iov[i].iov_len = 0; 1659 } 1660 1661 while (len) { 1662 assert(iovcnt > 0); 1663 error = vm_gla2gpa(ctx, vcpu, paging, gla, prot, &gpa, fault); 1664 if (error || *fault) 1665 return (error); 1666 1667 off = gpa & PAGE_MASK; 1668 n = min(len, PAGE_SIZE - off); 1669 1670 va = vm_map_gpa(ctx, gpa, n); 1671 if (va == NULL) 1672 return (EFAULT); 1673 1674 iov->iov_base = va; 1675 iov->iov_len = n; 1676 iov++; 1677 iovcnt--; 1678 1679 gla += n; 1680 len -= n; 1681 } 1682 return (0); 1683 } 1684 1685 void 1686 vm_copy_teardown(struct vmctx *ctx, int vcpu, struct iovec *iov, int iovcnt) 1687 { 1688 1689 return; 1690 } 1691 1692 void 1693 vm_copyin(struct vmctx *ctx, int vcpu, struct iovec *iov, void *vp, size_t len) 1694 { 1695 const char *src; 1696 char *dst; 1697 size_t n; 1698 1699 dst = vp; 1700 while (len) { 1701 assert(iov->iov_len); 1702 n = min(len, iov->iov_len); 1703 src = iov->iov_base; 1704 bcopy(src, dst, n); 1705 1706 iov++; 1707 dst += n; 1708 len -= n; 1709 } 1710 } 1711 1712 void 1713 vm_copyout(struct vmctx *ctx, int vcpu, const void *vp, struct iovec *iov, 1714 size_t len) 1715 { 1716 const char *src; 1717 char *dst; 1718 size_t n; 1719 1720 src = vp; 1721 while (len) { 1722 assert(iov->iov_len); 1723 n = min(len, iov->iov_len); 1724 dst = iov->iov_base; 1725 bcopy(src, dst, n); 1726 1727 iov++; 1728 src += n; 1729 len -= n; 1730 } 1731 } 1732 1733 static int 1734 vm_get_cpus(struct vmctx *ctx, int which, cpuset_t *cpus) 1735 { 1736 struct vm_cpuset vm_cpuset; 1737 int error; 1738 1739 bzero(&vm_cpuset, sizeof(struct vm_cpuset)); 1740 vm_cpuset.which = which; 1741 vm_cpuset.cpusetsize = sizeof(cpuset_t); 1742 vm_cpuset.cpus = cpus; 1743 1744 error = ioctl(ctx->fd, VM_GET_CPUS, &vm_cpuset); 1745 return (error); 1746 } 1747 1748 int 1749 vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus) 1750 { 1751 1752 return (vm_get_cpus(ctx, VM_ACTIVE_CPUS, cpus)); 1753 } 1754 1755 int 1756 vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus) 1757 { 1758 1759 return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus)); 1760 } 1761 1762 int 1763 vm_debug_cpus(struct vmctx *ctx, cpuset_t *cpus) 1764 { 1765 1766 return (vm_get_cpus(ctx, VM_DEBUG_CPUS, cpus)); 1767 } 1768 1769 int 1770 vm_activate_cpu(struct vmctx *ctx, int vcpu) 1771 { 1772 struct vm_activate_cpu ac; 1773 int error; 1774 1775 bzero(&ac, sizeof(struct vm_activate_cpu)); 1776 ac.vcpuid = vcpu; 1777 error = ioctl(ctx->fd, VM_ACTIVATE_CPU, &ac); 1778 return (error); 1779 } 1780 1781 int 1782 vm_suspend_cpu(struct vmctx *ctx, int vcpu) 1783 { 1784 struct vm_activate_cpu ac; 1785 int error; 1786 1787 bzero(&ac, sizeof(struct vm_activate_cpu)); 1788 ac.vcpuid = vcpu; 1789 error = ioctl(ctx->fd, VM_SUSPEND_CPU, &ac); 1790 return (error); 1791 } 1792 1793 int 1794 vm_resume_cpu(struct vmctx *ctx, int vcpu) 1795 { 1796 struct vm_activate_cpu ac; 1797 int error; 1798 1799 bzero(&ac, sizeof(struct vm_activate_cpu)); 1800 ac.vcpuid = vcpu; 1801 error = ioctl(ctx->fd, VM_RESUME_CPU, &ac); 1802 return (error); 1803 } 1804 1805 int 1806 vm_get_intinfo(struct vmctx *ctx, int vcpu, uint64_t *info1, uint64_t *info2) 1807 { 1808 struct vm_intinfo vmii; 1809 int error; 1810 1811 bzero(&vmii, sizeof(struct vm_intinfo)); 1812 vmii.vcpuid = vcpu; 1813 error = ioctl(ctx->fd, VM_GET_INTINFO, &vmii); 1814 if (error == 0) { 1815 *info1 = vmii.info1; 1816 *info2 = vmii.info2; 1817 } 1818 return (error); 1819 } 1820 1821 int 1822 vm_set_intinfo(struct vmctx *ctx, int vcpu, uint64_t info1) 1823 { 1824 struct vm_intinfo vmii; 1825 int error; 1826 1827 bzero(&vmii, sizeof(struct vm_intinfo)); 1828 vmii.vcpuid = vcpu; 1829 vmii.info1 = info1; 1830 error = ioctl(ctx->fd, VM_SET_INTINFO, &vmii); 1831 return (error); 1832 } 1833 1834 int 1835 vm_rtc_write(struct vmctx *ctx, int offset, uint8_t value) 1836 { 1837 struct vm_rtc_data rtcdata; 1838 int error; 1839 1840 bzero(&rtcdata, sizeof(struct vm_rtc_data)); 1841 rtcdata.offset = offset; 1842 rtcdata.value = value; 1843 error = ioctl(ctx->fd, VM_RTC_WRITE, &rtcdata); 1844 return (error); 1845 } 1846 1847 int 1848 vm_rtc_read(struct vmctx *ctx, int offset, uint8_t *retval) 1849 { 1850 struct vm_rtc_data rtcdata; 1851 int error; 1852 1853 bzero(&rtcdata, sizeof(struct vm_rtc_data)); 1854 rtcdata.offset = offset; 1855 error = ioctl(ctx->fd, VM_RTC_READ, &rtcdata); 1856 if (error == 0) 1857 *retval = rtcdata.value; 1858 return (error); 1859 } 1860 1861 int 1862 vm_rtc_settime(struct vmctx *ctx, time_t secs) 1863 { 1864 struct vm_rtc_time rtctime; 1865 int error; 1866 1867 bzero(&rtctime, sizeof(struct vm_rtc_time)); 1868 rtctime.secs = secs; 1869 error = ioctl(ctx->fd, VM_RTC_SETTIME, &rtctime); 1870 return (error); 1871 } 1872 1873 int 1874 vm_rtc_gettime(struct vmctx *ctx, time_t *secs) 1875 { 1876 struct vm_rtc_time rtctime; 1877 int error; 1878 1879 bzero(&rtctime, sizeof(struct vm_rtc_time)); 1880 error = ioctl(ctx->fd, VM_RTC_GETTIME, &rtctime); 1881 if (error == 0) 1882 *secs = rtctime.secs; 1883 return (error); 1884 } 1885 1886 int 1887 vm_restart_instruction(void *arg, int vcpu) 1888 { 1889 struct vmctx *ctx = arg; 1890 1891 return (ioctl(ctx->fd, VM_RESTART_INSTRUCTION, &vcpu)); 1892 } 1893 1894 int 1895 vm_set_topology(struct vmctx *ctx, 1896 uint16_t sockets, uint16_t cores, uint16_t threads, uint16_t maxcpus) 1897 { 1898 struct vm_cpu_topology topology; 1899 1900 bzero(&topology, sizeof (struct vm_cpu_topology)); 1901 topology.sockets = sockets; 1902 topology.cores = cores; 1903 topology.threads = threads; 1904 topology.maxcpus = maxcpus; 1905 return (ioctl(ctx->fd, VM_SET_TOPOLOGY, &topology)); 1906 } 1907 1908 int 1909 vm_get_topology(struct vmctx *ctx, 1910 uint16_t *sockets, uint16_t *cores, uint16_t *threads, uint16_t *maxcpus) 1911 { 1912 struct vm_cpu_topology topology; 1913 int error; 1914 1915 bzero(&topology, sizeof (struct vm_cpu_topology)); 1916 error = ioctl(ctx->fd, VM_GET_TOPOLOGY, &topology); 1917 if (error == 0) { 1918 *sockets = topology.sockets; 1919 *cores = topology.cores; 1920 *threads = topology.threads; 1921 *maxcpus = topology.maxcpus; 1922 } 1923 return (error); 1924 } 1925 1926 int 1927 vm_get_device_fd(struct vmctx *ctx) 1928 { 1929 1930 return (ctx->fd); 1931 } 1932 1933 #ifndef __FreeBSD__ 1934 int 1935 vm_pmtmr_set_location(struct vmctx *ctx, uint16_t ioport) 1936 { 1937 return (ioctl(ctx->fd, VM_PMTMR_LOCATE, ioport)); 1938 } 1939 1940 int 1941 vm_wrlock_cycle(struct vmctx *ctx) 1942 { 1943 if (ioctl(ctx->fd, VM_WRLOCK_CYCLE, 0) != 0) { 1944 return (errno); 1945 } 1946 return (0); 1947 } 1948 1949 int 1950 vm_get_run_state(struct vmctx *ctx, int vcpu, enum vcpu_run_state *state, 1951 uint8_t *sipi_vector) 1952 { 1953 struct vm_run_state data; 1954 1955 data.vcpuid = vcpu; 1956 if (ioctl(ctx->fd, VM_GET_RUN_STATE, &data) != 0) { 1957 return (errno); 1958 } 1959 1960 *state = data.state; 1961 *sipi_vector = data.sipi_vector; 1962 return (0); 1963 } 1964 1965 int 1966 vm_set_run_state(struct vmctx *ctx, int vcpu, enum vcpu_run_state state, 1967 uint8_t sipi_vector) 1968 { 1969 struct vm_run_state data; 1970 1971 data.vcpuid = vcpu; 1972 data.state = state; 1973 data.sipi_vector = sipi_vector; 1974 if (ioctl(ctx->fd, VM_SET_RUN_STATE, &data) != 0) { 1975 return (errno); 1976 } 1977 1978 return (0); 1979 } 1980 1981 #endif /* __FreeBSD__ */ 1982 1983 #ifdef __FreeBSD__ 1984 const cap_ioctl_t * 1985 vm_get_ioctls(size_t *len) 1986 { 1987 cap_ioctl_t *cmds; 1988 /* keep in sync with machine/vmm_dev.h */ 1989 static const cap_ioctl_t vm_ioctl_cmds[] = { VM_RUN, VM_SUSPEND, VM_REINIT, 1990 VM_ALLOC_MEMSEG, VM_GET_MEMSEG, VM_MMAP_MEMSEG, VM_MMAP_MEMSEG, 1991 VM_MMAP_GETNEXT, VM_MUNMAP_MEMSEG, VM_SET_REGISTER, VM_GET_REGISTER, 1992 VM_SET_SEGMENT_DESCRIPTOR, VM_GET_SEGMENT_DESCRIPTOR, 1993 VM_SET_REGISTER_SET, VM_GET_REGISTER_SET, 1994 VM_SET_KERNEMU_DEV, VM_GET_KERNEMU_DEV, 1995 VM_INJECT_EXCEPTION, VM_LAPIC_IRQ, VM_LAPIC_LOCAL_IRQ, 1996 VM_LAPIC_MSI, VM_IOAPIC_ASSERT_IRQ, VM_IOAPIC_DEASSERT_IRQ, 1997 VM_IOAPIC_PULSE_IRQ, VM_IOAPIC_PINCOUNT, VM_ISA_ASSERT_IRQ, 1998 VM_ISA_DEASSERT_IRQ, VM_ISA_PULSE_IRQ, VM_ISA_SET_IRQ_TRIGGER, 1999 VM_SET_CAPABILITY, VM_GET_CAPABILITY, VM_BIND_PPTDEV, 2000 VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO, VM_PPTDEV_MSI, 2001 VM_PPTDEV_MSIX, VM_UNMAP_PPTDEV_MMIO, VM_PPTDEV_DISABLE_MSIX, 2002 VM_INJECT_NMI, VM_STATS, VM_STAT_DESC, 2003 VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE, 2004 VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA, 2005 VM_GLA2GPA_NOFAULT, 2006 VM_ACTIVATE_CPU, VM_GET_CPUS, VM_SUSPEND_CPU, VM_RESUME_CPU, 2007 VM_SET_INTINFO, VM_GET_INTINFO, 2008 VM_RTC_WRITE, VM_RTC_READ, VM_RTC_SETTIME, VM_RTC_GETTIME, 2009 VM_RESTART_INSTRUCTION, VM_SET_TOPOLOGY, VM_GET_TOPOLOGY }; 2010 2011 if (len == NULL) { 2012 cmds = malloc(sizeof(vm_ioctl_cmds)); 2013 if (cmds == NULL) 2014 return (NULL); 2015 bcopy(vm_ioctl_cmds, cmds, sizeof(vm_ioctl_cmds)); 2016 return (cmds); 2017 } 2018 2019 *len = nitems(vm_ioctl_cmds); 2020 return (NULL); 2021 } 2022 #endif /* __FreeBSD__ */ 2023