xref: /illumos-gate/usr/src/cmd/bhyve/task_switch.c (revision e0c0d44e)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2014 Neel Natu <neel@freebsd.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 /*
29  * This file and its contents are supplied under the terms of the
30  * Common Development and Distribution License ("CDDL"), version 1.0.
31  * You may only use this file in accordance with the terms of version
32  * 1.0 of the CDDL.
33  *
34  * A full copy of the text of the CDDL should have accompanied this
35  * source.  A copy of the CDDL is also available via the Internet at
36  * http://www.illumos.org/license/CDDL.
37  *
38  * Copyright 2020 Oxide Computer Company
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include <sys/param.h>
45 #include <sys/_iovec.h>
46 #include <sys/mman.h>
47 
48 #include <x86/psl.h>
49 #include <x86/segments.h>
50 #include <x86/specialreg.h>
51 #include <machine/vmm.h>
52 
53 #include <assert.h>
54 #include <errno.h>
55 #include <stdbool.h>
56 #include <stdio.h>
57 #include <stdlib.h>
58 
59 #include <vmmapi.h>
60 
61 #include "bhyverun.h"
62 #include "debug.h"
63 
64 /*
65  * Using 'struct i386tss' is tempting but causes myriad sign extension
66  * issues because all of its fields are defined as signed integers.
67  */
68 struct tss32 {
69 	uint16_t	tss_link;
70 	uint16_t	rsvd1;
71 	uint32_t	tss_esp0;
72 	uint16_t	tss_ss0;
73 	uint16_t	rsvd2;
74 	uint32_t	tss_esp1;
75 	uint16_t	tss_ss1;
76 	uint16_t	rsvd3;
77 	uint32_t	tss_esp2;
78 	uint16_t	tss_ss2;
79 	uint16_t	rsvd4;
80 	uint32_t	tss_cr3;
81 	uint32_t	tss_eip;
82 	uint32_t	tss_eflags;
83 	uint32_t	tss_eax;
84 	uint32_t	tss_ecx;
85 	uint32_t	tss_edx;
86 	uint32_t	tss_ebx;
87 	uint32_t	tss_esp;
88 	uint32_t	tss_ebp;
89 	uint32_t	tss_esi;
90 	uint32_t	tss_edi;
91 	uint16_t	tss_es;
92 	uint16_t	rsvd5;
93 	uint16_t	tss_cs;
94 	uint16_t	rsvd6;
95 	uint16_t	tss_ss;
96 	uint16_t	rsvd7;
97 	uint16_t	tss_ds;
98 	uint16_t	rsvd8;
99 	uint16_t	tss_fs;
100 	uint16_t	rsvd9;
101 	uint16_t	tss_gs;
102 	uint16_t	rsvd10;
103 	uint16_t	tss_ldt;
104 	uint16_t	rsvd11;
105 	uint16_t	tss_trap;
106 	uint16_t	tss_iomap;
107 };
108 static_assert(sizeof(struct tss32) == 104, "compile-time assertion failed");
109 
110 #define	SEL_START(sel)	(((sel) & ~0x7))
111 #define	SEL_LIMIT(sel)	(((sel) | 0x7))
112 #define	TSS_BUSY(type)	(((type) & 0x2) != 0)
113 
114 static uint64_t
GETREG(struct vmctx * ctx,int vcpu,int reg)115 GETREG(struct vmctx *ctx, int vcpu, int reg)
116 {
117 	uint64_t val;
118 	int error;
119 
120 	error = vm_get_register(ctx, vcpu, reg, &val);
121 	assert(error == 0);
122 	return (val);
123 }
124 
125 static void
SETREG(struct vmctx * ctx,int vcpu,int reg,uint64_t val)126 SETREG(struct vmctx *ctx, int vcpu, int reg, uint64_t val)
127 {
128 	int error;
129 
130 	error = vm_set_register(ctx, vcpu, reg, val);
131 	assert(error == 0);
132 }
133 
134 static struct seg_desc
usd_to_seg_desc(struct user_segment_descriptor * usd)135 usd_to_seg_desc(struct user_segment_descriptor *usd)
136 {
137 	struct seg_desc seg_desc;
138 
139 	seg_desc.base = (u_int)USD_GETBASE(usd);
140 	if (usd->sd_gran)
141 		seg_desc.limit = (u_int)(USD_GETLIMIT(usd) << 12) | 0xfff;
142 	else
143 		seg_desc.limit = (u_int)USD_GETLIMIT(usd);
144 	seg_desc.access = usd->sd_type | usd->sd_dpl << 5 | usd->sd_p << 7;
145 	seg_desc.access |= usd->sd_xx << 12;
146 	seg_desc.access |= usd->sd_def32 << 14;
147 	seg_desc.access |= usd->sd_gran << 15;
148 
149 	return (seg_desc);
150 }
151 
152 /*
153  * Inject an exception with an error code that is a segment selector.
154  * The format of the error code is described in section 6.13, "Error Code",
155  * Intel SDM volume 3.
156  *
157  * Bit 0 (EXT) denotes whether the exception occurred during delivery
158  * of an external event like an interrupt.
159  *
160  * Bit 1 (IDT) indicates whether the selector points to a gate descriptor
161  * in the IDT.
162  *
163  * Bit 2(GDT/LDT) has the usual interpretation of Table Indicator (TI).
164  */
165 static void
sel_exception(struct vmctx * ctx,int vcpu,int vector,uint16_t sel,int ext)166 sel_exception(struct vmctx *ctx, int vcpu, int vector, uint16_t sel, int ext)
167 {
168 	/*
169 	 * Bit 2 from the selector is retained as-is in the error code.
170 	 *
171 	 * Bit 1 can be safely cleared because none of the selectors
172 	 * encountered during task switch emulation refer to a task
173 	 * gate in the IDT.
174 	 *
175 	 * Bit 0 is set depending on the value of 'ext'.
176 	 */
177 	sel &= ~0x3;
178 	if (ext)
179 		sel |= 0x1;
180 	vm_inject_fault(ctx, vcpu, vector, 1, sel);
181 }
182 
183 /*
184  * Return 0 if the selector 'sel' in within the limits of the GDT/LDT
185  * and non-zero otherwise.
186  */
187 static int
desc_table_limit_check(struct vmctx * ctx,int vcpu,uint16_t sel)188 desc_table_limit_check(struct vmctx *ctx, int vcpu, uint16_t sel)
189 {
190 	uint64_t base;
191 	uint32_t limit, access;
192 	int error, reg;
193 
194 	reg = ISLDT(sel) ? VM_REG_GUEST_LDTR : VM_REG_GUEST_GDTR;
195 	error = vm_get_desc(ctx, vcpu, reg, &base, &limit, &access);
196 	assert(error == 0);
197 
198 	if (reg == VM_REG_GUEST_LDTR) {
199 		if (SEG_DESC_UNUSABLE(access) || !SEG_DESC_PRESENT(access))
200 			return (-1);
201 	}
202 
203 	if (limit < SEL_LIMIT(sel))
204 		return (-1);
205 	else
206 		return (0);
207 }
208 
209 /*
210  * Read/write the segment descriptor 'desc' into the GDT/LDT slot referenced
211  * by the selector 'sel'.
212  *
213  * Returns 0 on success.
214  * Returns 1 if an exception was injected into the guest.
215  * Returns -1 otherwise.
216  */
217 static int
desc_table_rw(struct vmctx * ctx,int vcpu,struct vm_guest_paging * paging,uint16_t sel,struct user_segment_descriptor * desc,bool doread,int * faultptr)218 desc_table_rw(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
219     uint16_t sel, struct user_segment_descriptor *desc, bool doread,
220     int *faultptr)
221 {
222 	struct iovec iov[2];
223 	uint64_t base;
224 	uint32_t limit, access;
225 	int error, reg;
226 
227 	reg = ISLDT(sel) ? VM_REG_GUEST_LDTR : VM_REG_GUEST_GDTR;
228 	error = vm_get_desc(ctx, vcpu, reg, &base, &limit, &access);
229 	assert(error == 0);
230 	assert(limit >= SEL_LIMIT(sel));
231 
232 	error = vm_copy_setup(ctx, vcpu, paging, base + SEL_START(sel),
233 	    sizeof(*desc), doread ? PROT_READ : PROT_WRITE, iov, nitems(iov),
234 	    faultptr);
235 	if (error || *faultptr)
236 		return (error);
237 
238 	if (doread)
239 		vm_copyin(ctx, vcpu, iov, desc, sizeof(*desc));
240 	else
241 		vm_copyout(ctx, vcpu, desc, iov, sizeof(*desc));
242 	return (0);
243 }
244 
245 static int
desc_table_read(struct vmctx * ctx,int vcpu,struct vm_guest_paging * paging,uint16_t sel,struct user_segment_descriptor * desc,int * faultptr)246 desc_table_read(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
247     uint16_t sel, struct user_segment_descriptor *desc, int *faultptr)
248 {
249 	return (desc_table_rw(ctx, vcpu, paging, sel, desc, true, faultptr));
250 }
251 
252 static int
desc_table_write(struct vmctx * ctx,int vcpu,struct vm_guest_paging * paging,uint16_t sel,struct user_segment_descriptor * desc,int * faultptr)253 desc_table_write(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
254     uint16_t sel, struct user_segment_descriptor *desc, int *faultptr)
255 {
256 	return (desc_table_rw(ctx, vcpu, paging, sel, desc, false, faultptr));
257 }
258 
259 /*
260  * Read the TSS descriptor referenced by 'sel' into 'desc'.
261  *
262  * Returns 0 on success.
263  * Returns 1 if an exception was injected into the guest.
264  * Returns -1 otherwise.
265  */
266 static int
read_tss_descriptor(struct vmctx * ctx,int vcpu,struct vm_task_switch * ts,uint16_t sel,struct user_segment_descriptor * desc,int * faultptr)267 read_tss_descriptor(struct vmctx *ctx, int vcpu, struct vm_task_switch *ts,
268     uint16_t sel, struct user_segment_descriptor *desc, int *faultptr)
269 {
270 	struct vm_guest_paging sup_paging;
271 	int error;
272 
273 	assert(!ISLDT(sel));
274 	assert(IDXSEL(sel) != 0);
275 
276 	/* Fetch the new TSS descriptor */
277 	if (desc_table_limit_check(ctx, vcpu, sel)) {
278 		if (ts->reason == TSR_IRET)
279 			sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
280 		else
281 			sel_exception(ctx, vcpu, IDT_GP, sel, ts->ext);
282 		return (1);
283 	}
284 
285 	sup_paging = ts->paging;
286 	sup_paging.cpl = 0;		/* implicit supervisor mode */
287 	error = desc_table_read(ctx, vcpu, &sup_paging, sel, desc, faultptr);
288 	return (error);
289 }
290 
291 static bool
code_desc(int sd_type)292 code_desc(int sd_type)
293 {
294 	/* code descriptor */
295 	return ((sd_type & 0x18) == 0x18);
296 }
297 
298 static bool
stack_desc(int sd_type)299 stack_desc(int sd_type)
300 {
301 	/* writable data descriptor */
302 	return ((sd_type & 0x1A) == 0x12);
303 }
304 
305 static bool
data_desc(int sd_type)306 data_desc(int sd_type)
307 {
308 	/* data descriptor or a readable code descriptor */
309 	return ((sd_type & 0x18) == 0x10 || (sd_type & 0x1A) == 0x1A);
310 }
311 
312 static bool
ldt_desc(int sd_type)313 ldt_desc(int sd_type)
314 {
315 
316 	return (sd_type == SDT_SYSLDT);
317 }
318 
319 /*
320  * Validate the descriptor 'seg_desc' associated with 'segment'.
321  */
322 static int
validate_seg_desc(struct vmctx * ctx,int vcpu,struct vm_task_switch * ts,int segment,struct seg_desc * seg_desc,int * faultptr)323 validate_seg_desc(struct vmctx *ctx, int vcpu, struct vm_task_switch *ts,
324     int segment, struct seg_desc *seg_desc, int *faultptr)
325 {
326 	struct vm_guest_paging sup_paging;
327 	struct user_segment_descriptor usd;
328 	int error, idtvec;
329 	int cpl, dpl, rpl;
330 	uint16_t sel, cs;
331 	bool ldtseg, codeseg, stackseg, dataseg, conforming;
332 
333 	ldtseg = codeseg = stackseg = dataseg = false;
334 	switch (segment) {
335 	case VM_REG_GUEST_LDTR:
336 		ldtseg = true;
337 		break;
338 	case VM_REG_GUEST_CS:
339 		codeseg = true;
340 		break;
341 	case VM_REG_GUEST_SS:
342 		stackseg = true;
343 		break;
344 	case VM_REG_GUEST_DS:
345 	case VM_REG_GUEST_ES:
346 	case VM_REG_GUEST_FS:
347 	case VM_REG_GUEST_GS:
348 		dataseg = true;
349 		break;
350 	default:
351 		assert(0);
352 	}
353 
354 	/* Get the segment selector */
355 	sel = GETREG(ctx, vcpu, segment);
356 
357 	/* LDT selector must point into the GDT */
358 	if (ldtseg && ISLDT(sel)) {
359 		sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
360 		return (1);
361 	}
362 
363 	/* Descriptor table limit check */
364 	if (desc_table_limit_check(ctx, vcpu, sel)) {
365 		sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
366 		return (1);
367 	}
368 
369 	/* NULL selector */
370 	if (IDXSEL(sel) == 0) {
371 		/* Code and stack segment selectors cannot be NULL */
372 		if (codeseg || stackseg) {
373 			sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
374 			return (1);
375 		}
376 		seg_desc->base = 0;
377 		seg_desc->limit = 0;
378 		seg_desc->access = 0x10000;	/* unusable */
379 		return (0);
380 	}
381 
382 	/* Read the descriptor from the GDT/LDT */
383 	sup_paging = ts->paging;
384 	sup_paging.cpl = 0;	/* implicit supervisor mode */
385 	error = desc_table_read(ctx, vcpu, &sup_paging, sel, &usd, faultptr);
386 	if (error || *faultptr)
387 		return (error);
388 
389 	/* Verify that the descriptor type is compatible with the segment */
390 	if ((ldtseg && !ldt_desc(usd.sd_type)) ||
391 	    (codeseg && !code_desc(usd.sd_type)) ||
392 	    (dataseg && !data_desc(usd.sd_type)) ||
393 	    (stackseg && !stack_desc(usd.sd_type))) {
394 		sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
395 		return (1);
396 	}
397 
398 	/* Segment must be marked present */
399 	if (!usd.sd_p) {
400 		if (ldtseg)
401 			idtvec = IDT_TS;
402 		else if (stackseg)
403 			idtvec = IDT_SS;
404 		else
405 			idtvec = IDT_NP;
406 		sel_exception(ctx, vcpu, idtvec, sel, ts->ext);
407 		return (1);
408 	}
409 
410 	cs = GETREG(ctx, vcpu, VM_REG_GUEST_CS);
411 	cpl = cs & SEL_RPL_MASK;
412 	rpl = sel & SEL_RPL_MASK;
413 	dpl = usd.sd_dpl;
414 
415 	if (stackseg && (rpl != cpl || dpl != cpl)) {
416 		sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
417 		return (1);
418 	}
419 
420 	if (codeseg) {
421 		conforming = (usd.sd_type & 0x4) ? true : false;
422 		if ((conforming && (cpl < dpl)) ||
423 		    (!conforming && (cpl != dpl))) {
424 			sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
425 			return (1);
426 		}
427 	}
428 
429 	if (dataseg) {
430 		/*
431 		 * A data segment is always non-conforming except when it's
432 		 * descriptor is a readable, conforming code segment.
433 		 */
434 		if (code_desc(usd.sd_type) && (usd.sd_type & 0x4) != 0)
435 			conforming = true;
436 		else
437 			conforming = false;
438 
439 		if (!conforming && (rpl > dpl || cpl > dpl)) {
440 			sel_exception(ctx, vcpu, IDT_TS, sel, ts->ext);
441 			return (1);
442 		}
443 	}
444 	*seg_desc = usd_to_seg_desc(&usd);
445 	return (0);
446 }
447 
448 static void
tss32_save(struct vmctx * ctx,int vcpu,struct vm_task_switch * task_switch,uint32_t eip,struct tss32 * tss,struct iovec * iov)449 tss32_save(struct vmctx *ctx, int vcpu, struct vm_task_switch *task_switch,
450     uint32_t eip, struct tss32 *tss, struct iovec *iov)
451 {
452 
453 	/* General purpose registers */
454 	tss->tss_eax = GETREG(ctx, vcpu, VM_REG_GUEST_RAX);
455 	tss->tss_ecx = GETREG(ctx, vcpu, VM_REG_GUEST_RCX);
456 	tss->tss_edx = GETREG(ctx, vcpu, VM_REG_GUEST_RDX);
457 	tss->tss_ebx = GETREG(ctx, vcpu, VM_REG_GUEST_RBX);
458 	tss->tss_esp = GETREG(ctx, vcpu, VM_REG_GUEST_RSP);
459 	tss->tss_ebp = GETREG(ctx, vcpu, VM_REG_GUEST_RBP);
460 	tss->tss_esi = GETREG(ctx, vcpu, VM_REG_GUEST_RSI);
461 	tss->tss_edi = GETREG(ctx, vcpu, VM_REG_GUEST_RDI);
462 
463 	/* Segment selectors */
464 	tss->tss_es = GETREG(ctx, vcpu, VM_REG_GUEST_ES);
465 	tss->tss_cs = GETREG(ctx, vcpu, VM_REG_GUEST_CS);
466 	tss->tss_ss = GETREG(ctx, vcpu, VM_REG_GUEST_SS);
467 	tss->tss_ds = GETREG(ctx, vcpu, VM_REG_GUEST_DS);
468 	tss->tss_fs = GETREG(ctx, vcpu, VM_REG_GUEST_FS);
469 	tss->tss_gs = GETREG(ctx, vcpu, VM_REG_GUEST_GS);
470 
471 	/* eflags and eip */
472 	tss->tss_eflags = GETREG(ctx, vcpu, VM_REG_GUEST_RFLAGS);
473 	if (task_switch->reason == TSR_IRET)
474 		tss->tss_eflags &= ~PSL_NT;
475 	tss->tss_eip = eip;
476 
477 	/* Copy updated old TSS into guest memory */
478 	vm_copyout(ctx, vcpu, tss, iov, sizeof(struct tss32));
479 }
480 
481 static void
update_seg_desc(struct vmctx * ctx,int vcpu,int reg,struct seg_desc * sd)482 update_seg_desc(struct vmctx *ctx, int vcpu, int reg, struct seg_desc *sd)
483 {
484 	int error;
485 
486 	error = vm_set_desc(ctx, vcpu, reg, sd->base, sd->limit, sd->access);
487 	assert(error == 0);
488 }
489 
490 /*
491  * Update the vcpu registers to reflect the state of the new task.
492  */
493 static int
tss32_restore(struct vmctx * ctx,int vcpu,struct vm_task_switch * ts,uint16_t ot_sel,struct tss32 * tss,struct iovec * iov,int * faultptr)494 tss32_restore(struct vmctx *ctx, int vcpu, struct vm_task_switch *ts,
495     uint16_t ot_sel, struct tss32 *tss, struct iovec *iov, int *faultptr)
496 {
497 	struct seg_desc seg_desc, seg_desc2;
498 	uint64_t *pdpte, maxphyaddr, reserved;
499 	uint32_t eflags;
500 	int error, i;
501 	bool nested;
502 
503 	nested = false;
504 	if (ts->reason != TSR_IRET && ts->reason != TSR_JMP) {
505 		tss->tss_link = ot_sel;
506 		nested = true;
507 	}
508 
509 	eflags = tss->tss_eflags;
510 	if (nested)
511 		eflags |= PSL_NT;
512 
513 	/* LDTR */
514 	SETREG(ctx, vcpu, VM_REG_GUEST_LDTR, tss->tss_ldt);
515 
516 	/* PBDR */
517 	if (ts->paging.paging_mode != PAGING_MODE_FLAT) {
518 		if (ts->paging.paging_mode == PAGING_MODE_PAE) {
519 			/*
520 			 * XXX Assuming 36-bit MAXPHYADDR.
521 			 */
522 			maxphyaddr = (1UL << 36) - 1;
523 			pdpte = paddr_guest2host(ctx, tss->tss_cr3 & ~0x1f, 32);
524 			for (i = 0; i < 4; i++) {
525 				/* Check reserved bits if the PDPTE is valid */
526 				if (!(pdpte[i] & 0x1))
527 					continue;
528 				/*
529 				 * Bits 2:1, 8:5 and bits above the processor's
530 				 * maximum physical address are reserved.
531 				 */
532 				reserved = ~maxphyaddr | 0x1E6;
533 				if (pdpte[i] & reserved) {
534 					vm_inject_gp(ctx, vcpu);
535 					return (1);
536 				}
537 			}
538 			SETREG(ctx, vcpu, VM_REG_GUEST_PDPTE0, pdpte[0]);
539 			SETREG(ctx, vcpu, VM_REG_GUEST_PDPTE1, pdpte[1]);
540 			SETREG(ctx, vcpu, VM_REG_GUEST_PDPTE2, pdpte[2]);
541 			SETREG(ctx, vcpu, VM_REG_GUEST_PDPTE3, pdpte[3]);
542 		}
543 		SETREG(ctx, vcpu, VM_REG_GUEST_CR3, tss->tss_cr3);
544 		ts->paging.cr3 = tss->tss_cr3;
545 	}
546 
547 	/* eflags and eip */
548 	SETREG(ctx, vcpu, VM_REG_GUEST_RFLAGS, eflags);
549 	SETREG(ctx, vcpu, VM_REG_GUEST_RIP, tss->tss_eip);
550 
551 	/* General purpose registers */
552 	SETREG(ctx, vcpu, VM_REG_GUEST_RAX, tss->tss_eax);
553 	SETREG(ctx, vcpu, VM_REG_GUEST_RCX, tss->tss_ecx);
554 	SETREG(ctx, vcpu, VM_REG_GUEST_RDX, tss->tss_edx);
555 	SETREG(ctx, vcpu, VM_REG_GUEST_RBX, tss->tss_ebx);
556 	SETREG(ctx, vcpu, VM_REG_GUEST_RSP, tss->tss_esp);
557 	SETREG(ctx, vcpu, VM_REG_GUEST_RBP, tss->tss_ebp);
558 	SETREG(ctx, vcpu, VM_REG_GUEST_RSI, tss->tss_esi);
559 	SETREG(ctx, vcpu, VM_REG_GUEST_RDI, tss->tss_edi);
560 
561 	/* Segment selectors */
562 	SETREG(ctx, vcpu, VM_REG_GUEST_ES, tss->tss_es);
563 	SETREG(ctx, vcpu, VM_REG_GUEST_CS, tss->tss_cs);
564 	SETREG(ctx, vcpu, VM_REG_GUEST_SS, tss->tss_ss);
565 	SETREG(ctx, vcpu, VM_REG_GUEST_DS, tss->tss_ds);
566 	SETREG(ctx, vcpu, VM_REG_GUEST_FS, tss->tss_fs);
567 	SETREG(ctx, vcpu, VM_REG_GUEST_GS, tss->tss_gs);
568 
569 	/*
570 	 * If this is a nested task then write out the new TSS to update
571 	 * the previous link field.
572 	 */
573 	if (nested)
574 		vm_copyout(ctx, vcpu, tss, iov, sizeof(*tss));
575 
576 	/* Validate segment descriptors */
577 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_LDTR, &seg_desc,
578 	    faultptr);
579 	if (error || *faultptr)
580 		return (error);
581 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_LDTR, &seg_desc);
582 
583 	/*
584 	 * Section "Checks on Guest Segment Registers", Intel SDM, Vol 3.
585 	 *
586 	 * The SS and CS attribute checks on VM-entry are inter-dependent so
587 	 * we need to make sure that both segments are valid before updating
588 	 * either of them. This ensures that the VMCS state can pass the
589 	 * VM-entry checks so the guest can handle any exception injected
590 	 * during task switch emulation.
591 	 */
592 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_CS, &seg_desc,
593 	    faultptr);
594 	if (error || *faultptr)
595 		return (error);
596 
597 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_SS, &seg_desc2,
598 	    faultptr);
599 	if (error || *faultptr)
600 		return (error);
601 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_CS, &seg_desc);
602 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_SS, &seg_desc2);
603 	ts->paging.cpl = tss->tss_cs & SEL_RPL_MASK;
604 
605 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_DS, &seg_desc,
606 	    faultptr);
607 	if (error || *faultptr)
608 		return (error);
609 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_DS, &seg_desc);
610 
611 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_ES, &seg_desc,
612 	    faultptr);
613 	if (error || *faultptr)
614 		return (error);
615 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_ES, &seg_desc);
616 
617 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_FS, &seg_desc,
618 	    faultptr);
619 	if (error || *faultptr)
620 		return (error);
621 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_FS, &seg_desc);
622 
623 	error = validate_seg_desc(ctx, vcpu, ts, VM_REG_GUEST_GS, &seg_desc,
624 	    faultptr);
625 	if (error || *faultptr)
626 		return (error);
627 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_GS, &seg_desc);
628 
629 	return (0);
630 }
631 
632 
633 /*
634  * Copy of vie_alignment_check() from vmm_instruction_emul.c
635  */
636 static int
alignment_check(int cpl,int size,uint64_t cr0,uint64_t rf,uint64_t gla)637 alignment_check(int cpl, int size, uint64_t cr0, uint64_t rf, uint64_t gla)
638 {
639 	assert(size == 1 || size == 2 || size == 4 || size == 8);
640 	assert(cpl >= 0 && cpl <= 3);
641 
642 	if (cpl != 3 || (cr0 & CR0_AM) == 0 || (rf & PSL_AC) == 0)
643 		return (0);
644 
645 	return ((gla & (size - 1)) ? 1 : 0);
646 }
647 
648 /*
649  * Copy of vie_size2mask() from vmm_instruction_emul.c
650  */
651 static uint64_t
size2mask(int size)652 size2mask(int size)
653 {
654 	switch (size) {
655 	case 1:
656 		return (0xff);
657 	case 2:
658 		return (0xffff);
659 	case 4:
660 		return (0xffffffff);
661 	case 8:
662 		return (0xffffffffffffffff);
663 	default:
664 		assert(0);
665 		/* not reached */
666 		return (0);
667 	}
668 }
669 
670 /*
671  * Copy of vie_calculate_gla() from vmm_instruction_emul.c
672  */
673 static int
calculate_gla(enum vm_cpu_mode cpu_mode,enum vm_reg_name seg,struct seg_desc * desc,uint64_t offset,int length,int addrsize,int prot,uint64_t * gla)674 calculate_gla(enum vm_cpu_mode cpu_mode, enum vm_reg_name seg,
675     struct seg_desc *desc, uint64_t offset, int length, int addrsize,
676     int prot, uint64_t *gla)
677 {
678 	uint64_t firstoff, low_limit, high_limit, segbase;
679 	int glasize, type;
680 
681 	assert(seg >= VM_REG_GUEST_ES && seg <= VM_REG_GUEST_GS);
682 	assert((length == 1 || length == 2 || length == 4 || length == 8));
683 	assert((prot & ~(PROT_READ | PROT_WRITE)) == 0);
684 
685 	firstoff = offset;
686 	if (cpu_mode == CPU_MODE_64BIT) {
687 		assert(addrsize == 4 || addrsize == 8);
688 		glasize = 8;
689 	} else {
690 		assert(addrsize == 2 || addrsize == 4);
691 		glasize = 4;
692 		/*
693 		 * If the segment selector is loaded with a NULL selector
694 		 * then the descriptor is unusable and attempting to use
695 		 * it results in a #GP(0).
696 		 */
697 		if (SEG_DESC_UNUSABLE(desc->access))
698 			return (-1);
699 
700 		/*
701 		 * The processor generates a #NP exception when a segment
702 		 * register is loaded with a selector that points to a
703 		 * descriptor that is not present. If this was the case then
704 		 * it would have been checked before the VM-exit.
705 		 */
706 		assert(SEG_DESC_PRESENT(desc->access));
707 
708 		/*
709 		 * The descriptor type must indicate a code/data segment.
710 		 */
711 		type = SEG_DESC_TYPE(desc->access);
712 		assert(type >= 16 && type <= 31);
713 
714 		if (prot & PROT_READ) {
715 			/* #GP on a read access to a exec-only code segment */
716 			if ((type & 0xA) == 0x8)
717 				return (-1);
718 		}
719 
720 		if (prot & PROT_WRITE) {
721 			/*
722 			 * #GP on a write access to a code segment or a
723 			 * read-only data segment.
724 			 */
725 			if (type & 0x8)			/* code segment */
726 				return (-1);
727 
728 			if ((type & 0xA) == 0)		/* read-only data seg */
729 				return (-1);
730 		}
731 
732 		/*
733 		 * 'desc->limit' is fully expanded taking granularity into
734 		 * account.
735 		 */
736 		if ((type & 0xC) == 0x4) {
737 			/* expand-down data segment */
738 			low_limit = desc->limit + 1;
739 			high_limit = SEG_DESC_DEF32(desc->access) ?
740 			    0xffffffff : 0xffff;
741 		} else {
742 			/* code segment or expand-up data segment */
743 			low_limit = 0;
744 			high_limit = desc->limit;
745 		}
746 
747 		while (length > 0) {
748 			offset &= size2mask(addrsize);
749 			if (offset < low_limit || offset > high_limit)
750 				return (-1);
751 			offset++;
752 			length--;
753 		}
754 	}
755 
756 	/*
757 	 * In 64-bit mode all segments except %fs and %gs have a segment
758 	 * base address of 0.
759 	 */
760 	if (cpu_mode == CPU_MODE_64BIT && seg != VM_REG_GUEST_FS &&
761 	    seg != VM_REG_GUEST_GS) {
762 		segbase = 0;
763 	} else {
764 		segbase = desc->base;
765 	}
766 
767 	/*
768 	 * Truncate 'firstoff' to the effective address size before adding
769 	 * it to the segment base.
770 	 */
771 	firstoff &= size2mask(addrsize);
772 	*gla = (segbase + firstoff) & size2mask(glasize);
773 	return (0);
774 }
775 
776 /*
777  * Push an error code on the stack of the new task. This is needed if the
778  * task switch was triggered by a hardware exception that causes an error
779  * code to be saved (e.g. #PF).
780  */
781 static int
push_errcode(struct vmctx * ctx,int vcpu,struct vm_guest_paging * paging,int task_type,uint32_t errcode,int * faultptr)782 push_errcode(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
783     int task_type, uint32_t errcode, int *faultptr)
784 {
785 	struct iovec iov[2];
786 	struct seg_desc seg_desc;
787 	int stacksize, bytes, error;
788 	uint64_t gla, cr0, rflags;
789 	uint32_t esp;
790 	uint16_t stacksel;
791 
792 	*faultptr = 0;
793 
794 	cr0 = GETREG(ctx, vcpu, VM_REG_GUEST_CR0);
795 	rflags = GETREG(ctx, vcpu, VM_REG_GUEST_RFLAGS);
796 	stacksel = GETREG(ctx, vcpu, VM_REG_GUEST_SS);
797 
798 	error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_SS, &seg_desc.base,
799 	    &seg_desc.limit, &seg_desc.access);
800 	assert(error == 0);
801 
802 	/*
803 	 * Section "Error Code" in the Intel SDM vol 3: the error code is
804 	 * pushed on the stack as a doubleword or word (depending on the
805 	 * default interrupt, trap or task gate size).
806 	 */
807 	if (task_type == SDT_SYS386BSY || task_type == SDT_SYS386TSS)
808 		bytes = 4;
809 	else
810 		bytes = 2;
811 
812 	/*
813 	 * PUSH instruction from Intel SDM vol 2: the 'B' flag in the
814 	 * stack-segment descriptor determines the size of the stack
815 	 * pointer outside of 64-bit mode.
816 	 */
817 	if (SEG_DESC_DEF32(seg_desc.access))
818 		stacksize = 4;
819 	else
820 		stacksize = 2;
821 
822 	esp = GETREG(ctx, vcpu, VM_REG_GUEST_RSP);
823 	esp -= bytes;
824 
825 	if (calculate_gla(paging->cpu_mode, VM_REG_GUEST_SS,
826 	    &seg_desc, esp, bytes, stacksize, PROT_WRITE, &gla)) {
827 		sel_exception(ctx, vcpu, IDT_SS, stacksel, 1);
828 		*faultptr = 1;
829 		return (0);
830 	}
831 
832 	if (alignment_check(paging->cpl, bytes, cr0, rflags, gla)) {
833 		vm_inject_ac(ctx, vcpu, 1);
834 		*faultptr = 1;
835 		return (0);
836 	}
837 
838 	error = vm_copy_setup(ctx, vcpu, paging, gla, bytes, PROT_WRITE,
839 	    iov, nitems(iov), faultptr);
840 	if (error || *faultptr)
841 		return (error);
842 
843 	vm_copyout(ctx, vcpu, &errcode, iov, bytes);
844 	SETREG(ctx, vcpu, VM_REG_GUEST_RSP, esp);
845 	return (0);
846 }
847 
848 /*
849  * Evaluate return value from helper functions and potentially return to
850  * the VM run loop.
851  */
852 #define	CHKERR(error,fault)						\
853 	do {								\
854 		assert((error == 0) || (error == EFAULT));		\
855 		if (error)						\
856 			return (VMEXIT_ABORT);				\
857 		else if (fault)						\
858 			return (VMEXIT_CONTINUE);			\
859 	} while (0)
860 
861 int
vmexit_task_switch(struct vmctx * ctx,struct vm_exit * vmexit,int * pvcpu)862 vmexit_task_switch(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
863 {
864 	struct seg_desc nt;
865 	struct tss32 oldtss, newtss;
866 	struct vm_task_switch *task_switch;
867 	struct vm_guest_paging *paging, sup_paging;
868 	struct user_segment_descriptor nt_desc, ot_desc;
869 	struct iovec nt_iov[2], ot_iov[2];
870 	uint64_t cr0, ot_base;
871 	uint32_t eip, ot_lim, access;
872 	int error, ext, fault, minlimit, nt_type, ot_type, vcpu;
873 	enum task_switch_reason reason;
874 	uint16_t nt_sel, ot_sel;
875 
876 	task_switch = &vmexit->u.task_switch;
877 	nt_sel = task_switch->tsssel;
878 	ext = vmexit->u.task_switch.ext;
879 	reason = vmexit->u.task_switch.reason;
880 	paging = &vmexit->u.task_switch.paging;
881 	vcpu = *pvcpu;
882 
883 	assert(paging->cpu_mode == CPU_MODE_PROTECTED);
884 
885 	/*
886 	 * Calculate the instruction pointer to store in the old TSS.
887 	 */
888 	eip = vmexit->rip + vmexit->inst_length;
889 
890 	/*
891 	 * Section 4.6, "Access Rights" in Intel SDM Vol 3.
892 	 * The following page table accesses are implicitly supervisor mode:
893 	 * - accesses to GDT or LDT to load segment descriptors
894 	 * - accesses to the task state segment during task switch
895 	 */
896 	sup_paging = *paging;
897 	sup_paging.cpl = 0;	/* implicit supervisor mode */
898 
899 	/* Fetch the new TSS descriptor */
900 	error = read_tss_descriptor(ctx, vcpu, task_switch, nt_sel, &nt_desc,
901 	    &fault);
902 	CHKERR(error, fault);
903 
904 	nt = usd_to_seg_desc(&nt_desc);
905 
906 	/* Verify the type of the new TSS */
907 	nt_type = SEG_DESC_TYPE(nt.access);
908 	if (nt_type != SDT_SYS386BSY && nt_type != SDT_SYS386TSS &&
909 	    nt_type != SDT_SYS286BSY && nt_type != SDT_SYS286TSS) {
910 		sel_exception(ctx, vcpu, IDT_TS, nt_sel, ext);
911 		goto done;
912 	}
913 
914 	/* TSS descriptor must have present bit set */
915 	if (!SEG_DESC_PRESENT(nt.access)) {
916 		sel_exception(ctx, vcpu, IDT_NP, nt_sel, ext);
917 		goto done;
918 	}
919 
920 	/*
921 	 * TSS must have a minimum length of 104 bytes for a 32-bit TSS and
922 	 * 44 bytes for a 16-bit TSS.
923 	 */
924 	if (nt_type == SDT_SYS386BSY || nt_type == SDT_SYS386TSS)
925 		minlimit = 104 - 1;
926 	else if (nt_type == SDT_SYS286BSY || nt_type == SDT_SYS286TSS)
927 		minlimit = 44 - 1;
928 	else
929 		minlimit = 0;
930 
931 	assert(minlimit > 0);
932 	if (nt.limit < minlimit) {
933 		sel_exception(ctx, vcpu, IDT_TS, nt_sel, ext);
934 		goto done;
935 	}
936 
937 	/* TSS must be busy if task switch is due to IRET */
938 	if (reason == TSR_IRET && !TSS_BUSY(nt_type)) {
939 		sel_exception(ctx, vcpu, IDT_TS, nt_sel, ext);
940 		goto done;
941 	}
942 
943 	/*
944 	 * TSS must be available (not busy) if task switch reason is
945 	 * CALL, JMP, exception or interrupt.
946 	 */
947 	if (reason != TSR_IRET && TSS_BUSY(nt_type)) {
948 		sel_exception(ctx, vcpu, IDT_GP, nt_sel, ext);
949 		goto done;
950 	}
951 
952 	/* Fetch the new TSS */
953 	error = vm_copy_setup(ctx, vcpu, &sup_paging, nt.base, minlimit + 1,
954 	    PROT_READ | PROT_WRITE, nt_iov, nitems(nt_iov), &fault);
955 	CHKERR(error, fault);
956 	vm_copyin(ctx, vcpu, nt_iov, &newtss, minlimit + 1);
957 
958 	/* Get the old TSS selector from the guest's task register */
959 	ot_sel = GETREG(ctx, vcpu, VM_REG_GUEST_TR);
960 	if (ISLDT(ot_sel) || IDXSEL(ot_sel) == 0) {
961 		/*
962 		 * This might happen if a task switch was attempted without
963 		 * ever loading the task register with LTR. In this case the
964 		 * TR would contain the values from power-on:
965 		 * (sel = 0, base = 0, limit = 0xffff).
966 		 */
967 		sel_exception(ctx, vcpu, IDT_TS, ot_sel, task_switch->ext);
968 		goto done;
969 	}
970 
971 	/* Get the old TSS base and limit from the guest's task register */
972 	error = vm_get_desc(ctx, vcpu, VM_REG_GUEST_TR, &ot_base, &ot_lim,
973 	    &access);
974 	assert(error == 0);
975 	assert(!SEG_DESC_UNUSABLE(access) && SEG_DESC_PRESENT(access));
976 	ot_type = SEG_DESC_TYPE(access);
977 	assert(ot_type == SDT_SYS386BSY || ot_type == SDT_SYS286BSY);
978 
979 	/* Fetch the old TSS descriptor */
980 	error = read_tss_descriptor(ctx, vcpu, task_switch, ot_sel, &ot_desc,
981 	    &fault);
982 	CHKERR(error, fault);
983 
984 	/* Get the old TSS */
985 	error = vm_copy_setup(ctx, vcpu, &sup_paging, ot_base, minlimit + 1,
986 	    PROT_READ | PROT_WRITE, ot_iov, nitems(ot_iov), &fault);
987 	CHKERR(error, fault);
988 	vm_copyin(ctx, vcpu, ot_iov, &oldtss, minlimit + 1);
989 
990 	/*
991 	 * Clear the busy bit in the old TSS descriptor if the task switch
992 	 * due to an IRET or JMP instruction.
993 	 */
994 	if (reason == TSR_IRET || reason == TSR_JMP) {
995 		ot_desc.sd_type &= ~0x2;
996 		error = desc_table_write(ctx, vcpu, &sup_paging, ot_sel,
997 		    &ot_desc, &fault);
998 		CHKERR(error, fault);
999 	}
1000 
1001 	if (nt_type == SDT_SYS286BSY || nt_type == SDT_SYS286TSS) {
1002 		EPRINTLN("Task switch to 16-bit TSS not supported");
1003 		return (VMEXIT_ABORT);
1004 	}
1005 
1006 	/* Save processor state in old TSS */
1007 	tss32_save(ctx, vcpu, task_switch, eip, &oldtss, ot_iov);
1008 
1009 	/*
1010 	 * If the task switch was triggered for any reason other than IRET
1011 	 * then set the busy bit in the new TSS descriptor.
1012 	 */
1013 	if (reason != TSR_IRET) {
1014 		nt_desc.sd_type |= 0x2;
1015 		error = desc_table_write(ctx, vcpu, &sup_paging, nt_sel,
1016 		    &nt_desc, &fault);
1017 		CHKERR(error, fault);
1018 	}
1019 
1020 	/* Update task register to point at the new TSS */
1021 	SETREG(ctx, vcpu, VM_REG_GUEST_TR, nt_sel);
1022 
1023 	/* Update the hidden descriptor state of the task register */
1024 	nt = usd_to_seg_desc(&nt_desc);
1025 	update_seg_desc(ctx, vcpu, VM_REG_GUEST_TR, &nt);
1026 
1027 	/* Set CR0.TS */
1028 	cr0 = GETREG(ctx, vcpu, VM_REG_GUEST_CR0);
1029 	SETREG(ctx, vcpu, VM_REG_GUEST_CR0, cr0 | CR0_TS);
1030 
1031 	/*
1032 	 * We are now committed to the task switch. Any exceptions encountered
1033 	 * after this point will be handled in the context of the new task and
1034 	 * the saved instruction pointer will belong to the new task.
1035 	 */
1036 	error = vm_set_register(ctx, vcpu, VM_REG_GUEST_RIP, newtss.tss_eip);
1037 	assert(error == 0);
1038 
1039 	/* Load processor state from new TSS */
1040 	error = tss32_restore(ctx, vcpu, task_switch, ot_sel, &newtss, nt_iov,
1041 	    &fault);
1042 	CHKERR(error, fault);
1043 
1044 	/*
1045 	 * Section "Interrupt Tasks" in Intel SDM, Vol 3: if an exception
1046 	 * caused an error code to be generated, this error code is copied
1047 	 * to the stack of the new task.
1048 	 */
1049 	if (task_switch->errcode_valid) {
1050 		assert(task_switch->ext);
1051 		assert(task_switch->reason == TSR_IDT_GATE);
1052 		error = push_errcode(ctx, vcpu, &task_switch->paging, nt_type,
1053 		    task_switch->errcode, &fault);
1054 		CHKERR(error, fault);
1055 	}
1056 
1057 	/*
1058 	 * Treatment of virtual-NMI blocking if NMI is delivered through
1059 	 * a task gate.
1060 	 *
1061 	 * Section "Architectural State Before A VM Exit", Intel SDM, Vol3:
1062 	 * If the virtual NMIs VM-execution control is 1, VM entry injects
1063 	 * an NMI, and delivery of the NMI causes a task switch that causes
1064 	 * a VM exit, virtual-NMI blocking is in effect before the VM exit
1065 	 * commences.
1066 	 *
1067 	 * Thus, virtual-NMI blocking is in effect at the time of the task
1068 	 * switch VM exit.
1069 	 */
1070 
1071 	/*
1072 	 * Treatment of virtual-NMI unblocking on IRET from NMI handler task.
1073 	 *
1074 	 * Section "Changes to Instruction Behavior in VMX Non-Root Operation"
1075 	 * If "virtual NMIs" control is 1 IRET removes any virtual-NMI blocking.
1076 	 * This unblocking of virtual-NMI occurs even if IRET causes a fault.
1077 	 *
1078 	 * Thus, virtual-NMI blocking is cleared at the time of the task switch
1079 	 * VM exit.
1080 	 */
1081 
1082 	/*
1083 	 * If the task switch was triggered by an event delivered through
1084 	 * the IDT then extinguish the pending event from the vcpu's
1085 	 * exitintinfo.
1086 	 */
1087 	if (task_switch->reason == TSR_IDT_GATE) {
1088 		error = vm_set_intinfo(ctx, vcpu, 0);
1089 		assert(error == 0);
1090 	}
1091 
1092 	/*
1093 	 * XXX should inject debug exception if 'T' bit is 1
1094 	 */
1095 done:
1096 	return (VMEXIT_CONTINUE);
1097 }
1098