1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30 /*
31 * This file and its contents are supplied under the terms of the
32 * Common Development and Distribution License ("CDDL"), version 1.0.
33 * You may only use this file in accordance with the terms of version
34 * 1.0 of the CDDL.
35 *
36 * A full copy of the text of the CDDL should have accompanied this
37 * source. A copy of the CDDL is also available via the Internet at
38 * http://www.illumos.org/license/CDDL.
39 *
40 * Copyright 2015 Pluribus Networks Inc.
41 */
42
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD$");
45
46 #include <sys/param.h>
47 #include <sys/kernel.h>
48 #include <sys/types.h>
49 #include <sys/systm.h>
50 #include <sys/smp.h>
51 #include <sys/sysctl.h>
52 #ifndef __FreeBSD__
53 #include <sys/hma.h>
54 #endif
55
56 #include <vm/vm.h>
57 #include <vm/pmap.h>
58 #include <vm/vm_extern.h>
59
60 #include <machine/vmm.h>
61
62 #include "ept.h"
63
64 #define EPT_SUPPORTS_EXEC_ONLY(cap) ((cap) & (1UL << 0))
65 #define EPT_PWL4(cap) ((cap) & (1UL << 6))
66 #define EPT_MEMORY_TYPE_WB(cap) ((cap) & (1UL << 14))
67 #define EPT_PDE_SUPERPAGE(cap) ((cap) & (1UL << 16)) /* 2MB pages */
68 #define EPT_PDPTE_SUPERPAGE(cap) ((cap) & (1UL << 17)) /* 1GB pages */
69 #define INVEPT_SUPPORTED(cap) ((cap) & (1UL << 20))
70 #define AD_BITS_SUPPORTED(cap) ((cap) & (1UL << 21))
71 #define INVVPID_SUPPORTED(cap) ((cap) & (1UL << 32))
72
73 #define INVVPID_ALL_TYPES_MASK 0xF0000000000UL
74 #define INVVPID_ALL_TYPES_SUPPORTED(cap) \
75 (((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK)
76
77 #define INVEPT_ALL_TYPES_MASK 0x6000000UL
78 #define INVEPT_ALL_TYPES_SUPPORTED(cap) \
79 (((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK)
80
81 #define EPT_PWLEVELS 4 /* page walk levels */
82 #define EPT_ENABLE_AD_BITS (1 << 6)
83
84 SYSCTL_DECL(_hw_vmm);
85 SYSCTL_NODE(_hw_vmm, OID_AUTO, ept, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
86 NULL);
87
88 static int ept_enable_ad_bits;
89
90 static int ept_pmap_flags;
91
92 int
ept_init(int ipinum)93 ept_init(int ipinum)
94 {
95 int use_hw_ad_bits, use_superpages, use_exec_only;
96 uint64_t cap;
97
98 cap = rdmsr(MSR_VMX_EPT_VPID_CAP);
99
100 /*
101 * Verify that:
102 * - page walk length is 4 steps
103 * - extended page tables can be laid out in write-back memory
104 * - invvpid instruction with all possible types is supported
105 * - invept instruction with all possible types is supported
106 */
107 if (!EPT_PWL4(cap) ||
108 !EPT_MEMORY_TYPE_WB(cap) ||
109 !INVVPID_SUPPORTED(cap) ||
110 !INVVPID_ALL_TYPES_SUPPORTED(cap) ||
111 !INVEPT_SUPPORTED(cap) ||
112 !INVEPT_ALL_TYPES_SUPPORTED(cap))
113 return (EINVAL);
114
115 ept_pmap_flags = ipinum & PMAP_NESTED_IPIMASK;
116
117 use_superpages = 1;
118 TUNABLE_INT_FETCH("hw.vmm.ept.use_superpages", &use_superpages);
119 if (use_superpages && EPT_PDE_SUPERPAGE(cap))
120 ept_pmap_flags |= PMAP_PDE_SUPERPAGE; /* 2MB superpage */
121
122 use_hw_ad_bits = 1;
123 TUNABLE_INT_FETCH("hw.vmm.ept.use_hw_ad_bits", &use_hw_ad_bits);
124 if (use_hw_ad_bits && AD_BITS_SUPPORTED(cap))
125 ept_enable_ad_bits = 1;
126 else
127 ept_pmap_flags |= PMAP_EMULATE_AD_BITS;
128
129 use_exec_only = 1;
130 TUNABLE_INT_FETCH("hw.vmm.ept.use_exec_only", &use_exec_only);
131 if (use_exec_only && EPT_SUPPORTS_EXEC_ONLY(cap))
132 ept_pmap_flags |= PMAP_SUPPORTS_EXEC_ONLY;
133
134 return (0);
135 }
136
137 void
ept_invalidate_mappings(ulong_t eptp)138 ept_invalidate_mappings(ulong_t eptp)
139 {
140 hma_vmx_invept_allcpus((uintptr_t)eptp);
141 }
142
143 static int
ept_pinit(pmap_t pmap)144 ept_pinit(pmap_t pmap)
145 {
146
147 return (pmap_pinit_type(pmap, PT_EPT, ept_pmap_flags));
148 }
149
150 struct vmspace *
ept_vmspace_alloc(vm_offset_t min,vm_offset_t max)151 ept_vmspace_alloc(vm_offset_t min, vm_offset_t max)
152 {
153
154 return (vmspace_alloc(min, max, ept_pinit));
155 }
156
157 void
ept_vmspace_free(struct vmspace * vmspace)158 ept_vmspace_free(struct vmspace *vmspace)
159 {
160
161 vmspace_free(vmspace);
162 }
163
164 uint64_t
eptp(uint64_t pml4)165 eptp(uint64_t pml4)
166 {
167 uint64_t eptp_val;
168
169 eptp_val = pml4 | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK;
170 if (ept_enable_ad_bits)
171 eptp_val |= EPT_ENABLE_AD_BITS;
172
173 return (eptp_val);
174 }
175