1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11 /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
12
13 /*
14 * Copyright (c) 2018, Joyent, Inc.
15 * Copyright 2022 Oxide Computer Company
16 */
17
18 #include <sys/cpuvar.h>
19 #include <sys/debug.h>
20 #include <sys/kmem.h>
21 #include <sys/ksynch.h>
22 #include <sys/list.h>
23 #include <sys/types.h>
24 #include <sys/vmm.h>
25 #include <sys/vmm_kernel.h>
26 #include <sys/vmm_impl.h>
27 #include <sys/zone.h>
28
29 /*
30 * zone specific data
31 *
32 * Zone specific data is used to keep an association between zones and the vmm
33 * instances that may be running in them. This is used to ensure that vmm
34 * instances do not outlive their parent zone.
35 *
36 * Locking strategy
37 *
38 * The global vmm_zsd_lock is held while modifying vmm_zsd_list.
39 *
40 * The per zone vz_lock in vmm_zsd_t is held while reading or writing anything
41 * within in vmm_zsd_t instance. This is important to ensure that there's not
42 * an accidental VM creating as a zone is going down.
43 */
44
45 /*
46 * One of these per zone.
47 */
48 struct vmm_zsd {
49 list_t vz_vmms; /* vmm instances in the zone */
50 list_node_t vz_linkage; /* link to other zones */
51 boolean_t vz_active; /* B_FALSE early in shutdown callback */
52 zoneid_t vz_zoneid;
53 kmutex_t vz_lock;
54 };
55
56 static kmutex_t vmm_zsd_lock; /* Protects vmm_zsd_list */
57 static list_t vmm_zsd_list; /* Linkage between all zsd instances */
58
59 static zone_key_t vmm_zsd_key;
60
61 int
vmm_zsd_add_vm(vmm_softc_t * sc)62 vmm_zsd_add_vm(vmm_softc_t *sc)
63 {
64 vmm_zsd_t *zsd;
65
66 ASSERT(sc->vmm_zone != NULL);
67
68 mutex_enter(&vmm_zsd_lock);
69
70 for (zsd = list_head(&vmm_zsd_list); zsd != NULL;
71 zsd = list_next(&vmm_zsd_list, zsd)) {
72 if (zsd->vz_zoneid == sc->vmm_zone->zone_id) {
73 break;
74 }
75 }
76
77 VERIFY(zsd != NULL);
78 mutex_exit(&vmm_zsd_lock);
79
80 mutex_enter(&zsd->vz_lock);
81 if (!zsd->vz_active) {
82 mutex_exit(&zsd->vz_lock);
83 return (ENOSYS);
84 }
85
86 sc->vmm_zsd = zsd;
87 list_insert_tail(&zsd->vz_vmms, sc);
88
89 mutex_exit(&zsd->vz_lock);
90
91 return (0);
92 }
93
94 void
vmm_zsd_rem_vm(vmm_softc_t * sc)95 vmm_zsd_rem_vm(vmm_softc_t *sc)
96 {
97 vmm_zsd_t *zsd = sc->vmm_zsd;
98
99 mutex_enter(&zsd->vz_lock);
100
101 list_remove(&zsd->vz_vmms, sc);
102 sc->vmm_zsd = NULL;
103
104 mutex_exit(&zsd->vz_lock);
105 }
106
107 static void *
vmm_zsd_create(zoneid_t zid)108 vmm_zsd_create(zoneid_t zid)
109 {
110 vmm_zsd_t *zsd;
111 zone_t *zone;
112
113 zsd = kmem_zalloc(sizeof (*zsd), KM_SLEEP);
114
115 list_create(&zsd->vz_vmms, sizeof (vmm_softc_t),
116 offsetof(vmm_softc_t, vmm_zsd_linkage));
117
118 zsd->vz_zoneid = zid;
119
120 mutex_init(&zsd->vz_lock, NULL, MUTEX_DEFAULT, NULL);
121
122 /*
123 * If the vmm module is loaded while this zone is in the midst of
124 * shutting down, vmm_zsd_destroy() may be called without
125 * vmm_zsd_shutdown() ever being called. If it is shutting down, there
126 * is no sense in letting any in-flight VM creation succeed so set
127 * vz_active accordingly.
128 *
129 * zone_find_by_id_nolock() is used rather than zone_find_by_id()
130 * so that the zone is returned regardless of state.
131 */
132 zone = zone_find_by_id_nolock(zid);
133 VERIFY(zone != NULL);
134 zsd->vz_active = zone_status_get(zone) < ZONE_IS_SHUTTING_DOWN;
135
136 mutex_enter(&vmm_zsd_lock);
137 list_insert_tail(&vmm_zsd_list, zsd);
138 mutex_exit(&vmm_zsd_lock);
139
140 return (zsd);
141 }
142
143 /*
144 * Tells all running VMs in the zone to poweroff. This does not reclaim guest
145 * resources (memory, etc.).
146 */
147 static void
vmm_zsd_shutdown(zoneid_t zid,void * data)148 vmm_zsd_shutdown(zoneid_t zid, void *data)
149 {
150 vmm_zsd_t *zsd = data;
151 vmm_softc_t *sc;
152
153 mutex_enter(&zsd->vz_lock);
154
155 /*
156 * This may already be B_FALSE. See comment in vmm_zsd_create(). If it
157 * is already B_FALSE we will take a quick trip through the empty list.
158 */
159 zsd->vz_active = B_FALSE;
160
161 for (sc = list_head(&zsd->vz_vmms); sc != NULL;
162 sc = list_next(&zsd->vz_vmms, sc)) {
163 /* Send a poweroff to the VM, whether running or not. */
164 (void) vm_suspend(sc->vmm_vm, VM_SUSPEND_POWEROFF, -1);
165 }
166 mutex_exit(&zsd->vz_lock);
167 }
168
169 /*
170 * Reap all VMs that remain and free up guest resources.
171 */
172 static void
vmm_zsd_destroy(zoneid_t zid,void * data)173 vmm_zsd_destroy(zoneid_t zid, void *data)
174 {
175 vmm_zsd_t *zsd = data;
176 vmm_softc_t *sc;
177
178 mutex_enter(&vmm_zsd_lock);
179 list_remove(&vmm_zsd_list, zsd);
180 mutex_exit(&vmm_zsd_lock);
181
182 mutex_enter(&zsd->vz_lock);
183 ASSERT(!zsd->vz_active);
184
185 while ((sc = list_remove_head(&zsd->vz_vmms)) != NULL) {
186 vmm_zone_vm_destroy(sc);
187 }
188
189 mutex_exit(&zsd->vz_lock);
190 mutex_destroy(&zsd->vz_lock);
191
192 kmem_free(zsd, sizeof (*zsd));
193 }
194
195 void
vmm_zsd_init(void)196 vmm_zsd_init(void)
197 {
198 mutex_init(&vmm_zsd_lock, NULL, MUTEX_DEFAULT, NULL);
199 list_create(&vmm_zsd_list, sizeof (vmm_zsd_t),
200 offsetof(vmm_zsd_t, vz_linkage));
201 zone_key_create(&vmm_zsd_key, vmm_zsd_create, vmm_zsd_shutdown,
202 vmm_zsd_destroy);
203 }
204
205 void
vmm_zsd_fini(void)206 vmm_zsd_fini(void)
207 {
208 /* Calls vmm_zsd_destroy() on all zones. */
209 VERIFY0(zone_key_delete(vmm_zsd_key));
210
211 ASSERT(list_is_empty(&vmm_zsd_list));
212 list_destroy(&vmm_zsd_list);
213 mutex_destroy(&vmm_zsd_lock);
214 }
215