xref: /illumos-gate/usr/src/uts/intel/io/vmm/io/iommu.c (revision e760f150)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 /*
31  * This file and its contents are supplied under the terms of the
32  * Common Development and Distribution License ("CDDL"), version 1.0.
33  * You may only use this file in accordance with the terms of version
34  * 1.0 of the CDDL.
35  *
36  * A full copy of the text of the CDDL should have accompanied this
37  * source.  A copy of the CDDL is also available via the Internet at
38  * http://www.illumos.org/license/CDDL.
39  *
40  * Copyright 2022 Oxide Computer Company
41  */
42 
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD$");
45 
46 #include <sys/param.h>
47 #include <sys/bus.h>
48 #include <sys/eventhandler.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 
52 #include <dev/pci/pcivar.h>
53 #include <dev/pci/pcireg.h>
54 
55 #include <machine/cpu.h>
56 #include <machine/md_var.h>
57 
58 #include <sys/ddi.h>
59 #include <sys/sunddi.h>
60 #include <sys/pci.h>
61 
62 #include "vmm_util.h"
63 #include "iommu.h"
64 
65 
66 static kmutex_t iommu_lock;
67 
68 static uint_t iommu_refcnt;
69 ddi_modhandle_t iommu_modhdl;
70 static const struct iommu_ops *ops;
71 static void *host_domain;
72 
73 static volatile uint_t iommu_initted;
74 
75 static int
iommu_find_device(dev_info_t * dip,void * arg)76 iommu_find_device(dev_info_t *dip, void *arg)
77 {
78 	boolean_t add = (boolean_t)arg;
79 
80 	if (pcie_is_pci_device(dip)) {
81 		if (add)
82 			iommu_add_device(host_domain, pci_get_rid(dip));
83 		else
84 			iommu_remove_device(host_domain, pci_get_rid(dip));
85 	}
86 
87 	return (DDI_WALK_CONTINUE);
88 }
89 
90 static vm_paddr_t
vmm_mem_maxaddr(void)91 vmm_mem_maxaddr(void)
92 {
93 	return (ptoa(physmax + 1));
94 }
95 
96 static int
iommu_init(void)97 iommu_init(void)
98 {
99 	const char *mod_name;
100 	int error = 0;
101 
102 	ASSERT(MUTEX_HELD(&iommu_lock));
103 
104 	if (vmm_is_intel()) {
105 		mod_name = "misc/vmm_vtd";
106 	} else if (vmm_is_svm()) {
107 		/* Use the expected name for if/when this is ported */
108 		mod_name = "misc/vmm_amdvi";
109 	} else {
110 		return (ENXIO);
111 	}
112 
113 	/* Load the backend driver */
114 	iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error);
115 	if (iommu_modhdl == NULL) {
116 		return (error);
117 	}
118 
119 	/* Locate the iommu_ops struct */
120 	ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error);
121 	if (ops == NULL) {
122 		goto bail;
123 	}
124 
125 	/* Initialize the backend */
126 	error = ops->init();
127 	if (error != 0) {
128 		goto bail;
129 	}
130 
131 	/* Create a domain for the devices owned by the host */
132 	const vm_paddr_t maxaddr = vmm_mem_maxaddr();
133 	host_domain = ops->create_domain(maxaddr);
134 	if (host_domain == NULL) {
135 		goto bail;
136 	}
137 
138 	/* ... and populate it with 1:1 mappings for all of physical mem */
139 	iommu_create_mapping(host_domain, 0, 0, maxaddr);
140 
141 	ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE);
142 	ops->enable();
143 
144 	return (0);
145 
146 bail:
147 	if (ops != NULL) {
148 		ops->cleanup();
149 		ops = NULL;
150 	}
151 	if (iommu_modhdl != NULL) {
152 		(void) ddi_modclose(iommu_modhdl);
153 		iommu_modhdl = NULL;
154 	}
155 	return (error);
156 }
157 
158 static void
iommu_cleanup(void)159 iommu_cleanup(void)
160 {
161 	ASSERT(MUTEX_HELD(&iommu_lock));
162 	ASSERT3P(ops, !=, NULL);
163 	ASSERT0(iommu_refcnt);
164 
165 	ops->disable();
166 	ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE);
167 
168 	ops->destroy_domain(host_domain);
169 	host_domain = NULL;
170 
171 	ops->cleanup();
172 	ops = NULL;
173 
174 	(void) ddi_modclose(iommu_modhdl);
175 	iommu_modhdl = NULL;
176 }
177 
178 static bool
iommu_ref(void)179 iommu_ref(void)
180 {
181 	mutex_enter(&iommu_lock);
182 	if (ops == NULL) {
183 		int err = iommu_init();
184 
185 		if (err != 0) {
186 			VERIFY3P(ops, ==, NULL);
187 			mutex_exit(&iommu_lock);
188 			return (false);
189 		}
190 		VERIFY3P(ops, !=, NULL);
191 	}
192 	iommu_refcnt++;
193 	VERIFY3U(iommu_refcnt, <, UINT_MAX);
194 	mutex_exit(&iommu_lock);
195 
196 	return (true);
197 }
198 
199 static void
iommu_unref(void)200 iommu_unref(void)
201 {
202 	mutex_enter(&iommu_lock);
203 	VERIFY3U(iommu_refcnt, >, 0);
204 	iommu_refcnt--;
205 	if (iommu_refcnt == 0) {
206 		iommu_cleanup();
207 		VERIFY3P(ops, ==, NULL);
208 	}
209 	mutex_exit(&iommu_lock);
210 }
211 
212 void *
iommu_create_domain(vm_paddr_t maxaddr)213 iommu_create_domain(vm_paddr_t maxaddr)
214 {
215 	if (iommu_ref()) {
216 		return (ops->create_domain(maxaddr));
217 	} else {
218 		return (NULL);
219 	}
220 }
221 
222 void
iommu_destroy_domain(void * domain)223 iommu_destroy_domain(void *domain)
224 {
225 	ASSERT3P(domain, !=, NULL);
226 
227 	ops->destroy_domain(domain);
228 	iommu_unref();
229 }
230 
231 void
iommu_create_mapping(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)232 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
233 {
234 	uint64_t remaining = len;
235 
236 	ASSERT3P(domain, !=, NULL);
237 
238 	while (remaining > 0) {
239 		uint64_t mapped;
240 
241 		mapped = ops->create_mapping(domain, gpa, hpa, remaining);
242 		gpa += mapped;
243 		hpa += mapped;
244 		remaining -= mapped;
245 	}
246 }
247 
248 void
iommu_remove_mapping(void * domain,vm_paddr_t gpa,size_t len)249 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len)
250 {
251 	uint64_t remaining = len;
252 
253 	ASSERT3P(domain, !=, NULL);
254 
255 	while (remaining > 0) {
256 		uint64_t unmapped;
257 
258 		unmapped = ops->remove_mapping(domain, gpa, remaining);
259 		gpa += unmapped;
260 		remaining -= unmapped;
261 	}
262 }
263 
264 void *
iommu_host_domain(void)265 iommu_host_domain(void)
266 {
267 	return (host_domain);
268 }
269 
270 void
iommu_add_device(void * domain,uint16_t rid)271 iommu_add_device(void *domain, uint16_t rid)
272 {
273 	ASSERT3P(domain, !=, NULL);
274 
275 	ops->add_device(domain, rid);
276 }
277 
278 void
iommu_remove_device(void * domain,uint16_t rid)279 iommu_remove_device(void *domain, uint16_t rid)
280 {
281 	ASSERT3P(domain, !=, NULL);
282 
283 	ops->remove_device(domain, rid);
284 }
285 
286 void
iommu_invalidate_tlb(void * domain)287 iommu_invalidate_tlb(void *domain)
288 {
289 	ASSERT3P(domain, !=, NULL);
290 
291 	ops->invalidate_tlb(domain);
292 }
293