xref: /illumos-gate/usr/src/uts/i86pc/io/vmm/io/iommu.c (revision b0de25cb)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/bus.h>
36 #include <sys/eventhandler.h>
37 #include <sys/sysctl.h>
38 #include <sys/systm.h>
39 
40 #include <dev/pci/pcivar.h>
41 #include <dev/pci/pcireg.h>
42 
43 #include <machine/cpu.h>
44 #include <machine/md_var.h>
45 
46 #include <sys/ddi.h>
47 #include <sys/sunddi.h>
48 #include <sys/pci.h>
49 
50 #include "vmm_util.h"
51 #include "iommu.h"
52 
53 static int iommu_avail;
54 
55 static int iommu_enable = 1;
56 
57 static const struct iommu_ops *ops;
58 static void *host_domain;
59 #ifdef __FreeBSD__
60 static eventhandler_tag add_tag, delete_tag;
61 #endif
62 
63 #ifndef __FreeBSD__
64 static volatile uint_t iommu_initted;
65 #endif
66 
67 static __inline int
IOMMU_INIT(void)68 IOMMU_INIT(void)
69 {
70 	if (ops != NULL)
71 		return ((*ops->init)());
72 	else
73 		return (ENXIO);
74 }
75 
76 static __inline void
IOMMU_CLEANUP(void)77 IOMMU_CLEANUP(void)
78 {
79 	if (ops != NULL && iommu_avail)
80 		(*ops->cleanup)();
81 }
82 
83 static __inline void *
IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)84 IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
85 {
86 
87 	if (ops != NULL && iommu_avail)
88 		return ((*ops->create_domain)(maxaddr));
89 	else
90 		return (NULL);
91 }
92 
93 static __inline void
IOMMU_DESTROY_DOMAIN(void * dom)94 IOMMU_DESTROY_DOMAIN(void *dom)
95 {
96 
97 	if (ops != NULL && iommu_avail)
98 		(*ops->destroy_domain)(dom);
99 }
100 
101 static __inline uint64_t
IOMMU_CREATE_MAPPING(void * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)102 IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
103 {
104 
105 	if (ops != NULL && iommu_avail)
106 		return ((*ops->create_mapping)(domain, gpa, hpa, len));
107 	else
108 		return (len);		/* XXX */
109 }
110 
111 static __inline uint64_t
IOMMU_REMOVE_MAPPING(void * domain,vm_paddr_t gpa,uint64_t len)112 IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
113 {
114 
115 	if (ops != NULL && iommu_avail)
116 		return ((*ops->remove_mapping)(domain, gpa, len));
117 	else
118 		return (len);		/* XXX */
119 }
120 
121 static __inline void
IOMMU_ADD_DEVICE(void * domain,uint16_t rid)122 IOMMU_ADD_DEVICE(void *domain, uint16_t rid)
123 {
124 
125 	if (ops != NULL && iommu_avail)
126 		(*ops->add_device)(domain, rid);
127 }
128 
129 static __inline void
IOMMU_REMOVE_DEVICE(void * domain,uint16_t rid)130 IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid)
131 {
132 
133 	if (ops != NULL && iommu_avail)
134 		(*ops->remove_device)(domain, rid);
135 }
136 
137 static __inline void
IOMMU_INVALIDATE_TLB(void * domain)138 IOMMU_INVALIDATE_TLB(void *domain)
139 {
140 
141 	if (ops != NULL && iommu_avail)
142 		(*ops->invalidate_tlb)(domain);
143 }
144 
145 static __inline void
IOMMU_ENABLE(void)146 IOMMU_ENABLE(void)
147 {
148 
149 	if (ops != NULL && iommu_avail)
150 		(*ops->enable)();
151 }
152 
153 static __inline void
IOMMU_DISABLE(void)154 IOMMU_DISABLE(void)
155 {
156 
157 	if (ops != NULL && iommu_avail)
158 		(*ops->disable)();
159 }
160 
161 #ifdef __FreeBSD__
162 static void
iommu_pci_add(void * arg,device_t dev)163 iommu_pci_add(void *arg, device_t dev)
164 {
165 
166 	/* Add new devices to the host domain. */
167 	iommu_add_device(host_domain, pci_get_rid(dev));
168 }
169 
170 static void
iommu_pci_delete(void * arg,device_t dev)171 iommu_pci_delete(void *arg, device_t dev)
172 {
173 
174 	iommu_remove_device(host_domain, pci_get_rid(dev));
175 }
176 #endif
177 
178 #ifndef __FreeBSD__
179 static int
iommu_find_device(dev_info_t * dip,void * arg)180 iommu_find_device(dev_info_t *dip, void *arg)
181 {
182 	boolean_t add = (boolean_t)arg;
183 
184 	if (pcie_is_pci_device(dip)) {
185 		if (add)
186 			iommu_add_device(host_domain, pci_get_rid(dip));
187 		else
188 			iommu_remove_device(host_domain, pci_get_rid(dip));
189 	}
190 
191 	return (DDI_WALK_CONTINUE);
192 }
193 
194 static vm_paddr_t
vmm_mem_maxaddr(void)195 vmm_mem_maxaddr(void)
196 {
197 	return (ptoa(physmax + 1));
198 }
199 #endif
200 
201 static void
iommu_init(void)202 iommu_init(void)
203 {
204 	int error;
205 	vm_paddr_t maxaddr;
206 
207 	if (!iommu_enable)
208 		return;
209 
210 	if (vmm_is_intel())
211 		ops = &iommu_ops_intel;
212 	else if (vmm_is_svm())
213 		ops = &iommu_ops_amd;
214 	else
215 		ops = NULL;
216 
217 	error = IOMMU_INIT();
218 	if (error)
219 		return;
220 
221 	iommu_avail = 1;
222 
223 	/*
224 	 * Create a domain for the devices owned by the host
225 	 */
226 	maxaddr = vmm_mem_maxaddr();
227 	host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
228 	if (host_domain == NULL) {
229 		printf("iommu_init: unable to create a host domain");
230 		IOMMU_CLEANUP();
231 		ops = NULL;
232 		iommu_avail = 0;
233 		return;
234 	}
235 
236 	/*
237 	 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
238 	 * the host
239 	 */
240 	iommu_create_mapping(host_domain, 0, 0, maxaddr);
241 
242 	ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE);
243 	IOMMU_ENABLE();
244 
245 }
246 
247 void
iommu_cleanup(void)248 iommu_cleanup(void)
249 {
250 #ifdef __FreeBSD__
251 	if (add_tag != NULL) {
252 		EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
253 		add_tag = NULL;
254 	}
255 	if (delete_tag != NULL) {
256 		EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
257 		delete_tag = NULL;
258 	}
259 #else
260 	atomic_store_rel_int(&iommu_initted, 0);
261 #endif
262 	IOMMU_DISABLE();
263 #ifndef __FreeBSD__
264 	ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE);
265 #endif
266 	IOMMU_DESTROY_DOMAIN(host_domain);
267 	IOMMU_CLEANUP();
268 #ifndef __FreeBSD__
269 	ops = NULL;
270 #endif
271 }
272 
273 void *
iommu_create_domain(vm_paddr_t maxaddr)274 iommu_create_domain(vm_paddr_t maxaddr)
275 {
276 	if (iommu_initted < 2) {
277 		if (atomic_cmpset_int(&iommu_initted, 0, 1)) {
278 			iommu_init();
279 			atomic_store_rel_int(&iommu_initted, 2);
280 		} else
281 			while (iommu_initted == 1)
282 				cpu_spinwait();
283 	}
284 	return (IOMMU_CREATE_DOMAIN(maxaddr));
285 }
286 
287 void
iommu_destroy_domain(void * dom)288 iommu_destroy_domain(void *dom)
289 {
290 
291 	IOMMU_DESTROY_DOMAIN(dom);
292 }
293 
294 void
iommu_create_mapping(void * dom,vm_paddr_t gpa,vm_paddr_t hpa,size_t len)295 iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
296 {
297 	uint64_t mapped, remaining;
298 
299 	remaining = len;
300 
301 	while (remaining > 0) {
302 		mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
303 		gpa += mapped;
304 		hpa += mapped;
305 		remaining -= mapped;
306 	}
307 }
308 
309 void
iommu_remove_mapping(void * dom,vm_paddr_t gpa,size_t len)310 iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
311 {
312 	uint64_t unmapped, remaining;
313 
314 	remaining = len;
315 
316 	while (remaining > 0) {
317 		unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
318 		gpa += unmapped;
319 		remaining -= unmapped;
320 	}
321 }
322 
323 void *
iommu_host_domain(void)324 iommu_host_domain(void)
325 {
326 
327 	return (host_domain);
328 }
329 
330 void
iommu_add_device(void * dom,uint16_t rid)331 iommu_add_device(void *dom, uint16_t rid)
332 {
333 
334 	IOMMU_ADD_DEVICE(dom, rid);
335 }
336 
337 void
iommu_remove_device(void * dom,uint16_t rid)338 iommu_remove_device(void *dom, uint16_t rid)
339 {
340 
341 	IOMMU_REMOVE_DEVICE(dom, rid);
342 }
343 
344 void
iommu_invalidate_tlb(void * domain)345 iommu_invalidate_tlb(void *domain)
346 {
347 
348 	IOMMU_INVALIDATE_TLB(domain);
349 }
350