1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <sys/sunddi.h>
27 #include <sys/sunndi.h>
28 #include <sys/acpi/acpi.h>
29 #include <sys/acpica.h>
30 #include <sys/amd_iommu.h>
31 #include <sys/bootconf.h>
32 #include <sys/sysmacros.h>
33 #include <sys/ddidmareq.h>
34 
35 #include "amd_iommu_impl.h"
36 #include "amd_iommu_acpi.h"
37 #include "amd_iommu_page_tables.h"
38 
39 ddi_dma_attr_t amd_iommu_pgtable_dma_attr = {
40 	DMA_ATTR_V0,
41 	0U,				/* dma_attr_addr_lo */
42 	0xffffffffffffffffULL,		/* dma_attr_addr_hi */
43 	0xffffffffU,			/* dma_attr_count_max */
44 	(uint64_t)4096,			/* dma_attr_align */
45 	1,				/* dma_attr_burstsizes */
46 	64,				/* dma_attr_minxfer */
47 	0xffffffffU,			/* dma_attr_maxxfer */
48 	0xffffffffU,			/* dma_attr_seg */
49 	1,				/* dma_attr_sgllen, variable */
50 	64,				/* dma_attr_granular */
51 	0				/* dma_attr_flags */
52 };
53 
54 static amd_iommu_domain_t **amd_iommu_domain_table;
55 
56 static struct {
57 	int f_count;
58 	amd_iommu_page_table_t *f_list;
59 } amd_iommu_pgtable_freelist;
60 int amd_iommu_no_pgtable_freelist;
61 
62 /*ARGSUSED*/
63 static int
amd_iommu_get_src_bdf(amd_iommu_t * iommu,int32_t bdf,int32_t * src_bdfp)64 amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp)
65 {
66 	amd_iommu_acpi_ivhd_t *hinfop;
67 
68 	hinfop = amd_iommu_lookup_ivhd(bdf);
69 	if (hinfop == NULL) {
70 		if (bdf == -1) {
71 			*src_bdfp = bdf;
72 		} else {
73 			cmn_err(CE_WARN, "No IVHD entry for 0x%x", bdf);
74 			return (DDI_FAILURE);
75 		}
76 	} else if (hinfop->ach_src_deviceid == -1) {
77 		*src_bdfp = bdf;
78 	} else {
79 		*src_bdfp = hinfop->ach_src_deviceid;
80 	}
81 
82 	return (DDI_SUCCESS);
83 }
84 
85 /*ARGSUSED*/
86 static int
amd_iommu_get_domain(amd_iommu_t * iommu,dev_info_t * rdip,int alias,uint16_t deviceid,domain_id_t * domainid,const char * path)87 amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias,
88     uint16_t deviceid, domain_id_t *domainid, const char *path)
89 {
90 	const char *f = "amd_iommu_get_domain";
91 
92 	*domainid = AMD_IOMMU_INVALID_DOMAIN;
93 
94 	ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0);
95 
96 	switch (deviceid) {
97 		case AMD_IOMMU_INVALID_DOMAIN:
98 		case AMD_IOMMU_IDENTITY_DOMAIN:
99 		case AMD_IOMMU_PASSTHRU_DOMAIN:
100 		case AMD_IOMMU_SYS_DOMAIN:
101 			*domainid = AMD_IOMMU_SYS_DOMAIN;
102 			break;
103 		default:
104 			*domainid = deviceid;
105 			break;
106 	}
107 
108 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
109 		cmn_err(CE_NOTE, "%s: domainid for %s = %d",
110 		    f, path, *domainid);
111 	}
112 
113 	return (DDI_SUCCESS);
114 }
115 
116 static uint16_t
hash_domain(domain_id_t domainid)117 hash_domain(domain_id_t domainid)
118 {
119 	return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ);
120 }
121 
122 /*ARGSUSED*/
123 void
amd_iommu_init_page_tables(amd_iommu_t * iommu)124 amd_iommu_init_page_tables(amd_iommu_t *iommu)
125 {
126 	amd_iommu_domain_table = kmem_zalloc(
127 	    sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP);
128 }
129 
130 /*ARGSUSED*/
131 void
amd_iommu_fini_page_tables(amd_iommu_t * iommu)132 amd_iommu_fini_page_tables(amd_iommu_t *iommu)
133 {
134 	if (amd_iommu_domain_table) {
135 		kmem_free(amd_iommu_domain_table,
136 		    sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ);
137 		amd_iommu_domain_table = NULL;
138 	}
139 }
140 
141 static amd_iommu_domain_t *
amd_iommu_lookup_domain(amd_iommu_t * iommu,domain_id_t domainid,map_type_t type,int km_flags)142 amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid,
143     map_type_t type, int km_flags)
144 {
145 	uint16_t idx;
146 	amd_iommu_domain_t *dp;
147 	char name[AMD_IOMMU_VMEM_NAMELEN+1];
148 
149 	ASSERT(amd_iommu_domain_table);
150 
151 	idx = hash_domain(domainid);
152 
153 	for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) {
154 		if (dp->d_domainid == domainid)
155 			return (dp);
156 	}
157 
158 	ASSERT(type != AMD_IOMMU_INVALID_MAP);
159 
160 	dp = kmem_zalloc(sizeof (*dp), km_flags);
161 	if (dp == NULL)
162 		return (NULL);
163 	dp->d_domainid = domainid;
164 	dp->d_pgtable_root_4K = 0;	/* make this explicit */
165 
166 	if (type == AMD_IOMMU_VMEM_MAP) {
167 		uint64_t base;
168 		uint64_t size;
169 		(void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d",
170 		    iommu->aiomt_idx, domainid);
171 		base = MMU_PAGESIZE;
172 		size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE;
173 		dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size,
174 		    MMU_PAGESIZE, NULL, NULL, NULL, 0,
175 		    km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
176 		if (dp->d_vmem == NULL) {
177 			kmem_free(dp, sizeof (*dp));
178 			return (NULL);
179 		}
180 	} else {
181 		dp->d_vmem = NULL;
182 	}
183 
184 	dp->d_next = amd_iommu_domain_table[idx];
185 	dp->d_prev = NULL;
186 	amd_iommu_domain_table[idx] = dp;
187 	if (dp->d_next)
188 		dp->d_next->d_prev = dp;
189 	dp->d_ref = 0;
190 
191 
192 	return (dp);
193 }
194 
195 static void
amd_iommu_teardown_domain(amd_iommu_t * iommu,amd_iommu_domain_t * dp)196 amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp)
197 {
198 	uint16_t idx;
199 	int flags;
200 	amd_iommu_cmdargs_t cmdargs = {0};
201 	domain_id_t domainid = dp->d_domainid;
202 	const char *f = "amd_iommu_teardown_domain";
203 
204 	ASSERT(dp->d_ref == 0);
205 
206 	idx = hash_domain(dp->d_domainid);
207 
208 	if (dp->d_prev == NULL)
209 		amd_iommu_domain_table[idx] = dp->d_next;
210 	else
211 		dp->d_prev->d_next = dp->d_next;
212 
213 	if (dp->d_next)
214 		dp->d_next->d_prev = dp->d_prev;
215 
216 	if (dp->d_vmem != NULL) {
217 		vmem_destroy(dp->d_vmem);
218 		dp->d_vmem = NULL;
219 	}
220 
221 	kmem_free(dp, sizeof (*dp));
222 
223 	cmdargs.ca_domainid = (uint16_t)domainid;
224 	cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
225 	flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
226 	    AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
227 
228 	if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
229 	    &cmdargs, flags, 0) != DDI_SUCCESS) {
230 		cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
231 		    "Failed to invalidate domain in IOMMU HW cache",
232 		    f, iommu->aiomt_idx, cmdargs.ca_domainid);
233 	}
234 }
235 
236 static int
amd_iommu_get_deviceid(amd_iommu_t * iommu,dev_info_t * rdip,int32_t * deviceid,int * aliasp,const char * path)237 amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid,
238     int *aliasp, const char *path)
239 {
240 	int bus = -1;
241 	int device = -1;
242 	int func = -1;
243 	uint16_t bdf;
244 	int32_t src_bdf;
245 	dev_info_t *idip = iommu->aiomt_dip;
246 	const char *driver = ddi_driver_name(idip);
247 	int instance = ddi_get_instance(idip);
248 	dev_info_t *pci_dip;
249 	const char *f = "amd_iommu_get_deviceid";
250 
251 	/* be conservative. Always assume an alias */
252 	*aliasp = 1;
253 	*deviceid = 0;
254 
255 	/* Check for special special devices (rdip == NULL) */
256 	if (rdip == NULL) {
257 		if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) {
258 			cmn_err(CE_WARN,
259 			    "%s: %s%d: idx=%d, failed to get SRC BDF "
260 			    "for special-device",
261 			    f, driver, instance, iommu->aiomt_idx);
262 			return (DDI_DMA_NOMAPPING);
263 		}
264 		*deviceid = src_bdf;
265 		*aliasp = 1;
266 		return (DDI_SUCCESS);
267 	}
268 
269 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
270 		cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s",
271 		    f, path);
272 	}
273 
274 	pci_dip = amd_iommu_pci_dip(rdip, path);
275 	if (pci_dip == NULL) {
276 		cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
277 		    "for rdip=%p, path = %s",
278 		    f, driver, instance, iommu->aiomt_idx, (void *)rdip,
279 		    path);
280 		return (DDI_DMA_NOMAPPING);
281 	}
282 
283 	if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
284 		ndi_rele_devi(pci_dip);
285 		cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for "
286 		    "PCI dip (%p). rdip path = %s",
287 		    f, driver, instance, iommu->aiomt_idx,
288 		    (void *)pci_dip, path);
289 		return (DDI_DMA_NOMAPPING);
290 	}
291 
292 	ndi_rele_devi(pci_dip);
293 
294 	if (bus > UINT8_MAX || bus < 0 ||
295 	    device > UINT8_MAX || device < 0 ||
296 	    func > UINT8_MAX || func < 0) {
297 		cmn_err(CE_WARN, "%s: %s%d:  idx=%d, invalid BDF(%d,%d,%d) "
298 		    "for PCI dip (%p). rdip path = %s", f, driver, instance,
299 		    iommu->aiomt_idx,
300 		    bus, device, func,
301 		    (void *)pci_dip, path);
302 		return (DDI_DMA_NOMAPPING);
303 	}
304 
305 	bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
306 
307 	if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) {
308 		cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF "
309 		    "for PCI dip (%p) rdip path = %s.",
310 		    f, driver, instance, iommu->aiomt_idx, (void *)pci_dip,
311 		    path);
312 		return (DDI_DMA_NOMAPPING);
313 	}
314 
315 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
316 		cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s",
317 		    f, src_bdf, path);
318 	}
319 
320 	*deviceid = src_bdf;
321 	*aliasp = (src_bdf != bdf);
322 
323 	return (DDI_SUCCESS);
324 }
325 
326 /*ARGSUSED*/
327 static int
init_devtbl(amd_iommu_t * iommu,uint64_t * devtbl_entry,domain_id_t domainid,amd_iommu_domain_t * dp)328 init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid,
329     amd_iommu_domain_t *dp)
330 {
331 	uint64_t entry[4] = {0};
332 	int i;
333 
334 	/* If already passthru, don't touch */
335 	if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 &&
336 	    AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
337 		return (0);
338 	}
339 
340 	if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 &&
341 	    AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) {
342 
343 		ASSERT(dp->d_pgtable_root_4K ==
344 		    AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
345 		    AMD_IOMMU_DEVTBL_ROOT_PGTBL));
346 
347 		ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
348 		    AMD_IOMMU_DEVTBL_DOMAINID));
349 
350 		return (0);
351 	}
352 
353 	/* New devtbl entry for this domain. Bump up the domain ref-count */
354 	dp->d_ref++;
355 
356 	entry[3] = 0;
357 	entry[2] = 0;
358 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1);
359 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0);
360 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0);
361 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1);
362 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0);
363 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1);
364 	AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID,
365 	    (uint16_t)domainid);
366 	AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1);
367 	AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1);
368 	AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL,
369 	    dp->d_pgtable_root_4K);
370 	AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE,
371 	    AMD_IOMMU_PGTABLE_MAXLEVEL);
372 	AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV,
373 	    domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
374 	AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V,
375 	    domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
376 
377 	for (i = 1; i < 4; i++) {
378 		devtbl_entry[i] = entry[i];
379 	}
380 	devtbl_entry[0] = entry[0];
381 
382 	/* we did an actual init */
383 	return (1);
384 }
385 
386 void
amd_iommu_set_passthru(amd_iommu_t * iommu,dev_info_t * rdip)387 amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip)
388 {
389 	int32_t deviceid;
390 	int alias;
391 	uint64_t *devtbl_entry;
392 	amd_iommu_cmdargs_t cmdargs = {0};
393 	char *path;
394 	int pathfree;
395 	int V;
396 	int TV;
397 	int instance;
398 	const char *driver;
399 	const char *f = "amd_iommu_set_passthru";
400 
401 	if (rdip) {
402 		driver = ddi_driver_name(rdip);
403 		instance = ddi_get_instance(rdip);
404 	} else {
405 		driver = "special-device";
406 		instance = 0;
407 	}
408 
409 	path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
410 	if (path) {
411 		if (rdip)
412 			(void) ddi_pathname(rdip, path);
413 		else
414 			(void) strcpy(path, "special-device");
415 		pathfree = 1;
416 	} else {
417 		pathfree = 0;
418 		path = "<path-mem-alloc-failed>";
419 	}
420 
421 	if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
422 	    != DDI_SUCCESS) {
423 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
424 		    "Failed to get device ID for device %s.", f, driver,
425 		    instance,
426 		    iommu->aiomt_idx, (void *)rdip, path);
427 		goto out;
428 	}
429 
430 	/* No deviceid */
431 	if (deviceid == -1) {
432 		goto out;
433 	}
434 
435 	if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
436 	    iommu->aiomt_devtbl_sz) {
437 		cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
438 		    "for rdip (%p) exceeds device table size (%u), path=%s",
439 		    f, driver,
440 		    instance, iommu->aiomt_idx, deviceid, (void *)rdip,
441 		    iommu->aiomt_devtbl_sz, path);
442 		goto out;
443 	}
444 
445 	/*LINTED*/
446 	devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
447 	    [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
448 
449 	V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V);
450 	TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV);
451 
452 	/* Already passthru */
453 	if (V == 0 && TV == 0) {
454 		goto out;
455 	}
456 
457 	/* Existing translations */
458 	if (V == 1 && TV == 1) {
459 		goto out;
460 	}
461 
462 	/* Invalid setting */
463 	if (V == 0 && TV == 1) {
464 		goto out;
465 	}
466 
467 	AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0);
468 
469 	cmdargs.ca_deviceid = (uint16_t)deviceid;
470 	(void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
471 	    &cmdargs, 0, 0);
472 
473 out:
474 	if (pathfree)
475 		kmem_free(path, MAXPATHLEN);
476 }
477 
478 static int
amd_iommu_set_devtbl_entry(amd_iommu_t * iommu,dev_info_t * rdip,domain_id_t domainid,uint16_t deviceid,amd_iommu_domain_t * dp,const char * path)479 amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
480     domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
481     const char *path)
482 {
483 	uint64_t *devtbl_entry;
484 	amd_iommu_cmdargs_t cmdargs = {0};
485 	int error, flags;
486 	dev_info_t *idip = iommu->aiomt_dip;
487 	const char *driver = ddi_driver_name(idip);
488 	int instance = ddi_get_instance(idip);
489 	const char *f = "amd_iommu_set_devtbl_entry";
490 
491 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
492 		cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s",
493 		    f, path);
494 	}
495 
496 	if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
497 	    iommu->aiomt_devtbl_sz) {
498 		cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
499 		    "for rdip (%p) exceeds device table size (%u), path=%s",
500 		    f, driver,
501 		    instance, iommu->aiomt_idx, deviceid, (void *)rdip,
502 		    iommu->aiomt_devtbl_sz, path);
503 		return (DDI_DMA_NOMAPPING);
504 	}
505 
506 	/*LINTED*/
507 	devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
508 	    [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
509 
510 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
511 		cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
512 		    f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
513 	}
514 
515 	/*
516 	 * Flush internal caches, need to do this if we came up from
517 	 * fast boot
518 	 */
519 	cmdargs.ca_deviceid = deviceid;
520 	error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
521 	    &cmdargs, 0, 0);
522 	if (error != DDI_SUCCESS) {
523 		cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d"
524 		    "Failed to invalidate domain in IOMMU HW cache",
525 		    f, iommu->aiomt_idx, deviceid);
526 		return (error);
527 	}
528 
529 	cmdargs.ca_domainid = (uint16_t)domainid;
530 	cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
531 	flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
532 	    AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
533 
534 	error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
535 	    &cmdargs, flags, 0);
536 	if (error != DDI_SUCCESS) {
537 		cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
538 		    "Failed to invalidate translations in IOMMU HW cache",
539 		    f, iommu->aiomt_idx, cmdargs.ca_domainid);
540 		return (error);
541 	}
542 
543 	/* Initialize device table entry */
544 	if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {
545 		cmdargs.ca_deviceid = deviceid;
546 		error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
547 		    &cmdargs, 0, 0);
548 	}
549 
550 	return (error);
551 }
552 
553 int
amd_iommu_clear_devtbl_entry(amd_iommu_t * iommu,dev_info_t * rdip,domain_id_t domainid,uint16_t deviceid,amd_iommu_domain_t * dp,int * domain_freed,char * path)554 amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
555     domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
556     int *domain_freed, char *path)
557 {
558 	uint64_t *devtbl_entry;
559 	int error = DDI_SUCCESS;
560 	amd_iommu_cmdargs_t cmdargs = {0};
561 	const char *driver = ddi_driver_name(iommu->aiomt_dip);
562 	int instance = ddi_get_instance(iommu->aiomt_dip);
563 	const char *f = "amd_iommu_clear_devtbl_entry";
564 
565 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
566 		cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for "
567 		    "domainid = %d, deviceid = %u, path = %s",
568 		    f, domainid, deviceid, path);
569 	}
570 
571 	if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
572 	    iommu->aiomt_devtbl_sz) {
573 		cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
574 		    "for rdip (%p) exceeds device table size (%u), path = %s",
575 		    f, driver, instance,
576 		    iommu->aiomt_idx, deviceid, (void *)rdip,
577 		    iommu->aiomt_devtbl_sz, path);
578 		return (DDI_FAILURE);
579 	}
580 
581 	/*LINTED*/
582 	devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
583 	    [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
584 
585 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
586 		cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s",
587 		    f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
588 	}
589 
590 	if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
591 		/* Nothing to do */
592 		return (DDI_SUCCESS);
593 	}
594 
595 	ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
596 	    AMD_IOMMU_DEVTBL_ROOT_PGTBL));
597 
598 	ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
599 	    AMD_IOMMU_DEVTBL_DOMAINID));
600 
601 	AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0);
602 	AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0);
603 	AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1);
604 
605 	SYNC_FORDEV(iommu->aiomt_dmahdl);
606 
607 	dp->d_ref--;
608 	ASSERT(dp->d_ref >= 0);
609 
610 	if (dp->d_ref == 0) {
611 		*domain_freed = 1;
612 	}
613 
614 	cmdargs.ca_deviceid = deviceid;
615 	error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
616 	    &cmdargs, 0, 0);
617 	if (error != DDI_SUCCESS)
618 		error = DDI_FAILURE;
619 
620 	return (error);
621 }
622 
623 int
amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t * ampt)624 amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt)
625 {
626 	ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) *
627 	    AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP);
628 	return (DDI_SUCCESS);
629 }
630 
631 void
amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t * ampt)632 amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt)
633 {
634 	kmem_free(ampt->ampt_hash,
635 	    sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ);
636 	ampt->ampt_hash = NULL;
637 }
638 
639 static uint32_t
pt_hashfn(uint64_t pa_4K)640 pt_hashfn(uint64_t pa_4K)
641 {
642 	return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ);
643 }
644 
645 static void
amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t * pt)646 amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt)
647 {
648 	uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12;
649 	uint32_t idx = pt_hashfn(pa_4K);
650 
651 	ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
652 
653 	mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
654 
655 	pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx];
656 	pt->pt_prev = NULL;
657 	amd_iommu_page_table_hash.ampt_hash[idx] = pt;
658 	if (pt->pt_next)
659 		pt->pt_next->pt_prev = pt;
660 
661 	mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
662 }
663 
664 static void
amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t * pt)665 amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt)
666 {
667 	uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12);
668 	uint32_t idx = pt_hashfn(pa_4K);
669 
670 	ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
671 
672 	mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
673 
674 	if (pt->pt_next)
675 		pt->pt_next->pt_prev = pt->pt_prev;
676 
677 	if (pt->pt_prev)
678 		pt->pt_prev->pt_next = pt->pt_next;
679 	else
680 		amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next;
681 
682 	pt->pt_next = NULL;
683 	pt->pt_prev = NULL;
684 
685 	mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
686 }
687 
688 static amd_iommu_page_table_t *
amd_iommu_lookup_pgtable_hash(domain_id_t domainid,uint64_t pgtable_pa_4K)689 amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K)
690 {
691 	amd_iommu_page_table_t *pt;
692 	uint32_t idx = pt_hashfn(pgtable_pa_4K);
693 
694 	mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
695 	pt = amd_iommu_page_table_hash.ampt_hash[idx];
696 	for (; pt; pt = pt->pt_next) {
697 		if (domainid != pt->pt_domainid)
698 			continue;
699 		ASSERT((pt->pt_cookie.dmac_cookie_addr &
700 		    AMD_IOMMU_PGTABLE_ALIGN) == 0);
701 		if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) {
702 			break;
703 		}
704 	}
705 	mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
706 
707 	return (pt);
708 }
709 
710 /*ARGSUSED*/
711 static amd_iommu_page_table_t *
amd_iommu_lookup_pgtable(amd_iommu_t * iommu,amd_iommu_page_table_t * ppt,amd_iommu_domain_t * dp,int level,uint16_t index)712 amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt,
713     amd_iommu_domain_t *dp, int level, uint16_t index)
714 {
715 	uint64_t *pdtep;
716 	uint64_t pgtable_pa_4K;
717 
718 	ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
719 	ASSERT(dp);
720 
721 	if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
722 		ASSERT(ppt == NULL);
723 		ASSERT(index == 0);
724 		pgtable_pa_4K = dp->d_pgtable_root_4K;
725 	} else {
726 		ASSERT(ppt);
727 		pdtep = &(ppt->pt_pgtblva[index]);
728 		if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) {
729 			if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
730 				cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%"
731 				    PRIx64, *pdtep);
732 			}
733 			return (NULL);
734 		}
735 		pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR);
736 	}
737 
738 	return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K));
739 }
740 
741 static amd_iommu_page_table_t *
amd_iommu_alloc_from_freelist(void)742 amd_iommu_alloc_from_freelist(void)
743 {
744 	int i;
745 	uint64_t *pte_array;
746 	amd_iommu_page_table_t *pt;
747 
748 	if (amd_iommu_no_pgtable_freelist == 1)
749 		return (NULL);
750 
751 	if (amd_iommu_pgtable_freelist.f_count == 0)
752 		return (NULL);
753 
754 	pt = amd_iommu_pgtable_freelist.f_list;
755 	amd_iommu_pgtable_freelist.f_list = pt->pt_next;
756 	amd_iommu_pgtable_freelist.f_count--;
757 
758 	pte_array = pt->pt_pgtblva;
759 	for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
760 		ASSERT(pt->pt_pte_ref[i] == 0);
761 		ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
762 		    AMD_IOMMU_PTDE_PR)  == 0);
763 	}
764 
765 	return (pt);
766 }
767 
768 static int
amd_iommu_alloc_pgtable(amd_iommu_t * iommu,domain_id_t domainid,const char * path,amd_iommu_page_table_t ** ptp,int km_flags)769 amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid,
770     const char *path, amd_iommu_page_table_t **ptp, int km_flags)
771 {
772 	int err;
773 	uint_t ncookies;
774 	amd_iommu_page_table_t *pt;
775 	dev_info_t *idip = iommu->aiomt_dip;
776 	const char *driver = ddi_driver_name(idip);
777 	int instance = ddi_get_instance(idip);
778 	const char *f = "amd_iommu_alloc_pgtable";
779 
780 	*ptp = NULL;
781 
782 	pt = amd_iommu_alloc_from_freelist();
783 	if (pt)
784 		goto init_pgtable;
785 
786 	pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags);
787 	if (pt == NULL)
788 		return (DDI_DMA_NORESOURCES);
789 
790 	/*
791 	 * Each page table is 4K in size
792 	 */
793 	pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ;
794 
795 	/*
796 	 * Alloc a DMA handle. Use the IOMMU dip as we want this DMA
797 	 * to *not* enter the IOMMU - no recursive entrance.
798 	 */
799 	err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr,
800 	    km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
801 	    NULL, &pt->pt_dma_hdl);
802 	if (err != DDI_SUCCESS) {
803 		cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. "
804 		    "Cannot alloc DMA handle for IO Page Table",
805 		    f, driver, instance, domainid, path);
806 		kmem_free(pt, sizeof (amd_iommu_page_table_t));
807 		return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING);
808 	}
809 
810 	/*
811 	 * Alloc memory for IO Page Table.
812 	 * XXX remove size_t cast kludge
813 	 */
814 	err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz,
815 	    &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
816 	    km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
817 	    NULL, (caddr_t *)&pt->pt_pgtblva,
818 	    (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl);
819 	if (err != DDI_SUCCESS) {
820 		cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
821 		    "Cannot allocate DMA memory for IO Page table",
822 		    f, driver, instance, domainid, path);
823 		ddi_dma_free_handle(&pt->pt_dma_hdl);
824 		kmem_free(pt, sizeof (amd_iommu_page_table_t));
825 		return (DDI_DMA_NORESOURCES);
826 	}
827 
828 	/*
829 	 * The Page table DMA VA must be 4K aligned and
830 	 * size >= than requested memory.
831 	 *
832 	 */
833 	ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN)
834 	    == 0);
835 	ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
836 
837 	/*
838 	 * Now bind the handle
839 	 */
840 	err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL,
841 	    (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz,
842 	    DDI_DMA_READ | DDI_DMA_CONSISTENT,
843 	    km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
844 	    NULL, &pt->pt_cookie, &ncookies);
845 	if (err != DDI_DMA_MAPPED) {
846 		cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
847 		    "Cannot bind memory for DMA to IO Page Tables. "
848 		    "bufrealsz=%p",
849 		    f, driver, instance, domainid, path,
850 		    (void *)(uintptr_t)pt->pt_mem_realsz);
851 		ddi_dma_mem_free(&pt->pt_mem_hdl);
852 		ddi_dma_free_handle(&pt->pt_dma_hdl);
853 		kmem_free(pt, sizeof (amd_iommu_page_table_t));
854 		return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING :
855 		    err);
856 	}
857 
858 	/*
859 	 * We assume the DMA engine on the IOMMU is capable of handling the
860 	 * whole page table in a single cookie. If not and multiple cookies
861 	 * are needed we fail.
862 	 */
863 	if (ncookies != 1) {
864 		cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s "
865 		    "Cannot handle multiple "
866 		    "cookies for DMA to IO page Table, #cookies=%u",
867 		    f, driver, instance, domainid, path, ncookies);
868 		(void) ddi_dma_unbind_handle(pt->pt_dma_hdl);
869 		ddi_dma_mem_free(&pt->pt_mem_hdl);
870 		ddi_dma_free_handle(&pt->pt_dma_hdl);
871 		kmem_free(pt, sizeof (amd_iommu_page_table_t));
872 		return (DDI_DMA_NOMAPPING);
873 	}
874 
875 init_pgtable:
876 	/*
877 	 * The address in the cookie must be 4K aligned and >= table size
878 	 */
879 	ASSERT(pt->pt_cookie.dmac_cookie_addr != (uintptr_t)NULL);
880 	ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
881 	ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz);
882 	ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz);
883 	ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE);
884 	ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
885 	ASSERT(pt->pt_pgtblva);
886 
887 	pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN;
888 	pt->pt_level = 0x7;
889 	pt->pt_index = 0;
890 	pt->pt_ref = 0;
891 	pt->pt_next = NULL;
892 	pt->pt_prev = NULL;
893 	pt->pt_parent = NULL;
894 
895 	bzero(pt->pt_pgtblva, pt->pt_mem_realsz);
896 	SYNC_FORDEV(pt->pt_dma_hdl);
897 
898 	amd_iommu_insert_pgtable_hash(pt);
899 
900 	*ptp = pt;
901 
902 	return (DDI_SUCCESS);
903 }
904 
905 static int
amd_iommu_move_to_freelist(amd_iommu_page_table_t * pt)906 amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt)
907 {
908 	if (amd_iommu_no_pgtable_freelist == 1)
909 		return (DDI_FAILURE);
910 
911 	if (amd_iommu_pgtable_freelist.f_count ==
912 	    AMD_IOMMU_PGTABLE_FREELIST_MAX)
913 		return (DDI_FAILURE);
914 
915 	pt->pt_next = amd_iommu_pgtable_freelist.f_list;
916 	amd_iommu_pgtable_freelist.f_list = pt;
917 	amd_iommu_pgtable_freelist.f_count++;
918 
919 	return (DDI_SUCCESS);
920 }
921 
922 static void
amd_iommu_free_pgtable(amd_iommu_t * iommu,amd_iommu_page_table_t * pt)923 amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt)
924 {
925 	int i;
926 	uint64_t *pte_array;
927 	dev_info_t *dip = iommu->aiomt_dip;
928 	int instance = ddi_get_instance(dip);
929 	const char *driver = ddi_driver_name(dip);
930 	const char *f = "amd_iommu_free_pgtable";
931 
932 	ASSERT(pt->pt_ref == 0);
933 
934 	amd_iommu_remove_pgtable_hash(pt);
935 
936 	pte_array = pt->pt_pgtblva;
937 	for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
938 		ASSERT(pt->pt_pte_ref[i] == 0);
939 		ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
940 		    AMD_IOMMU_PTDE_PR)  == 0);
941 	}
942 
943 	if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS)
944 		return;
945 
946 	/* Unbind the handle */
947 	if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) {
948 		cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. "
949 		    "Failed to unbind handle: %p for IOMMU Page Table",
950 		    f, driver, instance, iommu->aiomt_idx, pt->pt_domainid,
951 		    (void *)pt->pt_dma_hdl);
952 	}
953 	/* Free the table memory allocated for DMA */
954 	ddi_dma_mem_free(&pt->pt_mem_hdl);
955 
956 	/* Free the DMA handle */
957 	ddi_dma_free_handle(&pt->pt_dma_hdl);
958 
959 	kmem_free(pt, sizeof (amd_iommu_page_table_t));
960 
961 }
962 
963 static int
init_pde(amd_iommu_page_table_t * ppt,amd_iommu_page_table_t * pt)964 init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt)
965 {
966 	uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]);
967 	uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
968 
969 	/* nothing to set. PDE is already set */
970 	if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) {
971 		ASSERT(PT_REF_VALID(ppt));
972 		ASSERT(PT_REF_VALID(pt));
973 		ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0);
974 		ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR)
975 		    == next_pgtable_pa_4K);
976 		return (DDI_SUCCESS);
977 	}
978 
979 	ppt->pt_ref++;
980 	ASSERT(PT_REF_VALID(ppt));
981 
982 	/* Page Directories are always RW */
983 	AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1);
984 	AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1);
985 	AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR,
986 	    next_pgtable_pa_4K);
987 	pt->pt_parent = ppt;
988 	AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL,
989 	    pt->pt_level);
990 	ppt->pt_pte_ref[pt->pt_index] = 0;
991 	AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1);
992 	SYNC_FORDEV(ppt->pt_dma_hdl);
993 	ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1);
994 
995 	return (DDI_SUCCESS);
996 }
997 
998 static int
init_pte(amd_iommu_page_table_t * pt,uint64_t pa,uint16_t index,struct ddi_dma_req * dmareq)999 init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index,
1000     struct ddi_dma_req *dmareq)
1001 {
1002 	uint64_t *ptep = &(pt->pt_pgtblva[index]);
1003 	uint64_t pa_4K = pa >> 12;
1004 	int R;
1005 	int W;
1006 
1007 	/* nothing to set if PTE is already set */
1008 	if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) {
1009 		/*
1010 		 * Adjust current permissions
1011 		 * DDI_DMA_WRITE means direction of DMA is MEM -> I/O
1012 		 * so that requires Memory READ permissions i.e. sense
1013 		 * is inverted.
1014 		 * Note: either or both of DD_DMA_READ/WRITE may be set
1015 		 */
1016 		if (amd_iommu_no_RW_perms == 0) {
1017 			R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR);
1018 			W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW);
1019 			if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) ||
1020 			    (dmareq->dmar_flags & DDI_DMA_RDWR))) {
1021 				AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1022 			}
1023 			if (W  == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) ||
1024 			    (dmareq->dmar_flags & DDI_DMA_RDWR))) {
1025 				AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1026 			}
1027 		}
1028 		ASSERT(PT_REF_VALID(pt));
1029 		pt->pt_pte_ref[index]++;
1030 		ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR)
1031 		    == pa_4K);
1032 		return (DDI_SUCCESS);
1033 	}
1034 
1035 	pt->pt_ref++;
1036 	ASSERT(PT_REF_VALID(pt));
1037 
1038 	/* see comment above about inverting sense of RD/WR */
1039 	if (amd_iommu_no_RW_perms == 0) {
1040 		AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0);
1041 		AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0);
1042 		if (dmareq->dmar_flags & DDI_DMA_RDWR) {
1043 			AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1044 			AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1045 		} else {
1046 			if (dmareq->dmar_flags & DDI_DMA_WRITE) {
1047 				AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1048 			}
1049 			if (dmareq->dmar_flags & DDI_DMA_READ) {
1050 				AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1051 			}
1052 		}
1053 	} else {
1054 		AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
1055 		AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
1056 	}
1057 
1058 	/* TODO what is correct for FC and U */
1059 	AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0);
1060 	AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0);
1061 	AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K);
1062 	AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0);
1063 	ASSERT(pt->pt_pte_ref[index] == 0);
1064 	pt->pt_pte_ref[index] = 1;
1065 	AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1);
1066 	SYNC_FORDEV(pt->pt_dma_hdl);
1067 	ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1);
1068 
1069 	return (DDI_SUCCESS);
1070 }
1071 
1072 
1073 static void
init_pt(amd_iommu_page_table_t * pt,amd_iommu_domain_t * dp,int level,uint16_t index)1074 init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp,
1075     int level, uint16_t index)
1076 {
1077 	ASSERT(dp);
1078 
1079 	if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1080 		dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
1081 	} else {
1082 		ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL);
1083 	}
1084 
1085 	pt->pt_domainid = dp->d_domainid;
1086 	pt->pt_level = level;
1087 	pt->pt_index = index;
1088 }
1089 
1090 static int
amd_iommu_setup_1_pgtable(amd_iommu_t * iommu,dev_info_t * rdip,struct ddi_dma_req * dmareq,domain_id_t domainid,amd_iommu_domain_t * dp,amd_iommu_page_table_t * ppt,uint16_t index,int level,uint64_t va,uint64_t pa,amd_iommu_page_table_t ** ptp,uint16_t * next_idxp,const char * path,int km_flags)1091 amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip,
1092     struct ddi_dma_req *dmareq,
1093     domain_id_t domainid, amd_iommu_domain_t *dp,
1094     amd_iommu_page_table_t *ppt,
1095     uint16_t index, int level, uint64_t va, uint64_t pa,
1096     amd_iommu_page_table_t **ptp,  uint16_t *next_idxp, const char *path,
1097     int km_flags)
1098 {
1099 	int error;
1100 	amd_iommu_page_table_t *pt;
1101 	const char *driver = ddi_driver_name(rdip);
1102 	int instance = ddi_get_instance(rdip);
1103 	const char *f = "amd_iommu_setup_1_pgtable";
1104 
1105 	*ptp = NULL;
1106 	*next_idxp = 0;
1107 	error = DDI_SUCCESS;
1108 
1109 	ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
1110 
1111 	ASSERT(dp);
1112 	if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1113 		ASSERT(ppt == NULL);
1114 		ASSERT(index == 0);
1115 	} else {
1116 		ASSERT(ppt);
1117 	}
1118 
1119 	/* Check if page table is already allocated */
1120 	if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) {
1121 		ASSERT(pt->pt_domainid == domainid);
1122 		ASSERT(pt->pt_level == level);
1123 		ASSERT(pt->pt_index == index);
1124 		goto out;
1125 	}
1126 
1127 	if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt,
1128 	    km_flags)) != DDI_SUCCESS) {
1129 		cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p "
1130 		    "path = %s", f, driver, instance, iommu->aiomt_idx,
1131 		    domainid, (void *)(uintptr_t)va, path);
1132 		return (error);
1133 	}
1134 
1135 	ASSERT(dp->d_domainid == domainid);
1136 
1137 	init_pt(pt, dp, level, index);
1138 
1139 out:
1140 	if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) {
1141 		error = init_pde(ppt, pt);
1142 	}
1143 
1144 	if (level == 1) {
1145 		ASSERT(error == DDI_SUCCESS);
1146 		error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq);
1147 	} else {
1148 		*next_idxp = AMD_IOMMU_VA_BITS(va, level);
1149 		*ptp = pt;
1150 	}
1151 
1152 	return (error);
1153 }
1154 
1155 typedef enum {
1156 	PDTE_NOT_TORN = 0x1,
1157 	PDTE_TORN_DOWN = 0x2,
1158 	PGTABLE_TORN_DOWN = 0x4
1159 } pdte_tear_t;
1160 
1161 static pdte_tear_t
amd_iommu_teardown_pdte(amd_iommu_t * iommu,amd_iommu_page_table_t * pt,int index)1162 amd_iommu_teardown_pdte(amd_iommu_t *iommu,
1163     amd_iommu_page_table_t *pt, int index)
1164 {
1165 	uint8_t next_level;
1166 	pdte_tear_t retval;
1167 	uint64_t *ptdep = &(pt->pt_pgtblva[index]);
1168 
1169 	next_level = AMD_IOMMU_REG_GET64(ptdep,
1170 	    AMD_IOMMU_PTDE_NXT_LVL);
1171 
1172 	if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) {
1173 		if (pt->pt_level == 1) {
1174 			ASSERT(next_level == 0);
1175 			/* PTE */
1176 			pt->pt_pte_ref[index]--;
1177 			if (pt->pt_pte_ref[index] != 0) {
1178 				return (PDTE_NOT_TORN);
1179 			}
1180 		} else {
1181 			ASSERT(next_level != 0 && next_level != 7);
1182 		}
1183 		ASSERT(pt->pt_pte_ref[index] == 0);
1184 		ASSERT(PT_REF_VALID(pt));
1185 
1186 		AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0);
1187 		SYNC_FORDEV(pt->pt_dma_hdl);
1188 		ASSERT(AMD_IOMMU_REG_GET64(ptdep,
1189 		    AMD_IOMMU_PTDE_PR) == 0);
1190 		pt->pt_ref--;
1191 		ASSERT(PT_REF_VALID(pt));
1192 		retval = PDTE_TORN_DOWN;
1193 	} else {
1194 		ASSERT(0);
1195 		ASSERT(pt->pt_pte_ref[index] == 0);
1196 		ASSERT(PT_REF_VALID(pt));
1197 		retval = PDTE_NOT_TORN;
1198 	}
1199 
1200 	if (pt->pt_ref == 0) {
1201 		amd_iommu_free_pgtable(iommu, pt);
1202 		return (PGTABLE_TORN_DOWN);
1203 	}
1204 
1205 	return (retval);
1206 }
1207 
1208 static int
amd_iommu_create_pgtables(amd_iommu_t * iommu,dev_info_t * rdip,struct ddi_dma_req * dmareq,uint64_t va,uint64_t pa,uint16_t deviceid,domain_id_t domainid,amd_iommu_domain_t * dp,const char * path,int km_flags)1209 amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
1210     struct ddi_dma_req *dmareq, uint64_t va,
1211     uint64_t pa, uint16_t deviceid, domain_id_t domainid,
1212     amd_iommu_domain_t *dp, const char *path, int km_flags)
1213 {
1214 	int level;
1215 	uint16_t index;
1216 	uint16_t next_idx;
1217 	amd_iommu_page_table_t *pt;
1218 	amd_iommu_page_table_t *ppt;
1219 	int error;
1220 	const char *driver = ddi_driver_name(rdip);
1221 	int instance = ddi_get_instance(rdip);
1222 	const char *f = "amd_iommu_create_pgtables";
1223 
1224 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1225 		cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
1226 		    "deviceid = %u, va = %p, pa = %p, path = %s",
1227 		    f, driver, instance,
1228 		    iommu->aiomt_idx, domainid, deviceid,
1229 		    (void *)(uintptr_t)va,
1230 		    (void *)(uintptr_t)pa, path);
1231 	}
1232 
1233 	if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
1234 		/* No need for pagetables. Just set up device table entry */
1235 		goto passthru;
1236 	}
1237 
1238 	index = 0;
1239 	ppt = NULL;
1240 	for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0;
1241 	    level--, pt = NULL, next_idx = 0) {
1242 		if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq,
1243 		    domainid, dp, ppt, index, level, va, pa, &pt,
1244 		    &next_idx, path, km_flags)) != DDI_SUCCESS) {
1245 			cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
1246 			    "deviceid=%u, va= %p, pa = %p, Failed to setup "
1247 			    "page table(s) at level = %d, path = %s.",
1248 			    f, driver, instance, iommu->aiomt_idx,
1249 			    domainid, deviceid, (void *)(uintptr_t)va,
1250 			    (void *)(uintptr_t)pa, level, path);
1251 			return (error);
1252 		}
1253 
1254 		if (level > 1) {
1255 			ASSERT(pt);
1256 			ASSERT(pt->pt_domainid == domainid);
1257 			ppt = pt;
1258 			index = next_idx;
1259 		} else {
1260 			ASSERT(level == 1);
1261 			ASSERT(pt == NULL);
1262 			ASSERT(next_idx == 0);
1263 			ppt = NULL;
1264 			index = 0;
1265 		}
1266 	}
1267 
1268 passthru:
1269 	if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid,
1270 	    dp, path)) != DDI_SUCCESS) {
1271 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, "
1272 		    "domainid=%d."
1273 		    "Failed to set device table entry for path %s.",
1274 		    f, driver, instance,
1275 		    iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path);
1276 		return (error);
1277 	}
1278 
1279 	SYNC_FORDEV(iommu->aiomt_dmahdl);
1280 
1281 	return (DDI_SUCCESS);
1282 }
1283 
1284 static int
amd_iommu_destroy_pgtables(amd_iommu_t * iommu,dev_info_t * rdip,uint64_t pageva,uint16_t deviceid,domain_id_t domainid,amd_iommu_domain_t * dp,map_type_t type,int * domain_freed,char * path)1285 amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
1286     uint64_t pageva, uint16_t deviceid, domain_id_t domainid,
1287     amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path)
1288 {
1289 	int level;
1290 	int flags;
1291 	amd_iommu_cmdargs_t cmdargs = {0};
1292 	uint16_t index;
1293 	uint16_t prev_index;
1294 	amd_iommu_page_table_t *pt;
1295 	amd_iommu_page_table_t *ppt;
1296 	pdte_tear_t retval;
1297 	int tear_level;
1298 	int invalidate_pte;
1299 	int invalidate_pde;
1300 	int error = DDI_FAILURE;
1301 	const char *driver = ddi_driver_name(iommu->aiomt_dip);
1302 	int instance = ddi_get_instance(iommu->aiomt_dip);
1303 	const char *f = "amd_iommu_destroy_pgtables";
1304 
1305 	tear_level = -1;
1306 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1307 		cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
1308 		    "deviceid = %u, va = %p, path = %s",
1309 		    f, driver, instance,
1310 		    iommu->aiomt_idx, domainid, deviceid,
1311 		    (void *)(uintptr_t)pageva, path);
1312 	}
1313 
1314 	if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
1315 		/*
1316 		 * there are no pagetables for the passthru domain.
1317 		 * Just the device table entry
1318 		 */
1319 		error = DDI_SUCCESS;
1320 		goto passthru;
1321 	}
1322 
1323 	ppt = NULL;
1324 	index = 0;
1325 	for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) {
1326 		pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index);
1327 		if (pt) {
1328 			ppt = pt;
1329 			index = AMD_IOMMU_VA_BITS(pageva, level);
1330 			continue;
1331 		}
1332 		break;
1333 	}
1334 
1335 	if (level == 0) {
1336 		uint64_t *ptep;
1337 		uint64_t pa_4K;
1338 
1339 		ASSERT(pt);
1340 		ASSERT(pt == ppt);
1341 		ASSERT(pt->pt_domainid == dp->d_domainid);
1342 
1343 		ptep = &(pt->pt_pgtblva[index]);
1344 
1345 		pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR);
1346 		if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1347 			ASSERT(pageva == (pa_4K << MMU_PAGESHIFT));
1348 		}
1349 	}
1350 
1351 	invalidate_pde = 0;
1352 	invalidate_pte = 0;
1353 	for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) {
1354 		prev_index = pt->pt_index;
1355 		ppt = pt->pt_parent;
1356 		retval = amd_iommu_teardown_pdte(iommu, pt, index);
1357 		switch (retval) {
1358 			case PDTE_NOT_TORN:
1359 				goto invalidate;
1360 			case PDTE_TORN_DOWN:
1361 				invalidate_pte = 1;
1362 				goto invalidate;
1363 			case PGTABLE_TORN_DOWN:
1364 				invalidate_pte = 1;
1365 				invalidate_pde = 1;
1366 				tear_level = level;
1367 				break;
1368 		}
1369 		index = prev_index;
1370 		pt = ppt;
1371 	}
1372 
1373 invalidate:
1374 	/*
1375 	 * Now teardown the IOMMU HW caches if applicable
1376 	 */
1377 	if (invalidate_pte) {
1378 		cmdargs.ca_domainid = (uint16_t)domainid;
1379 		if (amd_iommu_pageva_inval_all) {
1380 			cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
1381 			flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
1382 			    AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
1383 		} else if (invalidate_pde) {
1384 			cmdargs.ca_addr =
1385 			    (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level);
1386 			flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
1387 			    AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
1388 		} else {
1389 			cmdargs.ca_addr = (uintptr_t)pageva;
1390 			flags = 0;
1391 		}
1392 		if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
1393 		    &cmdargs, flags, 0) != DDI_SUCCESS) {
1394 			cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
1395 			    "rdip=%p. Failed to invalidate IOMMU HW cache "
1396 			    "for %s", f, driver, instance,
1397 			    iommu->aiomt_idx, domainid, (void *)rdip, path);
1398 			error = DDI_FAILURE;
1399 			goto out;
1400 		}
1401 	}
1402 
1403 passthru:
1404 	if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
1405 		error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid,
1406 		    deviceid, dp, domain_freed, path);
1407 	} else {
1408 		error = DDI_SUCCESS;
1409 	}
1410 
1411 out:
1412 	SYNC_FORDEV(iommu->aiomt_dmahdl);
1413 
1414 	return (error);
1415 }
1416 
1417 static int
cvt_bind_error(int error)1418 cvt_bind_error(int error)
1419 {
1420 	switch (error) {
1421 	case DDI_DMA_MAPPED:
1422 	case DDI_DMA_PARTIAL_MAP:
1423 	case DDI_DMA_NORESOURCES:
1424 	case DDI_DMA_NOMAPPING:
1425 		break;
1426 	default:
1427 		cmn_err(CE_PANIC, "Unsupported error code: %d", error);
1428 		/*NOTREACHED*/
1429 	}
1430 	return (error);
1431 }
1432 
1433 int
amd_iommu_map_pa2va(amd_iommu_t * iommu,dev_info_t * rdip,ddi_dma_attr_t * attrp,struct ddi_dma_req * dmareq,uint64_t start_pa,uint64_t pa_sz,map_type_t type,uint64_t * start_vap,int km_flags)1434 amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
1435     struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz,
1436     map_type_t type, uint64_t *start_vap, int km_flags)
1437 {
1438 	pfn_t pfn_start;
1439 	pfn_t pfn_end;
1440 	pfn_t pfn;
1441 	int alias;
1442 	int32_t deviceid;
1443 	domain_id_t domainid;
1444 	amd_iommu_domain_t *dp;
1445 	uint64_t end_pa;
1446 	uint64_t start_va;
1447 	uint64_t end_va;
1448 	uint64_t pg_start;
1449 	uint64_t pg_end;
1450 	uint64_t pg;
1451 	uint64_t va_sz;
1452 	char *path;
1453 	int error = DDI_DMA_NOMAPPING;
1454 	const char *driver = ddi_driver_name(iommu->aiomt_dip);
1455 	int instance = ddi_get_instance(iommu->aiomt_dip);
1456 	const char *f = "amd_iommu_map_pa2va";
1457 
1458 	ASSERT(pa_sz != 0);
1459 
1460 	*start_vap = 0;
1461 
1462 	ASSERT(rdip);
1463 
1464 	path = kmem_alloc(MAXPATHLEN, km_flags);
1465 	if (path == NULL) {
1466 		error = DDI_DMA_NORESOURCES;
1467 		goto out;
1468 	}
1469 	(void) ddi_pathname(rdip, path);
1470 
1471 	/*
1472 	 * First get deviceid
1473 	 */
1474 	if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
1475 	    != DDI_SUCCESS) {
1476 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
1477 		    "Failed to get device ID for %s.", f, driver, instance,
1478 		    iommu->aiomt_idx, (void *)rdip, path);
1479 		error = DDI_DMA_NOMAPPING;
1480 		goto out;
1481 	}
1482 
1483 	/*
1484 	 * Next get the domain for this rdip
1485 	 */
1486 	if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
1487 	    != DDI_SUCCESS) {
1488 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
1489 		    "Failed to get domain.", f, driver, instance,
1490 		    iommu->aiomt_idx, (void *)rdip, path);
1491 		error = DDI_DMA_NOMAPPING;
1492 		goto out;
1493 	}
1494 
1495 	dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags);
1496 	if (dp == NULL) {
1497 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
1498 		    "Failed to get device ID for %s.", f, driver, instance,
1499 		    iommu->aiomt_idx, domainid, (void *)rdip, path);
1500 		error = DDI_DMA_NORESOURCES;
1501 		goto out;
1502 	}
1503 
1504 	ASSERT(dp->d_domainid == domainid);
1505 
1506 	pfn_start = start_pa >> MMU_PAGESHIFT;
1507 
1508 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1509 		cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, "
1510 		    "pgshift = %d",
1511 		    (void *)(uintptr_t)start_pa,
1512 		    (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT),
1513 		    (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT);
1514 	}
1515 
1516 	end_pa = start_pa + pa_sz - 1;
1517 	pfn_end = end_pa >> MMU_PAGESHIFT;
1518 
1519 	if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1520 		start_va = start_pa;
1521 		end_va = end_pa;
1522 		va_sz = pa_sz;
1523 		*start_vap = start_va;
1524 	} else {
1525 		va_sz = mmu_ptob(pfn_end - pfn_start + 1);
1526 		start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz,
1527 		    MAX(attrp->dma_attr_align, MMU_PAGESIZE),
1528 		    0,
1529 		    attrp->dma_attr_seg + 1,
1530 		    (void *)(uintptr_t)attrp->dma_attr_addr_lo,
1531 		    (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1),
1532 		    AMD_IOMMU_SIZE_4G),	/* XXX rollover */
1533 		    km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1534 		if (start_va == 0) {
1535 			cmn_err(CE_WARN, "%s: No VA resources",
1536 			    amd_iommu_modname);
1537 			error = DDI_DMA_NORESOURCES;
1538 			goto out;
1539 		}
1540 		ASSERT((start_va & MMU_PAGEOFFSET) == 0);
1541 		end_va = start_va + va_sz - 1;
1542 		*start_vap = start_va + (start_pa & MMU_PAGEOFFSET);
1543 	}
1544 
1545 	pg_start = start_va >> MMU_PAGESHIFT;
1546 	pg_end = end_va >> MMU_PAGESHIFT;
1547 
1548 	pg = pg_start;
1549 	for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) {
1550 
1551 		if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1552 			cmn_err(CE_NOTE, "%s: attempting to create page tables "
1553 			    "for pfn = %p, va = %p, path = %s",
1554 			    f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT),
1555 			    (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path);
1556 
1557 		}
1558 
1559 		if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
1560 			ASSERT(pfn == pg);
1561 		}
1562 
1563 		if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq,
1564 		    pg << MMU_PAGESHIFT,
1565 		    pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path,
1566 		    km_flags)) != DDI_SUCCESS) {
1567 			cmn_err(CE_WARN, "Failed to create_pgtables");
1568 			goto out;
1569 		}
1570 
1571 		if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
1572 			cmn_err(CE_NOTE, "%s: successfully created page tables "
1573 			    "for pfn = %p, vapg = %p, path = %s",
1574 			    f, (void *)(uintptr_t)pfn,
1575 			    (void *)(uintptr_t)pg, path);
1576 		}
1577 
1578 	}
1579 	ASSERT(pg == pg_end + 1);
1580 
1581 
1582 	if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) {
1583 		cmn_err(CE_NOTE, "pa=%p, va=%p",
1584 		    (void *)(uintptr_t)start_pa,
1585 		    (void *)(uintptr_t)(*start_vap));
1586 	}
1587 	error = DDI_DMA_MAPPED;
1588 
1589 out:
1590 	kmem_free(path, MAXPATHLEN);
1591 	return (cvt_bind_error(error));
1592 }
1593 
1594 int
amd_iommu_unmap_va(amd_iommu_t * iommu,dev_info_t * rdip,uint64_t start_va,uint64_t va_sz,map_type_t type)1595 amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va,
1596     uint64_t va_sz, map_type_t type)
1597 {
1598 	uint64_t end_va;
1599 	uint64_t pg_start;
1600 	uint64_t pg_end;
1601 	uint64_t pg;
1602 	uint64_t actual_sz;
1603 	char *path;
1604 	int pathfree;
1605 	int alias;
1606 	int32_t deviceid;
1607 	domain_id_t domainid;
1608 	amd_iommu_domain_t *dp;
1609 	int error;
1610 	int domain_freed;
1611 	const char *driver = ddi_driver_name(iommu->aiomt_dip);
1612 	int instance = ddi_get_instance(iommu->aiomt_dip);
1613 	const char *f = "amd_iommu_unmap_va";
1614 
1615 	if (amd_iommu_no_unmap)
1616 		return (DDI_SUCCESS);
1617 
1618 	path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
1619 	if (path) {
1620 		(void) ddi_pathname(rdip, path);
1621 		pathfree = 1;
1622 	} else {
1623 		pathfree = 0;
1624 		path = "<path-mem-alloc-failed>";
1625 	}
1626 
1627 	/*
1628 	 * First get deviceid
1629 	 */
1630 	if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
1631 	    != DDI_SUCCESS) {
1632 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
1633 		    "Failed to get device ID for %s.", f, driver, instance,
1634 		    iommu->aiomt_idx, (void *)rdip, path);
1635 		error = DDI_FAILURE;
1636 		goto out;
1637 	}
1638 
1639 	/*
1640 	 * Next get the domain for this rdip
1641 	 */
1642 	if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
1643 	    != DDI_SUCCESS) {
1644 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
1645 		    "Failed to get domain.", f, driver, instance,
1646 		    iommu->aiomt_idx, (void *)rdip, path);
1647 		error = DDI_FAILURE;
1648 		goto out;
1649 	}
1650 
1651 	/* should never result in domain allocation/vmem_create */
1652 	dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP,
1653 	    KM_NOSLEEP);
1654 	if (dp == NULL) {
1655 		cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
1656 		    "Failed to get device ID for %s.", f, driver, instance,
1657 		    iommu->aiomt_idx, domainid, (void *)rdip, path);
1658 		error = DDI_FAILURE;
1659 		goto out;
1660 	}
1661 
1662 	ASSERT(dp->d_domainid == domainid);
1663 
1664 	pg_start = start_va >> MMU_PAGESHIFT;
1665 	end_va = start_va + va_sz - 1;
1666 	pg_end = end_va >> MMU_PAGESHIFT;
1667 	actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT;
1668 
1669 	domain_freed = 0;
1670 	for (pg = pg_start; pg <= pg_end; pg++) {
1671 		domain_freed = 0;
1672 		if (amd_iommu_destroy_pgtables(iommu, rdip,
1673 		    pg << MMU_PAGESHIFT, deviceid, domainid, dp, type,
1674 		    &domain_freed, path) != DDI_SUCCESS) {
1675 			error = DDI_FAILURE;
1676 			goto out;
1677 		}
1678 		if (domain_freed) {
1679 			ASSERT(pg == pg_end);
1680 			break;
1681 		}
1682 	}
1683 
1684 	/*
1685 	 * vmem_xalloc() must be paired with vmem_xfree
1686 	 */
1687 	if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) {
1688 		vmem_xfree(dp->d_vmem,
1689 		    (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz);
1690 	}
1691 
1692 	if (domain_freed)
1693 		amd_iommu_teardown_domain(iommu, dp);
1694 
1695 	error = DDI_SUCCESS;
1696 out:
1697 	if (pathfree)
1698 		kmem_free(path, MAXPATHLEN);
1699 	return (error);
1700 }
1701