xref: /illumos-gate/usr/src/uts/sun4u/io/pci/pci_reloc.c (revision d0662dbf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * PCI nexus DVMA relocation routines.
30  *
31  * These routines handle the interactions with the HAT layer to
32  * implement page relocation for page(s) which have active DMA handle
33  * bindings when DVMA is being used for those handles.
34  *
35  * The current modus operandi is as follows:
36  *
37  *   Object binding: register the appropriate callback for each page
38  *     of the kernel object while obtaining the PFN for the DVMA page.
39  *
40  *   Object unbinding: unregister the callback for each page of the
41  *     kernel object.
42  *
43  *   Relocation request:
44  *     1) Suspend the bus and sync the caches.
45  *     2) Remap the DVMA object using the new provided PFN.
46  *     3) Unsuspend the bus.
47  *
48  *  The relocation code runs with CPUs captured (idling in xc_loop())
49  *  so we can only acquire spinlocks at PIL >= 13 for synchronization
50  *  within those codepaths.
51  */
52 #include <sys/types.h>
53 #include <sys/kmem.h>
54 #include <sys/async.h>
55 #include <sys/sysmacros.h>
56 #include <sys/sunddi.h>
57 #include <sys/machsystm.h>
58 #include <sys/ddi_impldefs.h>
59 #include <sys/dvma.h>
60 #include <vm/hat.h>
61 #include <sys/pci/pci_obj.h>
62 
63 /*LINTLIBRARY*/
64 
65 void
pci_dvma_unregister_callbacks(pci_t * pci_p,ddi_dma_impl_t * mp)66 pci_dvma_unregister_callbacks(pci_t *pci_p, ddi_dma_impl_t *mp)
67 {
68 	ddi_dma_obj_t *dobj_p = &mp->dmai_object;
69 	struct as *as_p = dobj_p->dmao_obj.virt_obj.v_as;
70 	page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv;
71 	caddr_t vaddr = dobj_p->dmao_obj.virt_obj.v_addr;
72 	struct hat *hat_p;
73 	uint32_t offset;
74 	int i;
75 
76 	if (!PCI_DMA_CANRELOC(mp))
77 		return;
78 
79 	hat_p = (as_p == NULL)? kas.a_hat : as_p->a_hat;
80 	ASSERT(hat_p == kas.a_hat);
81 	ASSERT(pplist == NULL);
82 
83 	offset = mp->dmai_roffset;
84 	hat_delete_callback(vaddr, IOMMU_PAGE_SIZE - offset, mp, HAC_PAGELOCK,
85 	    MP_HAT_CB_COOKIE(mp, 0));
86 	vaddr = (caddr_t)(((uintptr_t)vaddr + IOMMU_PAGE_SIZE) &
87 	    IOMMU_PAGE_MASK);
88 	for (i = 1; i < mp->dmai_ndvmapages; i++) {
89 		hat_delete_callback(vaddr, IOMMU_PAGE_SIZE, mp, HAC_PAGELOCK,
90 		    MP_HAT_CB_COOKIE(mp, i));
91 		vaddr += IOMMU_PAGE_SIZE;
92 	}
93 	mp->dmai_flags &= ~DMAI_FLAGS_RELOC;
94 }
95 
96 static int
pci_dvma_postrelocator(caddr_t va,uint_t len,uint_t flags,void * mpvoid,pfn_t newpfn)97 pci_dvma_postrelocator(caddr_t va, uint_t len, uint_t flags, void *mpvoid,
98 	pfn_t newpfn)
99 {
100 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)mpvoid;
101 	dev_info_t *rdip = mp->dmai_rdip;
102 	ddi_dma_obj_t *dobj_p = &mp->dmai_object;
103 	page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv;
104 	caddr_t baseva = dobj_p->dmao_obj.virt_obj.v_addr;
105 	int index;
106 	size_t length = IOMMU_PTOB(1);
107 	off_t offset;
108 
109 	DEBUG0(DBG_RELOC, rdip, "postrelocator called\n");
110 
111 	if (flags == HAT_POSTUNSUSPEND) {
112 		mutex_enter(&pci_reloc_mutex);
113 		ASSERT(pci_reloc_thread == curthread);
114 		ASSERT(pci_reloc_presuspend > 0);
115 		if (--pci_reloc_presuspend == 0) {
116 			pci_reloc_thread = NULL;
117 			cv_broadcast(&pci_reloc_cv);
118 		}
119 		mutex_exit(&pci_reloc_mutex);
120 		return (0);
121 	}
122 
123 	ASSERT(flags == HAT_UNSUSPEND);
124 	ASSERT(pci_reloc_suspend > 0);
125 	pci_reloc_suspend--;
126 
127 	ASSERT(len <= length);
128 	ASSERT(pplist == NULL);	/* addr bind handle only */
129 	ASSERT(dobj_p->dmao_obj.virt_obj.v_as == &kas ||
130 	    dobj_p->dmao_obj.virt_obj.v_as == NULL);
131 	ASSERT(PCI_DMA_ISDVMA(mp));
132 	ASSERT(pci_reloc_thread == curthread);
133 
134 	offset = va - baseva;
135 	index = IOMMU_BTOPR(offset);
136 	ASSERT(index < mp->dmai_ndvmapages);
137 
138 	DEBUG3(DBG_RELOC, rdip, "index 0x%x, vaddr 0x%llx, baseva 0x%llx\n",
139 	    index, (int64_t)va, (int64_t)baseva);
140 
141 	if ((mp)->dmai_ndvmapages == 1) {
142 		DEBUG2(DBG_RELOC, rdip, "pfn remap (1) 0x%x -> 0x%x\n",
143 		    mp->dmai_pfnlst, newpfn);
144 		    mp->dmai_pfnlst = (void *)newpfn;
145 	} else {
146 		DEBUG3(DBG_RELOC, rdip, "pfn remap (%d) 0x%x -> 0x%x\n",
147 		    index, ((iopfn_t *)mp->dmai_pfnlst)[index], newpfn);
148 		((iopfn_t *)mp->dmai_pfnlst)[index] = (iopfn_t)newpfn;
149 	}
150 
151 	if (ddi_dma_mctl(rdip, rdip, (ddi_dma_handle_t)mp, DDI_DMA_REMAP,
152 	    &offset, &length, NULL, 0) != DDI_SUCCESS)
153 		return (EIO);
154 	if (ddi_ctlops(rdip, rdip, DDI_CTLOPS_UNQUIESCE, NULL, NULL) !=
155 	    DDI_SUCCESS)
156 		return (EIO);
157 
158 	return (0);
159 }
160 
161 /*
162  * Log a warning message if a callback is still registered on
163  * a page which is being freed.  This is indicative of a driver
164  * bug -- DMA handles are bound, and the memory is being freed by
165  * the VM subsystem without an unbind call on the handle first.
166  */
167 static int
pci_dma_relocerr(caddr_t va,uint_t len,uint_t errorcode,void * mpvoid)168 pci_dma_relocerr(caddr_t va, uint_t len, uint_t errorcode, void *mpvoid)
169 {
170 	int errlevel = pci_dma_panic_on_leak? CE_PANIC : CE_WARN;
171 	if (errorcode == HAT_CB_ERR_LEAKED) {
172 		cmn_err(errlevel, "object 0x%p has a bound DMA handle 0x%p\n",
173 			va, mpvoid);
174 		return (0);
175 	}
176 
177 	/* unknown error code, unhandled so panic */
178 	return (EINVAL);
179 }
180 
181 /*
182  * pci DVMA remap entry points
183  *
184  * Called in response to a DDI_DMA_REMAP DMA ctlops command.
185  * Remaps the region specified in the underlying IOMMU. Safe
186  * to assume that the bus was quiesced and ddi_dma_sync() was
187  * invoked by the caller before we got to this point.
188  */
189 int
pci_dvma_remap(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,off_t offset,size_t length)190 pci_dvma_remap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
191 	off_t offset, size_t length)
192 {
193 	pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
194 	iommu_t *iommu_p = pci_p->pci_iommu_p;
195 	dvma_addr_t dvma_pg;
196 	size_t npgs;
197 	int idx;
198 
199 	dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
200 	idx = IOMMU_BTOPR(offset);
201 	dvma_pg += idx;
202 	npgs = IOMMU_BTOPR(length);
203 
204 	DEBUG3(DBG_RELOC, mp->dmai_rdip,
205 	    "pci_dvma_remap: dvma_pg 0x%llx len 0x%llx idx 0x%x\n",
206 	    dvma_pg, length, idx);
207 
208 	ASSERT(pci_p->pci_pbm_p->pbm_quiesce_count > 0);
209 	iommu_remap_pages(iommu_p, mp, dvma_pg, npgs, idx);
210 
211 	return (DDI_SUCCESS);
212 }
213 
214 void
pci_fdvma_remap(ddi_dma_impl_t * mp,caddr_t kvaddr,dvma_addr_t dvma_pg,size_t npages,size_t index,pfn_t newpfn)215 pci_fdvma_remap(ddi_dma_impl_t *mp, caddr_t kvaddr, dvma_addr_t dvma_pg,
216 	size_t npages, size_t index, pfn_t newpfn)
217 {
218 	fdvma_t *fdvma_p = (fdvma_t *)mp->dmai_fdvma;
219 	pci_t *pci_p = (pci_t *)fdvma_p->softsp;
220 	iommu_t *iommu_p = pci_p->pci_iommu_p;
221 	dev_info_t *dip = pci_p->pci_dip;
222 	iopfn_t pfn = (iopfn_t)newpfn;
223 	dvma_addr_t pg_index = dvma_pg - iommu_p->dvma_base_pg;
224 	int i;
225 	uint64_t tte;
226 
227 	/* make sure we don't exceed reserved boundary */
228 	DEBUG3(DBG_FAST_DVMA, dip, "fast remap index=%x: %p, npgs=%x", index,
229 	    kvaddr, npages);
230 	if (index + npages > mp->dmai_ndvmapages) {
231 		cmn_err(pci_panic_on_fatal_errors ? CE_PANIC : CE_WARN,
232 			"%s%d: fdvma remap index(%lx)+pgs(%lx) exceeds limit\n",
233 			ddi_driver_name(dip), ddi_get_instance(dip),
234 			index, npages);
235 		return;
236 	}
237 
238 	for (i = 0; i < npages; i++, kvaddr += IOMMU_PAGE_SIZE) {
239 		DEBUG3(DBG_FAST_DVMA, dip, "remap dvma_pg %x -> pfn %x,"
240 		    " old tte 0x%llx\n", dvma_pg + i, pfn,
241 		    iommu_p->iommu_tsb_vaddr[pg_index + i]);
242 
243 		if (pfn == PFN_INVALID)
244 			goto bad_pfn;
245 
246 		if (i == 0)
247 			tte = MAKE_TTE_TEMPLATE(pfn, mp);
248 
249 		/* XXX assumes iommu and mmu has same page size */
250 		iommu_p->iommu_tsb_vaddr[pg_index + i] = tte | IOMMU_PTOB(pfn);
251 		IOMMU_PAGE_FLUSH(iommu_p, (dvma_pg + i));
252 	}
253 	return;
254 bad_pfn:
255 	cmn_err(CE_WARN, "%s%d: fdvma remap can't get page frame for vaddr %p",
256 		ddi_driver_name(dip), ddi_get_instance(dip), kvaddr);
257 }
258 
259 static int
pci_fdvma_prerelocator(caddr_t va,uint_t len,uint_t flags,void * mpvoid)260 pci_fdvma_prerelocator(caddr_t va, uint_t len, uint_t flags, void *mpvoid)
261 {
262 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)mpvoid;
263 	fdvma_t *fdvma_p = (fdvma_t *)mp->dmai_fdvma;
264 	caddr_t baseva, endva;
265 	int i;
266 
267 	/*
268 	 * It isn't safe to do relocation if all of the IOMMU
269 	 * mappings haven't yet been established at this index.
270 	 */
271 	for (i = 0; i < mp->dmai_ndvmapages; i++) {
272 		baseva = fdvma_p->kvbase[i];
273 		endva = baseva + IOMMU_PTOB(fdvma_p->pagecnt[i]);
274 		if (va >= baseva && va < endva)
275 			return (0);	/* found a valid index */
276 	}
277 	return (EAGAIN);
278 }
279 
280 static int
pci_fdvma_postrelocator(caddr_t va,uint_t len,uint_t flags,void * mpvoid,pfn_t pfn)281 pci_fdvma_postrelocator(caddr_t va, uint_t len, uint_t flags, void *mpvoid,
282 	pfn_t pfn)
283 {
284 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)mpvoid;
285 	dev_info_t *rdip = mp->dmai_rdip;
286 	fdvma_t *fdvma_p = (fdvma_t *)mp->dmai_fdvma;
287 	caddr_t baseva;
288 	dvma_addr_t dvma_pg;
289 	size_t length = PAGESIZE;
290 	int i;
291 
292 	DEBUG0(DBG_RELOC, rdip, "fdvma postrelocator called\n");
293 
294 	if (flags == HAT_POSTUNSUSPEND) {
295 		mutex_enter(&pci_reloc_mutex);
296 		ASSERT(pci_reloc_thread == curthread);
297 		if (--pci_reloc_presuspend == 0) {
298 			pci_reloc_thread = NULL;
299 			cv_broadcast(&pci_reloc_cv);
300 		}
301 		mutex_exit(&pci_reloc_mutex);
302 		return (0);
303 	}
304 
305 	pci_reloc_suspend--;
306 
307 	ASSERT(flags == HAT_UNSUSPEND);
308 	ASSERT(len <= length);
309 	ASSERT((mp->dmai_rflags & DMP_BYPASSNEXUS) != 0);
310 
311 	/*
312 	 * This virtual page can have multiple cookies that refer
313 	 * to it within the same handle. We must walk the whole
314 	 * table for this DMA handle finding all the cookies, and
315 	 * update all of them. Sigh.
316 	 */
317 	for (i = 0; i < mp->dmai_ndvmapages; i++) {
318 		caddr_t endva;
319 		int index;
320 
321 		baseva = fdvma_p->kvbase[i];
322 		endva = baseva + IOMMU_PTOB(fdvma_p->pagecnt[i]);
323 
324 		if (va >= baseva && va < endva) {
325 			index = i + IOMMU_BTOP(va - baseva);
326 			ASSERT(index < mp->dmai_ndvmapages);
327 
328 			DEBUG4(DBG_RELOC, rdip, "mp %p: index 0x%x, "
329 			    " vaddr 0x%llx, baseva 0x%llx\n", mp, index,
330 			    (int64_t)va, (int64_t)baseva);
331 
332 			dvma_pg = IOMMU_BTOP(mp->dmai_mapping) + index;
333 			pci_fdvma_remap(mp, va, dvma_pg, IOMMU_BTOP(length),
334 			    index, pfn);
335 		}
336 	}
337 
338 	if (ddi_ctlops(rdip, rdip, DDI_CTLOPS_UNQUIESCE, NULL, NULL) !=
339 	    DDI_SUCCESS)
340 		return (EIO);
341 
342 	return (0);
343 }
344 
345 void
pci_fdvma_unregister_callbacks(pci_t * pci_p,fdvma_t * fdvma_p,ddi_dma_impl_t * mp,uint_t index)346 pci_fdvma_unregister_callbacks(pci_t *pci_p, fdvma_t *fdvma_p,
347 	ddi_dma_impl_t *mp, uint_t index)
348 {
349 	size_t npgs = fdvma_p->pagecnt[index];
350 	caddr_t kva = fdvma_p->kvbase[index];
351 	int i;
352 
353 	ASSERT(index + npgs <= mp->dmai_ndvmapages);
354 	ASSERT(kva != NULL);
355 
356 	for (i = 0; i < npgs && pci_dvma_remap_enabled;
357 	    i++, kva += IOMMU_PAGE_SIZE)
358 		hat_delete_callback(kva, IOMMU_PAGE_SIZE, mp, HAC_PAGELOCK,
359 		    fdvma_p->cbcookie[index + i]);
360 }
361 
362 static int
pci_common_prerelocator(caddr_t va,uint_t len,uint_t flags,void * mpvoid)363 pci_common_prerelocator(caddr_t va, uint_t len, uint_t flags, void *mpvoid)
364 {
365 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)mpvoid;
366 	ddi_dma_handle_t h = (ddi_dma_handle_t)mpvoid;
367 	dev_info_t *rdip = mp->dmai_rdip;
368 	int ret;
369 
370 	DEBUG0(DBG_RELOC, rdip, "prerelocator called\n");
371 
372 	if (flags == HAT_PRESUSPEND) {
373 		if (!ddi_prop_exists(DDI_DEV_T_ANY, rdip, DDI_PROP_NOTPROM,
374 		    "dvma-remap-supported"))
375 			return (ENOTSUP);
376 		if (!PCI_DMA_ISMAPPED(mp))
377 			return (EAGAIN);
378 
379 		if (mp->dmai_rflags & DMP_BYPASSNEXUS) {
380 			ret = pci_fdvma_prerelocator(va, len, flags, mpvoid);
381 			if (ret != 0)
382 				return (ret);
383 		} else if (!PCI_DMA_ISDVMA(mp))
384 			return (EINVAL);
385 
386 		/*
387 		 * Acquire the exclusive right to relocate a PCI DMA page,
388 		 * since we later have to pause CPUs which could otherwise
389 		 * lead to all sorts of synchronization headaches.
390 		 */
391 		mutex_enter(&pci_reloc_mutex);
392 		if (pci_reloc_thread != curthread) {
393 			while (pci_reloc_thread != NULL) {
394 				cv_wait(&pci_reloc_cv, &pci_reloc_mutex);
395 			}
396 			pci_reloc_thread = curthread;
397 			ASSERT(pci_reloc_suspend == 0);
398 		}
399 		mutex_exit(&pci_reloc_mutex);
400 
401 		ASSERT(pci_reloc_thread == curthread);
402 		pci_reloc_presuspend++;
403 
404 		return (0);
405 	}
406 
407 	ASSERT(flags == HAT_SUSPEND);
408 	ASSERT(PCI_DMA_CANRELOC(mp));
409 	ASSERT(pci_reloc_thread == curthread);
410 	pci_reloc_suspend++;
411 
412 	if (ddi_ctlops(rdip, rdip, DDI_CTLOPS_QUIESCE, NULL, NULL) !=
413 	    DDI_SUCCESS)
414 		return (EIO);
415 	if (ddi_dma_sync(h, 0, 0, DDI_DMA_SYNC_FORKERNEL) != DDI_SUCCESS)
416 		return (EIO);
417 
418 	return (0);
419 }
420 
421 /*
422  * Register two callback types: one for normal DVMA and the
423  * other for fast DVMA, since each method has a different way
424  * of tracking the PFNs behind a handle.
425  */
426 void
pci_reloc_init(void)427 pci_reloc_init(void)
428 {
429 	int key = pci_reloc_getkey();
430 
431 	mutex_init(&pci_reloc_mutex, NULL, MUTEX_DEFAULT, NULL);
432 	cv_init(&pci_reloc_cv, NULL, CV_DEFAULT, NULL);
433 	pci_dvma_cbid = hat_register_callback(
434 		key + ('D'<<24 | 'V'<<16 | 'M'<<8 | 'A'),
435 		pci_common_prerelocator, pci_dvma_postrelocator,
436 		pci_dma_relocerr, 1);
437 	pci_fast_dvma_cbid = hat_register_callback(
438 		key + ('F'<<24 | 'D'<<16 | 'M'<<8 | 'A'),
439 		pci_common_prerelocator,
440 		pci_fdvma_postrelocator, pci_dma_relocerr, 1);
441 }
442 
443 void
pci_reloc_fini(void)444 pci_reloc_fini(void)
445 {
446 	cv_destroy(&pci_reloc_cv);
447 	mutex_destroy(&pci_reloc_mutex);
448 }
449