xref: /illumos-gate/usr/src/uts/sun4u/io/pci/pci_dma.c (revision 9a63ec27)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
27  */
28 
29 /*
30  * PCI nexus DVMA and DMA core routines:
31  *	dma_map/dma_bind_handle implementation
32  *	bypass and peer-to-peer support
33  *	fast track DVMA space allocation
34  *	runtime DVMA debug
35  */
36 #include <sys/types.h>
37 #include <sys/kmem.h>
38 #include <sys/async.h>
39 #include <sys/sysmacros.h>
40 #include <sys/sunddi.h>
41 #include <sys/machsystm.h>	/* lddphys() */
42 #include <sys/ddi_impldefs.h>
43 #include <vm/hat.h>
44 #include <sys/pci/pci_obj.h>
45 
46 /*LINTLIBRARY*/
47 
48 static void
pci_sc_pg_inv(dev_info_t * dip,sc_t * sc_p,ddi_dma_impl_t * mp,off_t off,size_t len)49 pci_sc_pg_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp, off_t off,
50 	size_t len)
51 {
52 	dvma_addr_t dvma_addr, pg_off;
53 	volatile uint64_t *invl_va = sc_p->sc_invl_reg;
54 
55 	if (!len)
56 		len = mp->dmai_size;
57 
58 	pg_off = mp->dmai_offset;			/* start min */
59 	dvma_addr = MAX(off, pg_off);			/* lo */
60 	pg_off += mp->dmai_size;			/* end max */
61 	pg_off = MIN(off + len, pg_off);		/* hi */
62 	if (dvma_addr >= pg_off) {			/* lo >= hi ? */
63 		DEBUG4(DBG_SC, dip, "%x+%x out of window [%x,%x)\n",
64 		    off, len, mp->dmai_offset,
65 		    mp->dmai_offset + mp->dmai_size);
66 		return;
67 	}
68 
69 	len = pg_off - dvma_addr;			/* sz = hi - lo */
70 	dvma_addr += mp->dmai_mapping;			/* start addr */
71 	pg_off = dvma_addr & IOMMU_PAGE_OFFSET;		/* offset in 1st pg */
72 	len = IOMMU_BTOPR(len + pg_off);		/* # of pages */
73 	dvma_addr ^= pg_off;
74 
75 	DEBUG2(DBG_SC, dip, "addr=%x+%x pages: \n", dvma_addr, len);
76 	for (; len; len--, dvma_addr += IOMMU_PAGE_SIZE) {
77 		DEBUG1(DBG_SC|DBG_CONT, dip, " %x", dvma_addr);
78 		*invl_va = (uint64_t)dvma_addr;
79 	}
80 	DEBUG0(DBG_SC|DBG_CONT, dip, "\n");
81 }
82 
83 static void
pci_dma_sync_flag_wait(ddi_dma_impl_t * mp,sc_t * sc_p,uint32_t onstack)84 pci_dma_sync_flag_wait(ddi_dma_impl_t *mp, sc_t *sc_p, uint32_t onstack)
85 {
86 	hrtime_t start_time;
87 	uint64_t loops = 0;
88 	uint64_t sync_flag_pa = SYNC_BUF_PA(mp);
89 	uint64_t sync_reg_pa = sc_p->sc_sync_reg_pa;
90 	uint8_t stack_buf[128];
91 
92 	stack_buf[0] = DDI_SUCCESS;
93 
94 	/* check for handle specific sync flag */
95 	if (sync_flag_pa)
96 		goto start;
97 
98 	sync_flag_pa = sc_p->sc_sync_flag_pa;
99 
100 	if (onstack) {
101 		sync_flag_pa = va_to_pa(stack_buf);
102 		sync_flag_pa += PCI_SYNC_FLAG_SIZE;
103 		sync_flag_pa >>= PCI_SYNC_FLAG_SZSHIFT;
104 		sync_flag_pa <<= PCI_SYNC_FLAG_SZSHIFT;
105 		goto start;
106 	}
107 	stack_buf[0] |= PCI_SYNC_FLAG_LOCKED;
108 	mutex_enter(&sc_p->sc_sync_mutex);
109 start:
110 	ASSERT(!(sync_flag_pa & PCI_SYNC_FLAG_SIZE - 1));
111 	stdphys(sync_flag_pa, 0);	/* reset sync flag to 0 */
112 					/* membar  #LoadStore|#StoreStore */
113 	stdphysio(sync_reg_pa, sync_flag_pa);
114 	start_time = gethrtime();
115 
116 	for (; gethrtime() - start_time < pci_sync_buf_timeout; loops++)
117 		if (lddphys(sync_flag_pa))
118 			goto done;
119 
120 	if (!lddphys(sync_flag_pa))
121 		stack_buf[0] |= PCI_SYNC_FLAG_FAILED;
122 done:
123 	DEBUG3(DBG_SC|DBG_CONT, 0, "flag wait loops=%lu ticks=%lu status=%x\n",
124 	    loops, gethrtime() - start_time, stack_buf[0]);
125 
126 	if (stack_buf[0] & PCI_SYNC_FLAG_LOCKED)
127 		mutex_exit(&sc_p->sc_sync_mutex);
128 
129 	if (stack_buf[0] & PCI_SYNC_FLAG_FAILED)
130 		cmn_err(CE_PANIC, "%p pci dma sync %lx %lx timeout!",
131 		    mp, sync_flag_pa, loops);
132 }
133 
134 /*
135  * Cache	RW	Before	During		After
136  *
137  * STREAMING	read	no/no	pg/no		ctx,pg/no
138  * STREAMING	write	no/no	pg/yes		ctx,pg/yes
139  * CONSISTENT	read	no/no	yes,no/no	yes,no/no
140  * CONSISTENT	write	no/no	yes,yes/yes	yes,yes/yes
141  *
142  * STREAMING	read	ctx,pg/no
143  * STREAMING	write	ctx,pg/yes
144  * CONSISTENT	read	yes,no/no
145  * CONSISTENT	write	yes,yes/yes
146  */
147 int
pci_dma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint32_t sync_flag)148 pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
149 	off_t off, size_t len, uint32_t sync_flag)
150 {
151 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
152 	int ret = ddi_get_instance(dip);
153 	pci_t *pci_p = get_pci_soft_state(ret);
154 	pbm_t *pbm_p = pci_p->pci_pbm_p;
155 	uint32_t dev_flag = mp->dmai_rflags;
156 	sc_t *sc_p;
157 
158 	DEBUG4(DBG_DMA_SYNC, dip, "%s%d flags=%x,%x\n", ddi_driver_name(rdip),
159 	    ddi_get_instance(rdip), dev_flag, sync_flag);
160 	DEBUG4(DBG_SC, dip, "dmai_mapping=%x, dmai_sz=%x off=%x len=%x\n",
161 	    mp->dmai_mapping, mp->dmai_size, off, len);
162 	DEBUG2(DBG_SC, dip, "mp=%p, ctx=%x\n", mp, MP2CTX(mp));
163 
164 	if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) {
165 		cmn_err(CE_WARN, "Unbound dma handle %p from %s%d", mp,
166 		    ddi_driver_name(rdip), ddi_get_instance(rdip));
167 		return (DDI_FAILURE);
168 	}
169 
170 	if (mp->dmai_flags & DMAI_FLAGS_NOSYNC)
171 		return (DDI_SUCCESS);
172 
173 	if (!(dev_flag & DDI_DMA_CONSISTENT))
174 		goto streaming;
175 
176 	if (sync_flag & PCI_DMA_SYNC_EXT) {
177 		if (sync_flag & (PCI_DMA_SYNC_BEFORE | PCI_DMA_SYNC_POST) ||
178 		    !(sync_flag & PCI_DMA_SYNC_WRITE))
179 			return (DDI_SUCCESS);
180 	} else {
181 		if (!(dev_flag & DDI_DMA_READ) ||
182 		    ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) ==
183 		    DDI_DMA_SYNC_FORDEV))
184 			return (DDI_SUCCESS);
185 	}
186 
187 	pci_pbm_dma_sync(pbm_p, pbm_p->pbm_sync_ino);
188 	return (DDI_SUCCESS);
189 
190 streaming:
191 	ASSERT(pci_stream_buf_exists && (pci_stream_buf_enable & 1 << ret));
192 	sc_p = pci_p->pci_sc_p;
193 	ret = DDI_FAILURE;
194 
195 	if (sync_flag & PCI_DMA_SYNC_EXT)
196 		goto ext;
197 
198 	if (mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts)
199 		ret = pci_sc_ctx_inv(dip, sc_p, mp);
200 	if (ret)
201 		pci_sc_pg_inv(dip, sc_p, mp, off, len);
202 
203 	if ((dev_flag & DDI_DMA_READ) &&
204 	    ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) != DDI_DMA_SYNC_FORDEV))
205 		goto wait;
206 
207 	return (DDI_SUCCESS);
208 ext:
209 	if (sync_flag & PCI_DMA_SYNC_BEFORE)
210 		return (DDI_SUCCESS);
211 	if (sync_flag & PCI_DMA_SYNC_BAR)
212 		goto wait_check;
213 	if (sync_flag & PCI_DMA_SYNC_AFTER &&
214 	    mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts)
215 		ret = pci_sc_ctx_inv(dip, sc_p, mp);
216 	if (ret)
217 		pci_sc_pg_inv(dip, sc_p, mp, off, len);
218 wait_check:
219 	if (sync_flag & PCI_DMA_SYNC_POST || !(sync_flag & PCI_DMA_SYNC_WRITE))
220 		return (DDI_SUCCESS);
221 wait:
222 	pci_dma_sync_flag_wait(mp, sc_p, sync_flag & PCI_DMA_SYNC_PRIVATE);
223 	return (DDI_SUCCESS);
224 }
225 
226 int
pci_dma_handle_clean(dev_info_t * rdip,ddi_dma_handle_t h)227 pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t h)
228 {
229 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
230 	if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0)
231 		return (DDI_FAILURE);
232 	mp->dmai_rflags |= DMP_NOSYNC;
233 	mp->dmai_flags |= DMAI_FLAGS_NOSYNC;
234 	return (DDI_SUCCESS);
235 }
236 
237 /*
238  * pci_dma_allocmp - Allocate a pci dma implementation structure
239  *
240  * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl
241  * to hold unmodified device limits. The ddi_dma_attr inside the
242  * ddi_dma_impl structure is augumented with system limits to enhance
243  * DVMA performance at runtime. The unaugumented device limits saved
244  * right after (accessed through the DEV_ATTR macro) is used
245  * strictly for peer-to-peer transfers which do not obey system limits.
246  *
247  * return: DDI_SUCCESS DDI_DMA_NORESOURCES
248  */
249 ddi_dma_impl_t *
pci_dma_allocmp(dev_info_t * dip,dev_info_t * rdip,int (* waitfp)(caddr_t),caddr_t arg)250 pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t),
251 	caddr_t arg)
252 {
253 	ddi_dma_impl_t *mp;
254 	int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
255 
256 	/* Caution: we don't use zalloc to enhance performance! */
257 	if ((mp = kmem_alloc(sizeof (pci_dma_hdl_t), sleep)) == 0) {
258 		DEBUG0(DBG_DMA_MAP, dip, "can't alloc dma_handle\n");
259 		if (waitfp != DDI_DMA_DONTWAIT) {
260 			DEBUG0(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n");
261 			ddi_set_callback(waitfp, arg, &pci_kmem_clid);
262 		}
263 		return (mp);
264 	}
265 
266 	mp->dmai_rdip = rdip;
267 	mp->dmai_flags = 0;
268 	mp->dmai_pfnlst = NULL;
269 	mp->dmai_winlst = NULL;
270 	mp->dmai_ncookies = 0;
271 	mp->dmai_curcookie = 0;
272 
273 	/*
274 	 * kmem_alloc debug: the following fields are not zero-ed
275 	 * mp->dmai_mapping = 0;
276 	 * mp->dmai_size = 0;
277 	 * mp->dmai_offset = 0;
278 	 * mp->dmai_minxfer = 0;
279 	 * mp->dmai_burstsizes = 0;
280 	 * mp->dmai_ndvmapages = 0;
281 	 * mp->dmai_pool/roffset = 0;
282 	 * mp->dmai_rflags = 0;
283 	 * mp->dmai_inuse/flags
284 	 * mp->dmai_nwin = 0;
285 	 * mp->dmai_winsize = 0;
286 	 * mp->dmai_nexus_private/tte = 0;
287 	 * mp->dmai_iopte/pfnlst
288 	 * mp->dmai_sbi/pfn0 = 0;
289 	 * mp->dmai_minfo/winlst/fdvma
290 	 * mp->dmai_rdip
291 	 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
292 	 * mp->dmai_cookie = 0;
293 	 */
294 
295 	mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;
296 	mp->dmai_attr.dma_attr_flags = (uint_t)0;
297 	mp->dmai_fault = 0;
298 	mp->dmai_fault_check = NULL;
299 	mp->dmai_fault_notify = NULL;
300 
301 	mp->dmai_error.err_ena = 0;
302 	mp->dmai_error.err_status = DDI_FM_OK;
303 	mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
304 	mp->dmai_error.err_ontrap = NULL;
305 	mp->dmai_error.err_fep = NULL;
306 	mp->dmai_error.err_cf = NULL;
307 	ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);
308 
309 	SYNC_BUF_PA(mp) = 0ull;
310 	return (mp);
311 }
312 
313 void
pci_dma_freemp(ddi_dma_impl_t * mp)314 pci_dma_freemp(ddi_dma_impl_t *mp)
315 {
316 	ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp);
317 	if (mp->dmai_ndvmapages > 1)
318 		pci_dma_freepfn(mp);
319 	if (mp->dmai_winlst)
320 		pci_dma_freewin(mp);
321 	kmem_free(mp, sizeof (pci_dma_hdl_t));
322 }
323 
324 void
pci_dma_freepfn(ddi_dma_impl_t * mp)325 pci_dma_freepfn(ddi_dma_impl_t *mp)
326 {
327 	void *addr = mp->dmai_pfnlst;
328 	ASSERT(!PCI_DMA_CANRELOC(mp));
329 	if (addr) {
330 		size_t npages = mp->dmai_ndvmapages;
331 		if (npages > 1)
332 			kmem_free(addr, npages * sizeof (iopfn_t));
333 		mp->dmai_pfnlst = NULL;
334 	}
335 	mp->dmai_ndvmapages = 0;
336 }
337 
338 /*
339  * pci_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits
340  *			and convert dmareq->dmar_limits to mp->dmai_attr
341  *
342  * ddi_dma_impl_t member modified     input
343  * ------------------------------------------------------------------------
344  * mp->dmai_minxfer		    - dev
345  * mp->dmai_burstsizes		    - dev
346  * mp->dmai_flags		    - no limit? peer-to-peer only?
347  *
348  * ddi_dma_attr member modified       input
349  * ------------------------------------------------------------------------
350  * mp->dmai_attr.dma_attr_addr_lo   - dev lo, sys lo
351  * mp->dmai_attr.dma_attr_addr_hi   - dev hi, sys hi
352  * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta
353  * mp->dmai_attr.dma_attr_seg       - 0         (no nocross   restriction)
354  * mp->dmai_attr.dma_attr_align     - 1		(no alignment restriction)
355  *
356  * The dlim_dmaspeed member of dmareq->dmar_limits is ignored.
357  */
358 ddi_dma_impl_t *
pci_dma_lmts2hdl(dev_info_t * dip,dev_info_t * rdip,iommu_t * iommu_p,ddi_dma_req_t * dmareq)359 pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p,
360 	ddi_dma_req_t *dmareq)
361 {
362 	ddi_dma_impl_t *mp;
363 	ddi_dma_attr_t *attr_p;
364 	uint64_t syslo		= iommu_p->iommu_dvma_base;
365 	uint64_t syshi		= iommu_p->iommu_dvma_end;
366 	uint64_t fasthi		= iommu_p->iommu_dvma_fast_end;
367 	ddi_dma_lim_t *lim_p	= dmareq->dmar_limits;
368 	uint32_t count_max	= lim_p->dlim_cntr_max;
369 	uint64_t lo		= lim_p->dlim_addr_lo;
370 	uint64_t hi		= lim_p->dlim_addr_hi;
371 	if (hi <= lo) {
372 		DEBUG0(DBG_DMA_MAP, dip, "Bad limits\n");
373 		return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING);
374 	}
375 	if (!count_max)
376 		count_max--;
377 
378 	if (!(mp = pci_dma_allocmp(dip, rdip, dmareq->dmar_fp,
379 	    dmareq->dmar_arg)))
380 		return (NULL);
381 
382 	/* store original dev input at the 2nd ddi_dma_attr */
383 	attr_p = DEV_ATTR(mp);
384 	SET_DMAATTR(attr_p, lo, hi, -1, count_max);
385 	SET_DMAALIGN(attr_p, 1);
386 
387 	lo = MAX(lo, syslo);
388 	hi = MIN(hi, syshi);
389 	if (hi <= lo)
390 		mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY;
391 	count_max = MIN(count_max, hi - lo);
392 
393 	if (DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1))
394 		mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT |
395 		    DMAI_FLAGS_NOSYSLIMIT;
396 	else {
397 		if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1))
398 			mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT;
399 	}
400 	if (PCI_DMA_NOCTX(rdip))
401 		mp->dmai_flags |= DMAI_FLAGS_NOCTX;
402 
403 	/* store augumented dev input to mp->dmai_attr */
404 	mp->dmai_minxfer	= lim_p->dlim_minxfer;
405 	mp->dmai_burstsizes	= lim_p->dlim_burstsizes;
406 	attr_p = &mp->dmai_attr;
407 	SET_DMAATTR(attr_p, lo, hi, -1, count_max);
408 	SET_DMAALIGN(attr_p, 1);
409 	return (mp);
410 }
411 
412 /*
413  * pci_dma_attr2hdl
414  *
415  * This routine is called from the alloc handle entry point to sanity check the
416  * dma attribute structure.
417  *
418  * use by: pci_dma_allochdl()
419  *
420  * return value:
421  *
422  *	DDI_SUCCESS		- on success
423  *	DDI_DMA_BADATTR		- attribute has invalid version number
424  *				  or address limits exclude dvma space
425  */
426 int
pci_dma_attr2hdl(pci_t * pci_p,ddi_dma_impl_t * mp)427 pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp)
428 {
429 	iommu_t *iommu_p = pci_p->pci_iommu_p;
430 	uint64_t syslo, syshi;
431 	ddi_dma_attr_t *attrp		= DEV_ATTR(mp);
432 	uint64_t hi		= attrp->dma_attr_addr_hi;
433 	uint64_t lo		= attrp->dma_attr_addr_lo;
434 	uint64_t align		= attrp->dma_attr_align;
435 	uint64_t nocross	= attrp->dma_attr_seg;
436 	uint64_t count_max	= attrp->dma_attr_count_max;
437 
438 	DEBUG3(DBG_DMA_ALLOCH, pci_p->pci_dip, "attrp=%p cntr_max=%x.%08x\n",
439 	    attrp, HI32(count_max), LO32(count_max));
440 	DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x lo=%x.%08x\n",
441 	    HI32(hi), LO32(hi), HI32(lo), LO32(lo));
442 	DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "seg=%x.%08x align=%x.%08x\n",
443 	    HI32(nocross), LO32(nocross), HI32(align), LO32(align));
444 
445 	if (!nocross)
446 		nocross--;
447 	if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */
448 
449 		DEBUG0(DBG_DMA_ALLOCH, pci_p->pci_dip, "bypass mode\n");
450 		/* if tomatillo ver <= 2.3 don't allow bypass */
451 		if (tomatillo_disallow_bypass)
452 			return (DDI_DMA_BADATTR);
453 
454 		mp->dmai_flags |= DMAI_FLAGS_BYPASSREQ;
455 		if (nocross != UINT64_MAX)
456 			return (DDI_DMA_BADATTR);
457 		if (align && (align > IOMMU_PAGE_SIZE))
458 			return (DDI_DMA_BADATTR);
459 		align = 1; /* align on 1 page boundary */
460 		syslo = iommu_p->iommu_dma_bypass_base;
461 		syshi = iommu_p->iommu_dma_bypass_end;
462 
463 	} else { /* IOMMU_XLATE or PEER_TO_PEER */
464 		align = MAX(align, IOMMU_PAGE_SIZE) - 1;
465 		if ((align & nocross) != align) {
466 			dev_info_t *rdip = mp->dmai_rdip;
467 			cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned",
468 			    NAMEINST(rdip));
469 			return (DDI_DMA_BADATTR);
470 		}
471 		align = IOMMU_BTOP(align + 1);
472 		syslo = iommu_p->iommu_dvma_base;
473 		syshi = iommu_p->iommu_dvma_end;
474 	}
475 	if (hi <= lo) {
476 		dev_info_t *rdip = mp->dmai_rdip;
477 		cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip));
478 		return (DDI_DMA_BADATTR);
479 	}
480 	lo = MAX(lo, syslo);
481 	hi = MIN(hi, syshi);
482 	if (!count_max)
483 		count_max--;
484 
485 	DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x, lo=%x.%08x\n",
486 	    HI32(hi), LO32(hi), HI32(lo), LO32(lo));
487 	if (hi <= lo) { /* peer transfers cannot have alignment & nocross */
488 		dev_info_t *rdip = mp->dmai_rdip;
489 		cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp);
490 		if ((nocross < UINT32_MAX) || (align > 1)) {
491 			cmn_err(CE_WARN, "%s%d peer only device bad attr",
492 			    NAMEINST(rdip));
493 			return (DDI_DMA_BADATTR);
494 		}
495 		mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY;
496 	} else /* set practical counter_max value */
497 		count_max = MIN(count_max, hi - lo);
498 
499 	if (DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align))
500 		mp->dmai_flags |= DMAI_FLAGS_NOSYSLIMIT |
501 		    DMAI_FLAGS_NOFASTLIMIT;
502 	else {
503 		syshi = iommu_p->iommu_dvma_fast_end;
504 		if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align))
505 			mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT;
506 	}
507 	if (PCI_DMA_NOCTX(mp->dmai_rdip))
508 		mp->dmai_flags |= DMAI_FLAGS_NOCTX;
509 
510 	mp->dmai_minxfer	= attrp->dma_attr_minxfer;
511 	mp->dmai_burstsizes	= attrp->dma_attr_burstsizes;
512 	attrp = &mp->dmai_attr;
513 	SET_DMAATTR(attrp, lo, hi, nocross, count_max);
514 	return (DDI_SUCCESS);
515 }
516 
517 /*
518  * set up consistent dma flags according to hardware capability
519  */
520 uint32_t
pci_dma_consist_check(uint32_t req_flags,pbm_t * pbm_p)521 pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p)
522 {
523 	if (!pci_stream_buf_enable || !pci_stream_buf_exists)
524 		req_flags |= DDI_DMA_CONSISTENT;
525 	if (req_flags & DDI_DMA_CONSISTENT && !pbm_p->pbm_sync_reg_pa)
526 		req_flags |= DMP_NOSYNC;
527 	return (req_flags);
528 }
529 
530 #define	TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end))
531 
532 /*
533  * pci_dma_type - determine which of the three types DMA (peer-to-peer,
534  *		iommu bypass, or iommu translate) we are asked to do.
535  *		Also checks pfn0 and rejects any non-peer-to-peer
536  *		requests for peer-only devices.
537  *
538  *	return values:
539  *		DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type
540  *		DDI_SUCCESS
541  *
542  *	dma handle members affected (set on exit):
543  *	mp->dmai_object		- dmareq->dmar_object
544  *	mp->dmai_rflags		- consistent?, nosync?, dmareq->dmar_flags
545  *	mp->dmai_flags   	- DMA type
546  *	mp->dmai_pfn0   	- 1st page pfn (if va/size pair and not shadow)
547  *	mp->dmai_roffset 	- initialized to starting IOMMU page offset
548  *	mp->dmai_ndvmapages	- # of total IOMMU pages of entire object
549  *	mp->pdh_sync_buf_pa	- dma sync buffer PA is DMA flow is supported
550  */
551 int
pci_dma_type(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)552 pci_dma_type(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
553 {
554 	dev_info_t *dip = pci_p->pci_dip;
555 	ddi_dma_obj_t *dobj_p = &dmareq->dmar_object;
556 	pbm_t *pbm_p = pci_p->pci_pbm_p;
557 	page_t **pplist;
558 	struct as *as_p;
559 	uint32_t offset;
560 	caddr_t vaddr;
561 	pfn_t pfn0;
562 
563 	mp->dmai_rflags = pci_dma_consist_check(dmareq->dmar_flags, pbm_p);
564 	mp->dmai_flags |= mp->dmai_rflags & DMP_NOSYNC ? DMAI_FLAGS_NOSYNC : 0;
565 
566 	switch (dobj_p->dmao_type) {
567 	case DMA_OTYP_BUFVADDR:
568 	case DMA_OTYP_VADDR: {
569 		vaddr = dobj_p->dmao_obj.virt_obj.v_addr;
570 		pplist = dobj_p->dmao_obj.virt_obj.v_priv;
571 		as_p = dobj_p->dmao_obj.virt_obj.v_as;
572 		if (as_p == NULL)
573 			as_p = &kas;
574 
575 		DEBUG2(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist);
576 		offset = (ulong_t)vaddr & IOMMU_PAGE_OFFSET;
577 
578 		if (pplist) {				/* shadow list */
579 			mp->dmai_flags |= DMAI_FLAGS_PGPFN;
580 			ASSERT(PAGE_LOCKED(*pplist));
581 			pfn0 = page_pptonum(*pplist);
582 		} else if (pci_dvma_remap_enabled && as_p == &kas &&
583 		    dobj_p->dmao_type != DMA_OTYP_BUFVADDR) {
584 			int (*waitfp)(caddr_t) = dmareq->dmar_fp;
585 			uint_t flags = ((waitfp == DDI_DMA_SLEEP)?
586 			    HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK;
587 			int ret;
588 
589 			ret = hat_add_callback(pci_dvma_cbid, vaddr,
590 			    IOMMU_PAGE_SIZE - offset, flags, mp, &pfn0,
591 			    MP_HAT_CB_COOKIE_PTR(mp, 0));
592 
593 			if (pfn0 == PFN_INVALID && ret == ENOMEM) {
594 				ASSERT(waitfp != DDI_DMA_SLEEP);
595 				if (waitfp != DDI_DMA_DONTWAIT) {
596 					ddi_set_callback(waitfp,
597 					    dmareq->dmar_arg,
598 					    &pci_kmem_clid);
599 					return (DDI_DMA_NORESOURCES);
600 					}
601 			}
602 			mp->dmai_flags |= DMAI_FLAGS_RELOC;
603 		} else
604 			pfn0 = hat_getpfnum(as_p->a_hat, vaddr);
605 		}
606 		break;
607 
608 	case DMA_OTYP_PAGES:
609 		offset = dobj_p->dmao_obj.pp_obj.pp_offset;
610 		mp->dmai_flags |= DMAI_FLAGS_PGPFN;
611 		pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp);
612 		ASSERT(PAGE_LOCKED(dobj_p->dmao_obj.pp_obj.pp_pp));
613 		break;
614 
615 	case DMA_OTYP_PADDR:
616 	default:
617 		cmn_err(CE_WARN, "%s%d requested unsupported dma type %x",
618 		    NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);
619 		return (DDI_DMA_NOMAPPING);
620 	}
621 	if (pfn0 == PFN_INVALID) {
622 		cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p",
623 		    NAMEINST(dip), dobj_p);
624 		return (DDI_DMA_NOMAPPING);
625 	}
626 	if (TGT_PFN_INBETWEEN(pfn0, pbm_p->pbm_base_pfn, pbm_p->pbm_last_pfn)) {
627 		mp->dmai_flags |= DMAI_FLAGS_PEER_TO_PEER;
628 		goto done;	/* leave bypass and dvma flag as 0 */
629 	}
630 	if (PCI_DMA_ISPEERONLY(mp)) {
631 		dev_info_t *rdip = mp->dmai_rdip;
632 		cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip));
633 		return (DDI_DMA_NOMAPPING);
634 	}
635 	mp->dmai_flags |= (mp->dmai_flags & DMAI_FLAGS_BYPASSREQ) ?
636 	    DMAI_FLAGS_BYPASS : DMAI_FLAGS_DVMA;
637 done:
638 	mp->dmai_object	 = *dobj_p;			/* whole object    */
639 	mp->dmai_pfn0	 = (void *)pfn0;		/* cache pfn0	   */
640 	mp->dmai_roffset = offset;			/* win0 pg0 offset */
641 	mp->dmai_ndvmapages = IOMMU_BTOPR(offset + mp->dmai_object.dmao_size);
642 
643 	return (DDI_SUCCESS);
644 }
645 
646 /*
647  * pci_dma_pgpfn - set up pfnlst array according to pages
648  *	VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES
649  */
650 /*ARGSUSED*/
651 static int
pci_dma_pgpfn(pci_t * pci_p,ddi_dma_impl_t * mp,uint_t npages)652 pci_dma_pgpfn(pci_t *pci_p, ddi_dma_impl_t *mp, uint_t npages)
653 {
654 	int i;
655 #ifdef DEBUG
656 	dev_info_t *dip = pci_p->pci_dip;
657 #endif
658 	switch (mp->dmai_object.dmao_type) {
659 	case DMA_OTYP_BUFVADDR:
660 	case DMA_OTYP_VADDR: {
661 		page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;
662 		DEBUG2(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=",
663 		    pplist, npages);
664 		for (i = 1; i < npages; i++) {
665 			iopfn_t pfn = page_pptonum(pplist[i]);
666 			ASSERT(PAGE_LOCKED(pplist[i]));
667 			PCI_SET_MP_PFN1(mp, i, pfn);
668 			DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
669 		}
670 		DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n");
671 		}
672 		break;
673 
674 	case DMA_OTYP_PAGES: {
675 		page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;
676 		DEBUG1(DBG_DMA_MAP, dip, "pp=%p pfns=", pp);
677 		for (i = 1; i < npages; i++, pp = pp->p_next) {
678 			iopfn_t pfn = page_pptonum(pp);
679 			ASSERT(PAGE_LOCKED(pp));
680 			PCI_SET_MP_PFN1(mp, i, pfn);
681 			DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
682 		}
683 		DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n");
684 		}
685 		break;
686 
687 	default:	/* check is already done by pci_dma_type */
688 		ASSERT(0);
689 		break;
690 	}
691 	return (DDI_SUCCESS);
692 }
693 
694 /*
695  * pci_dma_vapfn - set up pfnlst array according to VA
696  *	VA/size pair: <normal, bypass, peer-to-peer>
697  *	pfn0 is skipped as it is already done.
698  *	In this case, the cached pfn0 is used to fill pfnlst[0]
699  */
700 static int
pci_dma_vapfn(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint_t npages)701 pci_dma_vapfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,
702 	uint_t npages)
703 {
704 	dev_info_t *dip = pci_p->pci_dip;
705 	int i;
706 	caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;
707 	struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat;
708 	caddr_t sva;
709 	int needcb = 0;
710 
711 	sva = (caddr_t)(((uintptr_t)mp->dmai_object.dmao_obj.virt_obj.v_addr +
712 	    IOMMU_PAGE_SIZE) & IOMMU_PAGE_MASK);
713 
714 	if (pci_dvma_remap_enabled && hat_p == kas.a_hat &&
715 	    mp->dmai_object.dmao_type != DMA_OTYP_BUFVADDR)
716 		needcb = 1;
717 
718 	for (vaddr = sva, i = 1; i < npages; i++, vaddr += IOMMU_PAGE_SIZE) {
719 		pfn_t pfn;
720 
721 		if (needcb) {
722 			int (*waitfp)(caddr_t) = dmareq->dmar_fp;
723 			uint_t flags = ((waitfp == DDI_DMA_SLEEP)?
724 			    HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK;
725 			int ret;
726 
727 			ret = hat_add_callback(pci_dvma_cbid, vaddr,
728 			    IOMMU_PAGE_SIZE, flags, mp, &pfn,
729 			    MP_HAT_CB_COOKIE_PTR(mp, i));
730 
731 			if (pfn == PFN_INVALID && ret == ENOMEM) {
732 				ASSERT(waitfp != DDI_DMA_SLEEP);
733 				if (waitfp != DDI_DMA_DONTWAIT)
734 					ddi_set_callback(waitfp,
735 					    dmareq->dmar_arg, &pci_kmem_clid);
736 				return (DDI_DMA_NORESOURCES);
737 			}
738 		} else
739 			pfn = hat_getpfnum(hat_p, vaddr);
740 		if (pfn == PFN_INVALID)
741 			goto err_badpfn;
742 		PCI_SET_MP_PFN1(mp, i, (iopfn_t)pfn);
743 		DEBUG3(DBG_DMA_MAP, dip, "pci_dma_vapfn: mp=%p pfnlst[%x]=%x\n",
744 		    mp, i, (iopfn_t)pfn);
745 	}
746 	return (DDI_SUCCESS);
747 err_badpfn:
748 	cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr);
749 	return (DDI_DMA_NOMAPPING);
750 }
751 
752 /*
753  * pci_dma_pfn - Fills pfn list for all pages being DMA-ed.
754  *
755  * dependencies:
756  *	mp->dmai_ndvmapages	- set to total # of dma pages
757  *
758  * return value:
759  *	DDI_SUCCESS
760  *	DDI_DMA_NOMAPPING
761  */
762 int
pci_dma_pfn(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)763 pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
764 {
765 	uint32_t npages = mp->dmai_ndvmapages;
766 	int (*waitfp)(caddr_t) = dmareq->dmar_fp;
767 	int i, ret, peer = PCI_DMA_ISPTP(mp);
768 
769 	pbm_t *pbm_p = pci_p->pci_pbm_p;
770 	iopfn_t pfn_base = pbm_p->pbm_base_pfn;
771 	iopfn_t pfn_last = pbm_p->pbm_last_pfn;
772 	iopfn_t pfn_adj = peer ? pfn_base : 0;
773 
774 	DEBUG2(DBG_DMA_MAP, pci_p->pci_dip, "pci_dma_pfn: mp=%p pfn0=%x\n",
775 	    mp, MP_PFN0(mp) - pfn_adj);
776 	/* 1 page: no array alloc/fill, no mixed mode check */
777 	if (npages == 1) {
778 		PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj);
779 		return (DDI_SUCCESS);
780 	}
781 	/* allocate pfn array */
782 	if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (iopfn_t),
783 	    waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) {
784 		if (waitfp != DDI_DMA_DONTWAIT)
785 			ddi_set_callback(waitfp, dmareq->dmar_arg,
786 			    &pci_kmem_clid);
787 		return (DDI_DMA_NORESOURCES);
788 	}
789 	/* fill pfn array */
790 	PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj);	/* pfnlst[0] */
791 	if ((ret = PCI_DMA_ISPGPFN(mp) ? pci_dma_pgpfn(pci_p, mp, npages) :
792 	    pci_dma_vapfn(pci_p, dmareq, mp, npages)) != DDI_SUCCESS)
793 		goto err;
794 
795 	/* skip pfn0, check mixed mode and adjust peer to peer pfn */
796 	for (i = 1; i < npages; i++) {
797 		iopfn_t pfn = PCI_GET_MP_PFN1(mp, i);
798 		if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) {
799 			cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx",
800 			    NAMEINST(mp->dmai_rdip), MP_PFN0(mp), pfn);
801 			ret = DDI_DMA_NOMAPPING;	/* mixed mode */
802 			goto err;
803 		}
804 		DEBUG3(DBG_DMA_MAP, pci_p->pci_dip,
805 		    "pci_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj);
806 		if (pfn_adj)
807 			PCI_SET_MP_PFN1(mp, i, pfn - pfn_adj);
808 	}
809 	return (DDI_SUCCESS);
810 err:
811 	pci_dvma_unregister_callbacks(pci_p, mp);
812 	pci_dma_freepfn(mp);
813 	return (ret);
814 }
815 
816 /*
817  * pci_dvma_win() - trim requested DVMA size down to window size
818  *	The 1st window starts from offset and ends at page-aligned boundary.
819  *	From the 2nd window on, each window starts and ends at page-aligned
820  *	boundary except the last window ends at wherever requested.
821  *
822  *	accesses the following mp-> members:
823  *	mp->dmai_attr.dma_attr_count_max
824  *	mp->dmai_attr.dma_attr_seg
825  *	mp->dmai_roffset   - start offset of 1st window
826  *	mp->dmai_rflags (redzone)
827  *	mp->dmai_ndvmapages (for 1 page fast path)
828  *
829  *	sets the following mp-> members:
830  *	mp->dmai_size	   - xfer size, != winsize if 1st/last win  (not fixed)
831  *	mp->dmai_winsize   - window size (no redzone), n * page size    (fixed)
832  *	mp->dmai_nwin	   - # of DMA windows of entire object		(fixed)
833  *	mp->dmai_rflags	   - remove partial flag if nwin == 1		(fixed)
834  *	mp->dmai_winlst	   - NULL, window objects not used for DVMA	(fixed)
835  *
836  *	fixed - not changed across different DMA windows
837  */
838 /*ARGSUSED*/
839 int
pci_dvma_win(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)840 pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
841 {
842 	uint32_t redzone_sz	= HAS_REDZONE(mp) ? IOMMU_PAGE_SIZE : 0;
843 	size_t obj_sz	= mp->dmai_object.dmao_size;
844 	size_t xfer_sz;
845 	ulong_t pg_off;
846 
847 	if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {
848 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
849 		mp->dmai_size = obj_sz;
850 		mp->dmai_winsize = IOMMU_PAGE_SIZE;
851 		mp->dmai_nwin = 1;
852 		goto done;
853 	}
854 
855 	pg_off	= mp->dmai_roffset;
856 	xfer_sz	= obj_sz + redzone_sz;
857 
858 	/* include redzone in nocross check */
859 	{
860 		uint64_t nocross = mp->dmai_attr.dma_attr_seg;
861 		if (xfer_sz + pg_off - 1 > nocross)
862 			xfer_sz = nocross - pg_off + 1;
863 		if (redzone_sz && (xfer_sz <= redzone_sz)) {
864 			DEBUG5(DBG_DMA_MAP, pci_p->pci_dip,
865 			    "nocross too small %lx(%lx)+%lx+%x < %" PRIx64 "\n",
866 			    xfer_sz, obj_sz, pg_off, redzone_sz, nocross);
867 			return (DDI_DMA_TOOBIG);
868 		}
869 	}
870 	xfer_sz -= redzone_sz;	/* restore transfer size  */
871 	/* check counter max */
872 	{
873 		uint32_t count_max = mp->dmai_attr.dma_attr_count_max;
874 		if (xfer_sz - 1 > count_max)
875 			xfer_sz = count_max + 1;
876 	}
877 	if (xfer_sz >= obj_sz) {
878 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
879 		mp->dmai_size = xfer_sz;
880 		mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, IOMMU_PAGE_SIZE);
881 		mp->dmai_nwin = 1;
882 		goto done;
883 	}
884 	if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
885 		DEBUG4(DBG_DMA_MAP, pci_p->pci_dip,
886 		    "too big: %lx+%lx+%x > %lx\n",
887 		    obj_sz, pg_off, redzone_sz, xfer_sz);
888 		return (DDI_DMA_TOOBIG);
889 	}
890 
891 	xfer_sz = IOMMU_PTOB(IOMMU_BTOP(xfer_sz + pg_off)); /* page align */
892 	mp->dmai_size = xfer_sz - pg_off;	/* 1st window xferrable size */
893 	mp->dmai_winsize = xfer_sz;		/* redzone not in winsize */
894 	mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;
895 done:
896 	mp->dmai_winlst = NULL;
897 	dump_dma_handle(DBG_DMA_MAP, pci_p->pci_dip, mp);
898 	return (DDI_SUCCESS);
899 }
900 
901 /*
902  * fast track cache entry to iommu context, inserts 3 0 bits between
903  * upper 6-bits and lower 3-bits of the 9-bit cache entry
904  */
905 #define	IOMMU_FCE_TO_CTX(i)	(((i) << 3) | ((i) & 0x7) | 0x38)
906 
907 /*
908  * pci_dvma_map_fast - attempts to map fast trackable DVMA
909  */
910 int
pci_dvma_map_fast(iommu_t * iommu_p,ddi_dma_impl_t * mp)911 pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp)
912 {
913 	uint_t clustsz = pci_dvma_page_cache_clustsz;
914 	uint_t entries = pci_dvma_page_cache_entries;
915 	uint64_t *tte_addr;
916 	uint64_t tte = GET_TTE_TEMPLATE(mp);
917 	int i = iommu_p->iommu_dvma_addr_scan_start;
918 	uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i;
919 	iopfn_t *pfn_addr;
920 	dvma_addr_t dvma_pg;
921 	size_t npages = IOMMU_BTOP(mp->dmai_winsize);
922 #ifdef DEBUG
923 	dev_info_t *dip = mp->dmai_rdip;
924 #endif
925 	extern uint8_t ldstub(uint8_t *);
926 	ASSERT(IOMMU_PTOB(npages) == mp->dmai_winsize);
927 	ASSERT(npages + HAS_REDZONE(mp) <= clustsz);
928 
929 	for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
930 		;
931 	if (i >= entries) {
932 		lock_addr = iommu_p->iommu_dvma_cache_locks;
933 		i = 0;
934 		for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
935 			;
936 		if (i >= entries) {
937 #ifdef PCI_DMA_PROF
938 			pci_dvmaft_exhaust++;
939 #endif
940 			return (DDI_DMA_NORESOURCES);
941 		}
942 	}
943 	iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1);
944 	if (PCI_DMA_USECTX(mp)) {
945 		dvma_context_t ctx = IOMMU_FCE_TO_CTX(i);
946 		tte |= IOMMU_CTX2TTE(ctx);
947 		mp->dmai_flags |= DMAI_FLAGS_CONTEXT;
948 		DEBUG1(DBG_DMA_MAP, dip, "fast: ctx=0x%x\n", ctx);
949 	}
950 	i *= clustsz;
951 	tte_addr = iommu_p->iommu_tsb_vaddr + i;
952 	dvma_pg = iommu_p->dvma_base_pg + i;
953 #ifdef DEBUG
954 	for (i = 0; i < clustsz; i++)
955 		ASSERT(TTE_IS_INVALID(tte_addr[i]));
956 #endif
957 	*tte_addr = tte | IOMMU_PTOB(MP_PFN0(mp)); /* map page 0 */
958 	DEBUG5(DBG_DMA_MAP, dip, "fast %p:dvma_pg=%x tte0(%p)=%08x.%08x\n", mp,
959 	    dvma_pg, tte_addr, HI32(*tte_addr), LO32(*tte_addr));
960 	if (npages == 1)
961 		goto tte_done;
962 	pfn_addr = PCI_GET_MP_PFN1_ADDR(mp); /* short iommu_map_pages() */
963 	for (tte_addr++, i = 1; i < npages; i++, tte_addr++, pfn_addr++) {
964 		*tte_addr = tte | IOMMU_PTOB(*pfn_addr);
965 		DEBUG5(DBG_DMA_MAP, dip, "fast %p:tte(%p, %p)=%08x.%08x\n", mp,
966 		    tte_addr, pfn_addr, HI32(*tte_addr), LO32(*tte_addr));
967 	}
968 tte_done:
969 #ifdef PCI_DMA_PROF
970 	pci_dvmaft_success++;
971 #endif
972 	mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg);
973 	mp->dmai_offset = 0;
974 	mp->dmai_flags |= DMAI_FLAGS_FASTTRACK;
975 	PCI_SAVE_MP_TTE(mp, tte);	/* save TTE template for unmapping */
976 	if (DVMA_DBG_ON(iommu_p))
977 		pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
978 		    mp->dmai_size, mp);
979 	return (DDI_SUCCESS);
980 }
981 
982 /*
983  * pci_dvma_map: map non-fasttrack DMA
984  *		Use quantum cache if single page DMA.
985  */
986 int
pci_dvma_map(ddi_dma_impl_t * mp,ddi_dma_req_t * dmareq,iommu_t * iommu_p)987 pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p)
988 {
989 	uint_t npages = PCI_DMA_WINNPGS(mp);
990 	dvma_addr_t dvma_pg, dvma_pg_index;
991 	void *dvma_addr;
992 	uint64_t tte = GET_TTE_TEMPLATE(mp);
993 	int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP;
994 #ifdef DEBUG
995 	dev_info_t *dip = mp->dmai_rdip;
996 #endif
997 	/*
998 	 * allocate dvma space resource and map in the first window.
999 	 * (vmem_t *vmp, size_t size,
1000 	 *	size_t align, size_t phase, size_t nocross,
1001 	 *	void *minaddr, void *maxaddr, int vmflag)
1002 	 */
1003 	if ((npages == 1) && !HAS_REDZONE(mp) && HAS_NOSYSLIMIT(mp)) {
1004 		dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map,
1005 		    IOMMU_PAGE_SIZE, sleep);
1006 		mp->dmai_flags |= DMAI_FLAGS_VMEMCACHE;
1007 #ifdef PCI_DMA_PROF
1008 		pci_dvma_vmem_alloc++;
1009 #endif
1010 	} else {
1011 		dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map,
1012 		    IOMMU_PTOB(npages + HAS_REDZONE(mp)),
1013 		    MAX(mp->dmai_attr.dma_attr_align, IOMMU_PAGE_SIZE),
1014 		    0,
1015 		    mp->dmai_attr.dma_attr_seg + 1,
1016 		    (void *)mp->dmai_attr.dma_attr_addr_lo,
1017 		    (void *)(mp->dmai_attr.dma_attr_addr_hi + 1),
1018 		    sleep);
1019 #ifdef PCI_DMA_PROF
1020 		pci_dvma_vmem_xalloc++;
1021 #endif
1022 	}
1023 	dvma_pg = IOMMU_BTOP((ulong_t)dvma_addr);
1024 	dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg;
1025 	DEBUG2(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n",
1026 	    dvma_pg, dvma_pg_index);
1027 	if (dvma_pg == 0)
1028 		goto noresource;
1029 
1030 	/* allocate DVMA context */
1031 	if ((npages >= pci_context_minpages) && PCI_DMA_USECTX(mp)) {
1032 		dvma_context_t ctx;
1033 		if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) {
1034 			tte |= IOMMU_CTX2TTE(ctx);
1035 			mp->dmai_flags |= DMAI_FLAGS_CONTEXT;
1036 		}
1037 	}
1038 	mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg);
1039 	mp->dmai_offset = 0;
1040 	PCI_SAVE_MP_TTE(mp, tte);	/* mp->dmai_tte = tte */
1041 	iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0);
1042 	return (DDI_SUCCESS);
1043 noresource:
1044 	if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
1045 		DEBUG0(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n");
1046 		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
1047 		    &iommu_p->iommu_dvma_clid);
1048 	}
1049 	DEBUG0(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n");
1050 	return (DDI_DMA_NORESOURCES);
1051 }
1052 
1053 void
pci_dvma_unmap(iommu_t * iommu_p,ddi_dma_impl_t * mp)1054 pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp)
1055 {
1056 	size_t npages;
1057 	dvma_addr_t dvma_addr = (dvma_addr_t)mp->dmai_mapping;
1058 	dvma_addr_t dvma_pg = IOMMU_BTOP(dvma_addr);
1059 	dvma_addr = IOMMU_PTOB(dvma_pg);
1060 
1061 	if (mp->dmai_flags & DMAI_FLAGS_FASTTRACK) {
1062 		iopfn_t index = dvma_pg - iommu_p->dvma_base_pg;
1063 		ASSERT(index % pci_dvma_page_cache_clustsz == 0);
1064 		index /= pci_dvma_page_cache_clustsz;
1065 		ASSERT(index < pci_dvma_page_cache_entries);
1066 		iommu_p->iommu_dvma_cache_locks[index] = 0;
1067 #ifdef PCI_DMA_PROF
1068 		pci_dvmaft_free++;
1069 #endif
1070 		return;
1071 	}
1072 	npages = IOMMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp);
1073 	pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages);
1074 
1075 	if (mp->dmai_flags & DMAI_FLAGS_CONTEXT)
1076 		pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp));
1077 }
1078 
1079 void
pci_dma_sync_unmap(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp)1080 pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp)
1081 {
1082 	pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
1083 	iommu_t *iommu_p = pci_p->pci_iommu_p;
1084 	uint64_t sync_buf_save = SYNC_BUF_PA(mp);
1085 	uint32_t fast_track = mp->dmai_flags & DMAI_FLAGS_FASTTRACK;
1086 
1087 	if (fast_track) {
1088 		dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
1089 
1090 		SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg);
1091 		ASSERT(!(SYNC_BUF_PA(mp) & PCI_SYNC_FLAG_SIZE - 1));
1092 	}
1093 
1094 	if (pci_dvma_sync_before_unmap) {
1095 		pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0,
1096 		    DDI_DMA_SYNC_FORCPU);
1097 		iommu_unmap_window(iommu_p, mp);
1098 	} else {
1099 		iommu_unmap_window(iommu_p, mp);
1100 		pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0,
1101 		    DDI_DMA_SYNC_FORCPU);
1102 	}
1103 
1104 	if (fast_track)
1105 		SYNC_BUF_PA(mp) = sync_buf_save;
1106 }
1107 
1108 /*
1109  * DVMA mappings may have multiple windows, but each window always have
1110  * one segment.
1111  */
1112 int
pci_dvma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)1113 pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
1114 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
1115 	uint_t cache_flags)
1116 {
1117 	switch (cmd) {
1118 
1119 	case DDI_DMA_REMAP:
1120 		if (pci_dvma_remap_enabled)
1121 			return (pci_dvma_remap(dip, rdip, mp, *offp, *lenp));
1122 		return (DDI_FAILURE);
1123 
1124 	default:
1125 		DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1126 		    cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
1127 		break;
1128 	}
1129 	return (DDI_FAILURE);
1130 }
1131 
1132 void
pci_dma_freewin(ddi_dma_impl_t * mp)1133 pci_dma_freewin(ddi_dma_impl_t *mp)
1134 {
1135 	pci_dma_win_t *win_p = mp->dmai_winlst, *win2_p;
1136 	for (win2_p = win_p; win_p; win2_p = win_p) {
1137 		win_p = win2_p->win_next;
1138 		kmem_free(win2_p, sizeof (pci_dma_win_t) +
1139 		    sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies);
1140 	}
1141 	mp->dmai_nwin = 0;
1142 	mp->dmai_winlst = NULL;
1143 }
1144 
1145 /*
1146  * pci_dma_newwin - create a dma window object and cookies
1147  *
1148  *	After the initial scan in pci_dma_physwin(), which identifies
1149  *	a portion of the pfn array that belongs to a dma window,
1150  *	we are called to allocate and initialize representing memory
1151  *	resources. We know from the 1st scan the number of cookies
1152  *	or dma segment in this window so we can allocate a contiguous
1153  *	memory array for the dma cookies (The implementation of
1154  *	ddi_dma_nextcookie(9f) dictates dma cookies be contiguous).
1155  *
1156  *	A second round scan is done on the pfn array to identify
1157  *	each dma segment and initialize its corresponding dma cookie.
1158  *	We don't need to do all the safety checking and we know they
1159  *	all belong to the same dma window.
1160  *
1161  *	Input:	cookie_no - # of cookies identified by the 1st scan
1162  *		start_idx - subscript of the pfn array for the starting pfn
1163  *		end_idx   - subscript of the last pfn in dma window
1164  *		win_pp    - pointer to win_next member of previous window
1165  *	Return:	DDI_SUCCESS - with **win_pp as newly created window object
1166  *		DDI_DMA_NORESROUCE - caller frees all previous window objs
1167  *	Note:	Each cookie and window size are all initialized on page
1168  *		boundary. This is not true for the 1st cookie of the 1st
1169  *		window and the last cookie of the last window.
1170  *		We fix that later in upper layer which has access to size
1171  *		and offset info.
1172  *
1173  */
1174 static int
pci_dma_newwin(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint32_t cookie_no,uint32_t start_idx,uint32_t end_idx,pci_dma_win_t ** win_pp,uint64_t count_max,uint64_t bypass_prefix)1175 pci_dma_newwin(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, uint32_t cookie_no,
1176 	uint32_t start_idx, uint32_t end_idx, pci_dma_win_t **win_pp,
1177 	uint64_t count_max, uint64_t bypass_prefix)
1178 {
1179 	int (*waitfp)(caddr_t) = dmareq->dmar_fp;
1180 	ddi_dma_cookie_t *cookie_p;
1181 	uint32_t pfn_no = 1;
1182 	iopfn_t pfn = PCI_GET_MP_PFN(mp, start_idx);
1183 	iopfn_t prev_pfn = pfn;
1184 	uint64_t seg_pfn0 = pfn;
1185 	size_t sz = cookie_no * sizeof (ddi_dma_cookie_t);
1186 	pci_dma_win_t *win_p = kmem_alloc(sizeof (pci_dma_win_t) + sz,
1187 	    waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
1188 	if (!win_p)
1189 		goto noresource;
1190 
1191 	win_p->win_next = NULL;
1192 	win_p->win_ncookies = cookie_no;
1193 	win_p->win_curseg = 0;	/* start from segment 0 */
1194 	win_p->win_size = IOMMU_PTOB(end_idx - start_idx + 1);
1195 	/* win_p->win_offset is left uninitialized */
1196 
1197 	cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1198 	start_idx++;
1199 	for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) {
1200 		pfn = PCI_GET_MP_PFN1(mp, start_idx);
1201 		if ((pfn == prev_pfn + 1) &&
1202 		    (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max))
1203 			continue;
1204 
1205 		/* close up the cookie up to (including) prev_pfn */
1206 		MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix,
1207 		    IOMMU_PTOB(pfn_no));
1208 		DEBUG2(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",
1209 		    IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no);
1210 
1211 		cookie_p++;	/* advance to next available cookie cell */
1212 		pfn_no = 0;
1213 		seg_pfn0 = pfn;	/* start a new segment from current pfn */
1214 	}
1215 	MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix,
1216 	    IOMMU_PTOB(pfn_no));
1217 	DEBUG3(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",
1218 	    IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no, cookie_no);
1219 #ifdef DEBUG
1220 	cookie_p++;
1221 	ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no);
1222 #endif
1223 	*win_pp = win_p;
1224 	return (DDI_SUCCESS);
1225 noresource:
1226 	if (waitfp != DDI_DMA_DONTWAIT)
1227 		ddi_set_callback(waitfp, dmareq->dmar_arg, &pci_kmem_clid);
1228 	return (DDI_DMA_NORESOURCES);
1229 }
1230 
1231 /*
1232  * pci_dma_adjust - adjust 1st and last cookie and window sizes
1233  *	remove initial dma page offset from 1st cookie and window size
1234  *	remove last dma page remainder from last cookie and window size
1235  *	fill win_offset of each dma window according to just fixed up
1236  *		each window sizes
1237  *	pci_dma_win_t members modified:
1238  *	win_p->win_offset - this window's offset within entire DMA object
1239  *	win_p->win_size	  - xferrable size (in bytes) for this window
1240  *
1241  *	ddi_dma_impl_t members modified:
1242  *	mp->dmai_size	  - 1st window xferrable size
1243  *	mp->dmai_offset   - 0, which is the dma offset of the 1st window
1244  *
1245  *	ddi_dma_cookie_t members modified:
1246  *	cookie_p->dmac_size - 1st and last cookie remove offset or remainder
1247  *	cookie_p->dmac_laddress - 1st cookie add page offset
1248  */
1249 static void
pci_dma_adjust(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,pci_dma_win_t * win_p)1250 pci_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, pci_dma_win_t *win_p)
1251 {
1252 	ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1253 	size_t pg_offset = mp->dmai_roffset;
1254 	size_t win_offset = 0;
1255 
1256 	cookie_p->dmac_size -= pg_offset;
1257 	cookie_p->dmac_laddress |= pg_offset;
1258 	win_p->win_size -= pg_offset;
1259 	DEBUG1(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);
1260 
1261 	mp->dmai_size = win_p->win_size;
1262 	mp->dmai_offset = 0;
1263 
1264 	pg_offset += mp->dmai_object.dmao_size;
1265 	pg_offset &= IOMMU_PAGE_OFFSET;
1266 	if (pg_offset)
1267 		pg_offset = IOMMU_PAGE_SIZE - pg_offset;
1268 	DEBUG1(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);
1269 
1270 	for (; win_p->win_next; win_p = win_p->win_next) {
1271 		DEBUG1(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);
1272 		win_p->win_offset = win_offset;
1273 		win_offset += win_p->win_size;
1274 	}
1275 	/* last window */
1276 	win_p->win_offset = win_offset;
1277 	cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1278 	cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset;
1279 	win_p->win_size -= pg_offset;
1280 	ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);
1281 }
1282 
1283 /*
1284  * pci_dma_physwin() - carve up dma windows using physical addresses.
1285  *	Called to handle iommu bypass and pci peer-to-peer transfers.
1286  *	Calls pci_dma_newwin() to allocate window objects.
1287  *
1288  * Dependency: mp->dmai_pfnlst points to an array of pfns
1289  *
1290  * 1. Each dma window is represented by a pci_dma_win_t object.
1291  *	The object will be casted to ddi_dma_win_t and returned
1292  *	to leaf driver through the DDI interface.
1293  * 2. Each dma window can have several dma segments with each
1294  *	segment representing a physically contiguous either memory
1295  *	space (if we are doing an iommu bypass transfer) or pci address
1296  *	space (if we are doing a peer-to-peer transfer).
1297  * 3. Each segment has a DMA cookie to program the DMA engine.
1298  *	The cookies within each DMA window must be located in a
1299  *	contiguous array per ddi_dma_nextcookie(9f).
1300  * 4. The number of DMA segments within each DMA window cannot exceed
1301  *	mp->dmai_attr.dma_attr_sgllen. If the transfer size is
1302  *	too large to fit in the sgllen, the rest needs to be
1303  *	relocated to the next dma window.
1304  * 5. Peer-to-peer DMA segment follows device hi, lo, count_max,
1305  *	and nocross restrictions while bypass DMA follows the set of
1306  *	restrictions with system limits factored in.
1307  *
1308  * Return:
1309  *	mp->dmai_winlst	 - points to a link list of pci_dma_win_t objects.
1310  *		Each pci_dma_win_t object on the link list contains
1311  *		infomation such as its window size (# of pages),
1312  *		starting offset (also see Restriction), an array of
1313  *		DMA cookies, and # of cookies in the array.
1314  *	mp->dmai_pfnlst	 - NULL, the pfn list is freed to conserve memory.
1315  *	mp->dmai_nwin	 - # of total DMA windows on mp->dmai_winlst.
1316  *	mp->dmai_mapping - starting cookie address
1317  *	mp->dmai_rflags	 - consistent, nosync, no redzone
1318  *	mp->dmai_cookie	 - start of cookie table of the 1st DMA window
1319  *
1320  * Restriction:
1321  *	Each pci_dma_win_t object can theoratically start from any offset
1322  *	since the iommu is not involved. However, this implementation
1323  *	always make windows start from page aligned offset (except
1324  *	the 1st window, which follows the requested offset) due to the
1325  *	fact that we are handed a pfn list. This does require device's
1326  *	count_max and attr_seg to be at least IOMMU_PAGE_SIZE aligned.
1327  */
1328 int
pci_dma_physwin(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)1329 pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
1330 {
1331 	uint_t npages = mp->dmai_ndvmapages;
1332 	int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;
1333 	iopfn_t pfn_lo, pfn_hi, prev_pfn, bypass_pfn;
1334 	iopfn_t pfn = PCI_GET_MP_PFN(mp, 0);
1335 	uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0;
1336 	uint64_t count_max, bypass = PCI_DMA_BYPASS_PREFIX(mp, pfn);
1337 	pci_dma_win_t **win_pp = (pci_dma_win_t **)&mp->dmai_winlst;
1338 	ddi_dma_cookie_t *cookie0_p;
1339 
1340 	if (PCI_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */
1341 		ddi_dma_attr_t *dev_attr_p = DEV_ATTR(mp);
1342 		iopfn_t pfn_base = pci_p->pci_pbm_p->pbm_base_pfn;
1343 		iopfn_t pfn_last = pci_p->pci_pbm_p->pbm_last_pfn - pfn_base;
1344 		uint64_t nocross = dev_attr_p->dma_attr_seg;
1345 		if (nocross && (nocross < UINT32_MAX))
1346 			return (DDI_DMA_NOMAPPING);
1347 		if (dev_attr_p->dma_attr_align > IOMMU_PAGE_SIZE)
1348 			return (DDI_DMA_NOMAPPING);
1349 		pfn_lo = IOMMU_BTOP(dev_attr_p->dma_attr_addr_lo);
1350 		pfn_hi = IOMMU_BTOP(dev_attr_p->dma_attr_addr_hi);
1351 		pfn_hi = MIN(pfn_hi, pfn_last);
1352 		if ((pfn_lo > pfn_hi) || (pfn < pfn_lo))
1353 			return (DDI_DMA_NOMAPPING);
1354 		count_max = dev_attr_p->dma_attr_count_max;
1355 		count_max = MIN(count_max, nocross);
1356 		/*
1357 		 * the following count_max trim is not done because we are
1358 		 * making sure pfn_lo <= pfn <= pfn_hi inside the loop
1359 		 * count_max=MIN(count_max, IOMMU_PTOB(pfn_hi - pfn_lo + 1)-1);
1360 		 */
1361 	} else { /* bypass hi/lo/count_max have been processed by attr2hdl() */
1362 		count_max = mp->dmai_attr.dma_attr_count_max;
1363 		pfn_lo = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);
1364 		pfn_hi = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);
1365 	}
1366 
1367 	bypass_pfn = IOMMU_BTOP(bypass);
1368 
1369 	for (prev_pfn = (bypass_pfn | pfn), i = 1; i < npages;
1370 	    i++, prev_pfn = pfn, pfn_no++) {
1371 		pfn = bypass_pfn | PCI_GET_MP_PFN1(mp, i);
1372 		if ((pfn == prev_pfn + 1) &&
1373 		    (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max))
1374 			continue;
1375 		if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) {
1376 			ret = DDI_DMA_NOMAPPING;
1377 			goto err;
1378 		}
1379 		cookie_no++;
1380 		pfn_no = 0;
1381 		if (cookie_no < sgllen)
1382 			continue;
1383 
1384 		DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1385 		    win_pfn0_index, i - 1, cookie_no);
1386 		if (ret = pci_dma_newwin(dmareq, mp, cookie_no,
1387 		    win_pfn0_index, i - 1, win_pp, count_max, bypass))
1388 			goto err;
1389 
1390 		win_pp = &(*win_pp)->win_next;	/* win_pp = *(win_pp) */
1391 		win_no++;
1392 		win_pfn0_index = i;
1393 		cookie_no = 0;
1394 	}
1395 	if (pfn > pfn_hi) {
1396 		ret = DDI_DMA_NOMAPPING;
1397 		goto err;
1398 	}
1399 	cookie_no++;
1400 	DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1401 	    win_pfn0_index, i - 1, cookie_no);
1402 	if (ret = pci_dma_newwin(dmareq, mp, cookie_no, win_pfn0_index,
1403 	    i - 1, win_pp, count_max, bypass))
1404 		goto err;
1405 	win_no++;
1406 	pci_dma_adjust(dmareq, mp, mp->dmai_winlst);
1407 	mp->dmai_nwin = win_no;
1408 	mp->dmai_rflags |= DDI_DMA_CONSISTENT;
1409 	if (!pci_p->pci_pbm_p->pbm_sync_reg_pa) {
1410 		mp->dmai_rflags |= DMP_NOSYNC;
1411 		mp->dmai_flags |= DMAI_FLAGS_NOSYNC;
1412 	}
1413 	mp->dmai_rflags &= ~DDI_DMA_REDZONE;
1414 	cookie0_p = (ddi_dma_cookie_t *)(WINLST(mp) + 1);
1415 	mp->dmai_cookie = cookie0_p + 1;
1416 	mp->dmai_mapping = cookie0_p->dmac_laddress;
1417 	mp->dmai_ncookies = WINLST(mp)->win_ncookies;
1418 	mp->dmai_curcookie = 1;
1419 
1420 	pci_dma_freepfn(mp);
1421 	return (DDI_DMA_MAPPED);
1422 err:
1423 	pci_dma_freewin(mp);
1424 	return (ret);
1425 }
1426 
1427 /*ARGSUSED*/
1428 int
pci_dma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)1429 pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
1430 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
1431 	uint_t cache_flags)
1432 {
1433 	switch (cmd) {
1434 
1435 	case DDI_DMA_HTOC: {
1436 		off_t off = *offp;
1437 		ddi_dma_cookie_t *loop_cp, *cp;
1438 		pci_dma_win_t *win_p = mp->dmai_winlst;
1439 
1440 		if (off >= mp->dmai_object.dmao_size)
1441 			return (DDI_FAILURE);
1442 
1443 		/* locate window */
1444 		while (win_p->win_offset + win_p->win_size <= off)
1445 			win_p = win_p->win_next;
1446 
1447 		loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1);
1448 		mp->dmai_offset = win_p->win_offset;
1449 		mp->dmai_size   = win_p->win_size;
1450 		mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */
1451 
1452 		/* adjust cookie addr/len if we are not on cookie boundary */
1453 		off -= win_p->win_offset;	   /* offset within window */
1454 		for (; off >= loop_cp->dmac_size; loop_cp++)
1455 			off -= loop_cp->dmac_size; /* offset within cookie */
1456 
1457 		mp->dmai_cookie = loop_cp + 1;
1458 		win_p->win_curseg = loop_cp - cp;
1459 		cp = (ddi_dma_cookie_t *)objp;
1460 		MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off,
1461 		    loop_cp->dmac_size - off);
1462 
1463 		DEBUG2(DBG_DMA_CTL, dip,
1464 		    "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
1465 		    cp->dmac_laddress, cp->dmac_size);
1466 		}
1467 		return (DDI_SUCCESS);
1468 
1469 	case DDI_DMA_COFF: {
1470 		pci_dma_win_t *win_p;
1471 		ddi_dma_cookie_t *cp;
1472 		uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress;
1473 		size_t win_off;
1474 
1475 		for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) {
1476 			int i;
1477 			win_off = 0;
1478 			cp = (ddi_dma_cookie_t *)(win_p + 1);
1479 			for (i = 0; i < win_p->win_ncookies; i++, cp++) {
1480 				size_t sz = cp->dmac_size;
1481 
1482 				addr = cp->dmac_laddress;
1483 				if ((addr <= key) && (addr + sz >= key))
1484 					goto found;
1485 				win_off += sz;
1486 			}
1487 		}
1488 		return (DDI_FAILURE);
1489 found:
1490 		*objp = (caddr_t)(win_p->win_offset + win_off + (key - addr));
1491 		return (DDI_SUCCESS);
1492 		}
1493 
1494 	case DDI_DMA_REMAP:
1495 		return (DDI_FAILURE);
1496 
1497 	default:
1498 		DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1499 		    cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
1500 		break;
1501 	}
1502 	return (DDI_FAILURE);
1503 }
1504 
1505 static void
pci_dvma_debug_init(iommu_t * iommu_p)1506 pci_dvma_debug_init(iommu_t *iommu_p)
1507 {
1508 	size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec;
1509 	ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock));
1510 	cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p);
1511 
1512 	iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP);
1513 	iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP);
1514 
1515 	iommu_p->dvma_active_list = NULL;
1516 	iommu_p->dvma_alloc_rec_index = 0;
1517 	iommu_p->dvma_free_rec_index = 0;
1518 	iommu_p->dvma_active_count = 0;
1519 }
1520 
1521 void
pci_dvma_debug_fini(iommu_t * iommu_p)1522 pci_dvma_debug_fini(iommu_t *iommu_p)
1523 {
1524 	struct dvma_rec *prev, *ptr;
1525 	size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec;
1526 	uint64_t mask = ~(1ull << iommu_p->iommu_inst);
1527 	cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p);
1528 
1529 	kmem_free(iommu_p->dvma_alloc_rec, sz);
1530 	kmem_free(iommu_p->dvma_free_rec, sz);
1531 	iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL;
1532 
1533 	prev = iommu_p->dvma_active_list;
1534 	if (!prev)
1535 		return;
1536 	for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next)
1537 		kmem_free(prev, sizeof (struct dvma_rec));
1538 	kmem_free(prev, sizeof (struct dvma_rec));
1539 
1540 	iommu_p->dvma_active_list = NULL;
1541 	iommu_p->dvma_alloc_rec_index = 0;
1542 	iommu_p->dvma_free_rec_index = 0;
1543 	iommu_p->dvma_active_count = 0;
1544 
1545 	pci_dvma_debug_on  &= mask;
1546 	pci_dvma_debug_off &= mask;
1547 }
1548 
1549 void
pci_dvma_alloc_debug(iommu_t * iommu_p,char * address,uint_t len,ddi_dma_impl_t * mp)1550 pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len,
1551 	ddi_dma_impl_t *mp)
1552 {
1553 	struct dvma_rec *ptr;
1554 	mutex_enter(&iommu_p->dvma_debug_lock);
1555 
1556 	if (!iommu_p->dvma_alloc_rec)
1557 		pci_dvma_debug_init(iommu_p);
1558 	if (DVMA_DBG_OFF(iommu_p)) {
1559 		pci_dvma_debug_fini(iommu_p);
1560 		goto done;
1561 	}
1562 
1563 	ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index];
1564 	ptr->dvma_addr = address;
1565 	ptr->len = len;
1566 	ptr->mp = mp;
1567 	if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec)
1568 		iommu_p->dvma_alloc_rec_index = 0;
1569 
1570 	ptr = kmem_alloc(sizeof (struct dvma_rec), KM_SLEEP);
1571 	ptr->dvma_addr = address;
1572 	ptr->len = len;
1573 	ptr->mp = mp;
1574 
1575 	ptr->next = iommu_p->dvma_active_list;
1576 	iommu_p->dvma_active_list = ptr;
1577 	iommu_p->dvma_active_count++;
1578 done:
1579 	mutex_exit(&iommu_p->dvma_debug_lock);
1580 }
1581 
1582 void
pci_dvma_free_debug(iommu_t * iommu_p,char * address,uint_t len,ddi_dma_impl_t * mp)1583 pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len,
1584 	ddi_dma_impl_t *mp)
1585 {
1586 	struct dvma_rec *ptr, *ptr_save;
1587 	mutex_enter(&iommu_p->dvma_debug_lock);
1588 
1589 	if (!iommu_p->dvma_alloc_rec)
1590 		pci_dvma_debug_init(iommu_p);
1591 	if (DVMA_DBG_OFF(iommu_p)) {
1592 		pci_dvma_debug_fini(iommu_p);
1593 		goto done;
1594 	}
1595 
1596 	ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index];
1597 	ptr->dvma_addr = address;
1598 	ptr->len = len;
1599 	ptr->mp = mp;
1600 	if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec)
1601 		iommu_p->dvma_free_rec_index = 0;
1602 
1603 	ptr_save = iommu_p->dvma_active_list;
1604 	for (ptr = ptr_save; ptr; ptr = ptr->next) {
1605 		if ((ptr->dvma_addr == address) && (ptr->len = len))
1606 			break;
1607 		ptr_save = ptr;
1608 	}
1609 	if (!ptr) {
1610 		cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x",
1611 		    (long)address, len);
1612 		goto done;
1613 	}
1614 	if (ptr == iommu_p->dvma_active_list)
1615 		iommu_p->dvma_active_list = ptr->next;
1616 	else
1617 		ptr_save->next = ptr->next;
1618 	kmem_free(ptr, sizeof (struct dvma_rec));
1619 	iommu_p->dvma_active_count--;
1620 done:
1621 	mutex_exit(&iommu_p->dvma_debug_lock);
1622 }
1623 
1624 #ifdef DEBUG
1625 void
dump_dma_handle(uint64_t flag,dev_info_t * dip,ddi_dma_impl_t * hp)1626 dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp)
1627 {
1628 	DEBUG4(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n",
1629 	    hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size);
1630 	DEBUG4(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n",
1631 	    hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags,
1632 	    hp->dmai_nwin);
1633 	DEBUG4(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n",
1634 	    hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0);
1635 	DEBUG4(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n",
1636 	    hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr,
1637 	    hp->dmai_cookie);
1638 }
1639 #endif
1640 
1641 void
pci_vmem_do_free(iommu_t * iommu_p,void * base_addr,size_t npages,int vmemcache)1642 pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages,
1643     int vmemcache)
1644 {
1645 	vmem_t *map_p = iommu_p->iommu_dvma_map;
1646 
1647 	if (vmemcache) {
1648 		vmem_free(map_p, base_addr, IOMMU_PAGE_SIZE);
1649 #ifdef PCI_DMA_PROF
1650 		pci_dvma_vmem_free++;
1651 #endif
1652 		return;
1653 	}
1654 
1655 	vmem_xfree(map_p, base_addr, IOMMU_PTOB(npages));
1656 #ifdef PCI_DMA_PROF
1657 		pci_dvma_vmem_xfree++;
1658 #endif
1659 }
1660