xref: /illumos-gate/usr/src/uts/sun4/io/px/px_dma.c (revision 9a63ec27)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
51de45cd9Sgovinda  * Common Development and Distribution License (the "License").
61de45cd9Sgovinda  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22567c0b92SStephen Hanson  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
25cd21e7c5SGarrett D'Amore /*
26cd21e7c5SGarrett D'Amore  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
27cd21e7c5SGarrett D'Amore  */
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate  * PCI Express nexus DVMA and DMA core routines:
317c478bd9Sstevel@tonic-gate  *	dma_map/dma_bind_handle implementation
327c478bd9Sstevel@tonic-gate  *	bypass and peer-to-peer support
337c478bd9Sstevel@tonic-gate  *	fast track DVMA space allocation
347c478bd9Sstevel@tonic-gate  *	runtime DVMA debug
357c478bd9Sstevel@tonic-gate  */
367c478bd9Sstevel@tonic-gate #include <sys/types.h>
377c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
387c478bd9Sstevel@tonic-gate #include <sys/async.h>
397c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
407c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
417c478bd9Sstevel@tonic-gate #include <sys/ddi_impldefs.h>
427c478bd9Sstevel@tonic-gate #include "px_obj.h"
437c478bd9Sstevel@tonic-gate 
447c478bd9Sstevel@tonic-gate /*LINTLIBRARY*/
457c478bd9Sstevel@tonic-gate 
467c478bd9Sstevel@tonic-gate /*
477c478bd9Sstevel@tonic-gate  * px_dma_allocmp - Allocate a pci dma implementation structure
487c478bd9Sstevel@tonic-gate  *
497c478bd9Sstevel@tonic-gate  * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl
507c478bd9Sstevel@tonic-gate  * to hold unmodified device limits. The ddi_dma_attr inside the
517c478bd9Sstevel@tonic-gate  * ddi_dma_impl structure is augumented with system limits to enhance
527c478bd9Sstevel@tonic-gate  * DVMA performance at runtime. The unaugumented device limits saved
537c478bd9Sstevel@tonic-gate  * right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used
547c478bd9Sstevel@tonic-gate  * strictly for peer-to-peer transfers which do not obey system limits.
557c478bd9Sstevel@tonic-gate  *
567c478bd9Sstevel@tonic-gate  * return: DDI_SUCCESS DDI_DMA_NORESOURCES
577c478bd9Sstevel@tonic-gate  */
587c478bd9Sstevel@tonic-gate ddi_dma_impl_t *
px_dma_allocmp(dev_info_t * dip,dev_info_t * rdip,int (* waitfp)(caddr_t),caddr_t arg)597c478bd9Sstevel@tonic-gate px_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t),
607c478bd9Sstevel@tonic-gate 	caddr_t arg)
617c478bd9Sstevel@tonic-gate {
627c478bd9Sstevel@tonic-gate 	register ddi_dma_impl_t *mp;
637c478bd9Sstevel@tonic-gate 	int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
647c478bd9Sstevel@tonic-gate 
657c478bd9Sstevel@tonic-gate 	/* Caution: we don't use zalloc to enhance performance! */
667c478bd9Sstevel@tonic-gate 	if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) {
677c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip, "can't alloc dma_handle\n");
687c478bd9Sstevel@tonic-gate 		if (waitfp != DDI_DMA_DONTWAIT) {
697c478bd9Sstevel@tonic-gate 			DBG(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n");
707c478bd9Sstevel@tonic-gate 			ddi_set_callback(waitfp, arg, &px_kmem_clid);
717c478bd9Sstevel@tonic-gate 		}
727c478bd9Sstevel@tonic-gate 		return (mp);
737c478bd9Sstevel@tonic-gate 	}
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate 	mp->dmai_rdip = rdip;
767c478bd9Sstevel@tonic-gate 	mp->dmai_flags = 0;
777c478bd9Sstevel@tonic-gate 	mp->dmai_pfnlst = NULL;
787c478bd9Sstevel@tonic-gate 	mp->dmai_winlst = NULL;
79*9a63ec27SRobert Mustacchi 	mp->dmai_ncookies = 0;
80*9a63ec27SRobert Mustacchi 	mp->dmai_curcookie = 0;
817c478bd9Sstevel@tonic-gate 
827c478bd9Sstevel@tonic-gate 	/*
837c478bd9Sstevel@tonic-gate 	 * kmem_alloc debug: the following fields are not zero-ed
847c478bd9Sstevel@tonic-gate 	 * mp->dmai_mapping = 0;
857c478bd9Sstevel@tonic-gate 	 * mp->dmai_size = 0;
867c478bd9Sstevel@tonic-gate 	 * mp->dmai_offset = 0;
877c478bd9Sstevel@tonic-gate 	 * mp->dmai_minxfer = 0;
887c478bd9Sstevel@tonic-gate 	 * mp->dmai_burstsizes = 0;
897c478bd9Sstevel@tonic-gate 	 * mp->dmai_ndvmapages = 0;
907c478bd9Sstevel@tonic-gate 	 * mp->dmai_pool/roffset = 0;
917c478bd9Sstevel@tonic-gate 	 * mp->dmai_rflags = 0;
927c478bd9Sstevel@tonic-gate 	 * mp->dmai_inuse/flags
937c478bd9Sstevel@tonic-gate 	 * mp->dmai_nwin = 0;
947c478bd9Sstevel@tonic-gate 	 * mp->dmai_winsize = 0;
957c478bd9Sstevel@tonic-gate 	 * mp->dmai_nexus_private/tte = 0;
967c478bd9Sstevel@tonic-gate 	 * mp->dmai_iopte/pfnlst
977c478bd9Sstevel@tonic-gate 	 * mp->dmai_sbi/pfn0 = 0;
987c478bd9Sstevel@tonic-gate 	 * mp->dmai_minfo/winlst/fdvma
997c478bd9Sstevel@tonic-gate 	 * mp->dmai_rdip
1007c478bd9Sstevel@tonic-gate 	 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
1017c478bd9Sstevel@tonic-gate 	 * bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t));
1027c478bd9Sstevel@tonic-gate 	 * mp->dmai_cookie = 0;
1037c478bd9Sstevel@tonic-gate 	 */
1047c478bd9Sstevel@tonic-gate 
1057c478bd9Sstevel@tonic-gate 	mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;
1067c478bd9Sstevel@tonic-gate 	mp->dmai_attr.dma_attr_flags = (uint_t)0;
1077c478bd9Sstevel@tonic-gate 	mp->dmai_fault = 0;
1087c478bd9Sstevel@tonic-gate 	mp->dmai_fault_check = NULL;
1097c478bd9Sstevel@tonic-gate 	mp->dmai_fault_notify = NULL;
110b6ec8a57Svgadre 
111b6ec8a57Svgadre 	mp->dmai_error.err_ena = 0;
112b6ec8a57Svgadre 	mp->dmai_error.err_status = DDI_FM_OK;
113b6ec8a57Svgadre 	mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
114b6ec8a57Svgadre 	mp->dmai_error.err_ontrap = NULL;
115b6ec8a57Svgadre 	mp->dmai_error.err_fep = NULL;
11600d0963fSdilpreet 	mp->dmai_error.err_cf = NULL;
117b6ec8a57Svgadre 
11844961713Sgirish 	/*
1199fc8611eSDaniel Ice 	 * The bdf protection value is set to immediate child
1209fc8611eSDaniel Ice 	 * at first. It gets modified by switch/bridge drivers
1219fc8611eSDaniel Ice 	 * as the code traverses down the fabric topology.
1229fc8611eSDaniel Ice 	 *
1239fc8611eSDaniel Ice 	 * XXX No IOMMU protection for broken devices.
12444961713Sgirish 	 */
1259fc8611eSDaniel Ice 	ASSERT((intptr_t)ddi_get_parent_data(rdip) >> 1 == 0);
126c85864d8SKrishna Elango 	mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ?
127c85864d8SKrishna Elango 	    PCIE_INVALID_BDF : pcie_get_bdf_for_dma_xfer(dip, rdip);
12844961713Sgirish 
129567c0b92SStephen Hanson 	ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);
1307c478bd9Sstevel@tonic-gate 	return (mp);
1317c478bd9Sstevel@tonic-gate }
1327c478bd9Sstevel@tonic-gate 
1337c478bd9Sstevel@tonic-gate void
px_dma_freemp(ddi_dma_impl_t * mp)1347c478bd9Sstevel@tonic-gate px_dma_freemp(ddi_dma_impl_t *mp)
1357c478bd9Sstevel@tonic-gate {
136567c0b92SStephen Hanson 	ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp);
1377c478bd9Sstevel@tonic-gate 	if (mp->dmai_ndvmapages > 1)
1387c478bd9Sstevel@tonic-gate 		px_dma_freepfn(mp);
1397c478bd9Sstevel@tonic-gate 	if (mp->dmai_winlst)
1407c478bd9Sstevel@tonic-gate 		px_dma_freewin(mp);
1417c478bd9Sstevel@tonic-gate 	kmem_free(mp, sizeof (px_dma_hdl_t));
1427c478bd9Sstevel@tonic-gate }
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate void
px_dma_freepfn(ddi_dma_impl_t * mp)1457c478bd9Sstevel@tonic-gate px_dma_freepfn(ddi_dma_impl_t *mp)
1467c478bd9Sstevel@tonic-gate {
1477c478bd9Sstevel@tonic-gate 	void *addr = mp->dmai_pfnlst;
1487c478bd9Sstevel@tonic-gate 	if (addr) {
1497c478bd9Sstevel@tonic-gate 		size_t npages = mp->dmai_ndvmapages;
1507c478bd9Sstevel@tonic-gate 		if (npages > 1)
1517c478bd9Sstevel@tonic-gate 			kmem_free(addr, npages * sizeof (px_iopfn_t));
1527c478bd9Sstevel@tonic-gate 		mp->dmai_pfnlst = NULL;
1537c478bd9Sstevel@tonic-gate 	}
1547c478bd9Sstevel@tonic-gate 	mp->dmai_ndvmapages = 0;
1557c478bd9Sstevel@tonic-gate }
1567c478bd9Sstevel@tonic-gate 
1577c478bd9Sstevel@tonic-gate /*
1587c478bd9Sstevel@tonic-gate  * px_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits
1597c478bd9Sstevel@tonic-gate  *			and convert dmareq->dmar_limits to mp->dmai_attr
1607c478bd9Sstevel@tonic-gate  *
1617c478bd9Sstevel@tonic-gate  * ddi_dma_impl_t member modified     input
1627c478bd9Sstevel@tonic-gate  * ------------------------------------------------------------------------
1637c478bd9Sstevel@tonic-gate  * mp->dmai_minxfer		    - dev
1647c478bd9Sstevel@tonic-gate  * mp->dmai_burstsizes		    - dev
1657c478bd9Sstevel@tonic-gate  * mp->dmai_flags		    - no limit? peer-to-peer only?
1667c478bd9Sstevel@tonic-gate  *
1677c478bd9Sstevel@tonic-gate  * ddi_dma_attr member modified       input
1687c478bd9Sstevel@tonic-gate  * ------------------------------------------------------------------------
1697c478bd9Sstevel@tonic-gate  * mp->dmai_attr.dma_attr_addr_lo   - dev lo, sys lo
1707c478bd9Sstevel@tonic-gate  * mp->dmai_attr.dma_attr_addr_hi   - dev hi, sys hi
1717c478bd9Sstevel@tonic-gate  * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta
1727c478bd9Sstevel@tonic-gate  * mp->dmai_attr.dma_attr_seg       - 0         (no nocross   restriction)
1737c478bd9Sstevel@tonic-gate  * mp->dmai_attr.dma_attr_align     - 1         (no alignment restriction)
1747c478bd9Sstevel@tonic-gate  *
1757c478bd9Sstevel@tonic-gate  * The dlim_dmaspeed member of dmareq->dmar_limits is ignored.
1767c478bd9Sstevel@tonic-gate  */
1777c478bd9Sstevel@tonic-gate ddi_dma_impl_t *
px_dma_lmts2hdl(dev_info_t * dip,dev_info_t * rdip,px_mmu_t * mmu_p,ddi_dma_req_t * dmareq)1787c478bd9Sstevel@tonic-gate px_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, px_mmu_t *mmu_p,
1797c478bd9Sstevel@tonic-gate 	ddi_dma_req_t *dmareq)
1807c478bd9Sstevel@tonic-gate {
1817c478bd9Sstevel@tonic-gate 	ddi_dma_impl_t *mp;
1827c478bd9Sstevel@tonic-gate 	ddi_dma_attr_t *attr_p;
1837c478bd9Sstevel@tonic-gate 	uint64_t syslo		= mmu_p->mmu_dvma_base;
1847c478bd9Sstevel@tonic-gate 	uint64_t syshi		= mmu_p->mmu_dvma_end;
1857c478bd9Sstevel@tonic-gate 	uint64_t fasthi		= mmu_p->mmu_dvma_fast_end;
1867c478bd9Sstevel@tonic-gate 	ddi_dma_lim_t *lim_p	= dmareq->dmar_limits;
1877c478bd9Sstevel@tonic-gate 	uint32_t count_max	= lim_p->dlim_cntr_max;
1887c478bd9Sstevel@tonic-gate 	uint64_t lo		= lim_p->dlim_addr_lo;
1897c478bd9Sstevel@tonic-gate 	uint64_t hi		= lim_p->dlim_addr_hi;
1907c478bd9Sstevel@tonic-gate 	if (hi <= lo) {
1917c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip, "Bad limits\n");
1927c478bd9Sstevel@tonic-gate 		return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING);
1937c478bd9Sstevel@tonic-gate 	}
1947c478bd9Sstevel@tonic-gate 	if (!count_max)
1957c478bd9Sstevel@tonic-gate 		count_max--;
1967c478bd9Sstevel@tonic-gate 
1977c478bd9Sstevel@tonic-gate 	if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp,
1989fc8611eSDaniel Ice 	    dmareq->dmar_arg)))
1997c478bd9Sstevel@tonic-gate 		return (NULL);
2007c478bd9Sstevel@tonic-gate 
2017c478bd9Sstevel@tonic-gate 	/* store original dev input at the 2nd ddi_dma_attr */
20236fe4a92Segillett 	attr_p = PX_DEV_ATTR(mp);
2037c478bd9Sstevel@tonic-gate 	SET_DMAATTR(attr_p, lo, hi, -1, count_max);
2047c478bd9Sstevel@tonic-gate 	SET_DMAALIGN(attr_p, 1);
2057c478bd9Sstevel@tonic-gate 
2067c478bd9Sstevel@tonic-gate 	lo = MAX(lo, syslo);
2077c478bd9Sstevel@tonic-gate 	hi = MIN(hi, syshi);
2087c478bd9Sstevel@tonic-gate 	if (hi <= lo)
20936fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;
2107c478bd9Sstevel@tonic-gate 	count_max = MIN(count_max, hi - lo);
2117c478bd9Sstevel@tonic-gate 
21236fe4a92Segillett 	if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1))
21336fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT |
2149fc8611eSDaniel Ice 		    PX_DMAI_FLAGS_NOSYSLIMIT;
2157c478bd9Sstevel@tonic-gate 	else {
21636fe4a92Segillett 		if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1))
21736fe4a92Segillett 			mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;
2187c478bd9Sstevel@tonic-gate 	}
2197c478bd9Sstevel@tonic-gate 	if (PX_DMA_NOCTX(rdip))
22036fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;
2217c478bd9Sstevel@tonic-gate 
2227c478bd9Sstevel@tonic-gate 	/* store augumented dev input to mp->dmai_attr */
2237c478bd9Sstevel@tonic-gate 	mp->dmai_burstsizes	= lim_p->dlim_burstsizes;
2247c478bd9Sstevel@tonic-gate 	attr_p = &mp->dmai_attr;
2257c478bd9Sstevel@tonic-gate 	SET_DMAATTR(attr_p, lo, hi, -1, count_max);
2267c478bd9Sstevel@tonic-gate 	SET_DMAALIGN(attr_p, 1);
2277c478bd9Sstevel@tonic-gate 	return (mp);
2287c478bd9Sstevel@tonic-gate }
2297c478bd9Sstevel@tonic-gate 
2307c478bd9Sstevel@tonic-gate /*
2317c478bd9Sstevel@tonic-gate  * Called from px_attach to check for bypass dma support and set
2327c478bd9Sstevel@tonic-gate  * flags accordingly.
2337c478bd9Sstevel@tonic-gate  */
2347c478bd9Sstevel@tonic-gate int
px_dma_attach(px_t * px_p)2357c478bd9Sstevel@tonic-gate px_dma_attach(px_t *px_p)
2367c478bd9Sstevel@tonic-gate {
2377c478bd9Sstevel@tonic-gate 	uint64_t baddr;
2387c478bd9Sstevel@tonic-gate 
2397c478bd9Sstevel@tonic-gate 	if (px_lib_iommu_getbypass(px_p->px_dip, 0ull,
2409fc8611eSDaniel Ice 	    PCI_MAP_ATTR_WRITE|PCI_MAP_ATTR_READ,
2419fc8611eSDaniel Ice 	    &baddr) != DDI_ENOTSUP)
2427c478bd9Sstevel@tonic-gate 		/* ignore all other errors */
243b65731f1Skini 		px_p->px_dev_caps |= PX_BYPASS_DMA_ALLOWED;
2447c478bd9Sstevel@tonic-gate 
24522bbbd20Saa 	px_p->px_dma_sync_opt = ddi_prop_get_int(DDI_DEV_T_ANY,
24622bbbd20Saa 	    px_p->px_dip, DDI_PROP_DONTPASS, "dma-sync-options", 0);
24722bbbd20Saa 
24822bbbd20Saa 	if (px_p->px_dma_sync_opt != 0)
24922bbbd20Saa 		px_p->px_dev_caps |= PX_DMA_SYNC_REQUIRED;
25022bbbd20Saa 
2517c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
2527c478bd9Sstevel@tonic-gate }
2537c478bd9Sstevel@tonic-gate 
2547c478bd9Sstevel@tonic-gate /*
2557c478bd9Sstevel@tonic-gate  * px_dma_attr2hdl
2567c478bd9Sstevel@tonic-gate  *
2577c478bd9Sstevel@tonic-gate  * This routine is called from the alloc handle entry point to sanity check the
2587c478bd9Sstevel@tonic-gate  * dma attribute structure.
2597c478bd9Sstevel@tonic-gate  *
2607c478bd9Sstevel@tonic-gate  * use by: px_dma_allochdl()
2617c478bd9Sstevel@tonic-gate  *
2627c478bd9Sstevel@tonic-gate  * return value:
2637c478bd9Sstevel@tonic-gate  *
2647c478bd9Sstevel@tonic-gate  *	DDI_SUCCESS		- on success
2657c478bd9Sstevel@tonic-gate  *	DDI_DMA_BADATTR		- attribute has invalid version number
2667c478bd9Sstevel@tonic-gate  *				  or address limits exclude dvma space
2677c478bd9Sstevel@tonic-gate  */
2687c478bd9Sstevel@tonic-gate int
px_dma_attr2hdl(px_t * px_p,ddi_dma_impl_t * mp)2697c478bd9Sstevel@tonic-gate px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp)
2707c478bd9Sstevel@tonic-gate {
2717c478bd9Sstevel@tonic-gate 	px_mmu_t *mmu_p = px_p->px_mmu_p;
2727c478bd9Sstevel@tonic-gate 	uint64_t syslo, syshi;
2737c478bd9Sstevel@tonic-gate 	int	ret;
27436fe4a92Segillett 	ddi_dma_attr_t *attrp		= PX_DEV_ATTR(mp);
2757c478bd9Sstevel@tonic-gate 	uint64_t hi			= attrp->dma_attr_addr_hi;
2767c478bd9Sstevel@tonic-gate 	uint64_t lo			= attrp->dma_attr_addr_lo;
2777c478bd9Sstevel@tonic-gate 	uint64_t align			= attrp->dma_attr_align;
2787c478bd9Sstevel@tonic-gate 	uint64_t nocross		= attrp->dma_attr_seg;
2797c478bd9Sstevel@tonic-gate 	uint64_t count_max		= attrp->dma_attr_count_max;
2807c478bd9Sstevel@tonic-gate 
2817c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "attrp=%p cntr_max=%x.%08x\n",
2829fc8611eSDaniel Ice 	    attrp, HI32(count_max), LO32(count_max));
2837c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x lo=%x.%08x\n",
2849fc8611eSDaniel Ice 	    HI32(hi), LO32(hi), HI32(lo), LO32(lo));
2857c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "seg=%x.%08x align=%x.%08x\n",
2869fc8611eSDaniel Ice 	    HI32(nocross), LO32(nocross), HI32(align), LO32(align));
2877c478bd9Sstevel@tonic-gate 
2887c478bd9Sstevel@tonic-gate 	if (!nocross)
2897c478bd9Sstevel@tonic-gate 		nocross--;
2907c478bd9Sstevel@tonic-gate 	if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */
2917c478bd9Sstevel@tonic-gate 
2927c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_ALLOCH, px_p->px_dip, "bypass mode\n");
2937c478bd9Sstevel@tonic-gate 		/*
2947c478bd9Sstevel@tonic-gate 		 * If Bypass DMA is not supported, return error so that
2957c478bd9Sstevel@tonic-gate 		 * target driver can fall back to dvma mode of operation
2967c478bd9Sstevel@tonic-gate 		 */
297b65731f1Skini 		if (!(px_p->px_dev_caps & PX_BYPASS_DMA_ALLOWED))
2987c478bd9Sstevel@tonic-gate 			return (DDI_DMA_BADATTR);
29936fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ;
3007c478bd9Sstevel@tonic-gate 		if (nocross != UINT64_MAX)
3017c478bd9Sstevel@tonic-gate 			return (DDI_DMA_BADATTR);
3027c478bd9Sstevel@tonic-gate 		if (align && (align > MMU_PAGE_SIZE))
3037c478bd9Sstevel@tonic-gate 			return (DDI_DMA_BADATTR);
3047c478bd9Sstevel@tonic-gate 		align = 1; /* align on 1 page boundary */
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate 		/* do a range check and get the limits */
30725cf1a30Sjl 		ret = px_lib_dma_bypass_rngchk(px_p->px_dip, attrp,
3089fc8611eSDaniel Ice 		    &syslo, &syshi);
3097c478bd9Sstevel@tonic-gate 		if (ret != DDI_SUCCESS)
3107c478bd9Sstevel@tonic-gate 			return (ret);
3117c478bd9Sstevel@tonic-gate 	} else { /* MMU_XLATE or PEER_TO_PEER */
3127c478bd9Sstevel@tonic-gate 		align = MAX(align, MMU_PAGE_SIZE) - 1;
3137c478bd9Sstevel@tonic-gate 		if ((align & nocross) != align) {
3147c478bd9Sstevel@tonic-gate 			dev_info_t *rdip = mp->dmai_rdip;
3157c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned",
3169fc8611eSDaniel Ice 			    NAMEINST(rdip));
3177c478bd9Sstevel@tonic-gate 			return (DDI_DMA_BADATTR);
3187c478bd9Sstevel@tonic-gate 		}
3197c478bd9Sstevel@tonic-gate 		align = MMU_BTOP(align + 1);
3207c478bd9Sstevel@tonic-gate 		syslo = mmu_p->mmu_dvma_base;
3217c478bd9Sstevel@tonic-gate 		syshi = mmu_p->mmu_dvma_end;
3227c478bd9Sstevel@tonic-gate 	}
3237c478bd9Sstevel@tonic-gate 	if (hi <= lo) {
3247c478bd9Sstevel@tonic-gate 		dev_info_t *rdip = mp->dmai_rdip;
3257c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip));
3267c478bd9Sstevel@tonic-gate 		return (DDI_DMA_BADATTR);
3277c478bd9Sstevel@tonic-gate 	}
3287c478bd9Sstevel@tonic-gate 	lo = MAX(lo, syslo);
3297c478bd9Sstevel@tonic-gate 	hi = MIN(hi, syshi);
3307c478bd9Sstevel@tonic-gate 	if (!count_max)
3317c478bd9Sstevel@tonic-gate 		count_max--;
3327c478bd9Sstevel@tonic-gate 
3337c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x, lo=%x.%08x\n",
3349fc8611eSDaniel Ice 	    HI32(hi), LO32(hi), HI32(lo), LO32(lo));
335d85c7355Scjj 	if (hi <= lo) {
336d85c7355Scjj 		/*
337d85c7355Scjj 		 * If this is an IOMMU bypass access, the caller can't use
338d85c7355Scjj 		 * the required addresses, so fail it.  Otherwise, it's
339d85c7355Scjj 		 * peer-to-peer; ensure that the caller has no alignment or
340d85c7355Scjj 		 * segment size restrictions.
341d85c7355Scjj 		 */
342d85c7355Scjj 		if ((mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ||
343d85c7355Scjj 		    (nocross < UINT32_MAX) || (align > 1))
3447c478bd9Sstevel@tonic-gate 			return (DDI_DMA_BADATTR);
345d85c7355Scjj 
34636fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY;
3477c478bd9Sstevel@tonic-gate 	} else /* set practical counter_max value */
3487c478bd9Sstevel@tonic-gate 		count_max = MIN(count_max, hi - lo);
3497c478bd9Sstevel@tonic-gate 
35036fe4a92Segillett 	if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align))
35136fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT |
3529fc8611eSDaniel Ice 		    PX_DMAI_FLAGS_NOFASTLIMIT;
3537c478bd9Sstevel@tonic-gate 	else {
3547c478bd9Sstevel@tonic-gate 		syshi = mmu_p->mmu_dvma_fast_end;
35536fe4a92Segillett 		if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align))
35636fe4a92Segillett 			mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT;
3577c478bd9Sstevel@tonic-gate 	}
3587c478bd9Sstevel@tonic-gate 	if (PX_DMA_NOCTX(mp->dmai_rdip))
35936fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX;
3607c478bd9Sstevel@tonic-gate 
3617c478bd9Sstevel@tonic-gate 	mp->dmai_burstsizes	= attrp->dma_attr_burstsizes;
3627c478bd9Sstevel@tonic-gate 	attrp = &mp->dmai_attr;
3637c478bd9Sstevel@tonic-gate 	SET_DMAATTR(attrp, lo, hi, nocross, count_max);
3647c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
3657c478bd9Sstevel@tonic-gate }
3667c478bd9Sstevel@tonic-gate 
3677c478bd9Sstevel@tonic-gate #define	TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end))
3687c478bd9Sstevel@tonic-gate 
3697c478bd9Sstevel@tonic-gate /*
3707c478bd9Sstevel@tonic-gate  * px_dma_type - determine which of the three types DMA (peer-to-peer,
3717c478bd9Sstevel@tonic-gate  *		mmu bypass, or mmu translate) we are asked to do.
3727c478bd9Sstevel@tonic-gate  *		Also checks pfn0 and rejects any non-peer-to-peer
3737c478bd9Sstevel@tonic-gate  *		requests for peer-only devices.
3747c478bd9Sstevel@tonic-gate  *
3757c478bd9Sstevel@tonic-gate  *	return values:
3767c478bd9Sstevel@tonic-gate  *		DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type
3777c478bd9Sstevel@tonic-gate  *		DDI_SUCCESS
3787c478bd9Sstevel@tonic-gate  *
3797c478bd9Sstevel@tonic-gate  *	dma handle members affected (set on exit):
3807c478bd9Sstevel@tonic-gate  *	mp->dmai_object		- dmareq->dmar_object
3817c478bd9Sstevel@tonic-gate  *	mp->dmai_rflags		- consistent?, nosync?, dmareq->dmar_flags
3827c478bd9Sstevel@tonic-gate  *	mp->dmai_flags   	- DMA type
3837c478bd9Sstevel@tonic-gate  *	mp->dmai_pfn0   	- 1st page pfn (if va/size pair and not shadow)
3847c478bd9Sstevel@tonic-gate  *	mp->dmai_roffset 	- initialized to starting MMU page offset
3857c478bd9Sstevel@tonic-gate  *	mp->dmai_ndvmapages	- # of total MMU pages of entire object
3867c478bd9Sstevel@tonic-gate  */
3877c478bd9Sstevel@tonic-gate int
px_dma_type(px_t * px_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)3887c478bd9Sstevel@tonic-gate px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
3897c478bd9Sstevel@tonic-gate {
3907c478bd9Sstevel@tonic-gate 	dev_info_t *dip = px_p->px_dip;
3917c478bd9Sstevel@tonic-gate 	ddi_dma_obj_t *dobj_p = &dmareq->dmar_object;
3927c478bd9Sstevel@tonic-gate 	px_pec_t *pec_p = px_p->px_pec_p;
3937c478bd9Sstevel@tonic-gate 	uint32_t offset;
3947c478bd9Sstevel@tonic-gate 	pfn_t pfn0;
395cea92495Sgovinda 	uint_t redzone;
3967c478bd9Sstevel@tonic-gate 
39722bbbd20Saa 	mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
39822bbbd20Saa 
39922bbbd20Saa 	if (!(px_p->px_dev_caps & PX_DMA_SYNC_REQUIRED))
40022bbbd20Saa 		mp->dmai_rflags |= DMP_NOSYNC;
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate 	switch (dobj_p->dmao_type) {
4037c478bd9Sstevel@tonic-gate 	case DMA_OTYP_BUFVADDR:
4047c478bd9Sstevel@tonic-gate 	case DMA_OTYP_VADDR: {
4057c478bd9Sstevel@tonic-gate 		page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv;
4067c478bd9Sstevel@tonic-gate 		caddr_t vaddr = dobj_p->dmao_obj.virt_obj.v_addr;
4077c478bd9Sstevel@tonic-gate 
4087c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist);
4097c478bd9Sstevel@tonic-gate 		offset = (ulong_t)vaddr & MMU_PAGE_OFFSET;
4107c478bd9Sstevel@tonic-gate 		if (pplist) {				/* shadow list */
41136fe4a92Segillett 			mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;
4127c478bd9Sstevel@tonic-gate 			pfn0 = page_pptonum(*pplist);
4137c478bd9Sstevel@tonic-gate 		} else {
4147c478bd9Sstevel@tonic-gate 			struct as *as_p = dobj_p->dmao_obj.virt_obj.v_as;
4157c478bd9Sstevel@tonic-gate 			struct hat *hat_p = as_p ? as_p->a_hat : kas.a_hat;
4167c478bd9Sstevel@tonic-gate 			pfn0 = hat_getpfnum(hat_p, vaddr);
4177c478bd9Sstevel@tonic-gate 		}
4187c478bd9Sstevel@tonic-gate 		}
4197c478bd9Sstevel@tonic-gate 		break;
4207c478bd9Sstevel@tonic-gate 
4217c478bd9Sstevel@tonic-gate 	case DMA_OTYP_PAGES:
4227c478bd9Sstevel@tonic-gate 		offset = dobj_p->dmao_obj.pp_obj.pp_offset;
42336fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN;
4247c478bd9Sstevel@tonic-gate 		pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp);
4257c478bd9Sstevel@tonic-gate 		break;
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate 	case DMA_OTYP_PADDR:
4287c478bd9Sstevel@tonic-gate 	default:
4297c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "%s%d requested unsupported dma type %x",
4309fc8611eSDaniel Ice 		    NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);
4317c478bd9Sstevel@tonic-gate 		return (DDI_DMA_NOMAPPING);
4327c478bd9Sstevel@tonic-gate 	}
4337c478bd9Sstevel@tonic-gate 	if (pfn0 == PFN_INVALID) {
4347c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p",
4359fc8611eSDaniel Ice 		    NAMEINST(dip), dobj_p);
4367c478bd9Sstevel@tonic-gate 		return (DDI_DMA_NOMAPPING);
4377c478bd9Sstevel@tonic-gate 	}
4387c478bd9Sstevel@tonic-gate 	if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base32_pfn,
4399fc8611eSDaniel Ice 	    pec_p->pec_last32_pfn)) {
44036fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32;
4417c478bd9Sstevel@tonic-gate 		goto done;	/* leave bypass and dvma flag as 0 */
4427c478bd9Sstevel@tonic-gate 	} else if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base64_pfn,
4439fc8611eSDaniel Ice 	    pec_p->pec_last64_pfn)) {
44436fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64;
4457c478bd9Sstevel@tonic-gate 		goto done;	/* leave bypass and dvma flag as 0 */
4467c478bd9Sstevel@tonic-gate 	}
4477c478bd9Sstevel@tonic-gate 	if (PX_DMA_ISPEERONLY(mp)) {
4487c478bd9Sstevel@tonic-gate 		dev_info_t *rdip = mp->dmai_rdip;
4497c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip));
4507c478bd9Sstevel@tonic-gate 		return (DDI_DMA_NOMAPPING);
4517c478bd9Sstevel@tonic-gate 	}
452cea92495Sgovinda 
453cea92495Sgovinda 	redzone = (mp->dmai_rflags & DDI_DMA_REDZONE) ||
454cea92495Sgovinda 	    (mp->dmai_flags & PX_DMAI_FLAGS_MAP_BUFZONE) ?
455cea92495Sgovinda 	    PX_DMAI_FLAGS_REDZONE : 0;
456cea92495Sgovinda 
45736fe4a92Segillett 	mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ?
458cea92495Sgovinda 	    PX_DMAI_FLAGS_BYPASS : (PX_DMAI_FLAGS_DVMA | redzone);
4597c478bd9Sstevel@tonic-gate done:
4607c478bd9Sstevel@tonic-gate 	mp->dmai_object	 = *dobj_p;			/* whole object    */
4617c478bd9Sstevel@tonic-gate 	mp->dmai_pfn0	 = (void *)pfn0;		/* cache pfn0	   */
4627c478bd9Sstevel@tonic-gate 	mp->dmai_roffset = offset;			/* win0 pg0 offset */
4637c478bd9Sstevel@tonic-gate 	mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size);
4647c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
4657c478bd9Sstevel@tonic-gate }
4667c478bd9Sstevel@tonic-gate 
4677c478bd9Sstevel@tonic-gate /*
4687c478bd9Sstevel@tonic-gate  * px_dma_pgpfn - set up pfnlst array according to pages
4697c478bd9Sstevel@tonic-gate  *	VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES
4707c478bd9Sstevel@tonic-gate  */
4717c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4727c478bd9Sstevel@tonic-gate static int
px_dma_pgpfn(px_t * px_p,ddi_dma_impl_t * mp,uint_t npages)4737c478bd9Sstevel@tonic-gate px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)
4747c478bd9Sstevel@tonic-gate {
4757c478bd9Sstevel@tonic-gate 	int i;
4767c478bd9Sstevel@tonic-gate 	dev_info_t *dip = px_p->px_dip;
4777c478bd9Sstevel@tonic-gate 
4787c478bd9Sstevel@tonic-gate 	switch (mp->dmai_object.dmao_type) {
4797c478bd9Sstevel@tonic-gate 	case DMA_OTYP_BUFVADDR:
4807c478bd9Sstevel@tonic-gate 	case DMA_OTYP_VADDR: {
4817c478bd9Sstevel@tonic-gate 		page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;
4827c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=",
4839fc8611eSDaniel Ice 		    pplist, npages);
4847c478bd9Sstevel@tonic-gate 		for (i = 1; i < npages; i++) {
4857c478bd9Sstevel@tonic-gate 			px_iopfn_t pfn = page_pptonum(pplist[i]);
4867c478bd9Sstevel@tonic-gate 			PX_SET_MP_PFN1(mp, i, pfn);
4877c478bd9Sstevel@tonic-gate 			DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
4887c478bd9Sstevel@tonic-gate 		}
4897c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n");
4907c478bd9Sstevel@tonic-gate 		}
4917c478bd9Sstevel@tonic-gate 		break;
4927c478bd9Sstevel@tonic-gate 
4937c478bd9Sstevel@tonic-gate 	case DMA_OTYP_PAGES: {
4947c478bd9Sstevel@tonic-gate 		page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;
4957c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip, "pp=%p pfns=", pp);
4967c478bd9Sstevel@tonic-gate 		for (i = 1; i < npages; i++, pp = pp->p_next) {
4977c478bd9Sstevel@tonic-gate 			px_iopfn_t pfn = page_pptonum(pp);
4987c478bd9Sstevel@tonic-gate 			PX_SET_MP_PFN1(mp, i, pfn);
4997c478bd9Sstevel@tonic-gate 			DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
5007c478bd9Sstevel@tonic-gate 		}
5017c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n");
5027c478bd9Sstevel@tonic-gate 		}
5037c478bd9Sstevel@tonic-gate 		break;
5047c478bd9Sstevel@tonic-gate 
5057c478bd9Sstevel@tonic-gate 	default:	/* check is already done by px_dma_type */
5067c478bd9Sstevel@tonic-gate 		ASSERT(0);
5077c478bd9Sstevel@tonic-gate 		break;
5087c478bd9Sstevel@tonic-gate 	}
5097c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
5107c478bd9Sstevel@tonic-gate }
5117c478bd9Sstevel@tonic-gate 
5127c478bd9Sstevel@tonic-gate /*
5137c478bd9Sstevel@tonic-gate  * px_dma_vapfn - set up pfnlst array according to VA
5147c478bd9Sstevel@tonic-gate  *	VA/size pair: <normal, bypass, peer-to-peer>
5157c478bd9Sstevel@tonic-gate  *	pfn0 is skipped as it is already done.
5167c478bd9Sstevel@tonic-gate  *	In this case, the cached pfn0 is used to fill pfnlst[0]
5177c478bd9Sstevel@tonic-gate  */
5187c478bd9Sstevel@tonic-gate static int
px_dma_vapfn(px_t * px_p,ddi_dma_impl_t * mp,uint_t npages)5197c478bd9Sstevel@tonic-gate px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages)
5207c478bd9Sstevel@tonic-gate {
5217c478bd9Sstevel@tonic-gate 	dev_info_t *dip = px_p->px_dip;
5227c478bd9Sstevel@tonic-gate 	int i;
5237c478bd9Sstevel@tonic-gate 	caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;
5247c478bd9Sstevel@tonic-gate 	struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat;
5257c478bd9Sstevel@tonic-gate 
5267c478bd9Sstevel@tonic-gate 	vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE;
5277c478bd9Sstevel@tonic-gate 	for (i = 1; i < npages; i++, vaddr += MMU_PAGE_SIZE) {
5287c478bd9Sstevel@tonic-gate 		px_iopfn_t pfn = hat_getpfnum(hat_p, vaddr);
5297c478bd9Sstevel@tonic-gate 		if (pfn == PFN_INVALID)
5307c478bd9Sstevel@tonic-gate 			goto err_badpfn;
5317c478bd9Sstevel@tonic-gate 		PX_SET_MP_PFN1(mp, i, pfn);
5327c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_BINDH, dip, "px_dma_vapfn: mp=%p pfnlst[%x]=%x\n",
5339fc8611eSDaniel Ice 		    mp, i, pfn);
5347c478bd9Sstevel@tonic-gate 	}
5357c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
5367c478bd9Sstevel@tonic-gate err_badpfn:
5377c478bd9Sstevel@tonic-gate 	cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr);
5387c478bd9Sstevel@tonic-gate 	return (DDI_DMA_NOMAPPING);
5397c478bd9Sstevel@tonic-gate }
5407c478bd9Sstevel@tonic-gate 
5417c478bd9Sstevel@tonic-gate /*
5427c478bd9Sstevel@tonic-gate  * px_dma_pfn - Fills pfn list for all pages being DMA-ed.
5437c478bd9Sstevel@tonic-gate  *
5447c478bd9Sstevel@tonic-gate  * dependencies:
5457c478bd9Sstevel@tonic-gate  *	mp->dmai_ndvmapages	- set to total # of dma pages
5467c478bd9Sstevel@tonic-gate  *
5477c478bd9Sstevel@tonic-gate  * return value:
5487c478bd9Sstevel@tonic-gate  *	DDI_SUCCESS
5497c478bd9Sstevel@tonic-gate  *	DDI_DMA_NOMAPPING
5507c478bd9Sstevel@tonic-gate  */
5517c478bd9Sstevel@tonic-gate int
px_dma_pfn(px_t * px_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)5527c478bd9Sstevel@tonic-gate px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
5537c478bd9Sstevel@tonic-gate {
5547c478bd9Sstevel@tonic-gate 	uint32_t npages = mp->dmai_ndvmapages;
5557c478bd9Sstevel@tonic-gate 	int (*waitfp)(caddr_t) = dmareq->dmar_fp;
5567c478bd9Sstevel@tonic-gate 	int i, ret, peer = PX_DMA_ISPTP(mp);
5577c478bd9Sstevel@tonic-gate 	int peer32 = PX_DMA_ISPTP32(mp);
5587c478bd9Sstevel@tonic-gate 	dev_info_t *dip = px_p->px_dip;
5597c478bd9Sstevel@tonic-gate 
5607c478bd9Sstevel@tonic-gate 	px_pec_t *pec_p = px_p->px_pec_p;
5617c478bd9Sstevel@tonic-gate 	px_iopfn_t pfn_base = peer32 ? pec_p->pec_base32_pfn :
5629fc8611eSDaniel Ice 	    pec_p->pec_base64_pfn;
5637c478bd9Sstevel@tonic-gate 	px_iopfn_t pfn_last = peer32 ? pec_p->pec_last32_pfn :
5649fc8611eSDaniel Ice 	    pec_p->pec_last64_pfn;
5657c478bd9Sstevel@tonic-gate 	px_iopfn_t pfn_adj = peer ? pfn_base : 0;
5667c478bd9Sstevel@tonic-gate 
5677c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_BINDH, dip, "px_dma_pfn: mp=%p pfn0=%x\n",
5689fc8611eSDaniel Ice 	    mp, PX_MP_PFN0(mp) - pfn_adj);
5697c478bd9Sstevel@tonic-gate 	/* 1 page: no array alloc/fill, no mixed mode check */
5707c478bd9Sstevel@tonic-gate 	if (npages == 1) {
57136fe4a92Segillett 		PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);
5727c478bd9Sstevel@tonic-gate 		return (DDI_SUCCESS);
5737c478bd9Sstevel@tonic-gate 	}
5747c478bd9Sstevel@tonic-gate 	/* allocate pfn array */
5757c478bd9Sstevel@tonic-gate 	if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t),
5769fc8611eSDaniel Ice 	    waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) {
5777c478bd9Sstevel@tonic-gate 		if (waitfp != DDI_DMA_DONTWAIT)
5787c478bd9Sstevel@tonic-gate 			ddi_set_callback(waitfp, dmareq->dmar_arg,
5799fc8611eSDaniel Ice 			    &px_kmem_clid);
5807c478bd9Sstevel@tonic-gate 		return (DDI_DMA_NORESOURCES);
5817c478bd9Sstevel@tonic-gate 	}
5827c478bd9Sstevel@tonic-gate 	/* fill pfn array */
58336fe4a92Segillett 	PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj);	/* pfnlst[0] */
5847c478bd9Sstevel@tonic-gate 	if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) :
5859fc8611eSDaniel Ice 	    px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS)
5867c478bd9Sstevel@tonic-gate 		goto err;
5877c478bd9Sstevel@tonic-gate 
5887c478bd9Sstevel@tonic-gate 	/* skip pfn0, check mixed mode and adjust peer to peer pfn */
5897c478bd9Sstevel@tonic-gate 	for (i = 1; i < npages; i++) {
5907c478bd9Sstevel@tonic-gate 		px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i);
5917c478bd9Sstevel@tonic-gate 		if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) {
592b40cec45Skrishnae 			cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx",
5939fc8611eSDaniel Ice 			    NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn);
5947c478bd9Sstevel@tonic-gate 			ret = DDI_DMA_NOMAPPING;	/* mixed mode */
5957c478bd9Sstevel@tonic-gate 			goto err;
5967c478bd9Sstevel@tonic-gate 		}
5977c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip,
5989fc8611eSDaniel Ice 		    "px_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj);
5997c478bd9Sstevel@tonic-gate 		if (pfn_adj)
6007c478bd9Sstevel@tonic-gate 			PX_SET_MP_PFN1(mp, i, pfn - pfn_adj);
6017c478bd9Sstevel@tonic-gate 	}
6027c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
6037c478bd9Sstevel@tonic-gate err:
6047c478bd9Sstevel@tonic-gate 	px_dma_freepfn(mp);
6057c478bd9Sstevel@tonic-gate 	return (ret);
6067c478bd9Sstevel@tonic-gate }
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate /*
6097c478bd9Sstevel@tonic-gate  * px_dvma_win() - trim requested DVMA size down to window size
6107c478bd9Sstevel@tonic-gate  *	The 1st window starts from offset and ends at page-aligned boundary.
6117c478bd9Sstevel@tonic-gate  *	From the 2nd window on, each window starts and ends at page-aligned
6127c478bd9Sstevel@tonic-gate  *	boundary except the last window ends at wherever requested.
6137c478bd9Sstevel@tonic-gate  *
6147c478bd9Sstevel@tonic-gate  *	accesses the following mp-> members:
6157c478bd9Sstevel@tonic-gate  *	mp->dmai_attr.dma_attr_count_max
6167c478bd9Sstevel@tonic-gate  *	mp->dmai_attr.dma_attr_seg
6177c478bd9Sstevel@tonic-gate  *	mp->dmai_roffset   - start offset of 1st window
6187c478bd9Sstevel@tonic-gate  *	mp->dmai_rflags (redzone)
6197c478bd9Sstevel@tonic-gate  *	mp->dmai_ndvmapages (for 1 page fast path)
6207c478bd9Sstevel@tonic-gate  *
6217c478bd9Sstevel@tonic-gate  *	sets the following mp-> members:
6227c478bd9Sstevel@tonic-gate  *	mp->dmai_size	   - xfer size, != winsize if 1st/last win  (not fixed)
6237c478bd9Sstevel@tonic-gate  *	mp->dmai_winsize   - window size (no redzone), n * page size    (fixed)
6247c478bd9Sstevel@tonic-gate  *	mp->dmai_nwin	   - # of DMA windows of entire object		(fixed)
6257c478bd9Sstevel@tonic-gate  *	mp->dmai_rflags	   - remove partial flag if nwin == 1		(fixed)
6267c478bd9Sstevel@tonic-gate  *	mp->dmai_winlst	   - NULL, window objects not used for DVMA	(fixed)
6277c478bd9Sstevel@tonic-gate  *
6287c478bd9Sstevel@tonic-gate  *	fixed - not changed across different DMA windows
6297c478bd9Sstevel@tonic-gate  */
6307c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6317c478bd9Sstevel@tonic-gate int
px_dvma_win(px_t * px_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)6327c478bd9Sstevel@tonic-gate px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
6337c478bd9Sstevel@tonic-gate {
6341de45cd9Sgovinda 	uint32_t redzone_sz	= PX_HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0;
6357c478bd9Sstevel@tonic-gate 	size_t obj_sz		= mp->dmai_object.dmao_size;
6367c478bd9Sstevel@tonic-gate 	size_t xfer_sz;
6377c478bd9Sstevel@tonic-gate 	ulong_t pg_off;
6387c478bd9Sstevel@tonic-gate 
6397c478bd9Sstevel@tonic-gate 	if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {
6407c478bd9Sstevel@tonic-gate 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
6417c478bd9Sstevel@tonic-gate 		mp->dmai_size = obj_sz;
6427c478bd9Sstevel@tonic-gate 		mp->dmai_winsize = MMU_PAGE_SIZE;
6437c478bd9Sstevel@tonic-gate 		mp->dmai_nwin = 1;
6447c478bd9Sstevel@tonic-gate 		goto done;
6457c478bd9Sstevel@tonic-gate 	}
6467c478bd9Sstevel@tonic-gate 
6477c478bd9Sstevel@tonic-gate 	pg_off	= mp->dmai_roffset;
6487c478bd9Sstevel@tonic-gate 	xfer_sz	= obj_sz + redzone_sz;
6497c478bd9Sstevel@tonic-gate 
6509fc8611eSDaniel Ice 	/* include redzone in nocross check */	{
6517c478bd9Sstevel@tonic-gate 		uint64_t nocross = mp->dmai_attr.dma_attr_seg;
6527c478bd9Sstevel@tonic-gate 		if (xfer_sz + pg_off - 1 > nocross)
6537c478bd9Sstevel@tonic-gate 			xfer_sz = nocross - pg_off + 1;
6547c478bd9Sstevel@tonic-gate 		if (redzone_sz && (xfer_sz <= redzone_sz)) {
6557c478bd9Sstevel@tonic-gate 			DBG(DBG_DMA_MAP, px_p->px_dip,
6567c478bd9Sstevel@tonic-gate 			    "nocross too small: "
6577c478bd9Sstevel@tonic-gate 			    "%lx(%lx)+%lx+%lx < %llx\n",
6587c478bd9Sstevel@tonic-gate 			    xfer_sz, obj_sz, pg_off, redzone_sz, nocross);
6597c478bd9Sstevel@tonic-gate 			return (DDI_DMA_TOOBIG);
6607c478bd9Sstevel@tonic-gate 		}
6617c478bd9Sstevel@tonic-gate 	}
6627c478bd9Sstevel@tonic-gate 	xfer_sz -= redzone_sz;		/* restore transfer size  */
6639fc8611eSDaniel Ice 	/* check counter max */	{
6647c478bd9Sstevel@tonic-gate 		uint32_t count_max = mp->dmai_attr.dma_attr_count_max;
6657c478bd9Sstevel@tonic-gate 		if (xfer_sz - 1 > count_max)
6667c478bd9Sstevel@tonic-gate 			xfer_sz = count_max + 1;
6677c478bd9Sstevel@tonic-gate 	}
6687c478bd9Sstevel@tonic-gate 	if (xfer_sz >= obj_sz) {
6697c478bd9Sstevel@tonic-gate 		mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
6707c478bd9Sstevel@tonic-gate 		mp->dmai_size = xfer_sz;
6717c478bd9Sstevel@tonic-gate 		mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE);
6727c478bd9Sstevel@tonic-gate 		mp->dmai_nwin = 1;
6737c478bd9Sstevel@tonic-gate 		goto done;
6747c478bd9Sstevel@tonic-gate 	}
6757c478bd9Sstevel@tonic-gate 	if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
6767c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, px_p->px_dip, "too big: %lx+%lx+%lx > %lx\n",
6779fc8611eSDaniel Ice 		    obj_sz, pg_off, redzone_sz, xfer_sz);
6787c478bd9Sstevel@tonic-gate 		return (DDI_DMA_TOOBIG);
6797c478bd9Sstevel@tonic-gate 	}
6807c478bd9Sstevel@tonic-gate 
6817c478bd9Sstevel@tonic-gate 	xfer_sz = MMU_PTOB(MMU_BTOP(xfer_sz + pg_off)); /* page align */
6827c478bd9Sstevel@tonic-gate 	mp->dmai_size = xfer_sz - pg_off;	/* 1st window xferrable size */
6837c478bd9Sstevel@tonic-gate 	mp->dmai_winsize = xfer_sz;		/* redzone not in winsize */
6847c478bd9Sstevel@tonic-gate 	mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;
6857c478bd9Sstevel@tonic-gate done:
6867c478bd9Sstevel@tonic-gate 	mp->dmai_winlst = NULL;
6877c478bd9Sstevel@tonic-gate 	px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp);
6887c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
6897c478bd9Sstevel@tonic-gate }
6907c478bd9Sstevel@tonic-gate 
6917c478bd9Sstevel@tonic-gate /*
6927c478bd9Sstevel@tonic-gate  * fast track cache entry to mmu context, inserts 3 0 bits between
6937c478bd9Sstevel@tonic-gate  * upper 6-bits and lower 3-bits of the 9-bit cache entry
6947c478bd9Sstevel@tonic-gate  */
6957c478bd9Sstevel@tonic-gate #define	MMU_FCE_TO_CTX(i)	(((i) << 3) | ((i) & 0x7) | 0x38)
6967c478bd9Sstevel@tonic-gate 
6977c478bd9Sstevel@tonic-gate /*
6987c478bd9Sstevel@tonic-gate  * px_dvma_map_fast - attempts to map fast trackable DVMA
6997c478bd9Sstevel@tonic-gate  */
7007c478bd9Sstevel@tonic-gate /*ARGSUSED*/
7017c478bd9Sstevel@tonic-gate int
px_dvma_map_fast(px_mmu_t * mmu_p,ddi_dma_impl_t * mp)7027c478bd9Sstevel@tonic-gate px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)
7037c478bd9Sstevel@tonic-gate {
7047c478bd9Sstevel@tonic-gate 	uint_t clustsz = px_dvma_page_cache_clustsz;
7057c478bd9Sstevel@tonic-gate 	uint_t entries = px_dvma_page_cache_entries;
70625cf1a30Sjl 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
70725cf1a30Sjl 	    mp->dmai_attr.dma_attr_flags);
7087c478bd9Sstevel@tonic-gate 	int i = mmu_p->mmu_dvma_addr_scan_start;
7097c478bd9Sstevel@tonic-gate 	uint8_t *lock_addr = mmu_p->mmu_dvma_cache_locks + i;
7107c478bd9Sstevel@tonic-gate 	px_dvma_addr_t dvma_pg;
7117c478bd9Sstevel@tonic-gate 	size_t npages = MMU_BTOP(mp->dmai_winsize);
7121de45cd9Sgovinda 	dev_info_t *dip = mmu_p->mmu_px_p->px_dip;
7137c478bd9Sstevel@tonic-gate 
7147c478bd9Sstevel@tonic-gate 	extern uint8_t ldstub(uint8_t *);
7157c478bd9Sstevel@tonic-gate 	ASSERT(MMU_PTOB(npages) == mp->dmai_winsize);
7161de45cd9Sgovinda 	ASSERT(npages + PX_HAS_REDZONE(mp) <= clustsz);
7177c478bd9Sstevel@tonic-gate 
7189fc8611eSDaniel Ice 	for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
7199fc8611eSDaniel Ice 		;
7207c478bd9Sstevel@tonic-gate 	if (i >= entries) {
7217c478bd9Sstevel@tonic-gate 		lock_addr = mmu_p->mmu_dvma_cache_locks;
7227c478bd9Sstevel@tonic-gate 		i = 0;
7239fc8611eSDaniel Ice 		for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
7249fc8611eSDaniel Ice 			;
7257c478bd9Sstevel@tonic-gate 		if (i >= entries) {
7267c478bd9Sstevel@tonic-gate #ifdef	PX_DMA_PROF
7277c478bd9Sstevel@tonic-gate 			px_dvmaft_exhaust++;
7287c478bd9Sstevel@tonic-gate #endif	/* PX_DMA_PROF */
7297c478bd9Sstevel@tonic-gate 			return (DDI_DMA_NORESOURCES);
7307c478bd9Sstevel@tonic-gate 		}
7317c478bd9Sstevel@tonic-gate 	}
7327c478bd9Sstevel@tonic-gate 	mmu_p->mmu_dvma_addr_scan_start = (i + 1) & (entries - 1);
7337c478bd9Sstevel@tonic-gate 
7347c478bd9Sstevel@tonic-gate 	i *= clustsz;
7357c478bd9Sstevel@tonic-gate 	dvma_pg = mmu_p->dvma_base_pg + i;
7367c478bd9Sstevel@tonic-gate 
73744961713Sgirish 	if (px_lib_iommu_map(dip, PCI_TSBID(0, i), npages,
73844961713Sgirish 	    PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, 0,
73944961713Sgirish 	    MMU_MAP_PFN) != DDI_SUCCESS) {
7401de45cd9Sgovinda 		DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: "
7411de45cd9Sgovinda 		    "px_lib_iommu_map failed\n");
7421de45cd9Sgovinda 		return (DDI_FAILURE);
7431de45cd9Sgovinda 	}
7441de45cd9Sgovinda 
7451de45cd9Sgovinda 	if (!PX_MAP_BUFZONE(mp))
7461de45cd9Sgovinda 		goto done;
7471de45cd9Sgovinda 
7481de45cd9Sgovinda 	DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: redzone pg=%x\n", i + npages);
7491de45cd9Sgovinda 
7501de45cd9Sgovinda 	ASSERT(PX_HAS_REDZONE(mp));
7511de45cd9Sgovinda 
75244961713Sgirish 	if (px_lib_iommu_map(dip, PCI_TSBID(0, i + npages), 1,
75344961713Sgirish 	    PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, npages - 1,
75444961713Sgirish 	    MMU_MAP_PFN) != DDI_SUCCESS) {
7551de45cd9Sgovinda 		DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: "
7561de45cd9Sgovinda 		    "mapping REDZONE page failed\n");
7571de45cd9Sgovinda 
7581de45cd9Sgovinda 		(void) px_lib_iommu_demap(dip, PCI_TSBID(0, i), npages);
7597c478bd9Sstevel@tonic-gate 		return (DDI_FAILURE);
7601de45cd9Sgovinda 	}
7617c478bd9Sstevel@tonic-gate 
7621de45cd9Sgovinda done:
7637c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF
7647c478bd9Sstevel@tonic-gate 	px_dvmaft_success++;
7657c478bd9Sstevel@tonic-gate #endif
7667c478bd9Sstevel@tonic-gate 	mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);
7677c478bd9Sstevel@tonic-gate 	mp->dmai_offset = 0;
76836fe4a92Segillett 	mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK;
7697c478bd9Sstevel@tonic-gate 	PX_SAVE_MP_TTE(mp, attr);	/* save TTE template for unmapping */
77036fe4a92Segillett 	if (PX_DVMA_DBG_ON(mmu_p))
7717c478bd9Sstevel@tonic-gate 		px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping,
7729fc8611eSDaniel Ice 		    mp->dmai_size, mp);
7737c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
7747c478bd9Sstevel@tonic-gate }
7757c478bd9Sstevel@tonic-gate 
7767c478bd9Sstevel@tonic-gate /*
7777c478bd9Sstevel@tonic-gate  * px_dvma_map: map non-fasttrack DMA
7787c478bd9Sstevel@tonic-gate  *		Use quantum cache if single page DMA.
7797c478bd9Sstevel@tonic-gate  */
7807c478bd9Sstevel@tonic-gate int
px_dvma_map(ddi_dma_impl_t * mp,ddi_dma_req_t * dmareq,px_mmu_t * mmu_p)7817c478bd9Sstevel@tonic-gate px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p)
7827c478bd9Sstevel@tonic-gate {
7837c478bd9Sstevel@tonic-gate 	uint_t npages = PX_DMA_WINNPGS(mp);
7847c478bd9Sstevel@tonic-gate 	px_dvma_addr_t dvma_pg, dvma_pg_index;
7857c478bd9Sstevel@tonic-gate 	void *dvma_addr;
786ef2504f2SDaniel Ice 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
78725cf1a30Sjl 	    mp->dmai_attr.dma_attr_flags);
7887c478bd9Sstevel@tonic-gate 	int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP;
7897c478bd9Sstevel@tonic-gate 	dev_info_t *dip = mp->dmai_rdip;
7907c478bd9Sstevel@tonic-gate 	int	ret = DDI_SUCCESS;
7917c478bd9Sstevel@tonic-gate 
7927c478bd9Sstevel@tonic-gate 	/*
7937c478bd9Sstevel@tonic-gate 	 * allocate dvma space resource and map in the first window.
7947c478bd9Sstevel@tonic-gate 	 * (vmem_t *vmp, size_t size,
7957c478bd9Sstevel@tonic-gate 	 *	size_t align, size_t phase, size_t nocross,
7967c478bd9Sstevel@tonic-gate 	 *	void *minaddr, void *maxaddr, int vmflag)
7977c478bd9Sstevel@tonic-gate 	 */
7981de45cd9Sgovinda 	if ((npages == 1) && !PX_HAS_REDZONE(mp) && PX_HAS_NOSYSLIMIT(mp)) {
7997c478bd9Sstevel@tonic-gate 		dvma_addr = vmem_alloc(mmu_p->mmu_dvma_map,
8009fc8611eSDaniel Ice 		    MMU_PAGE_SIZE, sleep);
80136fe4a92Segillett 		mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE;
8027c478bd9Sstevel@tonic-gate #ifdef	PX_DMA_PROF
8037c478bd9Sstevel@tonic-gate 		px_dvma_vmem_alloc++;
8047c478bd9Sstevel@tonic-gate #endif	/* PX_DMA_PROF */
8057c478bd9Sstevel@tonic-gate 	} else {
8067c478bd9Sstevel@tonic-gate 		dvma_addr = vmem_xalloc(mmu_p->mmu_dvma_map,
8079fc8611eSDaniel Ice 		    MMU_PTOB(npages + PX_HAS_REDZONE(mp)),
8089fc8611eSDaniel Ice 		    MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE),
8099fc8611eSDaniel Ice 		    0,
8109fc8611eSDaniel Ice 		    mp->dmai_attr.dma_attr_seg + 1,
8119fc8611eSDaniel Ice 		    (void *)mp->dmai_attr.dma_attr_addr_lo,
8129fc8611eSDaniel Ice 		    (void *)(mp->dmai_attr.dma_attr_addr_hi + 1),
8139fc8611eSDaniel Ice 		    sleep);
8147c478bd9Sstevel@tonic-gate #ifdef	PX_DMA_PROF
8157c478bd9Sstevel@tonic-gate 		px_dvma_vmem_xalloc++;
8167c478bd9Sstevel@tonic-gate #endif	/* PX_DMA_PROF */
8177c478bd9Sstevel@tonic-gate 	}
8187c478bd9Sstevel@tonic-gate 	dvma_pg = MMU_BTOP((ulong_t)dvma_addr);
8197c478bd9Sstevel@tonic-gate 	dvma_pg_index = dvma_pg - mmu_p->dvma_base_pg;
8207c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n",
8219fc8611eSDaniel Ice 	    dvma_pg, dvma_pg_index);
8227c478bd9Sstevel@tonic-gate 	if (dvma_pg == 0)
8237c478bd9Sstevel@tonic-gate 		goto noresource;
8247c478bd9Sstevel@tonic-gate 
8257c478bd9Sstevel@tonic-gate 	mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg);
8267c478bd9Sstevel@tonic-gate 	mp->dmai_offset = 0;
827ef2504f2SDaniel Ice 	PX_SAVE_MP_TTE(mp, attr);	/* mp->dmai_tte = tte */
8287c478bd9Sstevel@tonic-gate 
8297c478bd9Sstevel@tonic-gate 	if ((ret = px_mmu_map_pages(mmu_p,
8307c478bd9Sstevel@tonic-gate 	    mp, dvma_pg, npages, 0)) != DDI_SUCCESS) {
83136fe4a92Segillett 		if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {
8327c478bd9Sstevel@tonic-gate 			vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr,
8337c478bd9Sstevel@tonic-gate 			    MMU_PAGE_SIZE);
8347c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF
8357c478bd9Sstevel@tonic-gate 			px_dvma_vmem_free++;
8367c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */
8377c478bd9Sstevel@tonic-gate 		} else {
8387c478bd9Sstevel@tonic-gate 			vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr,
8391de45cd9Sgovinda 			    MMU_PTOB(npages + PX_HAS_REDZONE(mp)));
8407c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF
8417c478bd9Sstevel@tonic-gate 			px_dvma_vmem_xfree++;
8427c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */
8437c478bd9Sstevel@tonic-gate 		}
8447c478bd9Sstevel@tonic-gate 	}
8457c478bd9Sstevel@tonic-gate 
8467c478bd9Sstevel@tonic-gate 	return (ret);
8477c478bd9Sstevel@tonic-gate noresource:
8487c478bd9Sstevel@tonic-gate 	if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
8497c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n");
8507c478bd9Sstevel@tonic-gate 		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
8519fc8611eSDaniel Ice 		    &mmu_p->mmu_dvma_clid);
8527c478bd9Sstevel@tonic-gate 	}
8537c478bd9Sstevel@tonic-gate 	DBG(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n");
8547c478bd9Sstevel@tonic-gate 	return (DDI_DMA_NORESOURCES);
8557c478bd9Sstevel@tonic-gate }
8567c478bd9Sstevel@tonic-gate 
8577c478bd9Sstevel@tonic-gate void
px_dvma_unmap(px_mmu_t * mmu_p,ddi_dma_impl_t * mp)8587c478bd9Sstevel@tonic-gate px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp)
8597c478bd9Sstevel@tonic-gate {
8607c478bd9Sstevel@tonic-gate 	px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping;
8617c478bd9Sstevel@tonic-gate 	px_dvma_addr_t dvma_pg = MMU_BTOP(dvma_addr);
8627c478bd9Sstevel@tonic-gate 	dvma_addr = MMU_PTOB(dvma_pg);
8637c478bd9Sstevel@tonic-gate 
86436fe4a92Segillett 	if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) {
8657c478bd9Sstevel@tonic-gate 		px_iopfn_t index = dvma_pg - mmu_p->dvma_base_pg;
8667c478bd9Sstevel@tonic-gate 		ASSERT(index % px_dvma_page_cache_clustsz == 0);
8677c478bd9Sstevel@tonic-gate 		index /= px_dvma_page_cache_clustsz;
8687c478bd9Sstevel@tonic-gate 		ASSERT(index < px_dvma_page_cache_entries);
8697c478bd9Sstevel@tonic-gate 		mmu_p->mmu_dvma_cache_locks[index] = 0;
8707c478bd9Sstevel@tonic-gate #ifdef	PX_DMA_PROF
8717c478bd9Sstevel@tonic-gate 		px_dvmaft_free++;
8727c478bd9Sstevel@tonic-gate #endif	/* PX_DMA_PROF */
8737c478bd9Sstevel@tonic-gate 		return;
8747c478bd9Sstevel@tonic-gate 	}
8757c478bd9Sstevel@tonic-gate 
87636fe4a92Segillett 	if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) {
8777c478bd9Sstevel@tonic-gate 		vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr,
8789fc8611eSDaniel Ice 		    MMU_PAGE_SIZE);
8797c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF
8807c478bd9Sstevel@tonic-gate 		px_dvma_vmem_free++;
8817c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */
8827c478bd9Sstevel@tonic-gate 	} else {
8831de45cd9Sgovinda 		size_t npages = MMU_BTOP(mp->dmai_winsize) + PX_HAS_REDZONE(mp);
8847c478bd9Sstevel@tonic-gate 		vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr,
8859fc8611eSDaniel Ice 		    MMU_PTOB(npages));
8867c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF
8877c478bd9Sstevel@tonic-gate 		px_dvma_vmem_xfree++;
8887c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */
8897c478bd9Sstevel@tonic-gate 	}
8907c478bd9Sstevel@tonic-gate }
8917c478bd9Sstevel@tonic-gate 
8927c478bd9Sstevel@tonic-gate /*
8937c478bd9Sstevel@tonic-gate  * DVMA mappings may have multiple windows, but each window always have
8947c478bd9Sstevel@tonic-gate  * one segment.
8957c478bd9Sstevel@tonic-gate  */
8967c478bd9Sstevel@tonic-gate int
px_dvma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)8977c478bd9Sstevel@tonic-gate px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
8987c478bd9Sstevel@tonic-gate 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
8997c478bd9Sstevel@tonic-gate 	uint_t cache_flags)
9007c478bd9Sstevel@tonic-gate {
9017c478bd9Sstevel@tonic-gate 	switch (cmd) {
9027c478bd9Sstevel@tonic-gate 	default:
9037c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
9049fc8611eSDaniel Ice 		    cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
9057c478bd9Sstevel@tonic-gate 		break;
9067c478bd9Sstevel@tonic-gate 	}
9077c478bd9Sstevel@tonic-gate 	return (DDI_FAILURE);
9087c478bd9Sstevel@tonic-gate }
9097c478bd9Sstevel@tonic-gate 
9107c478bd9Sstevel@tonic-gate void
px_dma_freewin(ddi_dma_impl_t * mp)9117c478bd9Sstevel@tonic-gate px_dma_freewin(ddi_dma_impl_t *mp)
9127c478bd9Sstevel@tonic-gate {
9137c478bd9Sstevel@tonic-gate 	px_dma_win_t *win_p = mp->dmai_winlst, *win2_p;
9147c478bd9Sstevel@tonic-gate 	for (win2_p = win_p; win_p; win2_p = win_p) {
9157c478bd9Sstevel@tonic-gate 		win_p = win2_p->win_next;
9167c478bd9Sstevel@tonic-gate 		kmem_free(win2_p, sizeof (px_dma_win_t) +
9179fc8611eSDaniel Ice 		    sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies);
9187c478bd9Sstevel@tonic-gate 	}
9197c478bd9Sstevel@tonic-gate 	mp->dmai_nwin = 0;
9207c478bd9Sstevel@tonic-gate 	mp->dmai_winlst = NULL;
9217c478bd9Sstevel@tonic-gate }
9227c478bd9Sstevel@tonic-gate 
9237c478bd9Sstevel@tonic-gate /*
9247c478bd9Sstevel@tonic-gate  * px_dma_newwin - create a dma window object and cookies
9257c478bd9Sstevel@tonic-gate  *
9267c478bd9Sstevel@tonic-gate  *	After the initial scan in px_dma_physwin(), which identifies
9277c478bd9Sstevel@tonic-gate  *	a portion of the pfn array that belongs to a dma window,
9287c478bd9Sstevel@tonic-gate  *	we are called to allocate and initialize representing memory
9297c478bd9Sstevel@tonic-gate  *	resources. We know from the 1st scan the number of cookies
9307c478bd9Sstevel@tonic-gate  *	or dma segment in this window so we can allocate a contiguous
9317c478bd9Sstevel@tonic-gate  *	memory array for the dma cookies (The implementation of
9327c478bd9Sstevel@tonic-gate  *	ddi_dma_nextcookie(9f) dictates dma cookies be contiguous).
9337c478bd9Sstevel@tonic-gate  *
9347c478bd9Sstevel@tonic-gate  *	A second round scan is done on the pfn array to identify
9357c478bd9Sstevel@tonic-gate  *	each dma segment and initialize its corresponding dma cookie.
9367c478bd9Sstevel@tonic-gate  *	We don't need to do all the safety checking and we know they
9377c478bd9Sstevel@tonic-gate  *	all belong to the same dma window.
9387c478bd9Sstevel@tonic-gate  *
9397c478bd9Sstevel@tonic-gate  *	Input:	cookie_no - # of cookies identified by the 1st scan
9407c478bd9Sstevel@tonic-gate  *		start_idx - subscript of the pfn array for the starting pfn
9417c478bd9Sstevel@tonic-gate  *		end_idx   - subscript of the last pfn in dma window
9427c478bd9Sstevel@tonic-gate  *		win_pp    - pointer to win_next member of previous window
9437c478bd9Sstevel@tonic-gate  *	Return:	DDI_SUCCESS - with **win_pp as newly created window object
9447c478bd9Sstevel@tonic-gate  *		DDI_DMA_NORESROUCE - caller frees all previous window objs
9457c478bd9Sstevel@tonic-gate  *	Note:	Each cookie and window size are all initialized on page
9467c478bd9Sstevel@tonic-gate  *		boundary. This is not true for the 1st cookie of the 1st
9477c478bd9Sstevel@tonic-gate  *		window and the last cookie of the last window.
9487c478bd9Sstevel@tonic-gate  *		We fix that later in upper layer which has access to size
9497c478bd9Sstevel@tonic-gate  *		and offset info.
9507c478bd9Sstevel@tonic-gate  *
9517c478bd9Sstevel@tonic-gate  */
9527c478bd9Sstevel@tonic-gate /*ARGSUSED*/
9537c478bd9Sstevel@tonic-gate static int
px_dma_newwin(dev_info_t * dip,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint32_t cookie_no,uint32_t start_idx,uint32_t end_idx,px_dma_win_t ** win_pp,uint64_t count_max,uint64_t bypass)9547c478bd9Sstevel@tonic-gate px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,
9557c478bd9Sstevel@tonic-gate 	uint32_t cookie_no, uint32_t start_idx, uint32_t end_idx,
9567c478bd9Sstevel@tonic-gate 	px_dma_win_t **win_pp, uint64_t count_max, uint64_t bypass)
9577c478bd9Sstevel@tonic-gate {
9587c478bd9Sstevel@tonic-gate 	int (*waitfp)(caddr_t) = dmareq->dmar_fp;
9597c478bd9Sstevel@tonic-gate 	ddi_dma_cookie_t *cookie_p;
9607c478bd9Sstevel@tonic-gate 	uint32_t pfn_no = 1;
9617c478bd9Sstevel@tonic-gate 	px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx);
9627c478bd9Sstevel@tonic-gate 	px_iopfn_t prev_pfn = pfn;
9637c478bd9Sstevel@tonic-gate 	uint64_t baddr, seg_pfn0 = pfn;
9647c478bd9Sstevel@tonic-gate 	size_t sz = cookie_no * sizeof (ddi_dma_cookie_t);
9657c478bd9Sstevel@tonic-gate 	px_dma_win_t *win_p = kmem_zalloc(sizeof (px_dma_win_t) + sz,
9669fc8611eSDaniel Ice 	    waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
96725cf1a30Sjl 	io_attributes_t	attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
96825cf1a30Sjl 	    mp->dmai_attr.dma_attr_flags);
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 	if (!win_p)
9717c478bd9Sstevel@tonic-gate 		goto noresource;
9727c478bd9Sstevel@tonic-gate 
9737c478bd9Sstevel@tonic-gate 	win_p->win_next = NULL;
9747c478bd9Sstevel@tonic-gate 	win_p->win_ncookies = cookie_no;
9757c478bd9Sstevel@tonic-gate 	win_p->win_curseg = 0;	/* start from segment 0 */
9767c478bd9Sstevel@tonic-gate 	win_p->win_size = MMU_PTOB(end_idx - start_idx + 1);
9777c478bd9Sstevel@tonic-gate 	/* win_p->win_offset is left uninitialized */
9787c478bd9Sstevel@tonic-gate 
9797c478bd9Sstevel@tonic-gate 	cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
9807c478bd9Sstevel@tonic-gate 	start_idx++;
9817c478bd9Sstevel@tonic-gate 	for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) {
9827c478bd9Sstevel@tonic-gate 		pfn = PX_GET_MP_PFN1(mp, start_idx);
9837c478bd9Sstevel@tonic-gate 		if ((pfn == prev_pfn + 1) &&
9849fc8611eSDaniel Ice 		    (MMU_PTOB(pfn_no + 1) - 1 <= count_max))
9857c478bd9Sstevel@tonic-gate 			continue;
9867c478bd9Sstevel@tonic-gate 
9877c478bd9Sstevel@tonic-gate 		/* close up the cookie up to (including) prev_pfn */
9887c478bd9Sstevel@tonic-gate 		baddr = MMU_PTOB(seg_pfn0);
989a616a11eSLida.Horn 		if (bypass) {
990a616a11eSLida.Horn 			if (px_lib_iommu_getbypass(dip, baddr, attr, &baddr)
991a616a11eSLida.Horn 			    == DDI_SUCCESS)
992a616a11eSLida.Horn 				baddr = px_lib_ro_bypass(dip, attr, baddr);
993a616a11eSLida.Horn 			else
994a616a11eSLida.Horn 				return (DDI_FAILURE);
995a616a11eSLida.Horn 		}
9967c478bd9Sstevel@tonic-gate 
9977c478bd9Sstevel@tonic-gate 		MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no));
9987c478bd9Sstevel@tonic-gate 		DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",
9999fc8611eSDaniel Ice 		    MMU_PTOB(seg_pfn0), pfn_no);
10007c478bd9Sstevel@tonic-gate 
10017c478bd9Sstevel@tonic-gate 		cookie_p++;	/* advance to next available cookie cell */
10027c478bd9Sstevel@tonic-gate 		pfn_no = 0;
10037c478bd9Sstevel@tonic-gate 		seg_pfn0 = pfn;	/* start a new segment from current pfn */
10047c478bd9Sstevel@tonic-gate 	}
10057c478bd9Sstevel@tonic-gate 
10067c478bd9Sstevel@tonic-gate 	baddr = MMU_PTOB(seg_pfn0);
1007a616a11eSLida.Horn 	if (bypass) {
1008a616a11eSLida.Horn 		if (px_lib_iommu_getbypass(dip, baddr, attr, &baddr)
1009a616a11eSLida.Horn 		    == DDI_SUCCESS)
1010a616a11eSLida.Horn 			baddr = px_lib_ro_bypass(dip, attr, baddr);
1011a616a11eSLida.Horn 		else
1012a616a11eSLida.Horn 			return (DDI_FAILURE);
1013a616a11eSLida.Horn 	}
10147c478bd9Sstevel@tonic-gate 
10157c478bd9Sstevel@tonic-gate 	MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no));
10167c478bd9Sstevel@tonic-gate 	DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",
10179fc8611eSDaniel Ice 	    MMU_PTOB(seg_pfn0), pfn_no, cookie_no);
10187c478bd9Sstevel@tonic-gate #ifdef	DEBUG
10197c478bd9Sstevel@tonic-gate 	cookie_p++;
10207c478bd9Sstevel@tonic-gate 	ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no);
10217c478bd9Sstevel@tonic-gate #endif	/* DEBUG */
10227c478bd9Sstevel@tonic-gate 	*win_pp = win_p;
10237c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
10247c478bd9Sstevel@tonic-gate noresource:
10257c478bd9Sstevel@tonic-gate 	if (waitfp != DDI_DMA_DONTWAIT)
10267c478bd9Sstevel@tonic-gate 		ddi_set_callback(waitfp, dmareq->dmar_arg, &px_kmem_clid);
10277c478bd9Sstevel@tonic-gate 	return (DDI_DMA_NORESOURCES);
10287c478bd9Sstevel@tonic-gate }
10297c478bd9Sstevel@tonic-gate 
10307c478bd9Sstevel@tonic-gate /*
10317c478bd9Sstevel@tonic-gate  * px_dma_adjust - adjust 1st and last cookie and window sizes
10327c478bd9Sstevel@tonic-gate  *	remove initial dma page offset from 1st cookie and window size
10337c478bd9Sstevel@tonic-gate  *	remove last dma page remainder from last cookie and window size
10347c478bd9Sstevel@tonic-gate  *	fill win_offset of each dma window according to just fixed up
10357c478bd9Sstevel@tonic-gate  *		each window sizes
10367c478bd9Sstevel@tonic-gate  *	px_dma_win_t members modified:
10377c478bd9Sstevel@tonic-gate  *	win_p->win_offset - this window's offset within entire DMA object
10387c478bd9Sstevel@tonic-gate  *	win_p->win_size	  - xferrable size (in bytes) for this window
10397c478bd9Sstevel@tonic-gate  *
10407c478bd9Sstevel@tonic-gate  *	ddi_dma_impl_t members modified:
10417c478bd9Sstevel@tonic-gate  *	mp->dmai_size	  - 1st window xferrable size
10427c478bd9Sstevel@tonic-gate  *	mp->dmai_offset   - 0, which is the dma offset of the 1st window
10437c478bd9Sstevel@tonic-gate  *
10447c478bd9Sstevel@tonic-gate  *	ddi_dma_cookie_t members modified:
10457c478bd9Sstevel@tonic-gate  *	cookie_p->dmac_size - 1st and last cookie remove offset or remainder
10467c478bd9Sstevel@tonic-gate  *	cookie_p->dmac_laddress - 1st cookie add page offset
10477c478bd9Sstevel@tonic-gate  */
10487c478bd9Sstevel@tonic-gate static void
px_dma_adjust(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,px_dma_win_t * win_p)10497c478bd9Sstevel@tonic-gate px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p)
10507c478bd9Sstevel@tonic-gate {
10517c478bd9Sstevel@tonic-gate 	ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
10527c478bd9Sstevel@tonic-gate 	size_t pg_offset = mp->dmai_roffset;
10537c478bd9Sstevel@tonic-gate 	size_t win_offset = 0;
10547c478bd9Sstevel@tonic-gate 
10557c478bd9Sstevel@tonic-gate 	cookie_p->dmac_size -= pg_offset;
10567c478bd9Sstevel@tonic-gate 	cookie_p->dmac_laddress |= pg_offset;
10577c478bd9Sstevel@tonic-gate 	win_p->win_size -= pg_offset;
10587c478bd9Sstevel@tonic-gate 	DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate 	mp->dmai_size = win_p->win_size;
10617c478bd9Sstevel@tonic-gate 	mp->dmai_offset = 0;
10627c478bd9Sstevel@tonic-gate 
10637c478bd9Sstevel@tonic-gate 	pg_offset += mp->dmai_object.dmao_size;
10647c478bd9Sstevel@tonic-gate 	pg_offset &= MMU_PAGE_OFFSET;
10657c478bd9Sstevel@tonic-gate 	if (pg_offset)
10667c478bd9Sstevel@tonic-gate 		pg_offset = MMU_PAGE_SIZE - pg_offset;
10677c478bd9Sstevel@tonic-gate 	DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);
10687c478bd9Sstevel@tonic-gate 
10697c478bd9Sstevel@tonic-gate 	for (; win_p->win_next; win_p = win_p->win_next) {
10707c478bd9Sstevel@tonic-gate 		DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);
10717c478bd9Sstevel@tonic-gate 		win_p->win_offset = win_offset;
10727c478bd9Sstevel@tonic-gate 		win_offset += win_p->win_size;
10737c478bd9Sstevel@tonic-gate 	}
10747c478bd9Sstevel@tonic-gate 	/* last window */
10757c478bd9Sstevel@tonic-gate 	win_p->win_offset = win_offset;
10767c478bd9Sstevel@tonic-gate 	cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
10777c478bd9Sstevel@tonic-gate 	cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset;
10787c478bd9Sstevel@tonic-gate 	win_p->win_size -= pg_offset;
10797c478bd9Sstevel@tonic-gate 	ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);
10807c478bd9Sstevel@tonic-gate }
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate /*
10837c478bd9Sstevel@tonic-gate  * px_dma_physwin() - carve up dma windows using physical addresses.
10847c478bd9Sstevel@tonic-gate  *	Called to handle mmu bypass and pci peer-to-peer transfers.
10857c478bd9Sstevel@tonic-gate  *	Calls px_dma_newwin() to allocate window objects.
10867c478bd9Sstevel@tonic-gate  *
10877c478bd9Sstevel@tonic-gate  * Dependency: mp->dmai_pfnlst points to an array of pfns
10887c478bd9Sstevel@tonic-gate  *
10897c478bd9Sstevel@tonic-gate  * 1. Each dma window is represented by a px_dma_win_t object.
10907c478bd9Sstevel@tonic-gate  *	The object will be casted to ddi_dma_win_t and returned
10917c478bd9Sstevel@tonic-gate  *	to leaf driver through the DDI interface.
10927c478bd9Sstevel@tonic-gate  * 2. Each dma window can have several dma segments with each
10937c478bd9Sstevel@tonic-gate  *	segment representing a physically contiguous either memory
10947c478bd9Sstevel@tonic-gate  *	space (if we are doing an mmu bypass transfer) or pci address
10957c478bd9Sstevel@tonic-gate  *	space (if we are doing a peer-to-peer transfer).
10967c478bd9Sstevel@tonic-gate  * 3. Each segment has a DMA cookie to program the DMA engine.
10977c478bd9Sstevel@tonic-gate  *	The cookies within each DMA window must be located in a
10987c478bd9Sstevel@tonic-gate  *	contiguous array per ddi_dma_nextcookie(9f).
10997c478bd9Sstevel@tonic-gate  * 4. The number of DMA segments within each DMA window cannot exceed
11007c478bd9Sstevel@tonic-gate  *	mp->dmai_attr.dma_attr_sgllen. If the transfer size is
11017c478bd9Sstevel@tonic-gate  *	too large to fit in the sgllen, the rest needs to be
11027c478bd9Sstevel@tonic-gate  *	relocated to the next dma window.
11037c478bd9Sstevel@tonic-gate  * 5. Peer-to-peer DMA segment follows device hi, lo, count_max,
11047c478bd9Sstevel@tonic-gate  *	and nocross restrictions while bypass DMA follows the set of
11057c478bd9Sstevel@tonic-gate  *	restrictions with system limits factored in.
11067c478bd9Sstevel@tonic-gate  *
11077c478bd9Sstevel@tonic-gate  * Return:
11087c478bd9Sstevel@tonic-gate  *	mp->dmai_winlst	 - points to a link list of px_dma_win_t objects.
11097c478bd9Sstevel@tonic-gate  *		Each px_dma_win_t object on the link list contains
11107c478bd9Sstevel@tonic-gate  *		infomation such as its window size (# of pages),
11117c478bd9Sstevel@tonic-gate  *		starting offset (also see Restriction), an array of
11127c478bd9Sstevel@tonic-gate  *		DMA cookies, and # of cookies in the array.
11137c478bd9Sstevel@tonic-gate  *	mp->dmai_pfnlst	 - NULL, the pfn list is freed to conserve memory.
11147c478bd9Sstevel@tonic-gate  *	mp->dmai_nwin	 - # of total DMA windows on mp->dmai_winlst.
11157c478bd9Sstevel@tonic-gate  *	mp->dmai_mapping - starting cookie address
11167c478bd9Sstevel@tonic-gate  *	mp->dmai_rflags	 - consistent, nosync, no redzone
11177c478bd9Sstevel@tonic-gate  *	mp->dmai_cookie	 - start of cookie table of the 1st DMA window
11187c478bd9Sstevel@tonic-gate  *
11197c478bd9Sstevel@tonic-gate  * Restriction:
11207c478bd9Sstevel@tonic-gate  *	Each px_dma_win_t object can theoratically start from any offset
11217c478bd9Sstevel@tonic-gate  *	since the mmu is not involved. However, this implementation
11227c478bd9Sstevel@tonic-gate  *	always make windows start from page aligned offset (except
11237c478bd9Sstevel@tonic-gate  *	the 1st window, which follows the requested offset) due to the
11247c478bd9Sstevel@tonic-gate  *	fact that we are handed a pfn list. This does require device's
11257c478bd9Sstevel@tonic-gate  *	count_max and attr_seg to be at least MMU_PAGE_SIZE aligned.
11267c478bd9Sstevel@tonic-gate  */
11277c478bd9Sstevel@tonic-gate int
px_dma_physwin(px_t * px_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)11287c478bd9Sstevel@tonic-gate px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
11297c478bd9Sstevel@tonic-gate {
11307c478bd9Sstevel@tonic-gate 	uint_t npages = mp->dmai_ndvmapages;
11317c478bd9Sstevel@tonic-gate 	int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;
11327c478bd9Sstevel@tonic-gate 	px_iopfn_t pfn_lo, pfn_hi, prev_pfn;
11337c478bd9Sstevel@tonic-gate 	px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0);
11347c478bd9Sstevel@tonic-gate 	uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0;
11357c478bd9Sstevel@tonic-gate 	uint64_t count_max, bypass_addr = 0;
11367c478bd9Sstevel@tonic-gate 	px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst;
11377c478bd9Sstevel@tonic-gate 	ddi_dma_cookie_t *cookie0_p;
113825cf1a30Sjl 	io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags,
113925cf1a30Sjl 	    mp->dmai_attr.dma_attr_flags);
11407c478bd9Sstevel@tonic-gate 	dev_info_t *dip = px_p->px_dip;
11417c478bd9Sstevel@tonic-gate 
11427c478bd9Sstevel@tonic-gate 	ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp));
11437c478bd9Sstevel@tonic-gate 	if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */
114436fe4a92Segillett 		ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp);
11457c478bd9Sstevel@tonic-gate 		uint64_t nocross = dev_attr_p->dma_attr_seg;
11467c478bd9Sstevel@tonic-gate 		px_pec_t *pec_p = px_p->px_pec_p;
11477c478bd9Sstevel@tonic-gate 		px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ?
11489fc8611eSDaniel Ice 		    pec_p->pec_last32_pfn - pec_p->pec_base32_pfn :
11499fc8611eSDaniel Ice 		    pec_p->pec_last64_pfn - pec_p->pec_base64_pfn;
11507c478bd9Sstevel@tonic-gate 
11517c478bd9Sstevel@tonic-gate 		if (nocross && (nocross < UINT32_MAX))
11527c478bd9Sstevel@tonic-gate 			return (DDI_DMA_NOMAPPING);
11537c478bd9Sstevel@tonic-gate 		if (dev_attr_p->dma_attr_align > MMU_PAGE_SIZE)
11547c478bd9Sstevel@tonic-gate 			return (DDI_DMA_NOMAPPING);
11557c478bd9Sstevel@tonic-gate 		pfn_lo = MMU_BTOP(dev_attr_p->dma_attr_addr_lo);
11567c478bd9Sstevel@tonic-gate 		pfn_hi = MMU_BTOP(dev_attr_p->dma_attr_addr_hi);
11577c478bd9Sstevel@tonic-gate 		pfn_hi = MIN(pfn_hi, pfn_last);
11587c478bd9Sstevel@tonic-gate 		if ((pfn_lo > pfn_hi) || (pfn < pfn_lo))
11597c478bd9Sstevel@tonic-gate 			return (DDI_DMA_NOMAPPING);
11607c478bd9Sstevel@tonic-gate 
11617c478bd9Sstevel@tonic-gate 		count_max = dev_attr_p->dma_attr_count_max;
11627c478bd9Sstevel@tonic-gate 		count_max = MIN(count_max, nocross);
11637c478bd9Sstevel@tonic-gate 		/*
11647c478bd9Sstevel@tonic-gate 		 * the following count_max trim is not done because we are
11657c478bd9Sstevel@tonic-gate 		 * making sure pfn_lo <= pfn <= pfn_hi inside the loop
11667c478bd9Sstevel@tonic-gate 		 * count_max=MIN(count_max, MMU_PTOB(pfn_hi - pfn_lo + 1)-1);
11677c478bd9Sstevel@tonic-gate 		 */
11687c478bd9Sstevel@tonic-gate 	} else { /* bypass hi/lo/count_max have been processed by attr2hdl() */
11697c478bd9Sstevel@tonic-gate 		count_max = mp->dmai_attr.dma_attr_count_max;
11707c478bd9Sstevel@tonic-gate 		pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);
11717c478bd9Sstevel@tonic-gate 		pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);
11727c478bd9Sstevel@tonic-gate 
11737c478bd9Sstevel@tonic-gate 		if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn),
11749fc8611eSDaniel Ice 		    attr, &bypass_addr) != DDI_SUCCESS) {
117503494a98SBill Taylor 			DBG(DBG_BYPASS, mp->dmai_rdip,
117603494a98SBill Taylor 			    "bypass cookie failure %lx\n", pfn);
11777c478bd9Sstevel@tonic-gate 			return (DDI_DMA_NOMAPPING);
11787c478bd9Sstevel@tonic-gate 		}
11797c478bd9Sstevel@tonic-gate 		pfn = MMU_BTOP(bypass_addr);
11807c478bd9Sstevel@tonic-gate 	}
11817c478bd9Sstevel@tonic-gate 
11827c478bd9Sstevel@tonic-gate 	/* pfn: absolute (bypass mode) or relative (p2p mode) */
11837c478bd9Sstevel@tonic-gate 	for (prev_pfn = pfn, i = 1; i < npages;
11847c478bd9Sstevel@tonic-gate 	    i++, prev_pfn = pfn, pfn_no++) {
11857c478bd9Sstevel@tonic-gate 		pfn = PX_GET_MP_PFN1(mp, i);
11867c478bd9Sstevel@tonic-gate 		if (bypass_addr) {
11877c478bd9Sstevel@tonic-gate 			if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), attr,
11889fc8611eSDaniel Ice 			    &bypass_addr) != DDI_SUCCESS) {
11897c478bd9Sstevel@tonic-gate 				ret = DDI_DMA_NOMAPPING;
11907c478bd9Sstevel@tonic-gate 				goto err;
11917c478bd9Sstevel@tonic-gate 			}
11927c478bd9Sstevel@tonic-gate 			pfn = MMU_BTOP(bypass_addr);
11937c478bd9Sstevel@tonic-gate 		}
11947c478bd9Sstevel@tonic-gate 		if ((pfn == prev_pfn + 1) &&
11959fc8611eSDaniel Ice 		    (MMU_PTOB(pfn_no + 1) - 1 <= count_max))
11967c478bd9Sstevel@tonic-gate 			continue;
11977c478bd9Sstevel@tonic-gate 		if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) {
11987c478bd9Sstevel@tonic-gate 			ret = DDI_DMA_NOMAPPING;
11997c478bd9Sstevel@tonic-gate 			goto err;
12007c478bd9Sstevel@tonic-gate 		}
12017c478bd9Sstevel@tonic-gate 		cookie_no++;
12027c478bd9Sstevel@tonic-gate 		pfn_no = 0;
12037c478bd9Sstevel@tonic-gate 		if (cookie_no < sgllen)
12047c478bd9Sstevel@tonic-gate 			continue;
12057c478bd9Sstevel@tonic-gate 
12067c478bd9Sstevel@tonic-gate 		DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
12079fc8611eSDaniel Ice 		    win_pfn0_index, i - 1, cookie_no);
12087c478bd9Sstevel@tonic-gate 		if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no,
12099fc8611eSDaniel Ice 		    win_pfn0_index, i - 1, win_pp, count_max, bypass_addr))
12107c478bd9Sstevel@tonic-gate 			goto err;
12117c478bd9Sstevel@tonic-gate 
12127c478bd9Sstevel@tonic-gate 		win_pp = &(*win_pp)->win_next;	/* win_pp = *(win_pp) */
12137c478bd9Sstevel@tonic-gate 		win_no++;
12147c478bd9Sstevel@tonic-gate 		win_pfn0_index = i;
12157c478bd9Sstevel@tonic-gate 		cookie_no = 0;
12167c478bd9Sstevel@tonic-gate 	}
12177c478bd9Sstevel@tonic-gate 	if (pfn > pfn_hi) {
12187c478bd9Sstevel@tonic-gate 		ret = DDI_DMA_NOMAPPING;
12197c478bd9Sstevel@tonic-gate 		goto err;
12207c478bd9Sstevel@tonic-gate 	}
12217c478bd9Sstevel@tonic-gate 	cookie_no++;
12227c478bd9Sstevel@tonic-gate 	DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
12239fc8611eSDaniel Ice 	    win_pfn0_index, i - 1, cookie_no);
12247c478bd9Sstevel@tonic-gate 	if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index,
12259fc8611eSDaniel Ice 	    i - 1, win_pp, count_max, bypass_addr))
12267c478bd9Sstevel@tonic-gate 		goto err;
12277c478bd9Sstevel@tonic-gate 	win_no++;
12287c478bd9Sstevel@tonic-gate 	px_dma_adjust(dmareq, mp, mp->dmai_winlst);
12297c478bd9Sstevel@tonic-gate 	mp->dmai_nwin = win_no;
12307c478bd9Sstevel@tonic-gate 	mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC;
12317c478bd9Sstevel@tonic-gate 	mp->dmai_rflags &= ~DDI_DMA_REDZONE;
123236fe4a92Segillett 	mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC;
123336fe4a92Segillett 	cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1);
1234*9a63ec27SRobert Mustacchi 	mp->dmai_cookie = cookie0_p + 1;
1235*9a63ec27SRobert Mustacchi 	mp->dmai_curcookie = 1;
1236*9a63ec27SRobert Mustacchi 	mp->dmai_ncookies = PX_WINLST(mp)->win_ncookies;
12377c478bd9Sstevel@tonic-gate 	mp->dmai_mapping = cookie0_p->dmac_laddress;
12387c478bd9Sstevel@tonic-gate 
12397c478bd9Sstevel@tonic-gate 	px_dma_freepfn(mp);
12407c478bd9Sstevel@tonic-gate 	return (DDI_DMA_MAPPED);
12417c478bd9Sstevel@tonic-gate err:
12427c478bd9Sstevel@tonic-gate 	px_dma_freewin(mp);
12437c478bd9Sstevel@tonic-gate 	return (ret);
12447c478bd9Sstevel@tonic-gate }
12457c478bd9Sstevel@tonic-gate 
12467c478bd9Sstevel@tonic-gate int
px_dma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)12477c478bd9Sstevel@tonic-gate px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
12487c478bd9Sstevel@tonic-gate 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
12497c478bd9Sstevel@tonic-gate 	uint_t cache_flags)
12507c478bd9Sstevel@tonic-gate {
12517c478bd9Sstevel@tonic-gate 	switch (cmd) {
12527c478bd9Sstevel@tonic-gate 	default:
12537c478bd9Sstevel@tonic-gate 		DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
12549fc8611eSDaniel Ice 		    cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
12557c478bd9Sstevel@tonic-gate 		break;
12567c478bd9Sstevel@tonic-gate 	}
12577c478bd9Sstevel@tonic-gate 	return (DDI_FAILURE);
12587c478bd9Sstevel@tonic-gate }
12597c478bd9Sstevel@tonic-gate 
12607c478bd9Sstevel@tonic-gate static void
px_dvma_debug_init(px_mmu_t * mmu_p)12617c478bd9Sstevel@tonic-gate px_dvma_debug_init(px_mmu_t *mmu_p)
12627c478bd9Sstevel@tonic-gate {
12637c478bd9Sstevel@tonic-gate 	size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec;
12647c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&mmu_p->dvma_debug_lock));
12657c478bd9Sstevel@tonic-gate 	cmn_err(CE_NOTE, "PCI Express DVMA %p stat ON", mmu_p);
12667c478bd9Sstevel@tonic-gate 
12677c478bd9Sstevel@tonic-gate 	mmu_p->dvma_alloc_rec = kmem_alloc(sz, KM_SLEEP);
12687c478bd9Sstevel@tonic-gate 	mmu_p->dvma_free_rec = kmem_alloc(sz, KM_SLEEP);
12697c478bd9Sstevel@tonic-gate 
12707c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_list = NULL;
12717c478bd9Sstevel@tonic-gate 	mmu_p->dvma_alloc_rec_index = 0;
12727c478bd9Sstevel@tonic-gate 	mmu_p->dvma_free_rec_index = 0;
12737c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_count = 0;
12747c478bd9Sstevel@tonic-gate }
12757c478bd9Sstevel@tonic-gate 
12767c478bd9Sstevel@tonic-gate void
px_dvma_debug_fini(px_mmu_t * mmu_p)12777c478bd9Sstevel@tonic-gate px_dvma_debug_fini(px_mmu_t *mmu_p)
12787c478bd9Sstevel@tonic-gate {
12797c478bd9Sstevel@tonic-gate 	struct px_dvma_rec *prev, *ptr;
12807c478bd9Sstevel@tonic-gate 	size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec;
12817c478bd9Sstevel@tonic-gate 	uint64_t mask = ~(1ull << mmu_p->mmu_inst);
12827c478bd9Sstevel@tonic-gate 	cmn_err(CE_NOTE, "PCI Express DVMA %p stat OFF", mmu_p);
12837c478bd9Sstevel@tonic-gate 
12841f7be8d9Sdanice 	if (mmu_p->dvma_alloc_rec) {
12851f7be8d9Sdanice 		kmem_free(mmu_p->dvma_alloc_rec, sz);
12861f7be8d9Sdanice 		mmu_p->dvma_alloc_rec = NULL;
12871f7be8d9Sdanice 	}
12881f7be8d9Sdanice 	if (mmu_p->dvma_free_rec) {
12891f7be8d9Sdanice 		kmem_free(mmu_p->dvma_free_rec, sz);
12901f7be8d9Sdanice 		mmu_p->dvma_free_rec = NULL;
12911f7be8d9Sdanice 	}
12927c478bd9Sstevel@tonic-gate 
12937c478bd9Sstevel@tonic-gate 	prev = mmu_p->dvma_active_list;
12947c478bd9Sstevel@tonic-gate 	if (!prev)
12957c478bd9Sstevel@tonic-gate 		return;
12967c478bd9Sstevel@tonic-gate 	for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next)
12977c478bd9Sstevel@tonic-gate 		kmem_free(prev, sizeof (struct px_dvma_rec));
12987c478bd9Sstevel@tonic-gate 	kmem_free(prev, sizeof (struct px_dvma_rec));
12997c478bd9Sstevel@tonic-gate 
13007c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_list = NULL;
13017c478bd9Sstevel@tonic-gate 	mmu_p->dvma_alloc_rec_index = 0;
13027c478bd9Sstevel@tonic-gate 	mmu_p->dvma_free_rec_index = 0;
13037c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_count = 0;
13047c478bd9Sstevel@tonic-gate 
13057c478bd9Sstevel@tonic-gate 	px_dvma_debug_off &= mask;
13067c478bd9Sstevel@tonic-gate 	px_dvma_debug_on &= mask;
13077c478bd9Sstevel@tonic-gate }
13087c478bd9Sstevel@tonic-gate 
13097c478bd9Sstevel@tonic-gate void
px_dvma_alloc_debug(px_mmu_t * mmu_p,char * address,uint_t len,ddi_dma_impl_t * mp)13107c478bd9Sstevel@tonic-gate px_dvma_alloc_debug(px_mmu_t *mmu_p, char *address, uint_t len,
13117c478bd9Sstevel@tonic-gate 	ddi_dma_impl_t *mp)
13127c478bd9Sstevel@tonic-gate {
13137c478bd9Sstevel@tonic-gate 	struct px_dvma_rec *ptr;
13147c478bd9Sstevel@tonic-gate 	mutex_enter(&mmu_p->dvma_debug_lock);
13157c478bd9Sstevel@tonic-gate 
13167c478bd9Sstevel@tonic-gate 	if (!mmu_p->dvma_alloc_rec)
13177c478bd9Sstevel@tonic-gate 		px_dvma_debug_init(mmu_p);
131836fe4a92Segillett 	if (PX_DVMA_DBG_OFF(mmu_p)) {
13197c478bd9Sstevel@tonic-gate 		px_dvma_debug_fini(mmu_p);
13207c478bd9Sstevel@tonic-gate 		goto done;
13217c478bd9Sstevel@tonic-gate 	}
13227c478bd9Sstevel@tonic-gate 
13237c478bd9Sstevel@tonic-gate 	ptr = &mmu_p->dvma_alloc_rec[mmu_p->dvma_alloc_rec_index];
13247c478bd9Sstevel@tonic-gate 	ptr->dvma_addr = address;
13257c478bd9Sstevel@tonic-gate 	ptr->len = len;
13267c478bd9Sstevel@tonic-gate 	ptr->mp = mp;
13277c478bd9Sstevel@tonic-gate 	if (++mmu_p->dvma_alloc_rec_index == px_dvma_debug_rec)
13287c478bd9Sstevel@tonic-gate 		mmu_p->dvma_alloc_rec_index = 0;
13297c478bd9Sstevel@tonic-gate 
13307c478bd9Sstevel@tonic-gate 	ptr = kmem_alloc(sizeof (struct px_dvma_rec), KM_SLEEP);
13317c478bd9Sstevel@tonic-gate 	ptr->dvma_addr = address;
13327c478bd9Sstevel@tonic-gate 	ptr->len = len;
13337c478bd9Sstevel@tonic-gate 	ptr->mp = mp;
13347c478bd9Sstevel@tonic-gate 
13357c478bd9Sstevel@tonic-gate 	ptr->next = mmu_p->dvma_active_list;
13367c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_list = ptr;
13377c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_count++;
13387c478bd9Sstevel@tonic-gate done:
13397c478bd9Sstevel@tonic-gate 	mutex_exit(&mmu_p->dvma_debug_lock);
13407c478bd9Sstevel@tonic-gate }
13417c478bd9Sstevel@tonic-gate 
13427c478bd9Sstevel@tonic-gate void
px_dvma_free_debug(px_mmu_t * mmu_p,char * address,uint_t len,ddi_dma_impl_t * mp)13437c478bd9Sstevel@tonic-gate px_dvma_free_debug(px_mmu_t *mmu_p, char *address, uint_t len,
13447c478bd9Sstevel@tonic-gate     ddi_dma_impl_t *mp)
13457c478bd9Sstevel@tonic-gate {
13467c478bd9Sstevel@tonic-gate 	struct px_dvma_rec *ptr, *ptr_save;
13477c478bd9Sstevel@tonic-gate 	mutex_enter(&mmu_p->dvma_debug_lock);
13487c478bd9Sstevel@tonic-gate 
13497c478bd9Sstevel@tonic-gate 	if (!mmu_p->dvma_alloc_rec)
13507c478bd9Sstevel@tonic-gate 		px_dvma_debug_init(mmu_p);
135136fe4a92Segillett 	if (PX_DVMA_DBG_OFF(mmu_p)) {
13527c478bd9Sstevel@tonic-gate 		px_dvma_debug_fini(mmu_p);
13537c478bd9Sstevel@tonic-gate 		goto done;
13547c478bd9Sstevel@tonic-gate 	}
13557c478bd9Sstevel@tonic-gate 
13567c478bd9Sstevel@tonic-gate 	ptr = &mmu_p->dvma_free_rec[mmu_p->dvma_free_rec_index];
13577c478bd9Sstevel@tonic-gate 	ptr->dvma_addr = address;
13587c478bd9Sstevel@tonic-gate 	ptr->len = len;
13597c478bd9Sstevel@tonic-gate 	ptr->mp = mp;
13607c478bd9Sstevel@tonic-gate 	if (++mmu_p->dvma_free_rec_index == px_dvma_debug_rec)
13617c478bd9Sstevel@tonic-gate 		mmu_p->dvma_free_rec_index = 0;
13627c478bd9Sstevel@tonic-gate 
13637c478bd9Sstevel@tonic-gate 	ptr_save = mmu_p->dvma_active_list;
13647c478bd9Sstevel@tonic-gate 	for (ptr = ptr_save; ptr; ptr = ptr->next) {
13657c478bd9Sstevel@tonic-gate 		if ((ptr->dvma_addr == address) && (ptr->len = len))
13667c478bd9Sstevel@tonic-gate 			break;
13677c478bd9Sstevel@tonic-gate 		ptr_save = ptr;
13687c478bd9Sstevel@tonic-gate 	}
13697c478bd9Sstevel@tonic-gate 	if (!ptr) {
13707c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x",
13719fc8611eSDaniel Ice 		    (long)address, len);
13727c478bd9Sstevel@tonic-gate 		goto done;
13737c478bd9Sstevel@tonic-gate 	}
13747c478bd9Sstevel@tonic-gate 	if (ptr == mmu_p->dvma_active_list)
13757c478bd9Sstevel@tonic-gate 		mmu_p->dvma_active_list = ptr->next;
13767c478bd9Sstevel@tonic-gate 	else
13777c478bd9Sstevel@tonic-gate 		ptr_save->next = ptr->next;
13787c478bd9Sstevel@tonic-gate 	kmem_free(ptr, sizeof (struct px_dvma_rec));
13797c478bd9Sstevel@tonic-gate 	mmu_p->dvma_active_count--;
13807c478bd9Sstevel@tonic-gate done:
13817c478bd9Sstevel@tonic-gate 	mutex_exit(&mmu_p->dvma_debug_lock);
13827c478bd9Sstevel@tonic-gate }
13837c478bd9Sstevel@tonic-gate 
13847c478bd9Sstevel@tonic-gate #ifdef	DEBUG
13857c478bd9Sstevel@tonic-gate void
px_dump_dma_handle(uint64_t flag,dev_info_t * dip,ddi_dma_impl_t * hp)13867c478bd9Sstevel@tonic-gate px_dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp)
13877c478bd9Sstevel@tonic-gate {
13887c478bd9Sstevel@tonic-gate 	DBG(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n",
13899fc8611eSDaniel Ice 	    hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size);
13907c478bd9Sstevel@tonic-gate 	DBG(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n",
13919fc8611eSDaniel Ice 	    hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags,
13929fc8611eSDaniel Ice 	    hp->dmai_nwin);
13937c478bd9Sstevel@tonic-gate 	DBG(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n",
13949fc8611eSDaniel Ice 	    hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0);
13957c478bd9Sstevel@tonic-gate 	DBG(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n",
13969fc8611eSDaniel Ice 	    hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr,
13979fc8611eSDaniel Ice 	    hp->dmai_cookie);
13987c478bd9Sstevel@tonic-gate }
13997c478bd9Sstevel@tonic-gate #endif	/* DEBUG */
1400