17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51de45cd9Sgovinda * Common Development and Distribution License (the "License").
61de45cd9Sgovinda * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
22567c0b92SStephen Hanson * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
25cd21e7c5SGarrett D'Amore /*
26cd21e7c5SGarrett D'Amore * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
27cd21e7c5SGarrett D'Amore */
287c478bd9Sstevel@tonic-gate
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate * PCI nexus DVMA and DMA core routines:
317c478bd9Sstevel@tonic-gate * dma_map/dma_bind_handle implementation
327c478bd9Sstevel@tonic-gate * bypass and peer-to-peer support
337c478bd9Sstevel@tonic-gate * fast track DVMA space allocation
347c478bd9Sstevel@tonic-gate * runtime DVMA debug
357c478bd9Sstevel@tonic-gate */
367c478bd9Sstevel@tonic-gate #include <sys/types.h>
377c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
387c478bd9Sstevel@tonic-gate #include <sys/async.h>
397c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
407c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
417c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> /* lddphys() */
427c478bd9Sstevel@tonic-gate #include <sys/ddi_impldefs.h>
437c478bd9Sstevel@tonic-gate #include <vm/hat.h>
447c478bd9Sstevel@tonic-gate #include <sys/pci/pci_obj.h>
457c478bd9Sstevel@tonic-gate
467c478bd9Sstevel@tonic-gate /*LINTLIBRARY*/
477c478bd9Sstevel@tonic-gate
487c478bd9Sstevel@tonic-gate static void
pci_sc_pg_inv(dev_info_t * dip,sc_t * sc_p,ddi_dma_impl_t * mp,off_t off,size_t len)497c478bd9Sstevel@tonic-gate pci_sc_pg_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp, off_t off,
507c478bd9Sstevel@tonic-gate size_t len)
517c478bd9Sstevel@tonic-gate {
527c478bd9Sstevel@tonic-gate dvma_addr_t dvma_addr, pg_off;
537c478bd9Sstevel@tonic-gate volatile uint64_t *invl_va = sc_p->sc_invl_reg;
547c478bd9Sstevel@tonic-gate
557c478bd9Sstevel@tonic-gate if (!len)
567c478bd9Sstevel@tonic-gate len = mp->dmai_size;
577c478bd9Sstevel@tonic-gate
587c478bd9Sstevel@tonic-gate pg_off = mp->dmai_offset; /* start min */
597c478bd9Sstevel@tonic-gate dvma_addr = MAX(off, pg_off); /* lo */
607c478bd9Sstevel@tonic-gate pg_off += mp->dmai_size; /* end max */
617c478bd9Sstevel@tonic-gate pg_off = MIN(off + len, pg_off); /* hi */
627c478bd9Sstevel@tonic-gate if (dvma_addr >= pg_off) { /* lo >= hi ? */
637c478bd9Sstevel@tonic-gate DEBUG4(DBG_SC, dip, "%x+%x out of window [%x,%x)\n",
64567c0b92SStephen Hanson off, len, mp->dmai_offset,
65567c0b92SStephen Hanson mp->dmai_offset + mp->dmai_size);
667c478bd9Sstevel@tonic-gate return;
677c478bd9Sstevel@tonic-gate }
687c478bd9Sstevel@tonic-gate
697c478bd9Sstevel@tonic-gate len = pg_off - dvma_addr; /* sz = hi - lo */
707c478bd9Sstevel@tonic-gate dvma_addr += mp->dmai_mapping; /* start addr */
717c478bd9Sstevel@tonic-gate pg_off = dvma_addr & IOMMU_PAGE_OFFSET; /* offset in 1st pg */
727c478bd9Sstevel@tonic-gate len = IOMMU_BTOPR(len + pg_off); /* # of pages */
737c478bd9Sstevel@tonic-gate dvma_addr ^= pg_off;
747c478bd9Sstevel@tonic-gate
757c478bd9Sstevel@tonic-gate DEBUG2(DBG_SC, dip, "addr=%x+%x pages: \n", dvma_addr, len);
767c478bd9Sstevel@tonic-gate for (; len; len--, dvma_addr += IOMMU_PAGE_SIZE) {
777c478bd9Sstevel@tonic-gate DEBUG1(DBG_SC|DBG_CONT, dip, " %x", dvma_addr);
787c478bd9Sstevel@tonic-gate *invl_va = (uint64_t)dvma_addr;
797c478bd9Sstevel@tonic-gate }
807c478bd9Sstevel@tonic-gate DEBUG0(DBG_SC|DBG_CONT, dip, "\n");
817c478bd9Sstevel@tonic-gate }
827c478bd9Sstevel@tonic-gate
837c478bd9Sstevel@tonic-gate static void
pci_dma_sync_flag_wait(ddi_dma_impl_t * mp,sc_t * sc_p,uint32_t onstack)847c478bd9Sstevel@tonic-gate pci_dma_sync_flag_wait(ddi_dma_impl_t *mp, sc_t *sc_p, uint32_t onstack)
857c478bd9Sstevel@tonic-gate {
867c478bd9Sstevel@tonic-gate hrtime_t start_time;
877c478bd9Sstevel@tonic-gate uint64_t loops = 0;
887c478bd9Sstevel@tonic-gate uint64_t sync_flag_pa = SYNC_BUF_PA(mp);
897c478bd9Sstevel@tonic-gate uint64_t sync_reg_pa = sc_p->sc_sync_reg_pa;
907c478bd9Sstevel@tonic-gate uint8_t stack_buf[128];
917c478bd9Sstevel@tonic-gate
927c478bd9Sstevel@tonic-gate stack_buf[0] = DDI_SUCCESS;
937c478bd9Sstevel@tonic-gate
947c478bd9Sstevel@tonic-gate /* check for handle specific sync flag */
957c478bd9Sstevel@tonic-gate if (sync_flag_pa)
967c478bd9Sstevel@tonic-gate goto start;
977c478bd9Sstevel@tonic-gate
987c478bd9Sstevel@tonic-gate sync_flag_pa = sc_p->sc_sync_flag_pa;
997c478bd9Sstevel@tonic-gate
1007c478bd9Sstevel@tonic-gate if (onstack) {
1017c478bd9Sstevel@tonic-gate sync_flag_pa = va_to_pa(stack_buf);
1027c478bd9Sstevel@tonic-gate sync_flag_pa += PCI_SYNC_FLAG_SIZE;
1037c478bd9Sstevel@tonic-gate sync_flag_pa >>= PCI_SYNC_FLAG_SZSHIFT;
1047c478bd9Sstevel@tonic-gate sync_flag_pa <<= PCI_SYNC_FLAG_SZSHIFT;
1057c478bd9Sstevel@tonic-gate goto start;
1067c478bd9Sstevel@tonic-gate }
1077c478bd9Sstevel@tonic-gate stack_buf[0] |= PCI_SYNC_FLAG_LOCKED;
1087c478bd9Sstevel@tonic-gate mutex_enter(&sc_p->sc_sync_mutex);
1097c478bd9Sstevel@tonic-gate start:
1107c478bd9Sstevel@tonic-gate ASSERT(!(sync_flag_pa & PCI_SYNC_FLAG_SIZE - 1));
1117c478bd9Sstevel@tonic-gate stdphys(sync_flag_pa, 0); /* reset sync flag to 0 */
1127c478bd9Sstevel@tonic-gate /* membar #LoadStore|#StoreStore */
1137c478bd9Sstevel@tonic-gate stdphysio(sync_reg_pa, sync_flag_pa);
1147c478bd9Sstevel@tonic-gate start_time = gethrtime();
1157c478bd9Sstevel@tonic-gate
1167c478bd9Sstevel@tonic-gate for (; gethrtime() - start_time < pci_sync_buf_timeout; loops++)
1177c478bd9Sstevel@tonic-gate if (lddphys(sync_flag_pa))
1187c478bd9Sstevel@tonic-gate goto done;
1197c478bd9Sstevel@tonic-gate
1207c478bd9Sstevel@tonic-gate if (!lddphys(sync_flag_pa))
1217c478bd9Sstevel@tonic-gate stack_buf[0] |= PCI_SYNC_FLAG_FAILED;
1227c478bd9Sstevel@tonic-gate done:
1237c478bd9Sstevel@tonic-gate DEBUG3(DBG_SC|DBG_CONT, 0, "flag wait loops=%lu ticks=%lu status=%x\n",
124567c0b92SStephen Hanson loops, gethrtime() - start_time, stack_buf[0]);
1257c478bd9Sstevel@tonic-gate
1267c478bd9Sstevel@tonic-gate if (stack_buf[0] & PCI_SYNC_FLAG_LOCKED)
1277c478bd9Sstevel@tonic-gate mutex_exit(&sc_p->sc_sync_mutex);
1287c478bd9Sstevel@tonic-gate
1297c478bd9Sstevel@tonic-gate if (stack_buf[0] & PCI_SYNC_FLAG_FAILED)
130f47a9c50Smathue cmn_err(CE_PANIC, "%p pci dma sync %lx %lx timeout!",
1317c478bd9Sstevel@tonic-gate mp, sync_flag_pa, loops);
1327c478bd9Sstevel@tonic-gate }
1337c478bd9Sstevel@tonic-gate
1347c478bd9Sstevel@tonic-gate /*
1357c478bd9Sstevel@tonic-gate * Cache RW Before During After
1367c478bd9Sstevel@tonic-gate *
1377c478bd9Sstevel@tonic-gate * STREAMING read no/no pg/no ctx,pg/no
1387c478bd9Sstevel@tonic-gate * STREAMING write no/no pg/yes ctx,pg/yes
1397c478bd9Sstevel@tonic-gate * CONSISTENT read no/no yes,no/no yes,no/no
1407c478bd9Sstevel@tonic-gate * CONSISTENT write no/no yes,yes/yes yes,yes/yes
1417c478bd9Sstevel@tonic-gate *
1427c478bd9Sstevel@tonic-gate * STREAMING read ctx,pg/no
1437c478bd9Sstevel@tonic-gate * STREAMING write ctx,pg/yes
1447c478bd9Sstevel@tonic-gate * CONSISTENT read yes,no/no
1457c478bd9Sstevel@tonic-gate * CONSISTENT write yes,yes/yes
1467c478bd9Sstevel@tonic-gate */
1477c478bd9Sstevel@tonic-gate int
pci_dma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint32_t sync_flag)1487c478bd9Sstevel@tonic-gate pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
1497c478bd9Sstevel@tonic-gate off_t off, size_t len, uint32_t sync_flag)
1507c478bd9Sstevel@tonic-gate {
1517c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1527c478bd9Sstevel@tonic-gate int ret = ddi_get_instance(dip);
1537c478bd9Sstevel@tonic-gate pci_t *pci_p = get_pci_soft_state(ret);
1547c478bd9Sstevel@tonic-gate pbm_t *pbm_p = pci_p->pci_pbm_p;
1557c478bd9Sstevel@tonic-gate uint32_t dev_flag = mp->dmai_rflags;
1567c478bd9Sstevel@tonic-gate sc_t *sc_p;
1577c478bd9Sstevel@tonic-gate
1587c478bd9Sstevel@tonic-gate DEBUG4(DBG_DMA_SYNC, dip, "%s%d flags=%x,%x\n", ddi_driver_name(rdip),
159567c0b92SStephen Hanson ddi_get_instance(rdip), dev_flag, sync_flag);
1607c478bd9Sstevel@tonic-gate DEBUG4(DBG_SC, dip, "dmai_mapping=%x, dmai_sz=%x off=%x len=%x\n",
161567c0b92SStephen Hanson mp->dmai_mapping, mp->dmai_size, off, len);
1627c478bd9Sstevel@tonic-gate DEBUG2(DBG_SC, dip, "mp=%p, ctx=%x\n", mp, MP2CTX(mp));
1637c478bd9Sstevel@tonic-gate
1647c478bd9Sstevel@tonic-gate if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) {
1657c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Unbound dma handle %p from %s%d", mp,
1667c478bd9Sstevel@tonic-gate ddi_driver_name(rdip), ddi_get_instance(rdip));
1677c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
1687c478bd9Sstevel@tonic-gate }
1697c478bd9Sstevel@tonic-gate
1707c478bd9Sstevel@tonic-gate if (mp->dmai_flags & DMAI_FLAGS_NOSYNC)
1717c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
1727c478bd9Sstevel@tonic-gate
1737c478bd9Sstevel@tonic-gate if (!(dev_flag & DDI_DMA_CONSISTENT))
1747c478bd9Sstevel@tonic-gate goto streaming;
1757c478bd9Sstevel@tonic-gate
1767c478bd9Sstevel@tonic-gate if (sync_flag & PCI_DMA_SYNC_EXT) {
1777c478bd9Sstevel@tonic-gate if (sync_flag & (PCI_DMA_SYNC_BEFORE | PCI_DMA_SYNC_POST) ||
1787c478bd9Sstevel@tonic-gate !(sync_flag & PCI_DMA_SYNC_WRITE))
1797c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
1807c478bd9Sstevel@tonic-gate } else {
1817c478bd9Sstevel@tonic-gate if (!(dev_flag & DDI_DMA_READ) ||
1827c478bd9Sstevel@tonic-gate ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) ==
1837c478bd9Sstevel@tonic-gate DDI_DMA_SYNC_FORDEV))
1847c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
1857c478bd9Sstevel@tonic-gate }
1867c478bd9Sstevel@tonic-gate
1877c478bd9Sstevel@tonic-gate pci_pbm_dma_sync(pbm_p, pbm_p->pbm_sync_ino);
1887c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
1897c478bd9Sstevel@tonic-gate
1907c478bd9Sstevel@tonic-gate streaming:
1917c478bd9Sstevel@tonic-gate ASSERT(pci_stream_buf_exists && (pci_stream_buf_enable & 1 << ret));
1927c478bd9Sstevel@tonic-gate sc_p = pci_p->pci_sc_p;
1937c478bd9Sstevel@tonic-gate ret = DDI_FAILURE;
1947c478bd9Sstevel@tonic-gate
1957c478bd9Sstevel@tonic-gate if (sync_flag & PCI_DMA_SYNC_EXT)
1967c478bd9Sstevel@tonic-gate goto ext;
1977c478bd9Sstevel@tonic-gate
1987c478bd9Sstevel@tonic-gate if (mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts)
1997c478bd9Sstevel@tonic-gate ret = pci_sc_ctx_inv(dip, sc_p, mp);
2007c478bd9Sstevel@tonic-gate if (ret)
2017c478bd9Sstevel@tonic-gate pci_sc_pg_inv(dip, sc_p, mp, off, len);
2027c478bd9Sstevel@tonic-gate
2037c478bd9Sstevel@tonic-gate if ((dev_flag & DDI_DMA_READ) &&
2047c478bd9Sstevel@tonic-gate ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) != DDI_DMA_SYNC_FORDEV))
2057c478bd9Sstevel@tonic-gate goto wait;
2067c478bd9Sstevel@tonic-gate
2077c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2087c478bd9Sstevel@tonic-gate ext:
2097c478bd9Sstevel@tonic-gate if (sync_flag & PCI_DMA_SYNC_BEFORE)
2107c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2117c478bd9Sstevel@tonic-gate if (sync_flag & PCI_DMA_SYNC_BAR)
2127c478bd9Sstevel@tonic-gate goto wait_check;
2137c478bd9Sstevel@tonic-gate if (sync_flag & PCI_DMA_SYNC_AFTER &&
214567c0b92SStephen Hanson mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts)
2157c478bd9Sstevel@tonic-gate ret = pci_sc_ctx_inv(dip, sc_p, mp);
2167c478bd9Sstevel@tonic-gate if (ret)
2177c478bd9Sstevel@tonic-gate pci_sc_pg_inv(dip, sc_p, mp, off, len);
2187c478bd9Sstevel@tonic-gate wait_check:
2197c478bd9Sstevel@tonic-gate if (sync_flag & PCI_DMA_SYNC_POST || !(sync_flag & PCI_DMA_SYNC_WRITE))
2207c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2217c478bd9Sstevel@tonic-gate wait:
2227c478bd9Sstevel@tonic-gate pci_dma_sync_flag_wait(mp, sc_p, sync_flag & PCI_DMA_SYNC_PRIVATE);
2237c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2247c478bd9Sstevel@tonic-gate }
2257c478bd9Sstevel@tonic-gate
2267c478bd9Sstevel@tonic-gate int
pci_dma_handle_clean(dev_info_t * rdip,ddi_dma_handle_t h)2277c478bd9Sstevel@tonic-gate pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t h)
2287c478bd9Sstevel@tonic-gate {
2297c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
2307c478bd9Sstevel@tonic-gate if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0)
2317c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
2327c478bd9Sstevel@tonic-gate mp->dmai_rflags |= DMP_NOSYNC;
2337c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOSYNC;
2347c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
2357c478bd9Sstevel@tonic-gate }
2367c478bd9Sstevel@tonic-gate
2377c478bd9Sstevel@tonic-gate /*
2387c478bd9Sstevel@tonic-gate * pci_dma_allocmp - Allocate a pci dma implementation structure
2397c478bd9Sstevel@tonic-gate *
2407c478bd9Sstevel@tonic-gate * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl
2417c478bd9Sstevel@tonic-gate * to hold unmodified device limits. The ddi_dma_attr inside the
2427c478bd9Sstevel@tonic-gate * ddi_dma_impl structure is augumented with system limits to enhance
2437c478bd9Sstevel@tonic-gate * DVMA performance at runtime. The unaugumented device limits saved
2447c478bd9Sstevel@tonic-gate * right after (accessed through the DEV_ATTR macro) is used
2457c478bd9Sstevel@tonic-gate * strictly for peer-to-peer transfers which do not obey system limits.
2467c478bd9Sstevel@tonic-gate *
2477c478bd9Sstevel@tonic-gate * return: DDI_SUCCESS DDI_DMA_NORESOURCES
2487c478bd9Sstevel@tonic-gate */
2497c478bd9Sstevel@tonic-gate ddi_dma_impl_t *
pci_dma_allocmp(dev_info_t * dip,dev_info_t * rdip,int (* waitfp)(caddr_t),caddr_t arg)2507c478bd9Sstevel@tonic-gate pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t),
2517c478bd9Sstevel@tonic-gate caddr_t arg)
2527c478bd9Sstevel@tonic-gate {
2537c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp;
2547c478bd9Sstevel@tonic-gate int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2557c478bd9Sstevel@tonic-gate
2567c478bd9Sstevel@tonic-gate /* Caution: we don't use zalloc to enhance performance! */
2577c478bd9Sstevel@tonic-gate if ((mp = kmem_alloc(sizeof (pci_dma_hdl_t), sleep)) == 0) {
2587c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP, dip, "can't alloc dma_handle\n");
2597c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT) {
2607c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n");
2617c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp, arg, &pci_kmem_clid);
2627c478bd9Sstevel@tonic-gate }
2637c478bd9Sstevel@tonic-gate return (mp);
2647c478bd9Sstevel@tonic-gate }
2657c478bd9Sstevel@tonic-gate
2667c478bd9Sstevel@tonic-gate mp->dmai_rdip = rdip;
2677c478bd9Sstevel@tonic-gate mp->dmai_flags = 0;
2687c478bd9Sstevel@tonic-gate mp->dmai_pfnlst = NULL;
2697c478bd9Sstevel@tonic-gate mp->dmai_winlst = NULL;
270*9a63ec27SRobert Mustacchi mp->dmai_ncookies = 0;
271*9a63ec27SRobert Mustacchi mp->dmai_curcookie = 0;
2727c478bd9Sstevel@tonic-gate
2737c478bd9Sstevel@tonic-gate /*
2747c478bd9Sstevel@tonic-gate * kmem_alloc debug: the following fields are not zero-ed
2757c478bd9Sstevel@tonic-gate * mp->dmai_mapping = 0;
2767c478bd9Sstevel@tonic-gate * mp->dmai_size = 0;
2777c478bd9Sstevel@tonic-gate * mp->dmai_offset = 0;
2787c478bd9Sstevel@tonic-gate * mp->dmai_minxfer = 0;
2797c478bd9Sstevel@tonic-gate * mp->dmai_burstsizes = 0;
2807c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages = 0;
2817c478bd9Sstevel@tonic-gate * mp->dmai_pool/roffset = 0;
2827c478bd9Sstevel@tonic-gate * mp->dmai_rflags = 0;
2837c478bd9Sstevel@tonic-gate * mp->dmai_inuse/flags
2847c478bd9Sstevel@tonic-gate * mp->dmai_nwin = 0;
2857c478bd9Sstevel@tonic-gate * mp->dmai_winsize = 0;
2867c478bd9Sstevel@tonic-gate * mp->dmai_nexus_private/tte = 0;
2877c478bd9Sstevel@tonic-gate * mp->dmai_iopte/pfnlst
2887c478bd9Sstevel@tonic-gate * mp->dmai_sbi/pfn0 = 0;
2897c478bd9Sstevel@tonic-gate * mp->dmai_minfo/winlst/fdvma
2907c478bd9Sstevel@tonic-gate * mp->dmai_rdip
2917c478bd9Sstevel@tonic-gate * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
2927c478bd9Sstevel@tonic-gate * mp->dmai_cookie = 0;
2937c478bd9Sstevel@tonic-gate */
2947c478bd9Sstevel@tonic-gate
2957c478bd9Sstevel@tonic-gate mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;
2967c478bd9Sstevel@tonic-gate mp->dmai_attr.dma_attr_flags = (uint_t)0;
2977c478bd9Sstevel@tonic-gate mp->dmai_fault = 0;
2987c478bd9Sstevel@tonic-gate mp->dmai_fault_check = NULL;
2997c478bd9Sstevel@tonic-gate mp->dmai_fault_notify = NULL;
3007c478bd9Sstevel@tonic-gate
3017c478bd9Sstevel@tonic-gate mp->dmai_error.err_ena = 0;
3027c478bd9Sstevel@tonic-gate mp->dmai_error.err_status = DDI_FM_OK;
3037c478bd9Sstevel@tonic-gate mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
3047c478bd9Sstevel@tonic-gate mp->dmai_error.err_ontrap = NULL;
3057c478bd9Sstevel@tonic-gate mp->dmai_error.err_fep = NULL;
30600d0963fSdilpreet mp->dmai_error.err_cf = NULL;
307567c0b92SStephen Hanson ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);
3087c478bd9Sstevel@tonic-gate
3097c478bd9Sstevel@tonic-gate SYNC_BUF_PA(mp) = 0ull;
3107c478bd9Sstevel@tonic-gate return (mp);
3117c478bd9Sstevel@tonic-gate }
3127c478bd9Sstevel@tonic-gate
3137c478bd9Sstevel@tonic-gate void
pci_dma_freemp(ddi_dma_impl_t * mp)3147c478bd9Sstevel@tonic-gate pci_dma_freemp(ddi_dma_impl_t *mp)
3157c478bd9Sstevel@tonic-gate {
316567c0b92SStephen Hanson ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp);
3177c478bd9Sstevel@tonic-gate if (mp->dmai_ndvmapages > 1)
3187c478bd9Sstevel@tonic-gate pci_dma_freepfn(mp);
3197c478bd9Sstevel@tonic-gate if (mp->dmai_winlst)
3207c478bd9Sstevel@tonic-gate pci_dma_freewin(mp);
3217c478bd9Sstevel@tonic-gate kmem_free(mp, sizeof (pci_dma_hdl_t));
3227c478bd9Sstevel@tonic-gate }
3237c478bd9Sstevel@tonic-gate
3247c478bd9Sstevel@tonic-gate void
pci_dma_freepfn(ddi_dma_impl_t * mp)3257c478bd9Sstevel@tonic-gate pci_dma_freepfn(ddi_dma_impl_t *mp)
3267c478bd9Sstevel@tonic-gate {
3277c478bd9Sstevel@tonic-gate void *addr = mp->dmai_pfnlst;
3287c478bd9Sstevel@tonic-gate ASSERT(!PCI_DMA_CANRELOC(mp));
3297c478bd9Sstevel@tonic-gate if (addr) {
3307c478bd9Sstevel@tonic-gate size_t npages = mp->dmai_ndvmapages;
3317c478bd9Sstevel@tonic-gate if (npages > 1)
3327c478bd9Sstevel@tonic-gate kmem_free(addr, npages * sizeof (iopfn_t));
3337c478bd9Sstevel@tonic-gate mp->dmai_pfnlst = NULL;
3347c478bd9Sstevel@tonic-gate }
3357c478bd9Sstevel@tonic-gate mp->dmai_ndvmapages = 0;
3367c478bd9Sstevel@tonic-gate }
3377c478bd9Sstevel@tonic-gate
3387c478bd9Sstevel@tonic-gate /*
3397c478bd9Sstevel@tonic-gate * pci_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits
3407c478bd9Sstevel@tonic-gate * and convert dmareq->dmar_limits to mp->dmai_attr
3417c478bd9Sstevel@tonic-gate *
3427c478bd9Sstevel@tonic-gate * ddi_dma_impl_t member modified input
3437c478bd9Sstevel@tonic-gate * ------------------------------------------------------------------------
3447c478bd9Sstevel@tonic-gate * mp->dmai_minxfer - dev
3457c478bd9Sstevel@tonic-gate * mp->dmai_burstsizes - dev
3467c478bd9Sstevel@tonic-gate * mp->dmai_flags - no limit? peer-to-peer only?
3477c478bd9Sstevel@tonic-gate *
3487c478bd9Sstevel@tonic-gate * ddi_dma_attr member modified input
3497c478bd9Sstevel@tonic-gate * ------------------------------------------------------------------------
3507c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo
3517c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi
3527c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta
3537c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction)
3547c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction)
3557c478bd9Sstevel@tonic-gate *
3567c478bd9Sstevel@tonic-gate * The dlim_dmaspeed member of dmareq->dmar_limits is ignored.
3577c478bd9Sstevel@tonic-gate */
3587c478bd9Sstevel@tonic-gate ddi_dma_impl_t *
pci_dma_lmts2hdl(dev_info_t * dip,dev_info_t * rdip,iommu_t * iommu_p,ddi_dma_req_t * dmareq)3597c478bd9Sstevel@tonic-gate pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p,
3607c478bd9Sstevel@tonic-gate ddi_dma_req_t *dmareq)
3617c478bd9Sstevel@tonic-gate {
3627c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp;
3637c478bd9Sstevel@tonic-gate ddi_dma_attr_t *attr_p;
3647c478bd9Sstevel@tonic-gate uint64_t syslo = iommu_p->iommu_dvma_base;
3657c478bd9Sstevel@tonic-gate uint64_t syshi = iommu_p->iommu_dvma_end;
3667c478bd9Sstevel@tonic-gate uint64_t fasthi = iommu_p->iommu_dvma_fast_end;
3677c478bd9Sstevel@tonic-gate ddi_dma_lim_t *lim_p = dmareq->dmar_limits;
3687c478bd9Sstevel@tonic-gate uint32_t count_max = lim_p->dlim_cntr_max;
3697c478bd9Sstevel@tonic-gate uint64_t lo = lim_p->dlim_addr_lo;
3707c478bd9Sstevel@tonic-gate uint64_t hi = lim_p->dlim_addr_hi;
3717c478bd9Sstevel@tonic-gate if (hi <= lo) {
3727c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP, dip, "Bad limits\n");
3737c478bd9Sstevel@tonic-gate return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING);
3747c478bd9Sstevel@tonic-gate }
3757c478bd9Sstevel@tonic-gate if (!count_max)
3767c478bd9Sstevel@tonic-gate count_max--;
3777c478bd9Sstevel@tonic-gate
3787c478bd9Sstevel@tonic-gate if (!(mp = pci_dma_allocmp(dip, rdip, dmareq->dmar_fp,
379567c0b92SStephen Hanson dmareq->dmar_arg)))
3807c478bd9Sstevel@tonic-gate return (NULL);
3817c478bd9Sstevel@tonic-gate
3827c478bd9Sstevel@tonic-gate /* store original dev input at the 2nd ddi_dma_attr */
3837c478bd9Sstevel@tonic-gate attr_p = DEV_ATTR(mp);
3847c478bd9Sstevel@tonic-gate SET_DMAATTR(attr_p, lo, hi, -1, count_max);
3857c478bd9Sstevel@tonic-gate SET_DMAALIGN(attr_p, 1);
3867c478bd9Sstevel@tonic-gate
3877c478bd9Sstevel@tonic-gate lo = MAX(lo, syslo);
3887c478bd9Sstevel@tonic-gate hi = MIN(hi, syshi);
3897c478bd9Sstevel@tonic-gate if (hi <= lo)
3907c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY;
3917c478bd9Sstevel@tonic-gate count_max = MIN(count_max, hi - lo);
3927c478bd9Sstevel@tonic-gate
3937c478bd9Sstevel@tonic-gate if (DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1))
3947c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT |
395567c0b92SStephen Hanson DMAI_FLAGS_NOSYSLIMIT;
3967c478bd9Sstevel@tonic-gate else {
3977c478bd9Sstevel@tonic-gate if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1))
3987c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT;
3997c478bd9Sstevel@tonic-gate }
4007c478bd9Sstevel@tonic-gate if (PCI_DMA_NOCTX(rdip))
4017c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOCTX;
4027c478bd9Sstevel@tonic-gate
4037c478bd9Sstevel@tonic-gate /* store augumented dev input to mp->dmai_attr */
4047c478bd9Sstevel@tonic-gate mp->dmai_minxfer = lim_p->dlim_minxfer;
4057c478bd9Sstevel@tonic-gate mp->dmai_burstsizes = lim_p->dlim_burstsizes;
4067c478bd9Sstevel@tonic-gate attr_p = &mp->dmai_attr;
4077c478bd9Sstevel@tonic-gate SET_DMAATTR(attr_p, lo, hi, -1, count_max);
4087c478bd9Sstevel@tonic-gate SET_DMAALIGN(attr_p, 1);
4097c478bd9Sstevel@tonic-gate return (mp);
4107c478bd9Sstevel@tonic-gate }
4117c478bd9Sstevel@tonic-gate
4127c478bd9Sstevel@tonic-gate /*
4137c478bd9Sstevel@tonic-gate * pci_dma_attr2hdl
4147c478bd9Sstevel@tonic-gate *
4157c478bd9Sstevel@tonic-gate * This routine is called from the alloc handle entry point to sanity check the
4167c478bd9Sstevel@tonic-gate * dma attribute structure.
4177c478bd9Sstevel@tonic-gate *
4187c478bd9Sstevel@tonic-gate * use by: pci_dma_allochdl()
4197c478bd9Sstevel@tonic-gate *
4207c478bd9Sstevel@tonic-gate * return value:
4217c478bd9Sstevel@tonic-gate *
4227c478bd9Sstevel@tonic-gate * DDI_SUCCESS - on success
4237c478bd9Sstevel@tonic-gate * DDI_DMA_BADATTR - attribute has invalid version number
4247c478bd9Sstevel@tonic-gate * or address limits exclude dvma space
4257c478bd9Sstevel@tonic-gate */
4267c478bd9Sstevel@tonic-gate int
pci_dma_attr2hdl(pci_t * pci_p,ddi_dma_impl_t * mp)4277c478bd9Sstevel@tonic-gate pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp)
4287c478bd9Sstevel@tonic-gate {
4297c478bd9Sstevel@tonic-gate iommu_t *iommu_p = pci_p->pci_iommu_p;
4307c478bd9Sstevel@tonic-gate uint64_t syslo, syshi;
4317c478bd9Sstevel@tonic-gate ddi_dma_attr_t *attrp = DEV_ATTR(mp);
4327c478bd9Sstevel@tonic-gate uint64_t hi = attrp->dma_attr_addr_hi;
4337c478bd9Sstevel@tonic-gate uint64_t lo = attrp->dma_attr_addr_lo;
4347c478bd9Sstevel@tonic-gate uint64_t align = attrp->dma_attr_align;
4357c478bd9Sstevel@tonic-gate uint64_t nocross = attrp->dma_attr_seg;
4367c478bd9Sstevel@tonic-gate uint64_t count_max = attrp->dma_attr_count_max;
4377c478bd9Sstevel@tonic-gate
4387c478bd9Sstevel@tonic-gate DEBUG3(DBG_DMA_ALLOCH, pci_p->pci_dip, "attrp=%p cntr_max=%x.%08x\n",
439567c0b92SStephen Hanson attrp, HI32(count_max), LO32(count_max));
4407c478bd9Sstevel@tonic-gate DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x lo=%x.%08x\n",
441567c0b92SStephen Hanson HI32(hi), LO32(hi), HI32(lo), LO32(lo));
4427c478bd9Sstevel@tonic-gate DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "seg=%x.%08x align=%x.%08x\n",
443567c0b92SStephen Hanson HI32(nocross), LO32(nocross), HI32(align), LO32(align));
4447c478bd9Sstevel@tonic-gate
4457c478bd9Sstevel@tonic-gate if (!nocross)
4467c478bd9Sstevel@tonic-gate nocross--;
4477c478bd9Sstevel@tonic-gate if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */
4487c478bd9Sstevel@tonic-gate
4497c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_ALLOCH, pci_p->pci_dip, "bypass mode\n");
4507c478bd9Sstevel@tonic-gate /* if tomatillo ver <= 2.3 don't allow bypass */
4517c478bd9Sstevel@tonic-gate if (tomatillo_disallow_bypass)
4527c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR);
4537c478bd9Sstevel@tonic-gate
4547c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_BYPASSREQ;
4557c478bd9Sstevel@tonic-gate if (nocross != UINT64_MAX)
4567c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR);
4577c478bd9Sstevel@tonic-gate if (align && (align > IOMMU_PAGE_SIZE))
4587c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR);
4597c478bd9Sstevel@tonic-gate align = 1; /* align on 1 page boundary */
4607c478bd9Sstevel@tonic-gate syslo = iommu_p->iommu_dma_bypass_base;
4617c478bd9Sstevel@tonic-gate syshi = iommu_p->iommu_dma_bypass_end;
4627c478bd9Sstevel@tonic-gate
4637c478bd9Sstevel@tonic-gate } else { /* IOMMU_XLATE or PEER_TO_PEER */
4647c478bd9Sstevel@tonic-gate align = MAX(align, IOMMU_PAGE_SIZE) - 1;
4657c478bd9Sstevel@tonic-gate if ((align & nocross) != align) {
4667c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip;
4677c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned",
468567c0b92SStephen Hanson NAMEINST(rdip));
4697c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR);
4707c478bd9Sstevel@tonic-gate }
4717c478bd9Sstevel@tonic-gate align = IOMMU_BTOP(align + 1);
4727c478bd9Sstevel@tonic-gate syslo = iommu_p->iommu_dvma_base;
4737c478bd9Sstevel@tonic-gate syshi = iommu_p->iommu_dvma_end;
4747c478bd9Sstevel@tonic-gate }
4757c478bd9Sstevel@tonic-gate if (hi <= lo) {
4767c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip;
4777c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip));
4787c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR);
4797c478bd9Sstevel@tonic-gate }
4807c478bd9Sstevel@tonic-gate lo = MAX(lo, syslo);
4817c478bd9Sstevel@tonic-gate hi = MIN(hi, syshi);
4827c478bd9Sstevel@tonic-gate if (!count_max)
4837c478bd9Sstevel@tonic-gate count_max--;
4847c478bd9Sstevel@tonic-gate
4857c478bd9Sstevel@tonic-gate DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x, lo=%x.%08x\n",
486567c0b92SStephen Hanson HI32(hi), LO32(hi), HI32(lo), LO32(lo));
4877c478bd9Sstevel@tonic-gate if (hi <= lo) { /* peer transfers cannot have alignment & nocross */
4887c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip;
4897c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp);
4907c478bd9Sstevel@tonic-gate if ((nocross < UINT32_MAX) || (align > 1)) {
4917c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d peer only device bad attr",
492567c0b92SStephen Hanson NAMEINST(rdip));
4937c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR);
4947c478bd9Sstevel@tonic-gate }
4957c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY;
4967c478bd9Sstevel@tonic-gate } else /* set practical counter_max value */
4977c478bd9Sstevel@tonic-gate count_max = MIN(count_max, hi - lo);
4987c478bd9Sstevel@tonic-gate
4997c478bd9Sstevel@tonic-gate if (DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align))
5007c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOSYSLIMIT |
501567c0b92SStephen Hanson DMAI_FLAGS_NOFASTLIMIT;
5027c478bd9Sstevel@tonic-gate else {
5037c478bd9Sstevel@tonic-gate syshi = iommu_p->iommu_dvma_fast_end;
5047c478bd9Sstevel@tonic-gate if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align))
5057c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT;
5067c478bd9Sstevel@tonic-gate }
5077c478bd9Sstevel@tonic-gate if (PCI_DMA_NOCTX(mp->dmai_rdip))
5087c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOCTX;
5097c478bd9Sstevel@tonic-gate
5107c478bd9Sstevel@tonic-gate mp->dmai_minxfer = attrp->dma_attr_minxfer;
5117c478bd9Sstevel@tonic-gate mp->dmai_burstsizes = attrp->dma_attr_burstsizes;
5127c478bd9Sstevel@tonic-gate attrp = &mp->dmai_attr;
5137c478bd9Sstevel@tonic-gate SET_DMAATTR(attrp, lo, hi, nocross, count_max);
5147c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
5157c478bd9Sstevel@tonic-gate }
5167c478bd9Sstevel@tonic-gate
5177c478bd9Sstevel@tonic-gate /*
5187c478bd9Sstevel@tonic-gate * set up consistent dma flags according to hardware capability
5197c478bd9Sstevel@tonic-gate */
5207c478bd9Sstevel@tonic-gate uint32_t
pci_dma_consist_check(uint32_t req_flags,pbm_t * pbm_p)5217c478bd9Sstevel@tonic-gate pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p)
5227c478bd9Sstevel@tonic-gate {
5237c478bd9Sstevel@tonic-gate if (!pci_stream_buf_enable || !pci_stream_buf_exists)
5247c478bd9Sstevel@tonic-gate req_flags |= DDI_DMA_CONSISTENT;
5257c478bd9Sstevel@tonic-gate if (req_flags & DDI_DMA_CONSISTENT && !pbm_p->pbm_sync_reg_pa)
5267c478bd9Sstevel@tonic-gate req_flags |= DMP_NOSYNC;
5277c478bd9Sstevel@tonic-gate return (req_flags);
5287c478bd9Sstevel@tonic-gate }
5297c478bd9Sstevel@tonic-gate
5307c478bd9Sstevel@tonic-gate #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end))
5317c478bd9Sstevel@tonic-gate
5327c478bd9Sstevel@tonic-gate /*
5337c478bd9Sstevel@tonic-gate * pci_dma_type - determine which of the three types DMA (peer-to-peer,
5347c478bd9Sstevel@tonic-gate * iommu bypass, or iommu translate) we are asked to do.
5357c478bd9Sstevel@tonic-gate * Also checks pfn0 and rejects any non-peer-to-peer
5367c478bd9Sstevel@tonic-gate * requests for peer-only devices.
5377c478bd9Sstevel@tonic-gate *
5387c478bd9Sstevel@tonic-gate * return values:
5397c478bd9Sstevel@tonic-gate * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type
5407c478bd9Sstevel@tonic-gate * DDI_SUCCESS
5417c478bd9Sstevel@tonic-gate *
5427c478bd9Sstevel@tonic-gate * dma handle members affected (set on exit):
5437c478bd9Sstevel@tonic-gate * mp->dmai_object - dmareq->dmar_object
5447c478bd9Sstevel@tonic-gate * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags
5457c478bd9Sstevel@tonic-gate * mp->dmai_flags - DMA type
5467c478bd9Sstevel@tonic-gate * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow)
5477c478bd9Sstevel@tonic-gate * mp->dmai_roffset - initialized to starting IOMMU page offset
5487c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages - # of total IOMMU pages of entire object
5497c478bd9Sstevel@tonic-gate * mp->pdh_sync_buf_pa - dma sync buffer PA is DMA flow is supported
5507c478bd9Sstevel@tonic-gate */
5517c478bd9Sstevel@tonic-gate int
pci_dma_type(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)5527c478bd9Sstevel@tonic-gate pci_dma_type(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
5537c478bd9Sstevel@tonic-gate {
5547c478bd9Sstevel@tonic-gate dev_info_t *dip = pci_p->pci_dip;
5557c478bd9Sstevel@tonic-gate ddi_dma_obj_t *dobj_p = &dmareq->dmar_object;
5567c478bd9Sstevel@tonic-gate pbm_t *pbm_p = pci_p->pci_pbm_p;
5577c478bd9Sstevel@tonic-gate page_t **pplist;
5587c478bd9Sstevel@tonic-gate struct as *as_p;
5597c478bd9Sstevel@tonic-gate uint32_t offset;
5607c478bd9Sstevel@tonic-gate caddr_t vaddr;
5617c478bd9Sstevel@tonic-gate pfn_t pfn0;
5627c478bd9Sstevel@tonic-gate
5637c478bd9Sstevel@tonic-gate mp->dmai_rflags = pci_dma_consist_check(dmareq->dmar_flags, pbm_p);
5647c478bd9Sstevel@tonic-gate mp->dmai_flags |= mp->dmai_rflags & DMP_NOSYNC ? DMAI_FLAGS_NOSYNC : 0;
5657c478bd9Sstevel@tonic-gate
5667c478bd9Sstevel@tonic-gate switch (dobj_p->dmao_type) {
5677c478bd9Sstevel@tonic-gate case DMA_OTYP_BUFVADDR:
5687c478bd9Sstevel@tonic-gate case DMA_OTYP_VADDR: {
5697c478bd9Sstevel@tonic-gate vaddr = dobj_p->dmao_obj.virt_obj.v_addr;
5707c478bd9Sstevel@tonic-gate pplist = dobj_p->dmao_obj.virt_obj.v_priv;
5717c478bd9Sstevel@tonic-gate as_p = dobj_p->dmao_obj.virt_obj.v_as;
5727c478bd9Sstevel@tonic-gate if (as_p == NULL)
5737c478bd9Sstevel@tonic-gate as_p = &kas;
5747c478bd9Sstevel@tonic-gate
5757c478bd9Sstevel@tonic-gate DEBUG2(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist);
5767c478bd9Sstevel@tonic-gate offset = (ulong_t)vaddr & IOMMU_PAGE_OFFSET;
5777c478bd9Sstevel@tonic-gate
5787c478bd9Sstevel@tonic-gate if (pplist) { /* shadow list */
5797c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_PGPFN;
5807c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(*pplist));
5817c478bd9Sstevel@tonic-gate pfn0 = page_pptonum(*pplist);
5827c478bd9Sstevel@tonic-gate } else if (pci_dvma_remap_enabled && as_p == &kas &&
583567c0b92SStephen Hanson dobj_p->dmao_type != DMA_OTYP_BUFVADDR) {
5847c478bd9Sstevel@tonic-gate int (*waitfp)(caddr_t) = dmareq->dmar_fp;
5857c478bd9Sstevel@tonic-gate uint_t flags = ((waitfp == DDI_DMA_SLEEP)?
586567c0b92SStephen Hanson HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK;
5877c478bd9Sstevel@tonic-gate int ret;
5887c478bd9Sstevel@tonic-gate
5897c478bd9Sstevel@tonic-gate ret = hat_add_callback(pci_dvma_cbid, vaddr,
590d0662dbfSelowe IOMMU_PAGE_SIZE - offset, flags, mp, &pfn0,
591d0662dbfSelowe MP_HAT_CB_COOKIE_PTR(mp, 0));
5927c478bd9Sstevel@tonic-gate
5937c478bd9Sstevel@tonic-gate if (pfn0 == PFN_INVALID && ret == ENOMEM) {
5947c478bd9Sstevel@tonic-gate ASSERT(waitfp != DDI_DMA_SLEEP);
5957c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT) {
5967c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp,
5977c478bd9Sstevel@tonic-gate dmareq->dmar_arg,
5987c478bd9Sstevel@tonic-gate &pci_kmem_clid);
5997c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES);
6007c478bd9Sstevel@tonic-gate }
6017c478bd9Sstevel@tonic-gate }
6027c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_RELOC;
6037c478bd9Sstevel@tonic-gate } else
6047c478bd9Sstevel@tonic-gate pfn0 = hat_getpfnum(as_p->a_hat, vaddr);
6057c478bd9Sstevel@tonic-gate }
6067c478bd9Sstevel@tonic-gate break;
6077c478bd9Sstevel@tonic-gate
6087c478bd9Sstevel@tonic-gate case DMA_OTYP_PAGES:
6097c478bd9Sstevel@tonic-gate offset = dobj_p->dmao_obj.pp_obj.pp_offset;
6107c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_PGPFN;
6117c478bd9Sstevel@tonic-gate pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp);
6127c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(dobj_p->dmao_obj.pp_obj.pp_pp));
6137c478bd9Sstevel@tonic-gate break;
6147c478bd9Sstevel@tonic-gate
6157c478bd9Sstevel@tonic-gate case DMA_OTYP_PADDR:
6167c478bd9Sstevel@tonic-gate default:
6177c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d requested unsupported dma type %x",
618567c0b92SStephen Hanson NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);
6197c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
6207c478bd9Sstevel@tonic-gate }
6217c478bd9Sstevel@tonic-gate if (pfn0 == PFN_INVALID) {
6227c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p",
623567c0b92SStephen Hanson NAMEINST(dip), dobj_p);
6247c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
6257c478bd9Sstevel@tonic-gate }
6267c478bd9Sstevel@tonic-gate if (TGT_PFN_INBETWEEN(pfn0, pbm_p->pbm_base_pfn, pbm_p->pbm_last_pfn)) {
6277c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_PEER_TO_PEER;
6287c478bd9Sstevel@tonic-gate goto done; /* leave bypass and dvma flag as 0 */
6297c478bd9Sstevel@tonic-gate }
6307c478bd9Sstevel@tonic-gate if (PCI_DMA_ISPEERONLY(mp)) {
6317c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip;
6327c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip));
6337c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
6347c478bd9Sstevel@tonic-gate }
6357c478bd9Sstevel@tonic-gate mp->dmai_flags |= (mp->dmai_flags & DMAI_FLAGS_BYPASSREQ) ?
636567c0b92SStephen Hanson DMAI_FLAGS_BYPASS : DMAI_FLAGS_DVMA;
6377c478bd9Sstevel@tonic-gate done:
6387c478bd9Sstevel@tonic-gate mp->dmai_object = *dobj_p; /* whole object */
6397c478bd9Sstevel@tonic-gate mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */
6407c478bd9Sstevel@tonic-gate mp->dmai_roffset = offset; /* win0 pg0 offset */
6417c478bd9Sstevel@tonic-gate mp->dmai_ndvmapages = IOMMU_BTOPR(offset + mp->dmai_object.dmao_size);
6427c478bd9Sstevel@tonic-gate
6437c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
6447c478bd9Sstevel@tonic-gate }
6457c478bd9Sstevel@tonic-gate
6467c478bd9Sstevel@tonic-gate /*
6477c478bd9Sstevel@tonic-gate * pci_dma_pgpfn - set up pfnlst array according to pages
6487c478bd9Sstevel@tonic-gate * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES
6497c478bd9Sstevel@tonic-gate */
6507c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6517c478bd9Sstevel@tonic-gate static int
pci_dma_pgpfn(pci_t * pci_p,ddi_dma_impl_t * mp,uint_t npages)6527c478bd9Sstevel@tonic-gate pci_dma_pgpfn(pci_t *pci_p, ddi_dma_impl_t *mp, uint_t npages)
6537c478bd9Sstevel@tonic-gate {
6547c478bd9Sstevel@tonic-gate int i;
6557c478bd9Sstevel@tonic-gate #ifdef DEBUG
6567c478bd9Sstevel@tonic-gate dev_info_t *dip = pci_p->pci_dip;
6577c478bd9Sstevel@tonic-gate #endif
6587c478bd9Sstevel@tonic-gate switch (mp->dmai_object.dmao_type) {
6597c478bd9Sstevel@tonic-gate case DMA_OTYP_BUFVADDR:
6607c478bd9Sstevel@tonic-gate case DMA_OTYP_VADDR: {
6617c478bd9Sstevel@tonic-gate page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;
6627c478bd9Sstevel@tonic-gate DEBUG2(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=",
663567c0b92SStephen Hanson pplist, npages);
6647c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++) {
6657c478bd9Sstevel@tonic-gate iopfn_t pfn = page_pptonum(pplist[i]);
6667c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pplist[i]));
6677c478bd9Sstevel@tonic-gate PCI_SET_MP_PFN1(mp, i, pfn);
6687c478bd9Sstevel@tonic-gate DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
6697c478bd9Sstevel@tonic-gate }
6707c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n");
6717c478bd9Sstevel@tonic-gate }
6727c478bd9Sstevel@tonic-gate break;
6737c478bd9Sstevel@tonic-gate
6747c478bd9Sstevel@tonic-gate case DMA_OTYP_PAGES: {
6757c478bd9Sstevel@tonic-gate page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;
6767c478bd9Sstevel@tonic-gate DEBUG1(DBG_DMA_MAP, dip, "pp=%p pfns=", pp);
6777c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++, pp = pp->p_next) {
6787c478bd9Sstevel@tonic-gate iopfn_t pfn = page_pptonum(pp);
6797c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp));
6807c478bd9Sstevel@tonic-gate PCI_SET_MP_PFN1(mp, i, pfn);
6817c478bd9Sstevel@tonic-gate DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
6827c478bd9Sstevel@tonic-gate }
6837c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n");
6847c478bd9Sstevel@tonic-gate }
6857c478bd9Sstevel@tonic-gate break;
6867c478bd9Sstevel@tonic-gate
6877c478bd9Sstevel@tonic-gate default: /* check is already done by pci_dma_type */
6887c478bd9Sstevel@tonic-gate ASSERT(0);
6897c478bd9Sstevel@tonic-gate break;
6907c478bd9Sstevel@tonic-gate }
6917c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
6927c478bd9Sstevel@tonic-gate }
6937c478bd9Sstevel@tonic-gate
6947c478bd9Sstevel@tonic-gate /*
6957c478bd9Sstevel@tonic-gate * pci_dma_vapfn - set up pfnlst array according to VA
6967c478bd9Sstevel@tonic-gate * VA/size pair: <normal, bypass, peer-to-peer>
6977c478bd9Sstevel@tonic-gate * pfn0 is skipped as it is already done.
6987c478bd9Sstevel@tonic-gate * In this case, the cached pfn0 is used to fill pfnlst[0]
6997c478bd9Sstevel@tonic-gate */
7007c478bd9Sstevel@tonic-gate static int
pci_dma_vapfn(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint_t npages)7017c478bd9Sstevel@tonic-gate pci_dma_vapfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,
7027c478bd9Sstevel@tonic-gate uint_t npages)
7037c478bd9Sstevel@tonic-gate {
7047c478bd9Sstevel@tonic-gate dev_info_t *dip = pci_p->pci_dip;
7057c478bd9Sstevel@tonic-gate int i;
7067c478bd9Sstevel@tonic-gate caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;
7077c478bd9Sstevel@tonic-gate struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat;
7087c478bd9Sstevel@tonic-gate caddr_t sva;
7097c478bd9Sstevel@tonic-gate int needcb = 0;
7107c478bd9Sstevel@tonic-gate
7117c478bd9Sstevel@tonic-gate sva = (caddr_t)(((uintptr_t)mp->dmai_object.dmao_obj.virt_obj.v_addr +
7127c478bd9Sstevel@tonic-gate IOMMU_PAGE_SIZE) & IOMMU_PAGE_MASK);
7137c478bd9Sstevel@tonic-gate
7147c478bd9Sstevel@tonic-gate if (pci_dvma_remap_enabled && hat_p == kas.a_hat &&
715567c0b92SStephen Hanson mp->dmai_object.dmao_type != DMA_OTYP_BUFVADDR)
7167c478bd9Sstevel@tonic-gate needcb = 1;
7177c478bd9Sstevel@tonic-gate
7187c478bd9Sstevel@tonic-gate for (vaddr = sva, i = 1; i < npages; i++, vaddr += IOMMU_PAGE_SIZE) {
7197c478bd9Sstevel@tonic-gate pfn_t pfn;
7207c478bd9Sstevel@tonic-gate
7217c478bd9Sstevel@tonic-gate if (needcb) {
7227c478bd9Sstevel@tonic-gate int (*waitfp)(caddr_t) = dmareq->dmar_fp;
7237c478bd9Sstevel@tonic-gate uint_t flags = ((waitfp == DDI_DMA_SLEEP)?
7247c478bd9Sstevel@tonic-gate HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK;
7257c478bd9Sstevel@tonic-gate int ret;
7267c478bd9Sstevel@tonic-gate
7277c478bd9Sstevel@tonic-gate ret = hat_add_callback(pci_dvma_cbid, vaddr,
728d0662dbfSelowe IOMMU_PAGE_SIZE, flags, mp, &pfn,
729d0662dbfSelowe MP_HAT_CB_COOKIE_PTR(mp, i));
730d0662dbfSelowe
7317c478bd9Sstevel@tonic-gate if (pfn == PFN_INVALID && ret == ENOMEM) {
7327c478bd9Sstevel@tonic-gate ASSERT(waitfp != DDI_DMA_SLEEP);
7337c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT)
7347c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp,
7357c478bd9Sstevel@tonic-gate dmareq->dmar_arg, &pci_kmem_clid);
7367c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES);
7377c478bd9Sstevel@tonic-gate }
7387c478bd9Sstevel@tonic-gate } else
7397c478bd9Sstevel@tonic-gate pfn = hat_getpfnum(hat_p, vaddr);
7407c478bd9Sstevel@tonic-gate if (pfn == PFN_INVALID)
7417c478bd9Sstevel@tonic-gate goto err_badpfn;
7427c478bd9Sstevel@tonic-gate PCI_SET_MP_PFN1(mp, i, (iopfn_t)pfn);
7437c478bd9Sstevel@tonic-gate DEBUG3(DBG_DMA_MAP, dip, "pci_dma_vapfn: mp=%p pfnlst[%x]=%x\n",
744567c0b92SStephen Hanson mp, i, (iopfn_t)pfn);
7457c478bd9Sstevel@tonic-gate }
7467c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
7477c478bd9Sstevel@tonic-gate err_badpfn:
7487c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr);
7497c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
7507c478bd9Sstevel@tonic-gate }
7517c478bd9Sstevel@tonic-gate
7527c478bd9Sstevel@tonic-gate /*
7537c478bd9Sstevel@tonic-gate * pci_dma_pfn - Fills pfn list for all pages being DMA-ed.
7547c478bd9Sstevel@tonic-gate *
7557c478bd9Sstevel@tonic-gate * dependencies:
7567c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages - set to total # of dma pages
7577c478bd9Sstevel@tonic-gate *
7587c478bd9Sstevel@tonic-gate * return value:
7597c478bd9Sstevel@tonic-gate * DDI_SUCCESS
7607c478bd9Sstevel@tonic-gate * DDI_DMA_NOMAPPING
7617c478bd9Sstevel@tonic-gate */
7627c478bd9Sstevel@tonic-gate int
pci_dma_pfn(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)7637c478bd9Sstevel@tonic-gate pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
7647c478bd9Sstevel@tonic-gate {
7657c478bd9Sstevel@tonic-gate uint32_t npages = mp->dmai_ndvmapages;
7667c478bd9Sstevel@tonic-gate int (*waitfp)(caddr_t) = dmareq->dmar_fp;
7677c478bd9Sstevel@tonic-gate int i, ret, peer = PCI_DMA_ISPTP(mp);
7687c478bd9Sstevel@tonic-gate
7697c478bd9Sstevel@tonic-gate pbm_t *pbm_p = pci_p->pci_pbm_p;
7707c478bd9Sstevel@tonic-gate iopfn_t pfn_base = pbm_p->pbm_base_pfn;
7717c478bd9Sstevel@tonic-gate iopfn_t pfn_last = pbm_p->pbm_last_pfn;
7727c478bd9Sstevel@tonic-gate iopfn_t pfn_adj = peer ? pfn_base : 0;
7737c478bd9Sstevel@tonic-gate
7747c478bd9Sstevel@tonic-gate DEBUG2(DBG_DMA_MAP, pci_p->pci_dip, "pci_dma_pfn: mp=%p pfn0=%x\n",
775567c0b92SStephen Hanson mp, MP_PFN0(mp) - pfn_adj);
7767c478bd9Sstevel@tonic-gate /* 1 page: no array alloc/fill, no mixed mode check */
7777c478bd9Sstevel@tonic-gate if (npages == 1) {
7787c478bd9Sstevel@tonic-gate PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj);
7797c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
7807c478bd9Sstevel@tonic-gate }
7817c478bd9Sstevel@tonic-gate /* allocate pfn array */
7827c478bd9Sstevel@tonic-gate if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (iopfn_t),
783567c0b92SStephen Hanson waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) {
7847c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT)
7857c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp, dmareq->dmar_arg,
786567c0b92SStephen Hanson &pci_kmem_clid);
7877c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES);
7887c478bd9Sstevel@tonic-gate }
7897c478bd9Sstevel@tonic-gate /* fill pfn array */
7907c478bd9Sstevel@tonic-gate PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */
7917c478bd9Sstevel@tonic-gate if ((ret = PCI_DMA_ISPGPFN(mp) ? pci_dma_pgpfn(pci_p, mp, npages) :
792567c0b92SStephen Hanson pci_dma_vapfn(pci_p, dmareq, mp, npages)) != DDI_SUCCESS)
7937c478bd9Sstevel@tonic-gate goto err;
7947c478bd9Sstevel@tonic-gate
7957c478bd9Sstevel@tonic-gate /* skip pfn0, check mixed mode and adjust peer to peer pfn */
7967c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++) {
7977c478bd9Sstevel@tonic-gate iopfn_t pfn = PCI_GET_MP_PFN1(mp, i);
7987c478bd9Sstevel@tonic-gate if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) {
799f47a9c50Smathue cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx",
800567c0b92SStephen Hanson NAMEINST(mp->dmai_rdip), MP_PFN0(mp), pfn);
8017c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING; /* mixed mode */
8027c478bd9Sstevel@tonic-gate goto err;
8037c478bd9Sstevel@tonic-gate }
8047c478bd9Sstevel@tonic-gate DEBUG3(DBG_DMA_MAP, pci_p->pci_dip,
805567c0b92SStephen Hanson "pci_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj);
8067c478bd9Sstevel@tonic-gate if (pfn_adj)
8077c478bd9Sstevel@tonic-gate PCI_SET_MP_PFN1(mp, i, pfn - pfn_adj);
8087c478bd9Sstevel@tonic-gate }
8097c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
8107c478bd9Sstevel@tonic-gate err:
8117c478bd9Sstevel@tonic-gate pci_dvma_unregister_callbacks(pci_p, mp);
8127c478bd9Sstevel@tonic-gate pci_dma_freepfn(mp);
8137c478bd9Sstevel@tonic-gate return (ret);
8147c478bd9Sstevel@tonic-gate }
8157c478bd9Sstevel@tonic-gate
8167c478bd9Sstevel@tonic-gate /*
8177c478bd9Sstevel@tonic-gate * pci_dvma_win() - trim requested DVMA size down to window size
8187c478bd9Sstevel@tonic-gate * The 1st window starts from offset and ends at page-aligned boundary.
8197c478bd9Sstevel@tonic-gate * From the 2nd window on, each window starts and ends at page-aligned
8207c478bd9Sstevel@tonic-gate * boundary except the last window ends at wherever requested.
8217c478bd9Sstevel@tonic-gate *
8227c478bd9Sstevel@tonic-gate * accesses the following mp-> members:
8237c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_count_max
8247c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_seg
8257c478bd9Sstevel@tonic-gate * mp->dmai_roffset - start offset of 1st window
8267c478bd9Sstevel@tonic-gate * mp->dmai_rflags (redzone)
8277c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages (for 1 page fast path)
8287c478bd9Sstevel@tonic-gate *
8297c478bd9Sstevel@tonic-gate * sets the following mp-> members:
8307c478bd9Sstevel@tonic-gate * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed)
8317c478bd9Sstevel@tonic-gate * mp->dmai_winsize - window size (no redzone), n * page size (fixed)
8327c478bd9Sstevel@tonic-gate * mp->dmai_nwin - # of DMA windows of entire object (fixed)
8337c478bd9Sstevel@tonic-gate * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed)
8347c478bd9Sstevel@tonic-gate * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed)
8357c478bd9Sstevel@tonic-gate *
8367c478bd9Sstevel@tonic-gate * fixed - not changed across different DMA windows
8377c478bd9Sstevel@tonic-gate */
8387c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8397c478bd9Sstevel@tonic-gate int
pci_dvma_win(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)8407c478bd9Sstevel@tonic-gate pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
8417c478bd9Sstevel@tonic-gate {
8427c478bd9Sstevel@tonic-gate uint32_t redzone_sz = HAS_REDZONE(mp) ? IOMMU_PAGE_SIZE : 0;
8437c478bd9Sstevel@tonic-gate size_t obj_sz = mp->dmai_object.dmao_size;
8447c478bd9Sstevel@tonic-gate size_t xfer_sz;
8457c478bd9Sstevel@tonic-gate ulong_t pg_off;
8467c478bd9Sstevel@tonic-gate
8477c478bd9Sstevel@tonic-gate if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {
8487c478bd9Sstevel@tonic-gate mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
8497c478bd9Sstevel@tonic-gate mp->dmai_size = obj_sz;
8507c478bd9Sstevel@tonic-gate mp->dmai_winsize = IOMMU_PAGE_SIZE;
8517c478bd9Sstevel@tonic-gate mp->dmai_nwin = 1;
8527c478bd9Sstevel@tonic-gate goto done;
8537c478bd9Sstevel@tonic-gate }
8547c478bd9Sstevel@tonic-gate
8557c478bd9Sstevel@tonic-gate pg_off = mp->dmai_roffset;
8567c478bd9Sstevel@tonic-gate xfer_sz = obj_sz + redzone_sz;
8577c478bd9Sstevel@tonic-gate
858567c0b92SStephen Hanson /* include redzone in nocross check */
859567c0b92SStephen Hanson {
8607c478bd9Sstevel@tonic-gate uint64_t nocross = mp->dmai_attr.dma_attr_seg;
8617c478bd9Sstevel@tonic-gate if (xfer_sz + pg_off - 1 > nocross)
8627c478bd9Sstevel@tonic-gate xfer_sz = nocross - pg_off + 1;
8637c478bd9Sstevel@tonic-gate if (redzone_sz && (xfer_sz <= redzone_sz)) {
8647c478bd9Sstevel@tonic-gate DEBUG5(DBG_DMA_MAP, pci_p->pci_dip,
8657c478bd9Sstevel@tonic-gate "nocross too small %lx(%lx)+%lx+%x < %" PRIx64 "\n",
8667c478bd9Sstevel@tonic-gate xfer_sz, obj_sz, pg_off, redzone_sz, nocross);
8677c478bd9Sstevel@tonic-gate return (DDI_DMA_TOOBIG);
8687c478bd9Sstevel@tonic-gate }
8697c478bd9Sstevel@tonic-gate }
870567c0b92SStephen Hanson xfer_sz -= redzone_sz; /* restore transfer size */
871567c0b92SStephen Hanson /* check counter max */
872567c0b92SStephen Hanson {
8737c478bd9Sstevel@tonic-gate uint32_t count_max = mp->dmai_attr.dma_attr_count_max;
8747c478bd9Sstevel@tonic-gate if (xfer_sz - 1 > count_max)
8757c478bd9Sstevel@tonic-gate xfer_sz = count_max + 1;
8767c478bd9Sstevel@tonic-gate }
8777c478bd9Sstevel@tonic-gate if (xfer_sz >= obj_sz) {
8787c478bd9Sstevel@tonic-gate mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
8797c478bd9Sstevel@tonic-gate mp->dmai_size = xfer_sz;
8807c478bd9Sstevel@tonic-gate mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, IOMMU_PAGE_SIZE);
8817c478bd9Sstevel@tonic-gate mp->dmai_nwin = 1;
8827c478bd9Sstevel@tonic-gate goto done;
8837c478bd9Sstevel@tonic-gate }
8847c478bd9Sstevel@tonic-gate if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
8857c478bd9Sstevel@tonic-gate DEBUG4(DBG_DMA_MAP, pci_p->pci_dip,
8867c478bd9Sstevel@tonic-gate "too big: %lx+%lx+%x > %lx\n",
8877c478bd9Sstevel@tonic-gate obj_sz, pg_off, redzone_sz, xfer_sz);
8887c478bd9Sstevel@tonic-gate return (DDI_DMA_TOOBIG);
8897c478bd9Sstevel@tonic-gate }
8907c478bd9Sstevel@tonic-gate
8917c478bd9Sstevel@tonic-gate xfer_sz = IOMMU_PTOB(IOMMU_BTOP(xfer_sz + pg_off)); /* page align */
8927c478bd9Sstevel@tonic-gate mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */
8937c478bd9Sstevel@tonic-gate mp->dmai_winsize = xfer_sz; /* redzone not in winsize */
8947c478bd9Sstevel@tonic-gate mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;
8957c478bd9Sstevel@tonic-gate done:
8967c478bd9Sstevel@tonic-gate mp->dmai_winlst = NULL;
8977c478bd9Sstevel@tonic-gate dump_dma_handle(DBG_DMA_MAP, pci_p->pci_dip, mp);
8987c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
8997c478bd9Sstevel@tonic-gate }
9007c478bd9Sstevel@tonic-gate
9017c478bd9Sstevel@tonic-gate /*
9027c478bd9Sstevel@tonic-gate * fast track cache entry to iommu context, inserts 3 0 bits between
9037c478bd9Sstevel@tonic-gate * upper 6-bits and lower 3-bits of the 9-bit cache entry
9047c478bd9Sstevel@tonic-gate */
9057c478bd9Sstevel@tonic-gate #define IOMMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38)
9067c478bd9Sstevel@tonic-gate
9077c478bd9Sstevel@tonic-gate /*
9087c478bd9Sstevel@tonic-gate * pci_dvma_map_fast - attempts to map fast trackable DVMA
9097c478bd9Sstevel@tonic-gate */
9107c478bd9Sstevel@tonic-gate int
pci_dvma_map_fast(iommu_t * iommu_p,ddi_dma_impl_t * mp)9117c478bd9Sstevel@tonic-gate pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp)
9127c478bd9Sstevel@tonic-gate {
9137c478bd9Sstevel@tonic-gate uint_t clustsz = pci_dvma_page_cache_clustsz;
9147c478bd9Sstevel@tonic-gate uint_t entries = pci_dvma_page_cache_entries;
9157c478bd9Sstevel@tonic-gate uint64_t *tte_addr;
9167c478bd9Sstevel@tonic-gate uint64_t tte = GET_TTE_TEMPLATE(mp);
9177c478bd9Sstevel@tonic-gate int i = iommu_p->iommu_dvma_addr_scan_start;
9187c478bd9Sstevel@tonic-gate uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i;
9197c478bd9Sstevel@tonic-gate iopfn_t *pfn_addr;
9207c478bd9Sstevel@tonic-gate dvma_addr_t dvma_pg;
9217c478bd9Sstevel@tonic-gate size_t npages = IOMMU_BTOP(mp->dmai_winsize);
9227c478bd9Sstevel@tonic-gate #ifdef DEBUG
9237c478bd9Sstevel@tonic-gate dev_info_t *dip = mp->dmai_rdip;
9247c478bd9Sstevel@tonic-gate #endif
9257c478bd9Sstevel@tonic-gate extern uint8_t ldstub(uint8_t *);
9267c478bd9Sstevel@tonic-gate ASSERT(IOMMU_PTOB(npages) == mp->dmai_winsize);
9277c478bd9Sstevel@tonic-gate ASSERT(npages + HAS_REDZONE(mp) <= clustsz);
9287c478bd9Sstevel@tonic-gate
929567c0b92SStephen Hanson for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
930567c0b92SStephen Hanson ;
9317c478bd9Sstevel@tonic-gate if (i >= entries) {
9327c478bd9Sstevel@tonic-gate lock_addr = iommu_p->iommu_dvma_cache_locks;
9337c478bd9Sstevel@tonic-gate i = 0;
934567c0b92SStephen Hanson for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
935567c0b92SStephen Hanson ;
9367c478bd9Sstevel@tonic-gate if (i >= entries) {
9377c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
9387c478bd9Sstevel@tonic-gate pci_dvmaft_exhaust++;
9397c478bd9Sstevel@tonic-gate #endif
9407c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES);
9417c478bd9Sstevel@tonic-gate }
9427c478bd9Sstevel@tonic-gate }
9437c478bd9Sstevel@tonic-gate iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1);
9447c478bd9Sstevel@tonic-gate if (PCI_DMA_USECTX(mp)) {
9457c478bd9Sstevel@tonic-gate dvma_context_t ctx = IOMMU_FCE_TO_CTX(i);
9467c478bd9Sstevel@tonic-gate tte |= IOMMU_CTX2TTE(ctx);
9477c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_CONTEXT;
9487c478bd9Sstevel@tonic-gate DEBUG1(DBG_DMA_MAP, dip, "fast: ctx=0x%x\n", ctx);
9497c478bd9Sstevel@tonic-gate }
9507c478bd9Sstevel@tonic-gate i *= clustsz;
9517c478bd9Sstevel@tonic-gate tte_addr = iommu_p->iommu_tsb_vaddr + i;
9527c478bd9Sstevel@tonic-gate dvma_pg = iommu_p->dvma_base_pg + i;
9537c478bd9Sstevel@tonic-gate #ifdef DEBUG
9547c478bd9Sstevel@tonic-gate for (i = 0; i < clustsz; i++)
9557c478bd9Sstevel@tonic-gate ASSERT(TTE_IS_INVALID(tte_addr[i]));
9567c478bd9Sstevel@tonic-gate #endif
9577c478bd9Sstevel@tonic-gate *tte_addr = tte | IOMMU_PTOB(MP_PFN0(mp)); /* map page 0 */
9587c478bd9Sstevel@tonic-gate DEBUG5(DBG_DMA_MAP, dip, "fast %p:dvma_pg=%x tte0(%p)=%08x.%08x\n", mp,
959567c0b92SStephen Hanson dvma_pg, tte_addr, HI32(*tte_addr), LO32(*tte_addr));
9607c478bd9Sstevel@tonic-gate if (npages == 1)
9617c478bd9Sstevel@tonic-gate goto tte_done;
9627c478bd9Sstevel@tonic-gate pfn_addr = PCI_GET_MP_PFN1_ADDR(mp); /* short iommu_map_pages() */
9637c478bd9Sstevel@tonic-gate for (tte_addr++, i = 1; i < npages; i++, tte_addr++, pfn_addr++) {
9647c478bd9Sstevel@tonic-gate *tte_addr = tte | IOMMU_PTOB(*pfn_addr);
9657c478bd9Sstevel@tonic-gate DEBUG5(DBG_DMA_MAP, dip, "fast %p:tte(%p, %p)=%08x.%08x\n", mp,
966567c0b92SStephen Hanson tte_addr, pfn_addr, HI32(*tte_addr), LO32(*tte_addr));
9677c478bd9Sstevel@tonic-gate }
9687c478bd9Sstevel@tonic-gate tte_done:
9697c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
9707c478bd9Sstevel@tonic-gate pci_dvmaft_success++;
9717c478bd9Sstevel@tonic-gate #endif
9727c478bd9Sstevel@tonic-gate mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg);
9737c478bd9Sstevel@tonic-gate mp->dmai_offset = 0;
9747c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_FASTTRACK;
9757c478bd9Sstevel@tonic-gate PCI_SAVE_MP_TTE(mp, tte); /* save TTE template for unmapping */
9767c478bd9Sstevel@tonic-gate if (DVMA_DBG_ON(iommu_p))
9777c478bd9Sstevel@tonic-gate pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
978567c0b92SStephen Hanson mp->dmai_size, mp);
9797c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
9807c478bd9Sstevel@tonic-gate }
9817c478bd9Sstevel@tonic-gate
9827c478bd9Sstevel@tonic-gate /*
9837c478bd9Sstevel@tonic-gate * pci_dvma_map: map non-fasttrack DMA
9847c478bd9Sstevel@tonic-gate * Use quantum cache if single page DMA.
9857c478bd9Sstevel@tonic-gate */
9867c478bd9Sstevel@tonic-gate int
pci_dvma_map(ddi_dma_impl_t * mp,ddi_dma_req_t * dmareq,iommu_t * iommu_p)9877c478bd9Sstevel@tonic-gate pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p)
9887c478bd9Sstevel@tonic-gate {
9897c478bd9Sstevel@tonic-gate uint_t npages = PCI_DMA_WINNPGS(mp);
9907c478bd9Sstevel@tonic-gate dvma_addr_t dvma_pg, dvma_pg_index;
9917c478bd9Sstevel@tonic-gate void *dvma_addr;
9927c478bd9Sstevel@tonic-gate uint64_t tte = GET_TTE_TEMPLATE(mp);
9937c478bd9Sstevel@tonic-gate int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP;
9947c478bd9Sstevel@tonic-gate #ifdef DEBUG
9957c478bd9Sstevel@tonic-gate dev_info_t *dip = mp->dmai_rdip;
9967c478bd9Sstevel@tonic-gate #endif
9977c478bd9Sstevel@tonic-gate /*
9987c478bd9Sstevel@tonic-gate * allocate dvma space resource and map in the first window.
9997c478bd9Sstevel@tonic-gate * (vmem_t *vmp, size_t size,
10007c478bd9Sstevel@tonic-gate * size_t align, size_t phase, size_t nocross,
10017c478bd9Sstevel@tonic-gate * void *minaddr, void *maxaddr, int vmflag)
10027c478bd9Sstevel@tonic-gate */
10031de45cd9Sgovinda if ((npages == 1) && !HAS_REDZONE(mp) && HAS_NOSYSLIMIT(mp)) {
10047c478bd9Sstevel@tonic-gate dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map,
1005567c0b92SStephen Hanson IOMMU_PAGE_SIZE, sleep);
10067c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_VMEMCACHE;
10077c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
10087c478bd9Sstevel@tonic-gate pci_dvma_vmem_alloc++;
10097c478bd9Sstevel@tonic-gate #endif
10107c478bd9Sstevel@tonic-gate } else {
10117c478bd9Sstevel@tonic-gate dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map,
1012567c0b92SStephen Hanson IOMMU_PTOB(npages + HAS_REDZONE(mp)),
1013567c0b92SStephen Hanson MAX(mp->dmai_attr.dma_attr_align, IOMMU_PAGE_SIZE),
1014567c0b92SStephen Hanson 0,
1015567c0b92SStephen Hanson mp->dmai_attr.dma_attr_seg + 1,
1016567c0b92SStephen Hanson (void *)mp->dmai_attr.dma_attr_addr_lo,
1017567c0b92SStephen Hanson (void *)(mp->dmai_attr.dma_attr_addr_hi + 1),
1018567c0b92SStephen Hanson sleep);
10197c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
10207c478bd9Sstevel@tonic-gate pci_dvma_vmem_xalloc++;
10217c478bd9Sstevel@tonic-gate #endif
10227c478bd9Sstevel@tonic-gate }
10237c478bd9Sstevel@tonic-gate dvma_pg = IOMMU_BTOP((ulong_t)dvma_addr);
10247c478bd9Sstevel@tonic-gate dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg;
10257c478bd9Sstevel@tonic-gate DEBUG2(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n",
1026567c0b92SStephen Hanson dvma_pg, dvma_pg_index);
10277c478bd9Sstevel@tonic-gate if (dvma_pg == 0)
10287c478bd9Sstevel@tonic-gate goto noresource;
10297c478bd9Sstevel@tonic-gate
10307c478bd9Sstevel@tonic-gate /* allocate DVMA context */
10317c478bd9Sstevel@tonic-gate if ((npages >= pci_context_minpages) && PCI_DMA_USECTX(mp)) {
10327c478bd9Sstevel@tonic-gate dvma_context_t ctx;
10337c478bd9Sstevel@tonic-gate if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) {
10347c478bd9Sstevel@tonic-gate tte |= IOMMU_CTX2TTE(ctx);
10357c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_CONTEXT;
10367c478bd9Sstevel@tonic-gate }
10377c478bd9Sstevel@tonic-gate }
10387c478bd9Sstevel@tonic-gate mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg);
10397c478bd9Sstevel@tonic-gate mp->dmai_offset = 0;
10407c478bd9Sstevel@tonic-gate PCI_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */
10417c478bd9Sstevel@tonic-gate iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0);
10427c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
10437c478bd9Sstevel@tonic-gate noresource:
10447c478bd9Sstevel@tonic-gate if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
10457c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n");
10467c478bd9Sstevel@tonic-gate ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
1047567c0b92SStephen Hanson &iommu_p->iommu_dvma_clid);
10487c478bd9Sstevel@tonic-gate }
10497c478bd9Sstevel@tonic-gate DEBUG0(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n");
10507c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES);
10517c478bd9Sstevel@tonic-gate }
10527c478bd9Sstevel@tonic-gate
10537c478bd9Sstevel@tonic-gate void
pci_dvma_unmap(iommu_t * iommu_p,ddi_dma_impl_t * mp)10547c478bd9Sstevel@tonic-gate pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp)
10557c478bd9Sstevel@tonic-gate {
10567c478bd9Sstevel@tonic-gate size_t npages;
10577c478bd9Sstevel@tonic-gate dvma_addr_t dvma_addr = (dvma_addr_t)mp->dmai_mapping;
10587c478bd9Sstevel@tonic-gate dvma_addr_t dvma_pg = IOMMU_BTOP(dvma_addr);
10597c478bd9Sstevel@tonic-gate dvma_addr = IOMMU_PTOB(dvma_pg);
10607c478bd9Sstevel@tonic-gate
10617c478bd9Sstevel@tonic-gate if (mp->dmai_flags & DMAI_FLAGS_FASTTRACK) {
10627c478bd9Sstevel@tonic-gate iopfn_t index = dvma_pg - iommu_p->dvma_base_pg;
10637c478bd9Sstevel@tonic-gate ASSERT(index % pci_dvma_page_cache_clustsz == 0);
10647c478bd9Sstevel@tonic-gate index /= pci_dvma_page_cache_clustsz;
10657c478bd9Sstevel@tonic-gate ASSERT(index < pci_dvma_page_cache_entries);
10667c478bd9Sstevel@tonic-gate iommu_p->iommu_dvma_cache_locks[index] = 0;
10677c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
10687c478bd9Sstevel@tonic-gate pci_dvmaft_free++;
10697c478bd9Sstevel@tonic-gate #endif
10707c478bd9Sstevel@tonic-gate return;
10717c478bd9Sstevel@tonic-gate }
10727c478bd9Sstevel@tonic-gate npages = IOMMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp);
10737c478bd9Sstevel@tonic-gate pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages);
10747c478bd9Sstevel@tonic-gate
10757c478bd9Sstevel@tonic-gate if (mp->dmai_flags & DMAI_FLAGS_CONTEXT)
10767c478bd9Sstevel@tonic-gate pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp));
10777c478bd9Sstevel@tonic-gate }
10787c478bd9Sstevel@tonic-gate
10797c478bd9Sstevel@tonic-gate void
pci_dma_sync_unmap(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp)10807c478bd9Sstevel@tonic-gate pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp)
10817c478bd9Sstevel@tonic-gate {
10827c478bd9Sstevel@tonic-gate pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
10837c478bd9Sstevel@tonic-gate iommu_t *iommu_p = pci_p->pci_iommu_p;
10847c478bd9Sstevel@tonic-gate uint64_t sync_buf_save = SYNC_BUF_PA(mp);
10857c478bd9Sstevel@tonic-gate uint32_t fast_track = mp->dmai_flags & DMAI_FLAGS_FASTTRACK;
10867c478bd9Sstevel@tonic-gate
10877c478bd9Sstevel@tonic-gate if (fast_track) {
10887c478bd9Sstevel@tonic-gate dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
10897c478bd9Sstevel@tonic-gate
10907c478bd9Sstevel@tonic-gate SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg);
10917c478bd9Sstevel@tonic-gate ASSERT(!(SYNC_BUF_PA(mp) & PCI_SYNC_FLAG_SIZE - 1));
10927c478bd9Sstevel@tonic-gate }
10937c478bd9Sstevel@tonic-gate
10947c478bd9Sstevel@tonic-gate if (pci_dvma_sync_before_unmap) {
109591e99212Ssuha pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0,
109691e99212Ssuha DDI_DMA_SYNC_FORCPU);
10977c478bd9Sstevel@tonic-gate iommu_unmap_window(iommu_p, mp);
10987c478bd9Sstevel@tonic-gate } else {
10997c478bd9Sstevel@tonic-gate iommu_unmap_window(iommu_p, mp);
110091e99212Ssuha pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0,
110191e99212Ssuha DDI_DMA_SYNC_FORCPU);
11027c478bd9Sstevel@tonic-gate }
11037c478bd9Sstevel@tonic-gate
11047c478bd9Sstevel@tonic-gate if (fast_track)
11057c478bd9Sstevel@tonic-gate SYNC_BUF_PA(mp) = sync_buf_save;
11067c478bd9Sstevel@tonic-gate }
11077c478bd9Sstevel@tonic-gate
11087c478bd9Sstevel@tonic-gate /*
11097c478bd9Sstevel@tonic-gate * DVMA mappings may have multiple windows, but each window always have
11107c478bd9Sstevel@tonic-gate * one segment.
11117c478bd9Sstevel@tonic-gate */
11127c478bd9Sstevel@tonic-gate int
pci_dvma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)11137c478bd9Sstevel@tonic-gate pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
11147c478bd9Sstevel@tonic-gate enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
11157c478bd9Sstevel@tonic-gate uint_t cache_flags)
11167c478bd9Sstevel@tonic-gate {
11177c478bd9Sstevel@tonic-gate switch (cmd) {
11187c478bd9Sstevel@tonic-gate
11197c478bd9Sstevel@tonic-gate case DDI_DMA_REMAP:
11207c478bd9Sstevel@tonic-gate if (pci_dvma_remap_enabled)
11217c478bd9Sstevel@tonic-gate return (pci_dvma_remap(dip, rdip, mp, *offp, *lenp));
11227c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
11237c478bd9Sstevel@tonic-gate
11247c478bd9Sstevel@tonic-gate default:
11257c478bd9Sstevel@tonic-gate DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1126567c0b92SStephen Hanson cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
11277c478bd9Sstevel@tonic-gate break;
11287c478bd9Sstevel@tonic-gate }
11297c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
11307c478bd9Sstevel@tonic-gate }
11317c478bd9Sstevel@tonic-gate
11327c478bd9Sstevel@tonic-gate void
pci_dma_freewin(ddi_dma_impl_t * mp)11337c478bd9Sstevel@tonic-gate pci_dma_freewin(ddi_dma_impl_t *mp)
11347c478bd9Sstevel@tonic-gate {
11357c478bd9Sstevel@tonic-gate pci_dma_win_t *win_p = mp->dmai_winlst, *win2_p;
11367c478bd9Sstevel@tonic-gate for (win2_p = win_p; win_p; win2_p = win_p) {
11377c478bd9Sstevel@tonic-gate win_p = win2_p->win_next;
11387c478bd9Sstevel@tonic-gate kmem_free(win2_p, sizeof (pci_dma_win_t) +
1139567c0b92SStephen Hanson sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies);
11407c478bd9Sstevel@tonic-gate }
11417c478bd9Sstevel@tonic-gate mp->dmai_nwin = 0;
11427c478bd9Sstevel@tonic-gate mp->dmai_winlst = NULL;
11437c478bd9Sstevel@tonic-gate }
11447c478bd9Sstevel@tonic-gate
11457c478bd9Sstevel@tonic-gate /*
11467c478bd9Sstevel@tonic-gate * pci_dma_newwin - create a dma window object and cookies
11477c478bd9Sstevel@tonic-gate *
11487c478bd9Sstevel@tonic-gate * After the initial scan in pci_dma_physwin(), which identifies
11497c478bd9Sstevel@tonic-gate * a portion of the pfn array that belongs to a dma window,
11507c478bd9Sstevel@tonic-gate * we are called to allocate and initialize representing memory
11517c478bd9Sstevel@tonic-gate * resources. We know from the 1st scan the number of cookies
11527c478bd9Sstevel@tonic-gate * or dma segment in this window so we can allocate a contiguous
11537c478bd9Sstevel@tonic-gate * memory array for the dma cookies (The implementation of
11547c478bd9Sstevel@tonic-gate * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous).
11557c478bd9Sstevel@tonic-gate *
11567c478bd9Sstevel@tonic-gate * A second round scan is done on the pfn array to identify
11577c478bd9Sstevel@tonic-gate * each dma segment and initialize its corresponding dma cookie.
11587c478bd9Sstevel@tonic-gate * We don't need to do all the safety checking and we know they
11597c478bd9Sstevel@tonic-gate * all belong to the same dma window.
11607c478bd9Sstevel@tonic-gate *
11617c478bd9Sstevel@tonic-gate * Input: cookie_no - # of cookies identified by the 1st scan
11627c478bd9Sstevel@tonic-gate * start_idx - subscript of the pfn array for the starting pfn
11637c478bd9Sstevel@tonic-gate * end_idx - subscript of the last pfn in dma window
11647c478bd9Sstevel@tonic-gate * win_pp - pointer to win_next member of previous window
11657c478bd9Sstevel@tonic-gate * Return: DDI_SUCCESS - with **win_pp as newly created window object
11667c478bd9Sstevel@tonic-gate * DDI_DMA_NORESROUCE - caller frees all previous window objs
11677c478bd9Sstevel@tonic-gate * Note: Each cookie and window size are all initialized on page
11687c478bd9Sstevel@tonic-gate * boundary. This is not true for the 1st cookie of the 1st
11697c478bd9Sstevel@tonic-gate * window and the last cookie of the last window.
11707c478bd9Sstevel@tonic-gate * We fix that later in upper layer which has access to size
11717c478bd9Sstevel@tonic-gate * and offset info.
11727c478bd9Sstevel@tonic-gate *
11737c478bd9Sstevel@tonic-gate */
11747c478bd9Sstevel@tonic-gate static int
pci_dma_newwin(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint32_t cookie_no,uint32_t start_idx,uint32_t end_idx,pci_dma_win_t ** win_pp,uint64_t count_max,uint64_t bypass_prefix)11757c478bd9Sstevel@tonic-gate pci_dma_newwin(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, uint32_t cookie_no,
11767c478bd9Sstevel@tonic-gate uint32_t start_idx, uint32_t end_idx, pci_dma_win_t **win_pp,
11777c478bd9Sstevel@tonic-gate uint64_t count_max, uint64_t bypass_prefix)
11787c478bd9Sstevel@tonic-gate {
11797c478bd9Sstevel@tonic-gate int (*waitfp)(caddr_t) = dmareq->dmar_fp;
11807c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cookie_p;
11817c478bd9Sstevel@tonic-gate uint32_t pfn_no = 1;
11827c478bd9Sstevel@tonic-gate iopfn_t pfn = PCI_GET_MP_PFN(mp, start_idx);
11837c478bd9Sstevel@tonic-gate iopfn_t prev_pfn = pfn;
11847c478bd9Sstevel@tonic-gate uint64_t seg_pfn0 = pfn;
11857c478bd9Sstevel@tonic-gate size_t sz = cookie_no * sizeof (ddi_dma_cookie_t);
11867c478bd9Sstevel@tonic-gate pci_dma_win_t *win_p = kmem_alloc(sizeof (pci_dma_win_t) + sz,
1187567c0b92SStephen Hanson waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
11887c478bd9Sstevel@tonic-gate if (!win_p)
11897c478bd9Sstevel@tonic-gate goto noresource;
11907c478bd9Sstevel@tonic-gate
11917c478bd9Sstevel@tonic-gate win_p->win_next = NULL;
11927c478bd9Sstevel@tonic-gate win_p->win_ncookies = cookie_no;
11937c478bd9Sstevel@tonic-gate win_p->win_curseg = 0; /* start from segment 0 */
11947c478bd9Sstevel@tonic-gate win_p->win_size = IOMMU_PTOB(end_idx - start_idx + 1);
11957c478bd9Sstevel@tonic-gate /* win_p->win_offset is left uninitialized */
11967c478bd9Sstevel@tonic-gate
11977c478bd9Sstevel@tonic-gate cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
11987c478bd9Sstevel@tonic-gate start_idx++;
11997c478bd9Sstevel@tonic-gate for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) {
12007c478bd9Sstevel@tonic-gate pfn = PCI_GET_MP_PFN1(mp, start_idx);
12017c478bd9Sstevel@tonic-gate if ((pfn == prev_pfn + 1) &&
1202567c0b92SStephen Hanson (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max))
12037c478bd9Sstevel@tonic-gate continue;
12047c478bd9Sstevel@tonic-gate
12057c478bd9Sstevel@tonic-gate /* close up the cookie up to (including) prev_pfn */
12067c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix,
1207567c0b92SStephen Hanson IOMMU_PTOB(pfn_no));
12087c478bd9Sstevel@tonic-gate DEBUG2(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",
1209567c0b92SStephen Hanson IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no);
12107c478bd9Sstevel@tonic-gate
12117c478bd9Sstevel@tonic-gate cookie_p++; /* advance to next available cookie cell */
12127c478bd9Sstevel@tonic-gate pfn_no = 0;
12137c478bd9Sstevel@tonic-gate seg_pfn0 = pfn; /* start a new segment from current pfn */
12147c478bd9Sstevel@tonic-gate }
12157c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix,
1216567c0b92SStephen Hanson IOMMU_PTOB(pfn_no));
12177c478bd9Sstevel@tonic-gate DEBUG3(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",
1218567c0b92SStephen Hanson IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no, cookie_no);
12197c478bd9Sstevel@tonic-gate #ifdef DEBUG
12207c478bd9Sstevel@tonic-gate cookie_p++;
12217c478bd9Sstevel@tonic-gate ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no);
12227c478bd9Sstevel@tonic-gate #endif
12237c478bd9Sstevel@tonic-gate *win_pp = win_p;
12247c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
12257c478bd9Sstevel@tonic-gate noresource:
12267c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT)
12277c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp, dmareq->dmar_arg, &pci_kmem_clid);
12287c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES);
12297c478bd9Sstevel@tonic-gate }
12307c478bd9Sstevel@tonic-gate
12317c478bd9Sstevel@tonic-gate /*
12327c478bd9Sstevel@tonic-gate * pci_dma_adjust - adjust 1st and last cookie and window sizes
12337c478bd9Sstevel@tonic-gate * remove initial dma page offset from 1st cookie and window size
12347c478bd9Sstevel@tonic-gate * remove last dma page remainder from last cookie and window size
12357c478bd9Sstevel@tonic-gate * fill win_offset of each dma window according to just fixed up
12367c478bd9Sstevel@tonic-gate * each window sizes
12377c478bd9Sstevel@tonic-gate * pci_dma_win_t members modified:
12387c478bd9Sstevel@tonic-gate * win_p->win_offset - this window's offset within entire DMA object
12397c478bd9Sstevel@tonic-gate * win_p->win_size - xferrable size (in bytes) for this window
12407c478bd9Sstevel@tonic-gate *
12417c478bd9Sstevel@tonic-gate * ddi_dma_impl_t members modified:
12427c478bd9Sstevel@tonic-gate * mp->dmai_size - 1st window xferrable size
12437c478bd9Sstevel@tonic-gate * mp->dmai_offset - 0, which is the dma offset of the 1st window
12447c478bd9Sstevel@tonic-gate *
12457c478bd9Sstevel@tonic-gate * ddi_dma_cookie_t members modified:
12467c478bd9Sstevel@tonic-gate * cookie_p->dmac_size - 1st and last cookie remove offset or remainder
12477c478bd9Sstevel@tonic-gate * cookie_p->dmac_laddress - 1st cookie add page offset
12487c478bd9Sstevel@tonic-gate */
12497c478bd9Sstevel@tonic-gate static void
pci_dma_adjust(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,pci_dma_win_t * win_p)12507c478bd9Sstevel@tonic-gate pci_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, pci_dma_win_t *win_p)
12517c478bd9Sstevel@tonic-gate {
12527c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
12537c478bd9Sstevel@tonic-gate size_t pg_offset = mp->dmai_roffset;
12547c478bd9Sstevel@tonic-gate size_t win_offset = 0;
12557c478bd9Sstevel@tonic-gate
12567c478bd9Sstevel@tonic-gate cookie_p->dmac_size -= pg_offset;
12577c478bd9Sstevel@tonic-gate cookie_p->dmac_laddress |= pg_offset;
12587c478bd9Sstevel@tonic-gate win_p->win_size -= pg_offset;
12597c478bd9Sstevel@tonic-gate DEBUG1(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);
12607c478bd9Sstevel@tonic-gate
12617c478bd9Sstevel@tonic-gate mp->dmai_size = win_p->win_size;
12627c478bd9Sstevel@tonic-gate mp->dmai_offset = 0;
12637c478bd9Sstevel@tonic-gate
12647c478bd9Sstevel@tonic-gate pg_offset += mp->dmai_object.dmao_size;
12657c478bd9Sstevel@tonic-gate pg_offset &= IOMMU_PAGE_OFFSET;
12667c478bd9Sstevel@tonic-gate if (pg_offset)
12677c478bd9Sstevel@tonic-gate pg_offset = IOMMU_PAGE_SIZE - pg_offset;
12687c478bd9Sstevel@tonic-gate DEBUG1(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);
12697c478bd9Sstevel@tonic-gate
12707c478bd9Sstevel@tonic-gate for (; win_p->win_next; win_p = win_p->win_next) {
12717c478bd9Sstevel@tonic-gate DEBUG1(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);
12727c478bd9Sstevel@tonic-gate win_p->win_offset = win_offset;
12737c478bd9Sstevel@tonic-gate win_offset += win_p->win_size;
12747c478bd9Sstevel@tonic-gate }
12757c478bd9Sstevel@tonic-gate /* last window */
12767c478bd9Sstevel@tonic-gate win_p->win_offset = win_offset;
12777c478bd9Sstevel@tonic-gate cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
12787c478bd9Sstevel@tonic-gate cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset;
12797c478bd9Sstevel@tonic-gate win_p->win_size -= pg_offset;
12807c478bd9Sstevel@tonic-gate ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);
12817c478bd9Sstevel@tonic-gate }
12827c478bd9Sstevel@tonic-gate
12837c478bd9Sstevel@tonic-gate /*
12847c478bd9Sstevel@tonic-gate * pci_dma_physwin() - carve up dma windows using physical addresses.
12857c478bd9Sstevel@tonic-gate * Called to handle iommu bypass and pci peer-to-peer transfers.
12867c478bd9Sstevel@tonic-gate * Calls pci_dma_newwin() to allocate window objects.
12877c478bd9Sstevel@tonic-gate *
12887c478bd9Sstevel@tonic-gate * Dependency: mp->dmai_pfnlst points to an array of pfns
12897c478bd9Sstevel@tonic-gate *
12907c478bd9Sstevel@tonic-gate * 1. Each dma window is represented by a pci_dma_win_t object.
12917c478bd9Sstevel@tonic-gate * The object will be casted to ddi_dma_win_t and returned
12927c478bd9Sstevel@tonic-gate * to leaf driver through the DDI interface.
12937c478bd9Sstevel@tonic-gate * 2. Each dma window can have several dma segments with each
12947c478bd9Sstevel@tonic-gate * segment representing a physically contiguous either memory
12957c478bd9Sstevel@tonic-gate * space (if we are doing an iommu bypass transfer) or pci address
12967c478bd9Sstevel@tonic-gate * space (if we are doing a peer-to-peer transfer).
12977c478bd9Sstevel@tonic-gate * 3. Each segment has a DMA cookie to program the DMA engine.
12987c478bd9Sstevel@tonic-gate * The cookies within each DMA window must be located in a
12997c478bd9Sstevel@tonic-gate * contiguous array per ddi_dma_nextcookie(9f).
13007c478bd9Sstevel@tonic-gate * 4. The number of DMA segments within each DMA window cannot exceed
13017c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_sgllen. If the transfer size is
13027c478bd9Sstevel@tonic-gate * too large to fit in the sgllen, the rest needs to be
13037c478bd9Sstevel@tonic-gate * relocated to the next dma window.
13047c478bd9Sstevel@tonic-gate * 5. Peer-to-peer DMA segment follows device hi, lo, count_max,
13057c478bd9Sstevel@tonic-gate * and nocross restrictions while bypass DMA follows the set of
13067c478bd9Sstevel@tonic-gate * restrictions with system limits factored in.
13077c478bd9Sstevel@tonic-gate *
13087c478bd9Sstevel@tonic-gate * Return:
13097c478bd9Sstevel@tonic-gate * mp->dmai_winlst - points to a link list of pci_dma_win_t objects.
13107c478bd9Sstevel@tonic-gate * Each pci_dma_win_t object on the link list contains
13117c478bd9Sstevel@tonic-gate * infomation such as its window size (# of pages),
13127c478bd9Sstevel@tonic-gate * starting offset (also see Restriction), an array of
13137c478bd9Sstevel@tonic-gate * DMA cookies, and # of cookies in the array.
13147c478bd9Sstevel@tonic-gate * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory.
13157c478bd9Sstevel@tonic-gate * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst.
13167c478bd9Sstevel@tonic-gate * mp->dmai_mapping - starting cookie address
13177c478bd9Sstevel@tonic-gate * mp->dmai_rflags - consistent, nosync, no redzone
13187c478bd9Sstevel@tonic-gate * mp->dmai_cookie - start of cookie table of the 1st DMA window
13197c478bd9Sstevel@tonic-gate *
13207c478bd9Sstevel@tonic-gate * Restriction:
13217c478bd9Sstevel@tonic-gate * Each pci_dma_win_t object can theoratically start from any offset
13227c478bd9Sstevel@tonic-gate * since the iommu is not involved. However, this implementation
13237c478bd9Sstevel@tonic-gate * always make windows start from page aligned offset (except
13247c478bd9Sstevel@tonic-gate * the 1st window, which follows the requested offset) due to the
13257c478bd9Sstevel@tonic-gate * fact that we are handed a pfn list. This does require device's
13267c478bd9Sstevel@tonic-gate * count_max and attr_seg to be at least IOMMU_PAGE_SIZE aligned.
13277c478bd9Sstevel@tonic-gate */
13287c478bd9Sstevel@tonic-gate int
pci_dma_physwin(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)13297c478bd9Sstevel@tonic-gate pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
13307c478bd9Sstevel@tonic-gate {
13317c478bd9Sstevel@tonic-gate uint_t npages = mp->dmai_ndvmapages;
13327c478bd9Sstevel@tonic-gate int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;
13337c478bd9Sstevel@tonic-gate iopfn_t pfn_lo, pfn_hi, prev_pfn, bypass_pfn;
13347c478bd9Sstevel@tonic-gate iopfn_t pfn = PCI_GET_MP_PFN(mp, 0);
13357c478bd9Sstevel@tonic-gate uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0;
13367c478bd9Sstevel@tonic-gate uint64_t count_max, bypass = PCI_DMA_BYPASS_PREFIX(mp, pfn);
13377c478bd9Sstevel@tonic-gate pci_dma_win_t **win_pp = (pci_dma_win_t **)&mp->dmai_winlst;
13387c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cookie0_p;
13397c478bd9Sstevel@tonic-gate
13407c478bd9Sstevel@tonic-gate if (PCI_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */
13417c478bd9Sstevel@tonic-gate ddi_dma_attr_t *dev_attr_p = DEV_ATTR(mp);
13427c478bd9Sstevel@tonic-gate iopfn_t pfn_base = pci_p->pci_pbm_p->pbm_base_pfn;
13437c478bd9Sstevel@tonic-gate iopfn_t pfn_last = pci_p->pci_pbm_p->pbm_last_pfn - pfn_base;
13447c478bd9Sstevel@tonic-gate uint64_t nocross = dev_attr_p->dma_attr_seg;
13457c478bd9Sstevel@tonic-gate if (nocross && (nocross < UINT32_MAX))
13467c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
13477c478bd9Sstevel@tonic-gate if (dev_attr_p->dma_attr_align > IOMMU_PAGE_SIZE)
13487c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
13497c478bd9Sstevel@tonic-gate pfn_lo = IOMMU_BTOP(dev_attr_p->dma_attr_addr_lo);
13507c478bd9Sstevel@tonic-gate pfn_hi = IOMMU_BTOP(dev_attr_p->dma_attr_addr_hi);
13517c478bd9Sstevel@tonic-gate pfn_hi = MIN(pfn_hi, pfn_last);
13527c478bd9Sstevel@tonic-gate if ((pfn_lo > pfn_hi) || (pfn < pfn_lo))
13537c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING);
13547c478bd9Sstevel@tonic-gate count_max = dev_attr_p->dma_attr_count_max;
13557c478bd9Sstevel@tonic-gate count_max = MIN(count_max, nocross);
13567c478bd9Sstevel@tonic-gate /*
13577c478bd9Sstevel@tonic-gate * the following count_max trim is not done because we are
13587c478bd9Sstevel@tonic-gate * making sure pfn_lo <= pfn <= pfn_hi inside the loop
13597c478bd9Sstevel@tonic-gate * count_max=MIN(count_max, IOMMU_PTOB(pfn_hi - pfn_lo + 1)-1);
13607c478bd9Sstevel@tonic-gate */
13617c478bd9Sstevel@tonic-gate } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */
13627c478bd9Sstevel@tonic-gate count_max = mp->dmai_attr.dma_attr_count_max;
13637c478bd9Sstevel@tonic-gate pfn_lo = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);
13647c478bd9Sstevel@tonic-gate pfn_hi = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);
13657c478bd9Sstevel@tonic-gate }
13667c478bd9Sstevel@tonic-gate
13677c478bd9Sstevel@tonic-gate bypass_pfn = IOMMU_BTOP(bypass);
13687c478bd9Sstevel@tonic-gate
13697c478bd9Sstevel@tonic-gate for (prev_pfn = (bypass_pfn | pfn), i = 1; i < npages;
13707c478bd9Sstevel@tonic-gate i++, prev_pfn = pfn, pfn_no++) {
13717c478bd9Sstevel@tonic-gate pfn = bypass_pfn | PCI_GET_MP_PFN1(mp, i);
13727c478bd9Sstevel@tonic-gate if ((pfn == prev_pfn + 1) &&
1373567c0b92SStephen Hanson (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max))
13747c478bd9Sstevel@tonic-gate continue;
13757c478bd9Sstevel@tonic-gate if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) {
13767c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING;
13777c478bd9Sstevel@tonic-gate goto err;
13787c478bd9Sstevel@tonic-gate }
13797c478bd9Sstevel@tonic-gate cookie_no++;
13807c478bd9Sstevel@tonic-gate pfn_no = 0;
13817c478bd9Sstevel@tonic-gate if (cookie_no < sgllen)
13827c478bd9Sstevel@tonic-gate continue;
13837c478bd9Sstevel@tonic-gate
13847c478bd9Sstevel@tonic-gate DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1385567c0b92SStephen Hanson win_pfn0_index, i - 1, cookie_no);
13867c478bd9Sstevel@tonic-gate if (ret = pci_dma_newwin(dmareq, mp, cookie_no,
1387567c0b92SStephen Hanson win_pfn0_index, i - 1, win_pp, count_max, bypass))
13887c478bd9Sstevel@tonic-gate goto err;
13897c478bd9Sstevel@tonic-gate
13907c478bd9Sstevel@tonic-gate win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */
13917c478bd9Sstevel@tonic-gate win_no++;
13927c478bd9Sstevel@tonic-gate win_pfn0_index = i;
13937c478bd9Sstevel@tonic-gate cookie_no = 0;
13947c478bd9Sstevel@tonic-gate }
13957c478bd9Sstevel@tonic-gate if (pfn > pfn_hi) {
13967c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING;
13977c478bd9Sstevel@tonic-gate goto err;
13987c478bd9Sstevel@tonic-gate }
13997c478bd9Sstevel@tonic-gate cookie_no++;
14007c478bd9Sstevel@tonic-gate DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1401567c0b92SStephen Hanson win_pfn0_index, i - 1, cookie_no);
14027c478bd9Sstevel@tonic-gate if (ret = pci_dma_newwin(dmareq, mp, cookie_no, win_pfn0_index,
1403567c0b92SStephen Hanson i - 1, win_pp, count_max, bypass))
14047c478bd9Sstevel@tonic-gate goto err;
14057c478bd9Sstevel@tonic-gate win_no++;
14067c478bd9Sstevel@tonic-gate pci_dma_adjust(dmareq, mp, mp->dmai_winlst);
14077c478bd9Sstevel@tonic-gate mp->dmai_nwin = win_no;
14087c478bd9Sstevel@tonic-gate mp->dmai_rflags |= DDI_DMA_CONSISTENT;
14097c478bd9Sstevel@tonic-gate if (!pci_p->pci_pbm_p->pbm_sync_reg_pa) {
14107c478bd9Sstevel@tonic-gate mp->dmai_rflags |= DMP_NOSYNC;
14117c478bd9Sstevel@tonic-gate mp->dmai_flags |= DMAI_FLAGS_NOSYNC;
14127c478bd9Sstevel@tonic-gate }
14137c478bd9Sstevel@tonic-gate mp->dmai_rflags &= ~DDI_DMA_REDZONE;
14147c478bd9Sstevel@tonic-gate cookie0_p = (ddi_dma_cookie_t *)(WINLST(mp) + 1);
1415*9a63ec27SRobert Mustacchi mp->dmai_cookie = cookie0_p + 1;
14167c478bd9Sstevel@tonic-gate mp->dmai_mapping = cookie0_p->dmac_laddress;
1417*9a63ec27SRobert Mustacchi mp->dmai_ncookies = WINLST(mp)->win_ncookies;
1418*9a63ec27SRobert Mustacchi mp->dmai_curcookie = 1;
14197c478bd9Sstevel@tonic-gate
14207c478bd9Sstevel@tonic-gate pci_dma_freepfn(mp);
14217c478bd9Sstevel@tonic-gate return (DDI_DMA_MAPPED);
14227c478bd9Sstevel@tonic-gate err:
14237c478bd9Sstevel@tonic-gate pci_dma_freewin(mp);
14247c478bd9Sstevel@tonic-gate return (ret);
14257c478bd9Sstevel@tonic-gate }
14267c478bd9Sstevel@tonic-gate
14277c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14287c478bd9Sstevel@tonic-gate int
pci_dma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)14297c478bd9Sstevel@tonic-gate pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
14307c478bd9Sstevel@tonic-gate enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
14317c478bd9Sstevel@tonic-gate uint_t cache_flags)
14327c478bd9Sstevel@tonic-gate {
14337c478bd9Sstevel@tonic-gate switch (cmd) {
14347c478bd9Sstevel@tonic-gate
14357c478bd9Sstevel@tonic-gate case DDI_DMA_HTOC: {
14367c478bd9Sstevel@tonic-gate off_t off = *offp;
14377c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *loop_cp, *cp;
14387c478bd9Sstevel@tonic-gate pci_dma_win_t *win_p = mp->dmai_winlst;
14397c478bd9Sstevel@tonic-gate
14407c478bd9Sstevel@tonic-gate if (off >= mp->dmai_object.dmao_size)
14417c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
14427c478bd9Sstevel@tonic-gate
14437c478bd9Sstevel@tonic-gate /* locate window */
14447c478bd9Sstevel@tonic-gate while (win_p->win_offset + win_p->win_size <= off)
14457c478bd9Sstevel@tonic-gate win_p = win_p->win_next;
14467c478bd9Sstevel@tonic-gate
14477c478bd9Sstevel@tonic-gate loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1);
14487c478bd9Sstevel@tonic-gate mp->dmai_offset = win_p->win_offset;
14497c478bd9Sstevel@tonic-gate mp->dmai_size = win_p->win_size;
14507c478bd9Sstevel@tonic-gate mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */
14517c478bd9Sstevel@tonic-gate
14527c478bd9Sstevel@tonic-gate /* adjust cookie addr/len if we are not on cookie boundary */
14537c478bd9Sstevel@tonic-gate off -= win_p->win_offset; /* offset within window */
14547c478bd9Sstevel@tonic-gate for (; off >= loop_cp->dmac_size; loop_cp++)
14557c478bd9Sstevel@tonic-gate off -= loop_cp->dmac_size; /* offset within cookie */
14567c478bd9Sstevel@tonic-gate
14577c478bd9Sstevel@tonic-gate mp->dmai_cookie = loop_cp + 1;
14587c478bd9Sstevel@tonic-gate win_p->win_curseg = loop_cp - cp;
14597c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)objp;
14607c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off,
1461567c0b92SStephen Hanson loop_cp->dmac_size - off);
14627c478bd9Sstevel@tonic-gate
14637c478bd9Sstevel@tonic-gate DEBUG2(DBG_DMA_CTL, dip,
1464567c0b92SStephen Hanson "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
1465567c0b92SStephen Hanson cp->dmac_laddress, cp->dmac_size);
14667c478bd9Sstevel@tonic-gate }
14677c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
14687c478bd9Sstevel@tonic-gate
14697c478bd9Sstevel@tonic-gate case DDI_DMA_COFF: {
14707c478bd9Sstevel@tonic-gate pci_dma_win_t *win_p;
14717c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp;
14727c478bd9Sstevel@tonic-gate uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress;
14737c478bd9Sstevel@tonic-gate size_t win_off;
14747c478bd9Sstevel@tonic-gate
14757c478bd9Sstevel@tonic-gate for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) {
14767c478bd9Sstevel@tonic-gate int i;
14777c478bd9Sstevel@tonic-gate win_off = 0;
14787c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)(win_p + 1);
14797c478bd9Sstevel@tonic-gate for (i = 0; i < win_p->win_ncookies; i++, cp++) {
14807c478bd9Sstevel@tonic-gate size_t sz = cp->dmac_size;
14817c478bd9Sstevel@tonic-gate
14827c478bd9Sstevel@tonic-gate addr = cp->dmac_laddress;
14837c478bd9Sstevel@tonic-gate if ((addr <= key) && (addr + sz >= key))
14847c478bd9Sstevel@tonic-gate goto found;
14857c478bd9Sstevel@tonic-gate win_off += sz;
14867c478bd9Sstevel@tonic-gate }
14877c478bd9Sstevel@tonic-gate }
14887c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
14897c478bd9Sstevel@tonic-gate found:
14907c478bd9Sstevel@tonic-gate *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr));
14917c478bd9Sstevel@tonic-gate return (DDI_SUCCESS);
14927c478bd9Sstevel@tonic-gate }
14937c478bd9Sstevel@tonic-gate
14947c478bd9Sstevel@tonic-gate case DDI_DMA_REMAP:
14957c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
14967c478bd9Sstevel@tonic-gate
14977c478bd9Sstevel@tonic-gate default:
14987c478bd9Sstevel@tonic-gate DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1499567c0b92SStephen Hanson cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
15007c478bd9Sstevel@tonic-gate break;
15017c478bd9Sstevel@tonic-gate }
15027c478bd9Sstevel@tonic-gate return (DDI_FAILURE);
15037c478bd9Sstevel@tonic-gate }
15047c478bd9Sstevel@tonic-gate
15057c478bd9Sstevel@tonic-gate static void
pci_dvma_debug_init(iommu_t * iommu_p)15067c478bd9Sstevel@tonic-gate pci_dvma_debug_init(iommu_t *iommu_p)
15077c478bd9Sstevel@tonic-gate {
15087c478bd9Sstevel@tonic-gate size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec;
15097c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock));
15107c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p);
15117c478bd9Sstevel@tonic-gate
15127c478bd9Sstevel@tonic-gate iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP);
15137c478bd9Sstevel@tonic-gate iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP);
15147c478bd9Sstevel@tonic-gate
15157c478bd9Sstevel@tonic-gate iommu_p->dvma_active_list = NULL;
15167c478bd9Sstevel@tonic-gate iommu_p->dvma_alloc_rec_index = 0;
15177c478bd9Sstevel@tonic-gate iommu_p->dvma_free_rec_index = 0;
15187c478bd9Sstevel@tonic-gate iommu_p->dvma_active_count = 0;
15197c478bd9Sstevel@tonic-gate }
15207c478bd9Sstevel@tonic-gate
15217c478bd9Sstevel@tonic-gate void
pci_dvma_debug_fini(iommu_t * iommu_p)15227c478bd9Sstevel@tonic-gate pci_dvma_debug_fini(iommu_t *iommu_p)
15237c478bd9Sstevel@tonic-gate {
15247c478bd9Sstevel@tonic-gate struct dvma_rec *prev, *ptr;
15257c478bd9Sstevel@tonic-gate size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec;
15267c478bd9Sstevel@tonic-gate uint64_t mask = ~(1ull << iommu_p->iommu_inst);
15277c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p);
15287c478bd9Sstevel@tonic-gate
15297c478bd9Sstevel@tonic-gate kmem_free(iommu_p->dvma_alloc_rec, sz);
15307c478bd9Sstevel@tonic-gate kmem_free(iommu_p->dvma_free_rec, sz);
15317c478bd9Sstevel@tonic-gate iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL;
15327c478bd9Sstevel@tonic-gate
15337c478bd9Sstevel@tonic-gate prev = iommu_p->dvma_active_list;
15347c478bd9Sstevel@tonic-gate if (!prev)
15357c478bd9Sstevel@tonic-gate return;
15367c478bd9Sstevel@tonic-gate for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next)
15377c478bd9Sstevel@tonic-gate kmem_free(prev, sizeof (struct dvma_rec));
15387c478bd9Sstevel@tonic-gate kmem_free(prev, sizeof (struct dvma_rec));
15397c478bd9Sstevel@tonic-gate
15407c478bd9Sstevel@tonic-gate iommu_p->dvma_active_list = NULL;
15417c478bd9Sstevel@tonic-gate iommu_p->dvma_alloc_rec_index = 0;
15427c478bd9Sstevel@tonic-gate iommu_p->dvma_free_rec_index = 0;
15437c478bd9Sstevel@tonic-gate iommu_p->dvma_active_count = 0;
15447c478bd9Sstevel@tonic-gate
15457c478bd9Sstevel@tonic-gate pci_dvma_debug_on &= mask;
15467c478bd9Sstevel@tonic-gate pci_dvma_debug_off &= mask;
15477c478bd9Sstevel@tonic-gate }
15487c478bd9Sstevel@tonic-gate
15497c478bd9Sstevel@tonic-gate void
pci_dvma_alloc_debug(iommu_t * iommu_p,char * address,uint_t len,ddi_dma_impl_t * mp)15507c478bd9Sstevel@tonic-gate pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len,
15517c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp)
15527c478bd9Sstevel@tonic-gate {
15537c478bd9Sstevel@tonic-gate struct dvma_rec *ptr;
15547c478bd9Sstevel@tonic-gate mutex_enter(&iommu_p->dvma_debug_lock);
15557c478bd9Sstevel@tonic-gate
15567c478bd9Sstevel@tonic-gate if (!iommu_p->dvma_alloc_rec)
15577c478bd9Sstevel@tonic-gate pci_dvma_debug_init(iommu_p);
15587c478bd9Sstevel@tonic-gate if (DVMA_DBG_OFF(iommu_p)) {
15597c478bd9Sstevel@tonic-gate pci_dvma_debug_fini(iommu_p);
15607c478bd9Sstevel@tonic-gate goto done;
15617c478bd9Sstevel@tonic-gate }
15627c478bd9Sstevel@tonic-gate
15637c478bd9Sstevel@tonic-gate ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index];
15647c478bd9Sstevel@tonic-gate ptr->dvma_addr = address;
15657c478bd9Sstevel@tonic-gate ptr->len = len;
15667c478bd9Sstevel@tonic-gate ptr->mp = mp;
15677c478bd9Sstevel@tonic-gate if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec)
15687c478bd9Sstevel@tonic-gate iommu_p->dvma_alloc_rec_index = 0;
15697c478bd9Sstevel@tonic-gate
15707c478bd9Sstevel@tonic-gate ptr = kmem_alloc(sizeof (struct dvma_rec), KM_SLEEP);
15717c478bd9Sstevel@tonic-gate ptr->dvma_addr = address;
15727c478bd9Sstevel@tonic-gate ptr->len = len;
15737c478bd9Sstevel@tonic-gate ptr->mp = mp;
15747c478bd9Sstevel@tonic-gate
15757c478bd9Sstevel@tonic-gate ptr->next = iommu_p->dvma_active_list;
15767c478bd9Sstevel@tonic-gate iommu_p->dvma_active_list = ptr;
15777c478bd9Sstevel@tonic-gate iommu_p->dvma_active_count++;
15787c478bd9Sstevel@tonic-gate done:
15797c478bd9Sstevel@tonic-gate mutex_exit(&iommu_p->dvma_debug_lock);
15807c478bd9Sstevel@tonic-gate }
15817c478bd9Sstevel@tonic-gate
15827c478bd9Sstevel@tonic-gate void
pci_dvma_free_debug(iommu_t * iommu_p,char * address,uint_t len,ddi_dma_impl_t * mp)15837c478bd9Sstevel@tonic-gate pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len,
15847c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp)
15857c478bd9Sstevel@tonic-gate {
15867c478bd9Sstevel@tonic-gate struct dvma_rec *ptr, *ptr_save;
15877c478bd9Sstevel@tonic-gate mutex_enter(&iommu_p->dvma_debug_lock);
15887c478bd9Sstevel@tonic-gate
15897c478bd9Sstevel@tonic-gate if (!iommu_p->dvma_alloc_rec)
15907c478bd9Sstevel@tonic-gate pci_dvma_debug_init(iommu_p);
15917c478bd9Sstevel@tonic-gate if (DVMA_DBG_OFF(iommu_p)) {
15927c478bd9Sstevel@tonic-gate pci_dvma_debug_fini(iommu_p);
15937c478bd9Sstevel@tonic-gate goto done;
15947c478bd9Sstevel@tonic-gate }
15957c478bd9Sstevel@tonic-gate
15967c478bd9Sstevel@tonic-gate ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index];
15977c478bd9Sstevel@tonic-gate ptr->dvma_addr = address;
15987c478bd9Sstevel@tonic-gate ptr->len = len;
15997c478bd9Sstevel@tonic-gate ptr->mp = mp;
16007c478bd9Sstevel@tonic-gate if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec)
16017c478bd9Sstevel@tonic-gate iommu_p->dvma_free_rec_index = 0;
16027c478bd9Sstevel@tonic-gate
16037c478bd9Sstevel@tonic-gate ptr_save = iommu_p->dvma_active_list;
16047c478bd9Sstevel@tonic-gate for (ptr = ptr_save; ptr; ptr = ptr->next) {
16057c478bd9Sstevel@tonic-gate if ((ptr->dvma_addr == address) && (ptr->len = len))
16067c478bd9Sstevel@tonic-gate break;
16077c478bd9Sstevel@tonic-gate ptr_save = ptr;
16087c478bd9Sstevel@tonic-gate }
16097c478bd9Sstevel@tonic-gate if (!ptr) {
16107c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x",
1611567c0b92SStephen Hanson (long)address, len);
16127c478bd9Sstevel@tonic-gate goto done;
16137c478bd9Sstevel@tonic-gate }
16147c478bd9Sstevel@tonic-gate if (ptr == iommu_p->dvma_active_list)
16157c478bd9Sstevel@tonic-gate iommu_p->dvma_active_list = ptr->next;
16167c478bd9Sstevel@tonic-gate else
16177c478bd9Sstevel@tonic-gate ptr_save->next = ptr->next;
16187c478bd9Sstevel@tonic-gate kmem_free(ptr, sizeof (struct dvma_rec));
16197c478bd9Sstevel@tonic-gate iommu_p->dvma_active_count--;
16207c478bd9Sstevel@tonic-gate done:
16217c478bd9Sstevel@tonic-gate mutex_exit(&iommu_p->dvma_debug_lock);
16227c478bd9Sstevel@tonic-gate }
16237c478bd9Sstevel@tonic-gate
16247c478bd9Sstevel@tonic-gate #ifdef DEBUG
16257c478bd9Sstevel@tonic-gate void
dump_dma_handle(uint64_t flag,dev_info_t * dip,ddi_dma_impl_t * hp)16267c478bd9Sstevel@tonic-gate dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp)
16277c478bd9Sstevel@tonic-gate {
16287c478bd9Sstevel@tonic-gate DEBUG4(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n",
1629567c0b92SStephen Hanson hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size);
16307c478bd9Sstevel@tonic-gate DEBUG4(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n",
1631567c0b92SStephen Hanson hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags,
1632567c0b92SStephen Hanson hp->dmai_nwin);
16337c478bd9Sstevel@tonic-gate DEBUG4(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n",
1634567c0b92SStephen Hanson hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0);
16357c478bd9Sstevel@tonic-gate DEBUG4(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n",
1636567c0b92SStephen Hanson hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr,
1637567c0b92SStephen Hanson hp->dmai_cookie);
16387c478bd9Sstevel@tonic-gate }
16397c478bd9Sstevel@tonic-gate #endif
16407c478bd9Sstevel@tonic-gate
16417c478bd9Sstevel@tonic-gate void
pci_vmem_do_free(iommu_t * iommu_p,void * base_addr,size_t npages,int vmemcache)16427c478bd9Sstevel@tonic-gate pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages,
16437c478bd9Sstevel@tonic-gate int vmemcache)
16447c478bd9Sstevel@tonic-gate {
16457c478bd9Sstevel@tonic-gate vmem_t *map_p = iommu_p->iommu_dvma_map;
16467c478bd9Sstevel@tonic-gate
16477c478bd9Sstevel@tonic-gate if (vmemcache) {
16487c478bd9Sstevel@tonic-gate vmem_free(map_p, base_addr, IOMMU_PAGE_SIZE);
16497c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
16507c478bd9Sstevel@tonic-gate pci_dvma_vmem_free++;
16517c478bd9Sstevel@tonic-gate #endif
16527c478bd9Sstevel@tonic-gate return;
16537c478bd9Sstevel@tonic-gate }
16547c478bd9Sstevel@tonic-gate
16557c478bd9Sstevel@tonic-gate vmem_xfree(map_p, base_addr, IOMMU_PTOB(npages));
16567c478bd9Sstevel@tonic-gate #ifdef PCI_DMA_PROF
16577c478bd9Sstevel@tonic-gate pci_dvma_vmem_xfree++;
16587c478bd9Sstevel@tonic-gate #endif
16597c478bd9Sstevel@tonic-gate }
1660