17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51de45cd9Sgovinda * Common Development and Distribution License (the "License"). 61de45cd9Sgovinda * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 2203494a98SBill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* 277c478bd9Sstevel@tonic-gate * PCI Express nexus DVMA and DMA core routines: 287c478bd9Sstevel@tonic-gate * dma_map/dma_bind_handle implementation 297c478bd9Sstevel@tonic-gate * bypass and peer-to-peer support 307c478bd9Sstevel@tonic-gate * fast track DVMA space allocation 317c478bd9Sstevel@tonic-gate * runtime DVMA debug 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate #include <sys/types.h> 347c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 357c478bd9Sstevel@tonic-gate #include <sys/async.h> 367c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 377c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 387c478bd9Sstevel@tonic-gate #include <sys/ddi_impldefs.h> 397c478bd9Sstevel@tonic-gate #include "px_obj.h" 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate /*LINTLIBRARY*/ 427c478bd9Sstevel@tonic-gate 437c478bd9Sstevel@tonic-gate /* 447c478bd9Sstevel@tonic-gate * px_dma_allocmp - Allocate a pci dma implementation structure 457c478bd9Sstevel@tonic-gate * 467c478bd9Sstevel@tonic-gate * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl 477c478bd9Sstevel@tonic-gate * to hold unmodified device limits. The ddi_dma_attr inside the 487c478bd9Sstevel@tonic-gate * ddi_dma_impl structure is augumented with system limits to enhance 497c478bd9Sstevel@tonic-gate * DVMA performance at runtime. The unaugumented device limits saved 507c478bd9Sstevel@tonic-gate * right after (accessed through (ddi_dma_attr_t *)(mp + 1)) is used 517c478bd9Sstevel@tonic-gate * strictly for peer-to-peer transfers which do not obey system limits. 527c478bd9Sstevel@tonic-gate * 537c478bd9Sstevel@tonic-gate * return: DDI_SUCCESS DDI_DMA_NORESOURCES 547c478bd9Sstevel@tonic-gate */ 557c478bd9Sstevel@tonic-gate ddi_dma_impl_t * 567c478bd9Sstevel@tonic-gate px_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t), 577c478bd9Sstevel@tonic-gate caddr_t arg) 587c478bd9Sstevel@tonic-gate { 597c478bd9Sstevel@tonic-gate register ddi_dma_impl_t *mp; 607c478bd9Sstevel@tonic-gate int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP; 617c478bd9Sstevel@tonic-gate 627c478bd9Sstevel@tonic-gate /* Caution: we don't use zalloc to enhance performance! */ 637c478bd9Sstevel@tonic-gate if ((mp = kmem_alloc(sizeof (px_dma_hdl_t), sleep)) == 0) { 647c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "can't alloc dma_handle\n"); 657c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT) { 667c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n"); 677c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp, arg, &px_kmem_clid); 687c478bd9Sstevel@tonic-gate } 697c478bd9Sstevel@tonic-gate return (mp); 707c478bd9Sstevel@tonic-gate } 717c478bd9Sstevel@tonic-gate 727c478bd9Sstevel@tonic-gate mp->dmai_rdip = rdip; 737c478bd9Sstevel@tonic-gate mp->dmai_flags = 0; 747c478bd9Sstevel@tonic-gate mp->dmai_pfnlst = NULL; 757c478bd9Sstevel@tonic-gate mp->dmai_winlst = NULL; 767c478bd9Sstevel@tonic-gate 777c478bd9Sstevel@tonic-gate /* 787c478bd9Sstevel@tonic-gate * kmem_alloc debug: the following fields are not zero-ed 797c478bd9Sstevel@tonic-gate * mp->dmai_mapping = 0; 807c478bd9Sstevel@tonic-gate * mp->dmai_size = 0; 817c478bd9Sstevel@tonic-gate * mp->dmai_offset = 0; 827c478bd9Sstevel@tonic-gate * mp->dmai_minxfer = 0; 837c478bd9Sstevel@tonic-gate * mp->dmai_burstsizes = 0; 847c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages = 0; 857c478bd9Sstevel@tonic-gate * mp->dmai_pool/roffset = 0; 867c478bd9Sstevel@tonic-gate * mp->dmai_rflags = 0; 877c478bd9Sstevel@tonic-gate * mp->dmai_inuse/flags 887c478bd9Sstevel@tonic-gate * mp->dmai_nwin = 0; 897c478bd9Sstevel@tonic-gate * mp->dmai_winsize = 0; 907c478bd9Sstevel@tonic-gate * mp->dmai_nexus_private/tte = 0; 917c478bd9Sstevel@tonic-gate * mp->dmai_iopte/pfnlst 927c478bd9Sstevel@tonic-gate * mp->dmai_sbi/pfn0 = 0; 937c478bd9Sstevel@tonic-gate * mp->dmai_minfo/winlst/fdvma 947c478bd9Sstevel@tonic-gate * mp->dmai_rdip 957c478bd9Sstevel@tonic-gate * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t)); 967c478bd9Sstevel@tonic-gate * bzero(&mp->dmai_attr, sizeof (ddi_dma_attr_t)); 977c478bd9Sstevel@tonic-gate * mp->dmai_cookie = 0; 987c478bd9Sstevel@tonic-gate */ 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION; 1017c478bd9Sstevel@tonic-gate mp->dmai_attr.dma_attr_flags = (uint_t)0; 1027c478bd9Sstevel@tonic-gate mp->dmai_fault = 0; 1037c478bd9Sstevel@tonic-gate mp->dmai_fault_check = NULL; 1047c478bd9Sstevel@tonic-gate mp->dmai_fault_notify = NULL; 105b6ec8a57Svgadre 106b6ec8a57Svgadre mp->dmai_error.err_ena = 0; 107b6ec8a57Svgadre mp->dmai_error.err_status = DDI_FM_OK; 108b6ec8a57Svgadre mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 109b6ec8a57Svgadre mp->dmai_error.err_ontrap = NULL; 110b6ec8a57Svgadre mp->dmai_error.err_fep = NULL; 11100d0963fSdilpreet mp->dmai_error.err_cf = NULL; 112b6ec8a57Svgadre 11344961713Sgirish /* 1149fc8611eSDaniel Ice * The bdf protection value is set to immediate child 1159fc8611eSDaniel Ice * at first. It gets modified by switch/bridge drivers 1169fc8611eSDaniel Ice * as the code traverses down the fabric topology. 1179fc8611eSDaniel Ice * 1189fc8611eSDaniel Ice * XXX No IOMMU protection for broken devices. 11944961713Sgirish */ 1209fc8611eSDaniel Ice ASSERT((intptr_t)ddi_get_parent_data(rdip) >> 1 == 0); 1219fc8611eSDaniel Ice mp->dmai_bdf = ((intptr_t)ddi_get_parent_data(rdip) == 1) ? 0 : 1229fc8611eSDaniel Ice pcie_get_bdf_for_dma_xfer(dip, rdip); 12344961713Sgirish 1247c478bd9Sstevel@tonic-gate return (mp); 1257c478bd9Sstevel@tonic-gate } 1267c478bd9Sstevel@tonic-gate 1277c478bd9Sstevel@tonic-gate void 1287c478bd9Sstevel@tonic-gate px_dma_freemp(ddi_dma_impl_t *mp) 1297c478bd9Sstevel@tonic-gate { 1307c478bd9Sstevel@tonic-gate if (mp->dmai_ndvmapages > 1) 1317c478bd9Sstevel@tonic-gate px_dma_freepfn(mp); 1327c478bd9Sstevel@tonic-gate if (mp->dmai_winlst) 1337c478bd9Sstevel@tonic-gate px_dma_freewin(mp); 1347c478bd9Sstevel@tonic-gate kmem_free(mp, sizeof (px_dma_hdl_t)); 1357c478bd9Sstevel@tonic-gate } 1367c478bd9Sstevel@tonic-gate 1377c478bd9Sstevel@tonic-gate void 1387c478bd9Sstevel@tonic-gate px_dma_freepfn(ddi_dma_impl_t *mp) 1397c478bd9Sstevel@tonic-gate { 1407c478bd9Sstevel@tonic-gate void *addr = mp->dmai_pfnlst; 1417c478bd9Sstevel@tonic-gate if (addr) { 1427c478bd9Sstevel@tonic-gate size_t npages = mp->dmai_ndvmapages; 1437c478bd9Sstevel@tonic-gate if (npages > 1) 1447c478bd9Sstevel@tonic-gate kmem_free(addr, npages * sizeof (px_iopfn_t)); 1457c478bd9Sstevel@tonic-gate mp->dmai_pfnlst = NULL; 1467c478bd9Sstevel@tonic-gate } 1477c478bd9Sstevel@tonic-gate mp->dmai_ndvmapages = 0; 1487c478bd9Sstevel@tonic-gate } 1497c478bd9Sstevel@tonic-gate 1507c478bd9Sstevel@tonic-gate /* 1517c478bd9Sstevel@tonic-gate * px_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits 1527c478bd9Sstevel@tonic-gate * and convert dmareq->dmar_limits to mp->dmai_attr 1537c478bd9Sstevel@tonic-gate * 1547c478bd9Sstevel@tonic-gate * ddi_dma_impl_t member modified input 1557c478bd9Sstevel@tonic-gate * ------------------------------------------------------------------------ 1567c478bd9Sstevel@tonic-gate * mp->dmai_minxfer - dev 1577c478bd9Sstevel@tonic-gate * mp->dmai_burstsizes - dev 1587c478bd9Sstevel@tonic-gate * mp->dmai_flags - no limit? peer-to-peer only? 1597c478bd9Sstevel@tonic-gate * 1607c478bd9Sstevel@tonic-gate * ddi_dma_attr member modified input 1617c478bd9Sstevel@tonic-gate * ------------------------------------------------------------------------ 1627c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo 1637c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi 1647c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta 1657c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction) 1667c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction) 1677c478bd9Sstevel@tonic-gate * 1687c478bd9Sstevel@tonic-gate * The dlim_dmaspeed member of dmareq->dmar_limits is ignored. 1697c478bd9Sstevel@tonic-gate */ 1707c478bd9Sstevel@tonic-gate ddi_dma_impl_t * 1717c478bd9Sstevel@tonic-gate px_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, px_mmu_t *mmu_p, 1727c478bd9Sstevel@tonic-gate ddi_dma_req_t *dmareq) 1737c478bd9Sstevel@tonic-gate { 1747c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp; 1757c478bd9Sstevel@tonic-gate ddi_dma_attr_t *attr_p; 1767c478bd9Sstevel@tonic-gate uint64_t syslo = mmu_p->mmu_dvma_base; 1777c478bd9Sstevel@tonic-gate uint64_t syshi = mmu_p->mmu_dvma_end; 1787c478bd9Sstevel@tonic-gate uint64_t fasthi = mmu_p->mmu_dvma_fast_end; 1797c478bd9Sstevel@tonic-gate ddi_dma_lim_t *lim_p = dmareq->dmar_limits; 1807c478bd9Sstevel@tonic-gate uint32_t count_max = lim_p->dlim_cntr_max; 1817c478bd9Sstevel@tonic-gate uint64_t lo = lim_p->dlim_addr_lo; 1827c478bd9Sstevel@tonic-gate uint64_t hi = lim_p->dlim_addr_hi; 1837c478bd9Sstevel@tonic-gate if (hi <= lo) { 1847c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "Bad limits\n"); 1857c478bd9Sstevel@tonic-gate return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING); 1867c478bd9Sstevel@tonic-gate } 1877c478bd9Sstevel@tonic-gate if (!count_max) 1887c478bd9Sstevel@tonic-gate count_max--; 1897c478bd9Sstevel@tonic-gate 1907c478bd9Sstevel@tonic-gate if (!(mp = px_dma_allocmp(dip, rdip, dmareq->dmar_fp, 1919fc8611eSDaniel Ice dmareq->dmar_arg))) 1927c478bd9Sstevel@tonic-gate return (NULL); 1937c478bd9Sstevel@tonic-gate 1947c478bd9Sstevel@tonic-gate /* store original dev input at the 2nd ddi_dma_attr */ 19536fe4a92Segillett attr_p = PX_DEV_ATTR(mp); 1967c478bd9Sstevel@tonic-gate SET_DMAATTR(attr_p, lo, hi, -1, count_max); 1977c478bd9Sstevel@tonic-gate SET_DMAALIGN(attr_p, 1); 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate lo = MAX(lo, syslo); 2007c478bd9Sstevel@tonic-gate hi = MIN(hi, syshi); 2017c478bd9Sstevel@tonic-gate if (hi <= lo) 20236fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY; 2037c478bd9Sstevel@tonic-gate count_max = MIN(count_max, hi - lo); 2047c478bd9Sstevel@tonic-gate 20536fe4a92Segillett if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1)) 20636fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT | 2079fc8611eSDaniel Ice PX_DMAI_FLAGS_NOSYSLIMIT; 2087c478bd9Sstevel@tonic-gate else { 20936fe4a92Segillett if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1)) 21036fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT; 2117c478bd9Sstevel@tonic-gate } 2127c478bd9Sstevel@tonic-gate if (PX_DMA_NOCTX(rdip)) 21336fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX; 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate /* store augumented dev input to mp->dmai_attr */ 2167c478bd9Sstevel@tonic-gate mp->dmai_burstsizes = lim_p->dlim_burstsizes; 2177c478bd9Sstevel@tonic-gate attr_p = &mp->dmai_attr; 2187c478bd9Sstevel@tonic-gate SET_DMAATTR(attr_p, lo, hi, -1, count_max); 2197c478bd9Sstevel@tonic-gate SET_DMAALIGN(attr_p, 1); 2207c478bd9Sstevel@tonic-gate return (mp); 2217c478bd9Sstevel@tonic-gate } 2227c478bd9Sstevel@tonic-gate 2237c478bd9Sstevel@tonic-gate /* 2247c478bd9Sstevel@tonic-gate * Called from px_attach to check for bypass dma support and set 2257c478bd9Sstevel@tonic-gate * flags accordingly. 2267c478bd9Sstevel@tonic-gate */ 2277c478bd9Sstevel@tonic-gate int 2287c478bd9Sstevel@tonic-gate px_dma_attach(px_t *px_p) 2297c478bd9Sstevel@tonic-gate { 2307c478bd9Sstevel@tonic-gate uint64_t baddr; 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate if (px_lib_iommu_getbypass(px_p->px_dip, 0ull, 2339fc8611eSDaniel Ice PCI_MAP_ATTR_WRITE|PCI_MAP_ATTR_READ, 2349fc8611eSDaniel Ice &baddr) != DDI_ENOTSUP) 2357c478bd9Sstevel@tonic-gate /* ignore all other errors */ 236b65731f1Skini px_p->px_dev_caps |= PX_BYPASS_DMA_ALLOWED; 2377c478bd9Sstevel@tonic-gate 23822bbbd20Saa px_p->px_dma_sync_opt = ddi_prop_get_int(DDI_DEV_T_ANY, 23922bbbd20Saa px_p->px_dip, DDI_PROP_DONTPASS, "dma-sync-options", 0); 24022bbbd20Saa 24122bbbd20Saa if (px_p->px_dma_sync_opt != 0) 24222bbbd20Saa px_p->px_dev_caps |= PX_DMA_SYNC_REQUIRED; 24322bbbd20Saa 2447c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 2457c478bd9Sstevel@tonic-gate } 2467c478bd9Sstevel@tonic-gate 2477c478bd9Sstevel@tonic-gate /* 2487c478bd9Sstevel@tonic-gate * px_dma_attr2hdl 2497c478bd9Sstevel@tonic-gate * 2507c478bd9Sstevel@tonic-gate * This routine is called from the alloc handle entry point to sanity check the 2517c478bd9Sstevel@tonic-gate * dma attribute structure. 2527c478bd9Sstevel@tonic-gate * 2537c478bd9Sstevel@tonic-gate * use by: px_dma_allochdl() 2547c478bd9Sstevel@tonic-gate * 2557c478bd9Sstevel@tonic-gate * return value: 2567c478bd9Sstevel@tonic-gate * 2577c478bd9Sstevel@tonic-gate * DDI_SUCCESS - on success 2587c478bd9Sstevel@tonic-gate * DDI_DMA_BADATTR - attribute has invalid version number 2597c478bd9Sstevel@tonic-gate * or address limits exclude dvma space 2607c478bd9Sstevel@tonic-gate */ 2617c478bd9Sstevel@tonic-gate int 2627c478bd9Sstevel@tonic-gate px_dma_attr2hdl(px_t *px_p, ddi_dma_impl_t *mp) 2637c478bd9Sstevel@tonic-gate { 2647c478bd9Sstevel@tonic-gate px_mmu_t *mmu_p = px_p->px_mmu_p; 2657c478bd9Sstevel@tonic-gate uint64_t syslo, syshi; 2667c478bd9Sstevel@tonic-gate int ret; 26736fe4a92Segillett ddi_dma_attr_t *attrp = PX_DEV_ATTR(mp); 2687c478bd9Sstevel@tonic-gate uint64_t hi = attrp->dma_attr_addr_hi; 2697c478bd9Sstevel@tonic-gate uint64_t lo = attrp->dma_attr_addr_lo; 2707c478bd9Sstevel@tonic-gate uint64_t align = attrp->dma_attr_align; 2717c478bd9Sstevel@tonic-gate uint64_t nocross = attrp->dma_attr_seg; 2727c478bd9Sstevel@tonic-gate uint64_t count_max = attrp->dma_attr_count_max; 2737c478bd9Sstevel@tonic-gate 2747c478bd9Sstevel@tonic-gate DBG(DBG_DMA_ALLOCH, px_p->px_dip, "attrp=%p cntr_max=%x.%08x\n", 2759fc8611eSDaniel Ice attrp, HI32(count_max), LO32(count_max)); 2767c478bd9Sstevel@tonic-gate DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x lo=%x.%08x\n", 2779fc8611eSDaniel Ice HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 2787c478bd9Sstevel@tonic-gate DBG(DBG_DMA_ALLOCH, px_p->px_dip, "seg=%x.%08x align=%x.%08x\n", 2799fc8611eSDaniel Ice HI32(nocross), LO32(nocross), HI32(align), LO32(align)); 2807c478bd9Sstevel@tonic-gate 2817c478bd9Sstevel@tonic-gate if (!nocross) 2827c478bd9Sstevel@tonic-gate nocross--; 2837c478bd9Sstevel@tonic-gate if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */ 2847c478bd9Sstevel@tonic-gate 2857c478bd9Sstevel@tonic-gate DBG(DBG_DMA_ALLOCH, px_p->px_dip, "bypass mode\n"); 2867c478bd9Sstevel@tonic-gate /* 2877c478bd9Sstevel@tonic-gate * If Bypass DMA is not supported, return error so that 2887c478bd9Sstevel@tonic-gate * target driver can fall back to dvma mode of operation 2897c478bd9Sstevel@tonic-gate */ 290b65731f1Skini if (!(px_p->px_dev_caps & PX_BYPASS_DMA_ALLOWED)) 2917c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR); 29236fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_BYPASSREQ; 2937c478bd9Sstevel@tonic-gate if (nocross != UINT64_MAX) 2947c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR); 2957c478bd9Sstevel@tonic-gate if (align && (align > MMU_PAGE_SIZE)) 2967c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR); 2977c478bd9Sstevel@tonic-gate align = 1; /* align on 1 page boundary */ 2987c478bd9Sstevel@tonic-gate 2997c478bd9Sstevel@tonic-gate /* do a range check and get the limits */ 30025cf1a30Sjl ret = px_lib_dma_bypass_rngchk(px_p->px_dip, attrp, 3019fc8611eSDaniel Ice &syslo, &syshi); 3027c478bd9Sstevel@tonic-gate if (ret != DDI_SUCCESS) 3037c478bd9Sstevel@tonic-gate return (ret); 3047c478bd9Sstevel@tonic-gate } else { /* MMU_XLATE or PEER_TO_PEER */ 3057c478bd9Sstevel@tonic-gate align = MAX(align, MMU_PAGE_SIZE) - 1; 3067c478bd9Sstevel@tonic-gate if ((align & nocross) != align) { 3077c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip; 3087c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned", 3099fc8611eSDaniel Ice NAMEINST(rdip)); 3107c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR); 3117c478bd9Sstevel@tonic-gate } 3127c478bd9Sstevel@tonic-gate align = MMU_BTOP(align + 1); 3137c478bd9Sstevel@tonic-gate syslo = mmu_p->mmu_dvma_base; 3147c478bd9Sstevel@tonic-gate syshi = mmu_p->mmu_dvma_end; 3157c478bd9Sstevel@tonic-gate } 3167c478bd9Sstevel@tonic-gate if (hi <= lo) { 3177c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip; 3187c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip)); 3197c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR); 3207c478bd9Sstevel@tonic-gate } 3217c478bd9Sstevel@tonic-gate lo = MAX(lo, syslo); 3227c478bd9Sstevel@tonic-gate hi = MIN(hi, syshi); 3237c478bd9Sstevel@tonic-gate if (!count_max) 3247c478bd9Sstevel@tonic-gate count_max--; 3257c478bd9Sstevel@tonic-gate 3267c478bd9Sstevel@tonic-gate DBG(DBG_DMA_ALLOCH, px_p->px_dip, "hi=%x.%08x, lo=%x.%08x\n", 3279fc8611eSDaniel Ice HI32(hi), LO32(hi), HI32(lo), LO32(lo)); 328d85c7355Scjj if (hi <= lo) { 329d85c7355Scjj /* 330d85c7355Scjj * If this is an IOMMU bypass access, the caller can't use 331d85c7355Scjj * the required addresses, so fail it. Otherwise, it's 332d85c7355Scjj * peer-to-peer; ensure that the caller has no alignment or 333d85c7355Scjj * segment size restrictions. 334d85c7355Scjj */ 335d85c7355Scjj if ((mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) || 336d85c7355Scjj (nocross < UINT32_MAX) || (align > 1)) 3377c478bd9Sstevel@tonic-gate return (DDI_DMA_BADATTR); 338d85c7355Scjj 33936fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_PEER_ONLY; 3407c478bd9Sstevel@tonic-gate } else /* set practical counter_max value */ 3417c478bd9Sstevel@tonic-gate count_max = MIN(count_max, hi - lo); 3427c478bd9Sstevel@tonic-gate 34336fe4a92Segillett if (PX_DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align)) 34436fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOSYSLIMIT | 3459fc8611eSDaniel Ice PX_DMAI_FLAGS_NOFASTLIMIT; 3467c478bd9Sstevel@tonic-gate else { 3477c478bd9Sstevel@tonic-gate syshi = mmu_p->mmu_dvma_fast_end; 34836fe4a92Segillett if (PX_DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align)) 34936fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOFASTLIMIT; 3507c478bd9Sstevel@tonic-gate } 3517c478bd9Sstevel@tonic-gate if (PX_DMA_NOCTX(mp->dmai_rdip)) 35236fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOCTX; 3537c478bd9Sstevel@tonic-gate 3547c478bd9Sstevel@tonic-gate mp->dmai_burstsizes = attrp->dma_attr_burstsizes; 3557c478bd9Sstevel@tonic-gate attrp = &mp->dmai_attr; 3567c478bd9Sstevel@tonic-gate SET_DMAATTR(attrp, lo, hi, nocross, count_max); 3577c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 3587c478bd9Sstevel@tonic-gate } 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end)) 3617c478bd9Sstevel@tonic-gate 3627c478bd9Sstevel@tonic-gate /* 3637c478bd9Sstevel@tonic-gate * px_dma_type - determine which of the three types DMA (peer-to-peer, 3647c478bd9Sstevel@tonic-gate * mmu bypass, or mmu translate) we are asked to do. 3657c478bd9Sstevel@tonic-gate * Also checks pfn0 and rejects any non-peer-to-peer 3667c478bd9Sstevel@tonic-gate * requests for peer-only devices. 3677c478bd9Sstevel@tonic-gate * 3687c478bd9Sstevel@tonic-gate * return values: 3697c478bd9Sstevel@tonic-gate * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type 3707c478bd9Sstevel@tonic-gate * DDI_SUCCESS 3717c478bd9Sstevel@tonic-gate * 3727c478bd9Sstevel@tonic-gate * dma handle members affected (set on exit): 3737c478bd9Sstevel@tonic-gate * mp->dmai_object - dmareq->dmar_object 3747c478bd9Sstevel@tonic-gate * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags 3757c478bd9Sstevel@tonic-gate * mp->dmai_flags - DMA type 3767c478bd9Sstevel@tonic-gate * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow) 3777c478bd9Sstevel@tonic-gate * mp->dmai_roffset - initialized to starting MMU page offset 3787c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages - # of total MMU pages of entire object 3797c478bd9Sstevel@tonic-gate */ 3807c478bd9Sstevel@tonic-gate int 3817c478bd9Sstevel@tonic-gate px_dma_type(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 3827c478bd9Sstevel@tonic-gate { 3837c478bd9Sstevel@tonic-gate dev_info_t *dip = px_p->px_dip; 3847c478bd9Sstevel@tonic-gate ddi_dma_obj_t *dobj_p = &dmareq->dmar_object; 3857c478bd9Sstevel@tonic-gate px_pec_t *pec_p = px_p->px_pec_p; 3867c478bd9Sstevel@tonic-gate uint32_t offset; 3877c478bd9Sstevel@tonic-gate pfn_t pfn0; 388cea92495Sgovinda uint_t redzone; 3897c478bd9Sstevel@tonic-gate 39022bbbd20Saa mp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 39122bbbd20Saa 39222bbbd20Saa if (!(px_p->px_dev_caps & PX_DMA_SYNC_REQUIRED)) 39322bbbd20Saa mp->dmai_rflags |= DMP_NOSYNC; 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate switch (dobj_p->dmao_type) { 3967c478bd9Sstevel@tonic-gate case DMA_OTYP_BUFVADDR: 3977c478bd9Sstevel@tonic-gate case DMA_OTYP_VADDR: { 3987c478bd9Sstevel@tonic-gate page_t **pplist = dobj_p->dmao_obj.virt_obj.v_priv; 3997c478bd9Sstevel@tonic-gate caddr_t vaddr = dobj_p->dmao_obj.virt_obj.v_addr; 4007c478bd9Sstevel@tonic-gate 4017c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist); 4027c478bd9Sstevel@tonic-gate offset = (ulong_t)vaddr & MMU_PAGE_OFFSET; 4037c478bd9Sstevel@tonic-gate if (pplist) { /* shadow list */ 40436fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN; 4057c478bd9Sstevel@tonic-gate pfn0 = page_pptonum(*pplist); 4067c478bd9Sstevel@tonic-gate } else { 4077c478bd9Sstevel@tonic-gate struct as *as_p = dobj_p->dmao_obj.virt_obj.v_as; 4087c478bd9Sstevel@tonic-gate struct hat *hat_p = as_p ? as_p->a_hat : kas.a_hat; 4097c478bd9Sstevel@tonic-gate pfn0 = hat_getpfnum(hat_p, vaddr); 4107c478bd9Sstevel@tonic-gate } 4117c478bd9Sstevel@tonic-gate } 4127c478bd9Sstevel@tonic-gate break; 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate case DMA_OTYP_PAGES: 4157c478bd9Sstevel@tonic-gate offset = dobj_p->dmao_obj.pp_obj.pp_offset; 41636fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_PGPFN; 4177c478bd9Sstevel@tonic-gate pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp); 4187c478bd9Sstevel@tonic-gate break; 4197c478bd9Sstevel@tonic-gate 4207c478bd9Sstevel@tonic-gate case DMA_OTYP_PADDR: 4217c478bd9Sstevel@tonic-gate default: 4227c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d requested unsupported dma type %x", 4239fc8611eSDaniel Ice NAMEINST(mp->dmai_rdip), dobj_p->dmao_type); 4247c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 4257c478bd9Sstevel@tonic-gate } 4267c478bd9Sstevel@tonic-gate if (pfn0 == PFN_INVALID) { 4277c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p", 4289fc8611eSDaniel Ice NAMEINST(dip), dobj_p); 4297c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 4307c478bd9Sstevel@tonic-gate } 4317c478bd9Sstevel@tonic-gate if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base32_pfn, 4329fc8611eSDaniel Ice pec_p->pec_last32_pfn)) { 43336fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP32; 4347c478bd9Sstevel@tonic-gate goto done; /* leave bypass and dvma flag as 0 */ 4357c478bd9Sstevel@tonic-gate } else if (TGT_PFN_INBETWEEN(pfn0, pec_p->pec_base64_pfn, 4369fc8611eSDaniel Ice pec_p->pec_last64_pfn)) { 43736fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_PTP|PX_DMAI_FLAGS_PTP64; 4387c478bd9Sstevel@tonic-gate goto done; /* leave bypass and dvma flag as 0 */ 4397c478bd9Sstevel@tonic-gate } 4407c478bd9Sstevel@tonic-gate if (PX_DMA_ISPEERONLY(mp)) { 4417c478bd9Sstevel@tonic-gate dev_info_t *rdip = mp->dmai_rdip; 4427c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip)); 4437c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 4447c478bd9Sstevel@tonic-gate } 445cea92495Sgovinda 446cea92495Sgovinda redzone = (mp->dmai_rflags & DDI_DMA_REDZONE) || 447cea92495Sgovinda (mp->dmai_flags & PX_DMAI_FLAGS_MAP_BUFZONE) ? 448cea92495Sgovinda PX_DMAI_FLAGS_REDZONE : 0; 449cea92495Sgovinda 45036fe4a92Segillett mp->dmai_flags |= (mp->dmai_flags & PX_DMAI_FLAGS_BYPASSREQ) ? 451cea92495Sgovinda PX_DMAI_FLAGS_BYPASS : (PX_DMAI_FLAGS_DVMA | redzone); 4527c478bd9Sstevel@tonic-gate done: 4537c478bd9Sstevel@tonic-gate mp->dmai_object = *dobj_p; /* whole object */ 4547c478bd9Sstevel@tonic-gate mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */ 4557c478bd9Sstevel@tonic-gate mp->dmai_roffset = offset; /* win0 pg0 offset */ 4567c478bd9Sstevel@tonic-gate mp->dmai_ndvmapages = MMU_BTOPR(offset + mp->dmai_object.dmao_size); 4577c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 4587c478bd9Sstevel@tonic-gate } 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate /* 4617c478bd9Sstevel@tonic-gate * px_dma_pgpfn - set up pfnlst array according to pages 4627c478bd9Sstevel@tonic-gate * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES 4637c478bd9Sstevel@tonic-gate */ 4647c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4657c478bd9Sstevel@tonic-gate static int 4667c478bd9Sstevel@tonic-gate px_dma_pgpfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages) 4677c478bd9Sstevel@tonic-gate { 4687c478bd9Sstevel@tonic-gate int i; 4697c478bd9Sstevel@tonic-gate dev_info_t *dip = px_p->px_dip; 4707c478bd9Sstevel@tonic-gate 4717c478bd9Sstevel@tonic-gate switch (mp->dmai_object.dmao_type) { 4727c478bd9Sstevel@tonic-gate case DMA_OTYP_BUFVADDR: 4737c478bd9Sstevel@tonic-gate case DMA_OTYP_VADDR: { 4747c478bd9Sstevel@tonic-gate page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv; 4757c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=", 4769fc8611eSDaniel Ice pplist, npages); 4777c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++) { 4787c478bd9Sstevel@tonic-gate px_iopfn_t pfn = page_pptonum(pplist[i]); 4797c478bd9Sstevel@tonic-gate PX_SET_MP_PFN1(mp, i, pfn); 4807c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 4817c478bd9Sstevel@tonic-gate } 4827c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 4837c478bd9Sstevel@tonic-gate } 4847c478bd9Sstevel@tonic-gate break; 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate case DMA_OTYP_PAGES: { 4877c478bd9Sstevel@tonic-gate page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next; 4887c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "pp=%p pfns=", pp); 4897c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++, pp = pp->p_next) { 4907c478bd9Sstevel@tonic-gate px_iopfn_t pfn = page_pptonum(pp); 4917c478bd9Sstevel@tonic-gate PX_SET_MP_PFN1(mp, i, pfn); 4927c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn); 4937c478bd9Sstevel@tonic-gate } 4947c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP|DBG_CONT, dip, "\n"); 4957c478bd9Sstevel@tonic-gate } 4967c478bd9Sstevel@tonic-gate break; 4977c478bd9Sstevel@tonic-gate 4987c478bd9Sstevel@tonic-gate default: /* check is already done by px_dma_type */ 4997c478bd9Sstevel@tonic-gate ASSERT(0); 5007c478bd9Sstevel@tonic-gate break; 5017c478bd9Sstevel@tonic-gate } 5027c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 5037c478bd9Sstevel@tonic-gate } 5047c478bd9Sstevel@tonic-gate 5057c478bd9Sstevel@tonic-gate /* 5067c478bd9Sstevel@tonic-gate * px_dma_vapfn - set up pfnlst array according to VA 5077c478bd9Sstevel@tonic-gate * VA/size pair: <normal, bypass, peer-to-peer> 5087c478bd9Sstevel@tonic-gate * pfn0 is skipped as it is already done. 5097c478bd9Sstevel@tonic-gate * In this case, the cached pfn0 is used to fill pfnlst[0] 5107c478bd9Sstevel@tonic-gate */ 5117c478bd9Sstevel@tonic-gate static int 5127c478bd9Sstevel@tonic-gate px_dma_vapfn(px_t *px_p, ddi_dma_impl_t *mp, uint_t npages) 5137c478bd9Sstevel@tonic-gate { 5147c478bd9Sstevel@tonic-gate dev_info_t *dip = px_p->px_dip; 5157c478bd9Sstevel@tonic-gate int i; 5167c478bd9Sstevel@tonic-gate caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as; 5177c478bd9Sstevel@tonic-gate struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat; 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate vaddr = mp->dmai_object.dmao_obj.virt_obj.v_addr + MMU_PAGE_SIZE; 5207c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++, vaddr += MMU_PAGE_SIZE) { 5217c478bd9Sstevel@tonic-gate px_iopfn_t pfn = hat_getpfnum(hat_p, vaddr); 5227c478bd9Sstevel@tonic-gate if (pfn == PFN_INVALID) 5237c478bd9Sstevel@tonic-gate goto err_badpfn; 5247c478bd9Sstevel@tonic-gate PX_SET_MP_PFN1(mp, i, pfn); 5257c478bd9Sstevel@tonic-gate DBG(DBG_DMA_BINDH, dip, "px_dma_vapfn: mp=%p pfnlst[%x]=%x\n", 5269fc8611eSDaniel Ice mp, i, pfn); 5277c478bd9Sstevel@tonic-gate } 5287c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 5297c478bd9Sstevel@tonic-gate err_badpfn: 5307c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr); 5317c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 5327c478bd9Sstevel@tonic-gate } 5337c478bd9Sstevel@tonic-gate 5347c478bd9Sstevel@tonic-gate /* 5357c478bd9Sstevel@tonic-gate * px_dma_pfn - Fills pfn list for all pages being DMA-ed. 5367c478bd9Sstevel@tonic-gate * 5377c478bd9Sstevel@tonic-gate * dependencies: 5387c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages - set to total # of dma pages 5397c478bd9Sstevel@tonic-gate * 5407c478bd9Sstevel@tonic-gate * return value: 5417c478bd9Sstevel@tonic-gate * DDI_SUCCESS 5427c478bd9Sstevel@tonic-gate * DDI_DMA_NOMAPPING 5437c478bd9Sstevel@tonic-gate */ 5447c478bd9Sstevel@tonic-gate int 5457c478bd9Sstevel@tonic-gate px_dma_pfn(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 5467c478bd9Sstevel@tonic-gate { 5477c478bd9Sstevel@tonic-gate uint32_t npages = mp->dmai_ndvmapages; 5487c478bd9Sstevel@tonic-gate int (*waitfp)(caddr_t) = dmareq->dmar_fp; 5497c478bd9Sstevel@tonic-gate int i, ret, peer = PX_DMA_ISPTP(mp); 5507c478bd9Sstevel@tonic-gate int peer32 = PX_DMA_ISPTP32(mp); 5517c478bd9Sstevel@tonic-gate dev_info_t *dip = px_p->px_dip; 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate px_pec_t *pec_p = px_p->px_pec_p; 5547c478bd9Sstevel@tonic-gate px_iopfn_t pfn_base = peer32 ? pec_p->pec_base32_pfn : 5559fc8611eSDaniel Ice pec_p->pec_base64_pfn; 5567c478bd9Sstevel@tonic-gate px_iopfn_t pfn_last = peer32 ? pec_p->pec_last32_pfn : 5579fc8611eSDaniel Ice pec_p->pec_last64_pfn; 5587c478bd9Sstevel@tonic-gate px_iopfn_t pfn_adj = peer ? pfn_base : 0; 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate DBG(DBG_DMA_BINDH, dip, "px_dma_pfn: mp=%p pfn0=%x\n", 5619fc8611eSDaniel Ice mp, PX_MP_PFN0(mp) - pfn_adj); 5627c478bd9Sstevel@tonic-gate /* 1 page: no array alloc/fill, no mixed mode check */ 5637c478bd9Sstevel@tonic-gate if (npages == 1) { 56436fe4a92Segillett PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); 5657c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 5667c478bd9Sstevel@tonic-gate } 5677c478bd9Sstevel@tonic-gate /* allocate pfn array */ 5687c478bd9Sstevel@tonic-gate if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (px_iopfn_t), 5699fc8611eSDaniel Ice waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) { 5707c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT) 5717c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp, dmareq->dmar_arg, 5729fc8611eSDaniel Ice &px_kmem_clid); 5737c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES); 5747c478bd9Sstevel@tonic-gate } 5757c478bd9Sstevel@tonic-gate /* fill pfn array */ 57636fe4a92Segillett PX_SET_MP_PFN(mp, 0, PX_MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */ 5777c478bd9Sstevel@tonic-gate if ((ret = PX_DMA_ISPGPFN(mp) ? px_dma_pgpfn(px_p, mp, npages) : 5789fc8611eSDaniel Ice px_dma_vapfn(px_p, mp, npages)) != DDI_SUCCESS) 5797c478bd9Sstevel@tonic-gate goto err; 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate /* skip pfn0, check mixed mode and adjust peer to peer pfn */ 5827c478bd9Sstevel@tonic-gate for (i = 1; i < npages; i++) { 5837c478bd9Sstevel@tonic-gate px_iopfn_t pfn = PX_GET_MP_PFN1(mp, i); 5847c478bd9Sstevel@tonic-gate if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) { 585b40cec45Skrishnae cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx", 5869fc8611eSDaniel Ice NAMEINST(mp->dmai_rdip), PX_MP_PFN0(mp), pfn); 5877c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING; /* mixed mode */ 5887c478bd9Sstevel@tonic-gate goto err; 5897c478bd9Sstevel@tonic-gate } 5907c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, 5919fc8611eSDaniel Ice "px_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj); 5927c478bd9Sstevel@tonic-gate if (pfn_adj) 5937c478bd9Sstevel@tonic-gate PX_SET_MP_PFN1(mp, i, pfn - pfn_adj); 5947c478bd9Sstevel@tonic-gate } 5957c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 5967c478bd9Sstevel@tonic-gate err: 5977c478bd9Sstevel@tonic-gate px_dma_freepfn(mp); 5987c478bd9Sstevel@tonic-gate return (ret); 5997c478bd9Sstevel@tonic-gate } 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate /* 6027c478bd9Sstevel@tonic-gate * px_dvma_win() - trim requested DVMA size down to window size 6037c478bd9Sstevel@tonic-gate * The 1st window starts from offset and ends at page-aligned boundary. 6047c478bd9Sstevel@tonic-gate * From the 2nd window on, each window starts and ends at page-aligned 6057c478bd9Sstevel@tonic-gate * boundary except the last window ends at wherever requested. 6067c478bd9Sstevel@tonic-gate * 6077c478bd9Sstevel@tonic-gate * accesses the following mp-> members: 6087c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_count_max 6097c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_seg 6107c478bd9Sstevel@tonic-gate * mp->dmai_roffset - start offset of 1st window 6117c478bd9Sstevel@tonic-gate * mp->dmai_rflags (redzone) 6127c478bd9Sstevel@tonic-gate * mp->dmai_ndvmapages (for 1 page fast path) 6137c478bd9Sstevel@tonic-gate * 6147c478bd9Sstevel@tonic-gate * sets the following mp-> members: 6157c478bd9Sstevel@tonic-gate * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed) 6167c478bd9Sstevel@tonic-gate * mp->dmai_winsize - window size (no redzone), n * page size (fixed) 6177c478bd9Sstevel@tonic-gate * mp->dmai_nwin - # of DMA windows of entire object (fixed) 6187c478bd9Sstevel@tonic-gate * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed) 6197c478bd9Sstevel@tonic-gate * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed) 6207c478bd9Sstevel@tonic-gate * 6217c478bd9Sstevel@tonic-gate * fixed - not changed across different DMA windows 6227c478bd9Sstevel@tonic-gate */ 6237c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6247c478bd9Sstevel@tonic-gate int 6257c478bd9Sstevel@tonic-gate px_dvma_win(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 6267c478bd9Sstevel@tonic-gate { 6271de45cd9Sgovinda uint32_t redzone_sz = PX_HAS_REDZONE(mp) ? MMU_PAGE_SIZE : 0; 6287c478bd9Sstevel@tonic-gate size_t obj_sz = mp->dmai_object.dmao_size; 6297c478bd9Sstevel@tonic-gate size_t xfer_sz; 6307c478bd9Sstevel@tonic-gate ulong_t pg_off; 6317c478bd9Sstevel@tonic-gate 6327c478bd9Sstevel@tonic-gate if ((mp->dmai_ndvmapages == 1) && !redzone_sz) { 6337c478bd9Sstevel@tonic-gate mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 6347c478bd9Sstevel@tonic-gate mp->dmai_size = obj_sz; 6357c478bd9Sstevel@tonic-gate mp->dmai_winsize = MMU_PAGE_SIZE; 6367c478bd9Sstevel@tonic-gate mp->dmai_nwin = 1; 6377c478bd9Sstevel@tonic-gate goto done; 6387c478bd9Sstevel@tonic-gate } 6397c478bd9Sstevel@tonic-gate 6407c478bd9Sstevel@tonic-gate pg_off = mp->dmai_roffset; 6417c478bd9Sstevel@tonic-gate xfer_sz = obj_sz + redzone_sz; 6427c478bd9Sstevel@tonic-gate 6439fc8611eSDaniel Ice /* include redzone in nocross check */ { 6447c478bd9Sstevel@tonic-gate uint64_t nocross = mp->dmai_attr.dma_attr_seg; 6457c478bd9Sstevel@tonic-gate if (xfer_sz + pg_off - 1 > nocross) 6467c478bd9Sstevel@tonic-gate xfer_sz = nocross - pg_off + 1; 6477c478bd9Sstevel@tonic-gate if (redzone_sz && (xfer_sz <= redzone_sz)) { 6487c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, px_p->px_dip, 6497c478bd9Sstevel@tonic-gate "nocross too small: " 6507c478bd9Sstevel@tonic-gate "%lx(%lx)+%lx+%lx < %llx\n", 6517c478bd9Sstevel@tonic-gate xfer_sz, obj_sz, pg_off, redzone_sz, nocross); 6527c478bd9Sstevel@tonic-gate return (DDI_DMA_TOOBIG); 6537c478bd9Sstevel@tonic-gate } 6547c478bd9Sstevel@tonic-gate } 6557c478bd9Sstevel@tonic-gate xfer_sz -= redzone_sz; /* restore transfer size */ 6569fc8611eSDaniel Ice /* check counter max */ { 6577c478bd9Sstevel@tonic-gate uint32_t count_max = mp->dmai_attr.dma_attr_count_max; 6587c478bd9Sstevel@tonic-gate if (xfer_sz - 1 > count_max) 6597c478bd9Sstevel@tonic-gate xfer_sz = count_max + 1; 6607c478bd9Sstevel@tonic-gate } 6617c478bd9Sstevel@tonic-gate if (xfer_sz >= obj_sz) { 6627c478bd9Sstevel@tonic-gate mp->dmai_rflags &= ~DDI_DMA_PARTIAL; 6637c478bd9Sstevel@tonic-gate mp->dmai_size = xfer_sz; 6647c478bd9Sstevel@tonic-gate mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, MMU_PAGE_SIZE); 6657c478bd9Sstevel@tonic-gate mp->dmai_nwin = 1; 6667c478bd9Sstevel@tonic-gate goto done; 6677c478bd9Sstevel@tonic-gate } 6687c478bd9Sstevel@tonic-gate if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 6697c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, px_p->px_dip, "too big: %lx+%lx+%lx > %lx\n", 6709fc8611eSDaniel Ice obj_sz, pg_off, redzone_sz, xfer_sz); 6717c478bd9Sstevel@tonic-gate return (DDI_DMA_TOOBIG); 6727c478bd9Sstevel@tonic-gate } 6737c478bd9Sstevel@tonic-gate 6747c478bd9Sstevel@tonic-gate xfer_sz = MMU_PTOB(MMU_BTOP(xfer_sz + pg_off)); /* page align */ 6757c478bd9Sstevel@tonic-gate mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */ 6767c478bd9Sstevel@tonic-gate mp->dmai_winsize = xfer_sz; /* redzone not in winsize */ 6777c478bd9Sstevel@tonic-gate mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz; 6787c478bd9Sstevel@tonic-gate done: 6797c478bd9Sstevel@tonic-gate mp->dmai_winlst = NULL; 6807c478bd9Sstevel@tonic-gate px_dump_dma_handle(DBG_DMA_MAP, px_p->px_dip, mp); 6817c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 6827c478bd9Sstevel@tonic-gate } 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate /* 6857c478bd9Sstevel@tonic-gate * fast track cache entry to mmu context, inserts 3 0 bits between 6867c478bd9Sstevel@tonic-gate * upper 6-bits and lower 3-bits of the 9-bit cache entry 6877c478bd9Sstevel@tonic-gate */ 6887c478bd9Sstevel@tonic-gate #define MMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38) 6897c478bd9Sstevel@tonic-gate 6907c478bd9Sstevel@tonic-gate /* 6917c478bd9Sstevel@tonic-gate * px_dvma_map_fast - attempts to map fast trackable DVMA 6927c478bd9Sstevel@tonic-gate */ 6937c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6947c478bd9Sstevel@tonic-gate int 6957c478bd9Sstevel@tonic-gate px_dvma_map_fast(px_mmu_t *mmu_p, ddi_dma_impl_t *mp) 6967c478bd9Sstevel@tonic-gate { 6977c478bd9Sstevel@tonic-gate uint_t clustsz = px_dvma_page_cache_clustsz; 6987c478bd9Sstevel@tonic-gate uint_t entries = px_dvma_page_cache_entries; 69925cf1a30Sjl io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 70025cf1a30Sjl mp->dmai_attr.dma_attr_flags); 7017c478bd9Sstevel@tonic-gate int i = mmu_p->mmu_dvma_addr_scan_start; 7027c478bd9Sstevel@tonic-gate uint8_t *lock_addr = mmu_p->mmu_dvma_cache_locks + i; 7037c478bd9Sstevel@tonic-gate px_dvma_addr_t dvma_pg; 7047c478bd9Sstevel@tonic-gate size_t npages = MMU_BTOP(mp->dmai_winsize); 7051de45cd9Sgovinda dev_info_t *dip = mmu_p->mmu_px_p->px_dip; 7067c478bd9Sstevel@tonic-gate 7077c478bd9Sstevel@tonic-gate extern uint8_t ldstub(uint8_t *); 7087c478bd9Sstevel@tonic-gate ASSERT(MMU_PTOB(npages) == mp->dmai_winsize); 7091de45cd9Sgovinda ASSERT(npages + PX_HAS_REDZONE(mp) <= clustsz); 7107c478bd9Sstevel@tonic-gate 7119fc8611eSDaniel Ice for (; i < entries && ldstub(lock_addr); i++, lock_addr++) 7129fc8611eSDaniel Ice ; 7137c478bd9Sstevel@tonic-gate if (i >= entries) { 7147c478bd9Sstevel@tonic-gate lock_addr = mmu_p->mmu_dvma_cache_locks; 7157c478bd9Sstevel@tonic-gate i = 0; 7169fc8611eSDaniel Ice for (; i < entries && ldstub(lock_addr); i++, lock_addr++) 7179fc8611eSDaniel Ice ; 7187c478bd9Sstevel@tonic-gate if (i >= entries) { 7197c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 7207c478bd9Sstevel@tonic-gate px_dvmaft_exhaust++; 7217c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 7227c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES); 7237c478bd9Sstevel@tonic-gate } 7247c478bd9Sstevel@tonic-gate } 7257c478bd9Sstevel@tonic-gate mmu_p->mmu_dvma_addr_scan_start = (i + 1) & (entries - 1); 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate i *= clustsz; 7287c478bd9Sstevel@tonic-gate dvma_pg = mmu_p->dvma_base_pg + i; 7297c478bd9Sstevel@tonic-gate 73044961713Sgirish if (px_lib_iommu_map(dip, PCI_TSBID(0, i), npages, 73144961713Sgirish PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, 0, 73244961713Sgirish MMU_MAP_PFN) != DDI_SUCCESS) { 7331de45cd9Sgovinda DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: " 7341de45cd9Sgovinda "px_lib_iommu_map failed\n"); 7351de45cd9Sgovinda return (DDI_FAILURE); 7361de45cd9Sgovinda } 7371de45cd9Sgovinda 7381de45cd9Sgovinda if (!PX_MAP_BUFZONE(mp)) 7391de45cd9Sgovinda goto done; 7401de45cd9Sgovinda 7411de45cd9Sgovinda DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: redzone pg=%x\n", i + npages); 7421de45cd9Sgovinda 7431de45cd9Sgovinda ASSERT(PX_HAS_REDZONE(mp)); 7441de45cd9Sgovinda 74544961713Sgirish if (px_lib_iommu_map(dip, PCI_TSBID(0, i + npages), 1, 74644961713Sgirish PX_ADD_ATTR_EXTNS(attr, mp->dmai_bdf), (void *)mp, npages - 1, 74744961713Sgirish MMU_MAP_PFN) != DDI_SUCCESS) { 7481de45cd9Sgovinda DBG(DBG_MAP_WIN, dip, "px_dvma_map_fast: " 7491de45cd9Sgovinda "mapping REDZONE page failed\n"); 7501de45cd9Sgovinda 7511de45cd9Sgovinda (void) px_lib_iommu_demap(dip, PCI_TSBID(0, i), npages); 7527c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 7531de45cd9Sgovinda } 7547c478bd9Sstevel@tonic-gate 7551de45cd9Sgovinda done: 7567c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 7577c478bd9Sstevel@tonic-gate px_dvmaft_success++; 7587c478bd9Sstevel@tonic-gate #endif 7597c478bd9Sstevel@tonic-gate mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg); 7607c478bd9Sstevel@tonic-gate mp->dmai_offset = 0; 76136fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_FASTTRACK; 7627c478bd9Sstevel@tonic-gate PX_SAVE_MP_TTE(mp, attr); /* save TTE template for unmapping */ 76336fe4a92Segillett if (PX_DVMA_DBG_ON(mmu_p)) 7647c478bd9Sstevel@tonic-gate px_dvma_alloc_debug(mmu_p, (char *)mp->dmai_mapping, 7659fc8611eSDaniel Ice mp->dmai_size, mp); 7667c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 7677c478bd9Sstevel@tonic-gate } 7687c478bd9Sstevel@tonic-gate 7697c478bd9Sstevel@tonic-gate /* 7707c478bd9Sstevel@tonic-gate * px_dvma_map: map non-fasttrack DMA 7717c478bd9Sstevel@tonic-gate * Use quantum cache if single page DMA. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate int 7747c478bd9Sstevel@tonic-gate px_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, px_mmu_t *mmu_p) 7757c478bd9Sstevel@tonic-gate { 7767c478bd9Sstevel@tonic-gate uint_t npages = PX_DMA_WINNPGS(mp); 7777c478bd9Sstevel@tonic-gate px_dvma_addr_t dvma_pg, dvma_pg_index; 7787c478bd9Sstevel@tonic-gate void *dvma_addr; 779*ef2504f2SDaniel Ice io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 78025cf1a30Sjl mp->dmai_attr.dma_attr_flags); 7817c478bd9Sstevel@tonic-gate int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP; 7827c478bd9Sstevel@tonic-gate dev_info_t *dip = mp->dmai_rdip; 7837c478bd9Sstevel@tonic-gate int ret = DDI_SUCCESS; 7847c478bd9Sstevel@tonic-gate 7857c478bd9Sstevel@tonic-gate /* 7867c478bd9Sstevel@tonic-gate * allocate dvma space resource and map in the first window. 7877c478bd9Sstevel@tonic-gate * (vmem_t *vmp, size_t size, 7887c478bd9Sstevel@tonic-gate * size_t align, size_t phase, size_t nocross, 7897c478bd9Sstevel@tonic-gate * void *minaddr, void *maxaddr, int vmflag) 7907c478bd9Sstevel@tonic-gate */ 7911de45cd9Sgovinda if ((npages == 1) && !PX_HAS_REDZONE(mp) && PX_HAS_NOSYSLIMIT(mp)) { 7927c478bd9Sstevel@tonic-gate dvma_addr = vmem_alloc(mmu_p->mmu_dvma_map, 7939fc8611eSDaniel Ice MMU_PAGE_SIZE, sleep); 79436fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_VMEMCACHE; 7957c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 7967c478bd9Sstevel@tonic-gate px_dvma_vmem_alloc++; 7977c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 7987c478bd9Sstevel@tonic-gate } else { 7997c478bd9Sstevel@tonic-gate dvma_addr = vmem_xalloc(mmu_p->mmu_dvma_map, 8009fc8611eSDaniel Ice MMU_PTOB(npages + PX_HAS_REDZONE(mp)), 8019fc8611eSDaniel Ice MAX(mp->dmai_attr.dma_attr_align, MMU_PAGE_SIZE), 8029fc8611eSDaniel Ice 0, 8039fc8611eSDaniel Ice mp->dmai_attr.dma_attr_seg + 1, 8049fc8611eSDaniel Ice (void *)mp->dmai_attr.dma_attr_addr_lo, 8059fc8611eSDaniel Ice (void *)(mp->dmai_attr.dma_attr_addr_hi + 1), 8069fc8611eSDaniel Ice sleep); 8077c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 8087c478bd9Sstevel@tonic-gate px_dvma_vmem_xalloc++; 8097c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 8107c478bd9Sstevel@tonic-gate } 8117c478bd9Sstevel@tonic-gate dvma_pg = MMU_BTOP((ulong_t)dvma_addr); 8127c478bd9Sstevel@tonic-gate dvma_pg_index = dvma_pg - mmu_p->dvma_base_pg; 8137c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n", 8149fc8611eSDaniel Ice dvma_pg, dvma_pg_index); 8157c478bd9Sstevel@tonic-gate if (dvma_pg == 0) 8167c478bd9Sstevel@tonic-gate goto noresource; 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate mp->dmai_mapping = mp->dmai_roffset | MMU_PTOB(dvma_pg); 8197c478bd9Sstevel@tonic-gate mp->dmai_offset = 0; 820*ef2504f2SDaniel Ice PX_SAVE_MP_TTE(mp, attr); /* mp->dmai_tte = tte */ 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate if ((ret = px_mmu_map_pages(mmu_p, 8237c478bd9Sstevel@tonic-gate mp, dvma_pg, npages, 0)) != DDI_SUCCESS) { 82436fe4a92Segillett if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) { 8257c478bd9Sstevel@tonic-gate vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr, 8267c478bd9Sstevel@tonic-gate MMU_PAGE_SIZE); 8277c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 8287c478bd9Sstevel@tonic-gate px_dvma_vmem_free++; 8297c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 8307c478bd9Sstevel@tonic-gate } else { 8317c478bd9Sstevel@tonic-gate vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr, 8321de45cd9Sgovinda MMU_PTOB(npages + PX_HAS_REDZONE(mp))); 8337c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 8347c478bd9Sstevel@tonic-gate px_dvma_vmem_xfree++; 8357c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 8367c478bd9Sstevel@tonic-gate } 8377c478bd9Sstevel@tonic-gate } 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate return (ret); 8407c478bd9Sstevel@tonic-gate noresource: 8417c478bd9Sstevel@tonic-gate if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 8427c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n"); 8437c478bd9Sstevel@tonic-gate ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 8449fc8611eSDaniel Ice &mmu_p->mmu_dvma_clid); 8457c478bd9Sstevel@tonic-gate } 8467c478bd9Sstevel@tonic-gate DBG(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n"); 8477c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES); 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate void 8517c478bd9Sstevel@tonic-gate px_dvma_unmap(px_mmu_t *mmu_p, ddi_dma_impl_t *mp) 8527c478bd9Sstevel@tonic-gate { 8537c478bd9Sstevel@tonic-gate px_dvma_addr_t dvma_addr = (px_dvma_addr_t)mp->dmai_mapping; 8547c478bd9Sstevel@tonic-gate px_dvma_addr_t dvma_pg = MMU_BTOP(dvma_addr); 8557c478bd9Sstevel@tonic-gate dvma_addr = MMU_PTOB(dvma_pg); 8567c478bd9Sstevel@tonic-gate 85736fe4a92Segillett if (mp->dmai_flags & PX_DMAI_FLAGS_FASTTRACK) { 8587c478bd9Sstevel@tonic-gate px_iopfn_t index = dvma_pg - mmu_p->dvma_base_pg; 8597c478bd9Sstevel@tonic-gate ASSERT(index % px_dvma_page_cache_clustsz == 0); 8607c478bd9Sstevel@tonic-gate index /= px_dvma_page_cache_clustsz; 8617c478bd9Sstevel@tonic-gate ASSERT(index < px_dvma_page_cache_entries); 8627c478bd9Sstevel@tonic-gate mmu_p->mmu_dvma_cache_locks[index] = 0; 8637c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 8647c478bd9Sstevel@tonic-gate px_dvmaft_free++; 8657c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 8667c478bd9Sstevel@tonic-gate return; 8677c478bd9Sstevel@tonic-gate } 8687c478bd9Sstevel@tonic-gate 86936fe4a92Segillett if (mp->dmai_flags & PX_DMAI_FLAGS_VMEMCACHE) { 8707c478bd9Sstevel@tonic-gate vmem_free(mmu_p->mmu_dvma_map, (void *)dvma_addr, 8719fc8611eSDaniel Ice MMU_PAGE_SIZE); 8727c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 8737c478bd9Sstevel@tonic-gate px_dvma_vmem_free++; 8747c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 8757c478bd9Sstevel@tonic-gate } else { 8761de45cd9Sgovinda size_t npages = MMU_BTOP(mp->dmai_winsize) + PX_HAS_REDZONE(mp); 8777c478bd9Sstevel@tonic-gate vmem_xfree(mmu_p->mmu_dvma_map, (void *)dvma_addr, 8789fc8611eSDaniel Ice MMU_PTOB(npages)); 8797c478bd9Sstevel@tonic-gate #ifdef PX_DMA_PROF 8807c478bd9Sstevel@tonic-gate px_dvma_vmem_xfree++; 8817c478bd9Sstevel@tonic-gate #endif /* PX_DMA_PROF */ 8827c478bd9Sstevel@tonic-gate } 8837c478bd9Sstevel@tonic-gate } 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate /* 8867c478bd9Sstevel@tonic-gate * DVMA mappings may have multiple windows, but each window always have 8877c478bd9Sstevel@tonic-gate * one segment. 8887c478bd9Sstevel@tonic-gate */ 8897c478bd9Sstevel@tonic-gate int 8907c478bd9Sstevel@tonic-gate px_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 8917c478bd9Sstevel@tonic-gate enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 8927c478bd9Sstevel@tonic-gate uint_t cache_flags) 8937c478bd9Sstevel@tonic-gate { 8947c478bd9Sstevel@tonic-gate switch (cmd) { 8957c478bd9Sstevel@tonic-gate case DDI_DMA_SYNC: 8967c478bd9Sstevel@tonic-gate return (px_lib_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 8977c478bd9Sstevel@tonic-gate *offp, *lenp, cache_flags)); 8987c478bd9Sstevel@tonic-gate 8997c478bd9Sstevel@tonic-gate case DDI_DMA_HTOC: { 9007c478bd9Sstevel@tonic-gate int ret; 9017c478bd9Sstevel@tonic-gate off_t wo_off, off = *offp; /* wo_off: wnd's obj offset */ 9027c478bd9Sstevel@tonic-gate uint_t win_size = mp->dmai_winsize; 9037c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp; 9047c478bd9Sstevel@tonic-gate 9057c478bd9Sstevel@tonic-gate if (off >= mp->dmai_object.dmao_size) { 9067c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx", 9079fc8611eSDaniel Ice NAMEINST(mp->dmai_rdip), off); 9087c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 9097c478bd9Sstevel@tonic-gate } 9107c478bd9Sstevel@tonic-gate off += mp->dmai_roffset; 9117c478bd9Sstevel@tonic-gate ret = px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 9127c478bd9Sstevel@tonic-gate off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */ 9137c478bd9Sstevel@tonic-gate if (ret) 9147c478bd9Sstevel@tonic-gate return (ret); 9157c478bd9Sstevel@tonic-gate DBG(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n", 9169fc8611eSDaniel Ice cp->dmac_address, cp->dmac_size, off, *offp); 9177c478bd9Sstevel@tonic-gate 9187c478bd9Sstevel@tonic-gate /* adjust cookie addr/len if we are not on window boundary */ 9197c478bd9Sstevel@tonic-gate ASSERT((off % win_size) == (off - 9209fc8611eSDaniel Ice (PX_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off)); 9217c478bd9Sstevel@tonic-gate off = PX_DMA_CURWIN(mp) ? off % win_size : *offp; 9227c478bd9Sstevel@tonic-gate ASSERT(cp->dmac_size > off); 9237c478bd9Sstevel@tonic-gate cp->dmac_laddress += off; 9247c478bd9Sstevel@tonic-gate cp->dmac_size -= off; 9257c478bd9Sstevel@tonic-gate DBG(DBG_DMA_CTL, dip, "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n", 9269fc8611eSDaniel Ice mp, cp->dmac_address, cp->dmac_size, off, wo_off); 9277c478bd9Sstevel@tonic-gate } 9287c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 9297c478bd9Sstevel@tonic-gate 9307c478bd9Sstevel@tonic-gate case DDI_DMA_REPWIN: 9317c478bd9Sstevel@tonic-gate *offp = mp->dmai_offset; 9327c478bd9Sstevel@tonic-gate *lenp = mp->dmai_size; 9337c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 9347c478bd9Sstevel@tonic-gate 9357c478bd9Sstevel@tonic-gate case DDI_DMA_MOVWIN: { 9367c478bd9Sstevel@tonic-gate off_t off = *offp; 9377c478bd9Sstevel@tonic-gate if (off >= mp->dmai_object.dmao_size) 9387c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 9397c478bd9Sstevel@tonic-gate off += mp->dmai_roffset; 9407c478bd9Sstevel@tonic-gate return (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 9417c478bd9Sstevel@tonic-gate off / mp->dmai_winsize, offp, lenp, 9427c478bd9Sstevel@tonic-gate (ddi_dma_cookie_t *)objp, NULL)); 9437c478bd9Sstevel@tonic-gate } 9447c478bd9Sstevel@tonic-gate 9457c478bd9Sstevel@tonic-gate case DDI_DMA_NEXTWIN: { 9467c478bd9Sstevel@tonic-gate px_window_t win = PX_DMA_CURWIN(mp); 9477c478bd9Sstevel@tonic-gate if (offp) { 9487c478bd9Sstevel@tonic-gate if (*(px_window_t *)offp != win) { 9497c478bd9Sstevel@tonic-gate /* window not active */ 9507c478bd9Sstevel@tonic-gate *(px_window_t *)objp = win; /* return cur win */ 9517c478bd9Sstevel@tonic-gate return (DDI_DMA_STALE); 9527c478bd9Sstevel@tonic-gate } 9537c478bd9Sstevel@tonic-gate win++; 9547c478bd9Sstevel@tonic-gate } else /* map win 0 */ 9557c478bd9Sstevel@tonic-gate win = 0; 9567c478bd9Sstevel@tonic-gate if (win >= mp->dmai_nwin) { 9577c478bd9Sstevel@tonic-gate *(px_window_t *)objp = win - 1; 9587c478bd9Sstevel@tonic-gate return (DDI_DMA_DONE); 9597c478bd9Sstevel@tonic-gate } 9607c478bd9Sstevel@tonic-gate if (px_dma_win(dip, rdip, (ddi_dma_handle_t)mp, 9617c478bd9Sstevel@tonic-gate win, 0, 0, 0, 0)) { 9627c478bd9Sstevel@tonic-gate *(px_window_t *)objp = win - 1; 9637c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 9647c478bd9Sstevel@tonic-gate } 9657c478bd9Sstevel@tonic-gate *(px_window_t *)objp = win; 9667c478bd9Sstevel@tonic-gate } 9677c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate case DDI_DMA_NEXTSEG: 9707c478bd9Sstevel@tonic-gate if (*(px_window_t *)offp != PX_DMA_CURWIN(mp)) 9717c478bd9Sstevel@tonic-gate return (DDI_DMA_STALE); 9727c478bd9Sstevel@tonic-gate if (lenp) /* only 1 seg allowed */ 9737c478bd9Sstevel@tonic-gate return (DDI_DMA_DONE); 9747c478bd9Sstevel@tonic-gate 9757c478bd9Sstevel@tonic-gate /* return mp as seg 0 */ 9767c478bd9Sstevel@tonic-gate *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 9777c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate case DDI_DMA_SEGTOC: 9807c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping, 9819fc8611eSDaniel Ice mp->dmai_size); 9827c478bd9Sstevel@tonic-gate *offp = mp->dmai_offset; 9837c478bd9Sstevel@tonic-gate *lenp = mp->dmai_size; 9847c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 9857c478bd9Sstevel@tonic-gate 9867c478bd9Sstevel@tonic-gate case DDI_DMA_COFF: { 9877c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp; 9887c478bd9Sstevel@tonic-gate if (cp->dmac_address < mp->dmai_mapping || 9899fc8611eSDaniel Ice (cp->dmac_address + cp->dmac_size) > 9909fc8611eSDaniel Ice (mp->dmai_mapping + mp->dmai_size)) 9917c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 9927c478bd9Sstevel@tonic-gate *objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping + 9939fc8611eSDaniel Ice mp->dmai_offset); 9947c478bd9Sstevel@tonic-gate } 9957c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 9967c478bd9Sstevel@tonic-gate default: 9977c478bd9Sstevel@tonic-gate DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 9989fc8611eSDaniel Ice cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 9997c478bd9Sstevel@tonic-gate break; 10007c478bd9Sstevel@tonic-gate } 10017c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 10027c478bd9Sstevel@tonic-gate } 10037c478bd9Sstevel@tonic-gate 10047c478bd9Sstevel@tonic-gate void 10057c478bd9Sstevel@tonic-gate px_dma_freewin(ddi_dma_impl_t *mp) 10067c478bd9Sstevel@tonic-gate { 10077c478bd9Sstevel@tonic-gate px_dma_win_t *win_p = mp->dmai_winlst, *win2_p; 10087c478bd9Sstevel@tonic-gate for (win2_p = win_p; win_p; win2_p = win_p) { 10097c478bd9Sstevel@tonic-gate win_p = win2_p->win_next; 10107c478bd9Sstevel@tonic-gate kmem_free(win2_p, sizeof (px_dma_win_t) + 10119fc8611eSDaniel Ice sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies); 10127c478bd9Sstevel@tonic-gate } 10137c478bd9Sstevel@tonic-gate mp->dmai_nwin = 0; 10147c478bd9Sstevel@tonic-gate mp->dmai_winlst = NULL; 10157c478bd9Sstevel@tonic-gate } 10167c478bd9Sstevel@tonic-gate 10177c478bd9Sstevel@tonic-gate /* 10187c478bd9Sstevel@tonic-gate * px_dma_newwin - create a dma window object and cookies 10197c478bd9Sstevel@tonic-gate * 10207c478bd9Sstevel@tonic-gate * After the initial scan in px_dma_physwin(), which identifies 10217c478bd9Sstevel@tonic-gate * a portion of the pfn array that belongs to a dma window, 10227c478bd9Sstevel@tonic-gate * we are called to allocate and initialize representing memory 10237c478bd9Sstevel@tonic-gate * resources. We know from the 1st scan the number of cookies 10247c478bd9Sstevel@tonic-gate * or dma segment in this window so we can allocate a contiguous 10257c478bd9Sstevel@tonic-gate * memory array for the dma cookies (The implementation of 10267c478bd9Sstevel@tonic-gate * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous). 10277c478bd9Sstevel@tonic-gate * 10287c478bd9Sstevel@tonic-gate * A second round scan is done on the pfn array to identify 10297c478bd9Sstevel@tonic-gate * each dma segment and initialize its corresponding dma cookie. 10307c478bd9Sstevel@tonic-gate * We don't need to do all the safety checking and we know they 10317c478bd9Sstevel@tonic-gate * all belong to the same dma window. 10327c478bd9Sstevel@tonic-gate * 10337c478bd9Sstevel@tonic-gate * Input: cookie_no - # of cookies identified by the 1st scan 10347c478bd9Sstevel@tonic-gate * start_idx - subscript of the pfn array for the starting pfn 10357c478bd9Sstevel@tonic-gate * end_idx - subscript of the last pfn in dma window 10367c478bd9Sstevel@tonic-gate * win_pp - pointer to win_next member of previous window 10377c478bd9Sstevel@tonic-gate * Return: DDI_SUCCESS - with **win_pp as newly created window object 10387c478bd9Sstevel@tonic-gate * DDI_DMA_NORESROUCE - caller frees all previous window objs 10397c478bd9Sstevel@tonic-gate * Note: Each cookie and window size are all initialized on page 10407c478bd9Sstevel@tonic-gate * boundary. This is not true for the 1st cookie of the 1st 10417c478bd9Sstevel@tonic-gate * window and the last cookie of the last window. 10427c478bd9Sstevel@tonic-gate * We fix that later in upper layer which has access to size 10437c478bd9Sstevel@tonic-gate * and offset info. 10447c478bd9Sstevel@tonic-gate * 10457c478bd9Sstevel@tonic-gate */ 10467c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10477c478bd9Sstevel@tonic-gate static int 10487c478bd9Sstevel@tonic-gate px_dma_newwin(dev_info_t *dip, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, 10497c478bd9Sstevel@tonic-gate uint32_t cookie_no, uint32_t start_idx, uint32_t end_idx, 10507c478bd9Sstevel@tonic-gate px_dma_win_t **win_pp, uint64_t count_max, uint64_t bypass) 10517c478bd9Sstevel@tonic-gate { 10527c478bd9Sstevel@tonic-gate int (*waitfp)(caddr_t) = dmareq->dmar_fp; 10537c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cookie_p; 10547c478bd9Sstevel@tonic-gate uint32_t pfn_no = 1; 10557c478bd9Sstevel@tonic-gate px_iopfn_t pfn = PX_GET_MP_PFN(mp, start_idx); 10567c478bd9Sstevel@tonic-gate px_iopfn_t prev_pfn = pfn; 10577c478bd9Sstevel@tonic-gate uint64_t baddr, seg_pfn0 = pfn; 10587c478bd9Sstevel@tonic-gate size_t sz = cookie_no * sizeof (ddi_dma_cookie_t); 10597c478bd9Sstevel@tonic-gate px_dma_win_t *win_p = kmem_zalloc(sizeof (px_dma_win_t) + sz, 10609fc8611eSDaniel Ice waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP); 106125cf1a30Sjl io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 106225cf1a30Sjl mp->dmai_attr.dma_attr_flags); 10637c478bd9Sstevel@tonic-gate 10647c478bd9Sstevel@tonic-gate if (!win_p) 10657c478bd9Sstevel@tonic-gate goto noresource; 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate win_p->win_next = NULL; 10687c478bd9Sstevel@tonic-gate win_p->win_ncookies = cookie_no; 10697c478bd9Sstevel@tonic-gate win_p->win_curseg = 0; /* start from segment 0 */ 10707c478bd9Sstevel@tonic-gate win_p->win_size = MMU_PTOB(end_idx - start_idx + 1); 10717c478bd9Sstevel@tonic-gate /* win_p->win_offset is left uninitialized */ 10727c478bd9Sstevel@tonic-gate 10737c478bd9Sstevel@tonic-gate cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 10747c478bd9Sstevel@tonic-gate start_idx++; 10757c478bd9Sstevel@tonic-gate for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) { 10767c478bd9Sstevel@tonic-gate pfn = PX_GET_MP_PFN1(mp, start_idx); 10777c478bd9Sstevel@tonic-gate if ((pfn == prev_pfn + 1) && 10789fc8611eSDaniel Ice (MMU_PTOB(pfn_no + 1) - 1 <= count_max)) 10797c478bd9Sstevel@tonic-gate continue; 10807c478bd9Sstevel@tonic-gate 10817c478bd9Sstevel@tonic-gate /* close up the cookie up to (including) prev_pfn */ 10827c478bd9Sstevel@tonic-gate baddr = MMU_PTOB(seg_pfn0); 1083a616a11eSLida.Horn if (bypass) { 1084a616a11eSLida.Horn if (px_lib_iommu_getbypass(dip, baddr, attr, &baddr) 1085a616a11eSLida.Horn == DDI_SUCCESS) 1086a616a11eSLida.Horn baddr = px_lib_ro_bypass(dip, attr, baddr); 1087a616a11eSLida.Horn else 1088a616a11eSLida.Horn return (DDI_FAILURE); 1089a616a11eSLida.Horn } 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no)); 10927c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n", 10939fc8611eSDaniel Ice MMU_PTOB(seg_pfn0), pfn_no); 10947c478bd9Sstevel@tonic-gate 10957c478bd9Sstevel@tonic-gate cookie_p++; /* advance to next available cookie cell */ 10967c478bd9Sstevel@tonic-gate pfn_no = 0; 10977c478bd9Sstevel@tonic-gate seg_pfn0 = pfn; /* start a new segment from current pfn */ 10987c478bd9Sstevel@tonic-gate } 10997c478bd9Sstevel@tonic-gate 11007c478bd9Sstevel@tonic-gate baddr = MMU_PTOB(seg_pfn0); 1101a616a11eSLida.Horn if (bypass) { 1102a616a11eSLida.Horn if (px_lib_iommu_getbypass(dip, baddr, attr, &baddr) 1103a616a11eSLida.Horn == DDI_SUCCESS) 1104a616a11eSLida.Horn baddr = px_lib_ro_bypass(dip, attr, baddr); 1105a616a11eSLida.Horn else 1106a616a11eSLida.Horn return (DDI_FAILURE); 1107a616a11eSLida.Horn } 11087c478bd9Sstevel@tonic-gate 11097c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE(cookie_p, baddr, MMU_PTOB(pfn_no)); 11107c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n", 11119fc8611eSDaniel Ice MMU_PTOB(seg_pfn0), pfn_no, cookie_no); 11127c478bd9Sstevel@tonic-gate #ifdef DEBUG 11137c478bd9Sstevel@tonic-gate cookie_p++; 11147c478bd9Sstevel@tonic-gate ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no); 11157c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 11167c478bd9Sstevel@tonic-gate *win_pp = win_p; 11177c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 11187c478bd9Sstevel@tonic-gate noresource: 11197c478bd9Sstevel@tonic-gate if (waitfp != DDI_DMA_DONTWAIT) 11207c478bd9Sstevel@tonic-gate ddi_set_callback(waitfp, dmareq->dmar_arg, &px_kmem_clid); 11217c478bd9Sstevel@tonic-gate return (DDI_DMA_NORESOURCES); 11227c478bd9Sstevel@tonic-gate } 11237c478bd9Sstevel@tonic-gate 11247c478bd9Sstevel@tonic-gate /* 11257c478bd9Sstevel@tonic-gate * px_dma_adjust - adjust 1st and last cookie and window sizes 11267c478bd9Sstevel@tonic-gate * remove initial dma page offset from 1st cookie and window size 11277c478bd9Sstevel@tonic-gate * remove last dma page remainder from last cookie and window size 11287c478bd9Sstevel@tonic-gate * fill win_offset of each dma window according to just fixed up 11297c478bd9Sstevel@tonic-gate * each window sizes 11307c478bd9Sstevel@tonic-gate * px_dma_win_t members modified: 11317c478bd9Sstevel@tonic-gate * win_p->win_offset - this window's offset within entire DMA object 11327c478bd9Sstevel@tonic-gate * win_p->win_size - xferrable size (in bytes) for this window 11337c478bd9Sstevel@tonic-gate * 11347c478bd9Sstevel@tonic-gate * ddi_dma_impl_t members modified: 11357c478bd9Sstevel@tonic-gate * mp->dmai_size - 1st window xferrable size 11367c478bd9Sstevel@tonic-gate * mp->dmai_offset - 0, which is the dma offset of the 1st window 11377c478bd9Sstevel@tonic-gate * 11387c478bd9Sstevel@tonic-gate * ddi_dma_cookie_t members modified: 11397c478bd9Sstevel@tonic-gate * cookie_p->dmac_size - 1st and last cookie remove offset or remainder 11407c478bd9Sstevel@tonic-gate * cookie_p->dmac_laddress - 1st cookie add page offset 11417c478bd9Sstevel@tonic-gate */ 11427c478bd9Sstevel@tonic-gate static void 11437c478bd9Sstevel@tonic-gate px_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, px_dma_win_t *win_p) 11447c478bd9Sstevel@tonic-gate { 11457c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 11467c478bd9Sstevel@tonic-gate size_t pg_offset = mp->dmai_roffset; 11477c478bd9Sstevel@tonic-gate size_t win_offset = 0; 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate cookie_p->dmac_size -= pg_offset; 11507c478bd9Sstevel@tonic-gate cookie_p->dmac_laddress |= pg_offset; 11517c478bd9Sstevel@tonic-gate win_p->win_size -= pg_offset; 11527c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset); 11537c478bd9Sstevel@tonic-gate 11547c478bd9Sstevel@tonic-gate mp->dmai_size = win_p->win_size; 11557c478bd9Sstevel@tonic-gate mp->dmai_offset = 0; 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate pg_offset += mp->dmai_object.dmao_size; 11587c478bd9Sstevel@tonic-gate pg_offset &= MMU_PAGE_OFFSET; 11597c478bd9Sstevel@tonic-gate if (pg_offset) 11607c478bd9Sstevel@tonic-gate pg_offset = MMU_PAGE_SIZE - pg_offset; 11617c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset); 11627c478bd9Sstevel@tonic-gate 11637c478bd9Sstevel@tonic-gate for (; win_p->win_next; win_p = win_p->win_next) { 11647c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset); 11657c478bd9Sstevel@tonic-gate win_p->win_offset = win_offset; 11667c478bd9Sstevel@tonic-gate win_offset += win_p->win_size; 11677c478bd9Sstevel@tonic-gate } 11687c478bd9Sstevel@tonic-gate /* last window */ 11697c478bd9Sstevel@tonic-gate win_p->win_offset = win_offset; 11707c478bd9Sstevel@tonic-gate cookie_p = (ddi_dma_cookie_t *)(win_p + 1); 11717c478bd9Sstevel@tonic-gate cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset; 11727c478bd9Sstevel@tonic-gate win_p->win_size -= pg_offset; 11737c478bd9Sstevel@tonic-gate ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size); 11747c478bd9Sstevel@tonic-gate } 11757c478bd9Sstevel@tonic-gate 11767c478bd9Sstevel@tonic-gate /* 11777c478bd9Sstevel@tonic-gate * px_dma_physwin() - carve up dma windows using physical addresses. 11787c478bd9Sstevel@tonic-gate * Called to handle mmu bypass and pci peer-to-peer transfers. 11797c478bd9Sstevel@tonic-gate * Calls px_dma_newwin() to allocate window objects. 11807c478bd9Sstevel@tonic-gate * 11817c478bd9Sstevel@tonic-gate * Dependency: mp->dmai_pfnlst points to an array of pfns 11827c478bd9Sstevel@tonic-gate * 11837c478bd9Sstevel@tonic-gate * 1. Each dma window is represented by a px_dma_win_t object. 11847c478bd9Sstevel@tonic-gate * The object will be casted to ddi_dma_win_t and returned 11857c478bd9Sstevel@tonic-gate * to leaf driver through the DDI interface. 11867c478bd9Sstevel@tonic-gate * 2. Each dma window can have several dma segments with each 11877c478bd9Sstevel@tonic-gate * segment representing a physically contiguous either memory 11887c478bd9Sstevel@tonic-gate * space (if we are doing an mmu bypass transfer) or pci address 11897c478bd9Sstevel@tonic-gate * space (if we are doing a peer-to-peer transfer). 11907c478bd9Sstevel@tonic-gate * 3. Each segment has a DMA cookie to program the DMA engine. 11917c478bd9Sstevel@tonic-gate * The cookies within each DMA window must be located in a 11927c478bd9Sstevel@tonic-gate * contiguous array per ddi_dma_nextcookie(9f). 11937c478bd9Sstevel@tonic-gate * 4. The number of DMA segments within each DMA window cannot exceed 11947c478bd9Sstevel@tonic-gate * mp->dmai_attr.dma_attr_sgllen. If the transfer size is 11957c478bd9Sstevel@tonic-gate * too large to fit in the sgllen, the rest needs to be 11967c478bd9Sstevel@tonic-gate * relocated to the next dma window. 11977c478bd9Sstevel@tonic-gate * 5. Peer-to-peer DMA segment follows device hi, lo, count_max, 11987c478bd9Sstevel@tonic-gate * and nocross restrictions while bypass DMA follows the set of 11997c478bd9Sstevel@tonic-gate * restrictions with system limits factored in. 12007c478bd9Sstevel@tonic-gate * 12017c478bd9Sstevel@tonic-gate * Return: 12027c478bd9Sstevel@tonic-gate * mp->dmai_winlst - points to a link list of px_dma_win_t objects. 12037c478bd9Sstevel@tonic-gate * Each px_dma_win_t object on the link list contains 12047c478bd9Sstevel@tonic-gate * infomation such as its window size (# of pages), 12057c478bd9Sstevel@tonic-gate * starting offset (also see Restriction), an array of 12067c478bd9Sstevel@tonic-gate * DMA cookies, and # of cookies in the array. 12077c478bd9Sstevel@tonic-gate * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory. 12087c478bd9Sstevel@tonic-gate * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst. 12097c478bd9Sstevel@tonic-gate * mp->dmai_mapping - starting cookie address 12107c478bd9Sstevel@tonic-gate * mp->dmai_rflags - consistent, nosync, no redzone 12117c478bd9Sstevel@tonic-gate * mp->dmai_cookie - start of cookie table of the 1st DMA window 12127c478bd9Sstevel@tonic-gate * 12137c478bd9Sstevel@tonic-gate * Restriction: 12147c478bd9Sstevel@tonic-gate * Each px_dma_win_t object can theoratically start from any offset 12157c478bd9Sstevel@tonic-gate * since the mmu is not involved. However, this implementation 12167c478bd9Sstevel@tonic-gate * always make windows start from page aligned offset (except 12177c478bd9Sstevel@tonic-gate * the 1st window, which follows the requested offset) due to the 12187c478bd9Sstevel@tonic-gate * fact that we are handed a pfn list. This does require device's 12197c478bd9Sstevel@tonic-gate * count_max and attr_seg to be at least MMU_PAGE_SIZE aligned. 12207c478bd9Sstevel@tonic-gate */ 12217c478bd9Sstevel@tonic-gate int 12227c478bd9Sstevel@tonic-gate px_dma_physwin(px_t *px_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp) 12237c478bd9Sstevel@tonic-gate { 12247c478bd9Sstevel@tonic-gate uint_t npages = mp->dmai_ndvmapages; 12257c478bd9Sstevel@tonic-gate int ret, sgllen = mp->dmai_attr.dma_attr_sgllen; 12267c478bd9Sstevel@tonic-gate px_iopfn_t pfn_lo, pfn_hi, prev_pfn; 12277c478bd9Sstevel@tonic-gate px_iopfn_t pfn = PX_GET_MP_PFN(mp, 0); 12287c478bd9Sstevel@tonic-gate uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0; 12297c478bd9Sstevel@tonic-gate uint64_t count_max, bypass_addr = 0; 12307c478bd9Sstevel@tonic-gate px_dma_win_t **win_pp = (px_dma_win_t **)&mp->dmai_winlst; 12317c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cookie0_p; 123225cf1a30Sjl io_attributes_t attr = PX_GET_TTE_ATTR(mp->dmai_rflags, 123325cf1a30Sjl mp->dmai_attr.dma_attr_flags); 12347c478bd9Sstevel@tonic-gate dev_info_t *dip = px_p->px_dip; 12357c478bd9Sstevel@tonic-gate 12367c478bd9Sstevel@tonic-gate ASSERT(PX_DMA_ISPTP(mp) || PX_DMA_ISBYPASS(mp)); 12377c478bd9Sstevel@tonic-gate if (PX_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */ 123836fe4a92Segillett ddi_dma_attr_t *dev_attr_p = PX_DEV_ATTR(mp); 12397c478bd9Sstevel@tonic-gate uint64_t nocross = dev_attr_p->dma_attr_seg; 12407c478bd9Sstevel@tonic-gate px_pec_t *pec_p = px_p->px_pec_p; 12417c478bd9Sstevel@tonic-gate px_iopfn_t pfn_last = PX_DMA_ISPTP32(mp) ? 12429fc8611eSDaniel Ice pec_p->pec_last32_pfn - pec_p->pec_base32_pfn : 12439fc8611eSDaniel Ice pec_p->pec_last64_pfn - pec_p->pec_base64_pfn; 12447c478bd9Sstevel@tonic-gate 12457c478bd9Sstevel@tonic-gate if (nocross && (nocross < UINT32_MAX)) 12467c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 12477c478bd9Sstevel@tonic-gate if (dev_attr_p->dma_attr_align > MMU_PAGE_SIZE) 12487c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 12497c478bd9Sstevel@tonic-gate pfn_lo = MMU_BTOP(dev_attr_p->dma_attr_addr_lo); 12507c478bd9Sstevel@tonic-gate pfn_hi = MMU_BTOP(dev_attr_p->dma_attr_addr_hi); 12517c478bd9Sstevel@tonic-gate pfn_hi = MIN(pfn_hi, pfn_last); 12527c478bd9Sstevel@tonic-gate if ((pfn_lo > pfn_hi) || (pfn < pfn_lo)) 12537c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 12547c478bd9Sstevel@tonic-gate 12557c478bd9Sstevel@tonic-gate count_max = dev_attr_p->dma_attr_count_max; 12567c478bd9Sstevel@tonic-gate count_max = MIN(count_max, nocross); 12577c478bd9Sstevel@tonic-gate /* 12587c478bd9Sstevel@tonic-gate * the following count_max trim is not done because we are 12597c478bd9Sstevel@tonic-gate * making sure pfn_lo <= pfn <= pfn_hi inside the loop 12607c478bd9Sstevel@tonic-gate * count_max=MIN(count_max, MMU_PTOB(pfn_hi - pfn_lo + 1)-1); 12617c478bd9Sstevel@tonic-gate */ 12627c478bd9Sstevel@tonic-gate } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */ 12637c478bd9Sstevel@tonic-gate count_max = mp->dmai_attr.dma_attr_count_max; 12647c478bd9Sstevel@tonic-gate pfn_lo = MMU_BTOP(mp->dmai_attr.dma_attr_addr_lo); 12657c478bd9Sstevel@tonic-gate pfn_hi = MMU_BTOP(mp->dmai_attr.dma_attr_addr_hi); 12667c478bd9Sstevel@tonic-gate 12677c478bd9Sstevel@tonic-gate if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), 12689fc8611eSDaniel Ice attr, &bypass_addr) != DDI_SUCCESS) { 126903494a98SBill Taylor DBG(DBG_BYPASS, mp->dmai_rdip, 127003494a98SBill Taylor "bypass cookie failure %lx\n", pfn); 12717c478bd9Sstevel@tonic-gate return (DDI_DMA_NOMAPPING); 12727c478bd9Sstevel@tonic-gate } 12737c478bd9Sstevel@tonic-gate pfn = MMU_BTOP(bypass_addr); 12747c478bd9Sstevel@tonic-gate } 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate /* pfn: absolute (bypass mode) or relative (p2p mode) */ 12777c478bd9Sstevel@tonic-gate for (prev_pfn = pfn, i = 1; i < npages; 12787c478bd9Sstevel@tonic-gate i++, prev_pfn = pfn, pfn_no++) { 12797c478bd9Sstevel@tonic-gate pfn = PX_GET_MP_PFN1(mp, i); 12807c478bd9Sstevel@tonic-gate if (bypass_addr) { 12817c478bd9Sstevel@tonic-gate if (px_lib_iommu_getbypass(dip, MMU_PTOB(pfn), attr, 12829fc8611eSDaniel Ice &bypass_addr) != DDI_SUCCESS) { 12837c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING; 12847c478bd9Sstevel@tonic-gate goto err; 12857c478bd9Sstevel@tonic-gate } 12867c478bd9Sstevel@tonic-gate pfn = MMU_BTOP(bypass_addr); 12877c478bd9Sstevel@tonic-gate } 12887c478bd9Sstevel@tonic-gate if ((pfn == prev_pfn + 1) && 12899fc8611eSDaniel Ice (MMU_PTOB(pfn_no + 1) - 1 <= count_max)) 12907c478bd9Sstevel@tonic-gate continue; 12917c478bd9Sstevel@tonic-gate if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) { 12927c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING; 12937c478bd9Sstevel@tonic-gate goto err; 12947c478bd9Sstevel@tonic-gate } 12957c478bd9Sstevel@tonic-gate cookie_no++; 12967c478bd9Sstevel@tonic-gate pfn_no = 0; 12977c478bd9Sstevel@tonic-gate if (cookie_no < sgllen) 12987c478bd9Sstevel@tonic-gate continue; 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 13019fc8611eSDaniel Ice win_pfn0_index, i - 1, cookie_no); 13027c478bd9Sstevel@tonic-gate if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, 13039fc8611eSDaniel Ice win_pfn0_index, i - 1, win_pp, count_max, bypass_addr)) 13047c478bd9Sstevel@tonic-gate goto err; 13057c478bd9Sstevel@tonic-gate 13067c478bd9Sstevel@tonic-gate win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */ 13077c478bd9Sstevel@tonic-gate win_no++; 13087c478bd9Sstevel@tonic-gate win_pfn0_index = i; 13097c478bd9Sstevel@tonic-gate cookie_no = 0; 13107c478bd9Sstevel@tonic-gate } 13117c478bd9Sstevel@tonic-gate if (pfn > pfn_hi) { 13127c478bd9Sstevel@tonic-gate ret = DDI_DMA_NOMAPPING; 13137c478bd9Sstevel@tonic-gate goto err; 13147c478bd9Sstevel@tonic-gate } 13157c478bd9Sstevel@tonic-gate cookie_no++; 13167c478bd9Sstevel@tonic-gate DBG(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n", 13179fc8611eSDaniel Ice win_pfn0_index, i - 1, cookie_no); 13187c478bd9Sstevel@tonic-gate if (ret = px_dma_newwin(dip, dmareq, mp, cookie_no, win_pfn0_index, 13199fc8611eSDaniel Ice i - 1, win_pp, count_max, bypass_addr)) 13207c478bd9Sstevel@tonic-gate goto err; 13217c478bd9Sstevel@tonic-gate win_no++; 13227c478bd9Sstevel@tonic-gate px_dma_adjust(dmareq, mp, mp->dmai_winlst); 13237c478bd9Sstevel@tonic-gate mp->dmai_nwin = win_no; 13247c478bd9Sstevel@tonic-gate mp->dmai_rflags |= DDI_DMA_CONSISTENT | DMP_NOSYNC; 13257c478bd9Sstevel@tonic-gate mp->dmai_rflags &= ~DDI_DMA_REDZONE; 132636fe4a92Segillett mp->dmai_flags |= PX_DMAI_FLAGS_NOSYNC; 132736fe4a92Segillett cookie0_p = (ddi_dma_cookie_t *)(PX_WINLST(mp) + 1); 132836fe4a92Segillett mp->dmai_cookie = PX_WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0; 13297c478bd9Sstevel@tonic-gate mp->dmai_mapping = cookie0_p->dmac_laddress; 13307c478bd9Sstevel@tonic-gate 13317c478bd9Sstevel@tonic-gate px_dma_freepfn(mp); 13327c478bd9Sstevel@tonic-gate return (DDI_DMA_MAPPED); 13337c478bd9Sstevel@tonic-gate err: 13347c478bd9Sstevel@tonic-gate px_dma_freewin(mp); 13357c478bd9Sstevel@tonic-gate return (ret); 13367c478bd9Sstevel@tonic-gate } 13377c478bd9Sstevel@tonic-gate 13387c478bd9Sstevel@tonic-gate int 13397c478bd9Sstevel@tonic-gate px_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp, 13407c478bd9Sstevel@tonic-gate enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp, 13417c478bd9Sstevel@tonic-gate uint_t cache_flags) 13427c478bd9Sstevel@tonic-gate { 13437c478bd9Sstevel@tonic-gate switch (cmd) { 13447c478bd9Sstevel@tonic-gate case DDI_DMA_SYNC: 13457c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 13467c478bd9Sstevel@tonic-gate 13477c478bd9Sstevel@tonic-gate case DDI_DMA_HTOC: { 13487c478bd9Sstevel@tonic-gate off_t off = *offp; 13497c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *loop_cp, *cp; 13507c478bd9Sstevel@tonic-gate px_dma_win_t *win_p = mp->dmai_winlst; 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate if (off >= mp->dmai_object.dmao_size) 13537c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate /* locate window */ 13567c478bd9Sstevel@tonic-gate while (win_p->win_offset + win_p->win_size <= off) 13577c478bd9Sstevel@tonic-gate win_p = win_p->win_next; 13587c478bd9Sstevel@tonic-gate 13597c478bd9Sstevel@tonic-gate loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1); 13607c478bd9Sstevel@tonic-gate mp->dmai_offset = win_p->win_offset; 13617c478bd9Sstevel@tonic-gate mp->dmai_size = win_p->win_size; 13627c478bd9Sstevel@tonic-gate mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */ 13637c478bd9Sstevel@tonic-gate 13647c478bd9Sstevel@tonic-gate /* adjust cookie addr/len if we are not on cookie boundary */ 13657c478bd9Sstevel@tonic-gate off -= win_p->win_offset; /* offset within window */ 13667c478bd9Sstevel@tonic-gate for (; off >= loop_cp->dmac_size; loop_cp++) 13677c478bd9Sstevel@tonic-gate off -= loop_cp->dmac_size; /* offset within cookie */ 13687c478bd9Sstevel@tonic-gate 13697c478bd9Sstevel@tonic-gate mp->dmai_cookie = loop_cp + 1; 13707c478bd9Sstevel@tonic-gate win_p->win_curseg = loop_cp - cp; 13717c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)objp; 13727c478bd9Sstevel@tonic-gate MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off, 13739fc8611eSDaniel Ice loop_cp->dmac_size - off); 13747c478bd9Sstevel@tonic-gate 13757c478bd9Sstevel@tonic-gate DBG(DBG_DMA_CTL, dip, 13769fc8611eSDaniel Ice "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 13779fc8611eSDaniel Ice cp->dmac_laddress, cp->dmac_size); 13787c478bd9Sstevel@tonic-gate } 13797c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate case DDI_DMA_REPWIN: 13827c478bd9Sstevel@tonic-gate *offp = mp->dmai_offset; 13837c478bd9Sstevel@tonic-gate *lenp = mp->dmai_size; 13847c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 13857c478bd9Sstevel@tonic-gate 13867c478bd9Sstevel@tonic-gate case DDI_DMA_MOVWIN: { 13877c478bd9Sstevel@tonic-gate off_t off = *offp; 13887c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp; 13897c478bd9Sstevel@tonic-gate px_dma_win_t *win_p = mp->dmai_winlst; 13907c478bd9Sstevel@tonic-gate 13917c478bd9Sstevel@tonic-gate if (off >= mp->dmai_object.dmao_size) 13927c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 13937c478bd9Sstevel@tonic-gate 13947c478bd9Sstevel@tonic-gate /* locate window */ 13957c478bd9Sstevel@tonic-gate while (win_p->win_offset + win_p->win_size <= off) 13967c478bd9Sstevel@tonic-gate win_p = win_p->win_next; 13977c478bd9Sstevel@tonic-gate 13987c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)(win_p + 1); 13997c478bd9Sstevel@tonic-gate mp->dmai_offset = win_p->win_offset; 14007c478bd9Sstevel@tonic-gate mp->dmai_size = win_p->win_size; 14017c478bd9Sstevel@tonic-gate mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 14027c478bd9Sstevel@tonic-gate mp->dmai_cookie = cp + 1; 14037c478bd9Sstevel@tonic-gate win_p->win_curseg = 0; 14047c478bd9Sstevel@tonic-gate 14057c478bd9Sstevel@tonic-gate *(ddi_dma_cookie_t *)objp = *cp; 14067c478bd9Sstevel@tonic-gate *offp = win_p->win_offset; 14077c478bd9Sstevel@tonic-gate *lenp = win_p->win_size; 14087c478bd9Sstevel@tonic-gate DBG(DBG_DMA_CTL, dip, 14099fc8611eSDaniel Ice "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n", 14109fc8611eSDaniel Ice cp->dmac_laddress, cp->dmac_size); 14117c478bd9Sstevel@tonic-gate } 14127c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 14137c478bd9Sstevel@tonic-gate 14147c478bd9Sstevel@tonic-gate case DDI_DMA_NEXTWIN: { 14157c478bd9Sstevel@tonic-gate px_dma_win_t *win_p = *(px_dma_win_t **)offp; 14167c478bd9Sstevel@tonic-gate px_dma_win_t **nw_pp = (px_dma_win_t **)objp; 14177c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp; 14187c478bd9Sstevel@tonic-gate if (!win_p) { 14197c478bd9Sstevel@tonic-gate *nw_pp = mp->dmai_winlst; 14207c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 14217c478bd9Sstevel@tonic-gate } 14227c478bd9Sstevel@tonic-gate 14237c478bd9Sstevel@tonic-gate if (win_p->win_offset != mp->dmai_offset) 14247c478bd9Sstevel@tonic-gate return (DDI_DMA_STALE); 14257c478bd9Sstevel@tonic-gate if (!win_p->win_next) 14267c478bd9Sstevel@tonic-gate return (DDI_DMA_DONE); 14277c478bd9Sstevel@tonic-gate win_p = win_p->win_next; 14287c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)(win_p + 1); 14297c478bd9Sstevel@tonic-gate mp->dmai_offset = win_p->win_offset; 14307c478bd9Sstevel@tonic-gate mp->dmai_size = win_p->win_size; 14317c478bd9Sstevel@tonic-gate mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */ 14327c478bd9Sstevel@tonic-gate mp->dmai_cookie = cp + 1; 14337c478bd9Sstevel@tonic-gate win_p->win_curseg = 0; 14347c478bd9Sstevel@tonic-gate *nw_pp = win_p; 14357c478bd9Sstevel@tonic-gate } 14367c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 14377c478bd9Sstevel@tonic-gate 14387c478bd9Sstevel@tonic-gate case DDI_DMA_NEXTSEG: { 14397c478bd9Sstevel@tonic-gate px_dma_win_t *w_p = *(px_dma_win_t **)offp; 14407c478bd9Sstevel@tonic-gate if (w_p->win_offset != mp->dmai_offset) 14417c478bd9Sstevel@tonic-gate return (DDI_DMA_STALE); 14427c478bd9Sstevel@tonic-gate if (w_p->win_curseg + 1 >= w_p->win_ncookies) 14437c478bd9Sstevel@tonic-gate return (DDI_DMA_DONE); 14447c478bd9Sstevel@tonic-gate w_p->win_curseg++; 14457c478bd9Sstevel@tonic-gate } 14467c478bd9Sstevel@tonic-gate *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp; 14477c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 14487c478bd9Sstevel@tonic-gate 14497c478bd9Sstevel@tonic-gate case DDI_DMA_SEGTOC: { 14507c478bd9Sstevel@tonic-gate px_dma_win_t *win_p = mp->dmai_winlst; 14517c478bd9Sstevel@tonic-gate off_t off = mp->dmai_offset; 14527c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp; 14537c478bd9Sstevel@tonic-gate int i; 14547c478bd9Sstevel@tonic-gate 14557c478bd9Sstevel@tonic-gate /* locate active window */ 14569fc8611eSDaniel Ice for (; win_p->win_offset != off; win_p = win_p->win_next) 14579fc8611eSDaniel Ice ; 14587c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)(win_p + 1); 14597c478bd9Sstevel@tonic-gate for (i = 0; i < win_p->win_curseg; i++, cp++) 14607c478bd9Sstevel@tonic-gate off += cp->dmac_size; 14617c478bd9Sstevel@tonic-gate *offp = off; 14627c478bd9Sstevel@tonic-gate *lenp = cp->dmac_size; 14637c478bd9Sstevel@tonic-gate *(ddi_dma_cookie_t *)objp = *cp; /* copy cookie */ 14647c478bd9Sstevel@tonic-gate } 14657c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 14667c478bd9Sstevel@tonic-gate 14677c478bd9Sstevel@tonic-gate case DDI_DMA_COFF: { 14687c478bd9Sstevel@tonic-gate px_dma_win_t *win_p; 14697c478bd9Sstevel@tonic-gate ddi_dma_cookie_t *cp; 14707c478bd9Sstevel@tonic-gate uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress; 14717c478bd9Sstevel@tonic-gate size_t win_off; 14727c478bd9Sstevel@tonic-gate 14737c478bd9Sstevel@tonic-gate for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) { 14747c478bd9Sstevel@tonic-gate int i; 14757c478bd9Sstevel@tonic-gate win_off = 0; 14767c478bd9Sstevel@tonic-gate cp = (ddi_dma_cookie_t *)(win_p + 1); 14777c478bd9Sstevel@tonic-gate for (i = 0; i < win_p->win_ncookies; i++, cp++) { 14787c478bd9Sstevel@tonic-gate size_t sz = cp->dmac_size; 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate addr = cp->dmac_laddress; 14817c478bd9Sstevel@tonic-gate if ((addr <= key) && (addr + sz >= key)) 14827c478bd9Sstevel@tonic-gate goto found; 14837c478bd9Sstevel@tonic-gate win_off += sz; 14847c478bd9Sstevel@tonic-gate } 14857c478bd9Sstevel@tonic-gate } 14867c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 14877c478bd9Sstevel@tonic-gate found: 14887c478bd9Sstevel@tonic-gate *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr)); 14897c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 14907c478bd9Sstevel@tonic-gate } 14917c478bd9Sstevel@tonic-gate default: 14927c478bd9Sstevel@tonic-gate DBG(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n", 14939fc8611eSDaniel Ice cmd, ddi_driver_name(rdip), ddi_get_instance(rdip)); 14947c478bd9Sstevel@tonic-gate break; 14957c478bd9Sstevel@tonic-gate } 14967c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 14977c478bd9Sstevel@tonic-gate } 14987c478bd9Sstevel@tonic-gate 14997c478bd9Sstevel@tonic-gate static void 15007c478bd9Sstevel@tonic-gate px_dvma_debug_init(px_mmu_t *mmu_p) 15017c478bd9Sstevel@tonic-gate { 15027c478bd9Sstevel@tonic-gate size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec; 15037c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&mmu_p->dvma_debug_lock)); 15047c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "PCI Express DVMA %p stat ON", mmu_p); 15057c478bd9Sstevel@tonic-gate 15067c478bd9Sstevel@tonic-gate mmu_p->dvma_alloc_rec = kmem_alloc(sz, KM_SLEEP); 15077c478bd9Sstevel@tonic-gate mmu_p->dvma_free_rec = kmem_alloc(sz, KM_SLEEP); 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate mmu_p->dvma_active_list = NULL; 15107c478bd9Sstevel@tonic-gate mmu_p->dvma_alloc_rec_index = 0; 15117c478bd9Sstevel@tonic-gate mmu_p->dvma_free_rec_index = 0; 15127c478bd9Sstevel@tonic-gate mmu_p->dvma_active_count = 0; 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate 15157c478bd9Sstevel@tonic-gate void 15167c478bd9Sstevel@tonic-gate px_dvma_debug_fini(px_mmu_t *mmu_p) 15177c478bd9Sstevel@tonic-gate { 15187c478bd9Sstevel@tonic-gate struct px_dvma_rec *prev, *ptr; 15197c478bd9Sstevel@tonic-gate size_t sz = sizeof (struct px_dvma_rec) * px_dvma_debug_rec; 15207c478bd9Sstevel@tonic-gate uint64_t mask = ~(1ull << mmu_p->mmu_inst); 15217c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "PCI Express DVMA %p stat OFF", mmu_p); 15227c478bd9Sstevel@tonic-gate 15231f7be8d9Sdanice if (mmu_p->dvma_alloc_rec) { 15241f7be8d9Sdanice kmem_free(mmu_p->dvma_alloc_rec, sz); 15251f7be8d9Sdanice mmu_p->dvma_alloc_rec = NULL; 15261f7be8d9Sdanice } 15271f7be8d9Sdanice if (mmu_p->dvma_free_rec) { 15281f7be8d9Sdanice kmem_free(mmu_p->dvma_free_rec, sz); 15291f7be8d9Sdanice mmu_p->dvma_free_rec = NULL; 15301f7be8d9Sdanice } 15317c478bd9Sstevel@tonic-gate 15327c478bd9Sstevel@tonic-gate prev = mmu_p->dvma_active_list; 15337c478bd9Sstevel@tonic-gate if (!prev) 15347c478bd9Sstevel@tonic-gate return; 15357c478bd9Sstevel@tonic-gate for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next) 15367c478bd9Sstevel@tonic-gate kmem_free(prev, sizeof (struct px_dvma_rec)); 15377c478bd9Sstevel@tonic-gate kmem_free(prev, sizeof (struct px_dvma_rec)); 15387c478bd9Sstevel@tonic-gate 15397c478bd9Sstevel@tonic-gate mmu_p->dvma_active_list = NULL; 15407c478bd9Sstevel@tonic-gate mmu_p->dvma_alloc_rec_index = 0; 15417c478bd9Sstevel@tonic-gate mmu_p->dvma_free_rec_index = 0; 15427c478bd9Sstevel@tonic-gate mmu_p->dvma_active_count = 0; 15437c478bd9Sstevel@tonic-gate 15447c478bd9Sstevel@tonic-gate px_dvma_debug_off &= mask; 15457c478bd9Sstevel@tonic-gate px_dvma_debug_on &= mask; 15467c478bd9Sstevel@tonic-gate } 15477c478bd9Sstevel@tonic-gate 15487c478bd9Sstevel@tonic-gate void 15497c478bd9Sstevel@tonic-gate px_dvma_alloc_debug(px_mmu_t *mmu_p, char *address, uint_t len, 15507c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp) 15517c478bd9Sstevel@tonic-gate { 15527c478bd9Sstevel@tonic-gate struct px_dvma_rec *ptr; 15537c478bd9Sstevel@tonic-gate mutex_enter(&mmu_p->dvma_debug_lock); 15547c478bd9Sstevel@tonic-gate 15557c478bd9Sstevel@tonic-gate if (!mmu_p->dvma_alloc_rec) 15567c478bd9Sstevel@tonic-gate px_dvma_debug_init(mmu_p); 155736fe4a92Segillett if (PX_DVMA_DBG_OFF(mmu_p)) { 15587c478bd9Sstevel@tonic-gate px_dvma_debug_fini(mmu_p); 15597c478bd9Sstevel@tonic-gate goto done; 15607c478bd9Sstevel@tonic-gate } 15617c478bd9Sstevel@tonic-gate 15627c478bd9Sstevel@tonic-gate ptr = &mmu_p->dvma_alloc_rec[mmu_p->dvma_alloc_rec_index]; 15637c478bd9Sstevel@tonic-gate ptr->dvma_addr = address; 15647c478bd9Sstevel@tonic-gate ptr->len = len; 15657c478bd9Sstevel@tonic-gate ptr->mp = mp; 15667c478bd9Sstevel@tonic-gate if (++mmu_p->dvma_alloc_rec_index == px_dvma_debug_rec) 15677c478bd9Sstevel@tonic-gate mmu_p->dvma_alloc_rec_index = 0; 15687c478bd9Sstevel@tonic-gate 15697c478bd9Sstevel@tonic-gate ptr = kmem_alloc(sizeof (struct px_dvma_rec), KM_SLEEP); 15707c478bd9Sstevel@tonic-gate ptr->dvma_addr = address; 15717c478bd9Sstevel@tonic-gate ptr->len = len; 15727c478bd9Sstevel@tonic-gate ptr->mp = mp; 15737c478bd9Sstevel@tonic-gate 15747c478bd9Sstevel@tonic-gate ptr->next = mmu_p->dvma_active_list; 15757c478bd9Sstevel@tonic-gate mmu_p->dvma_active_list = ptr; 15767c478bd9Sstevel@tonic-gate mmu_p->dvma_active_count++; 15777c478bd9Sstevel@tonic-gate done: 15787c478bd9Sstevel@tonic-gate mutex_exit(&mmu_p->dvma_debug_lock); 15797c478bd9Sstevel@tonic-gate } 15807c478bd9Sstevel@tonic-gate 15817c478bd9Sstevel@tonic-gate void 15827c478bd9Sstevel@tonic-gate px_dvma_free_debug(px_mmu_t *mmu_p, char *address, uint_t len, 15837c478bd9Sstevel@tonic-gate ddi_dma_impl_t *mp) 15847c478bd9Sstevel@tonic-gate { 15857c478bd9Sstevel@tonic-gate struct px_dvma_rec *ptr, *ptr_save; 15867c478bd9Sstevel@tonic-gate mutex_enter(&mmu_p->dvma_debug_lock); 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate if (!mmu_p->dvma_alloc_rec) 15897c478bd9Sstevel@tonic-gate px_dvma_debug_init(mmu_p); 159036fe4a92Segillett if (PX_DVMA_DBG_OFF(mmu_p)) { 15917c478bd9Sstevel@tonic-gate px_dvma_debug_fini(mmu_p); 15927c478bd9Sstevel@tonic-gate goto done; 15937c478bd9Sstevel@tonic-gate } 15947c478bd9Sstevel@tonic-gate 15957c478bd9Sstevel@tonic-gate ptr = &mmu_p->dvma_free_rec[mmu_p->dvma_free_rec_index]; 15967c478bd9Sstevel@tonic-gate ptr->dvma_addr = address; 15977c478bd9Sstevel@tonic-gate ptr->len = len; 15987c478bd9Sstevel@tonic-gate ptr->mp = mp; 15997c478bd9Sstevel@tonic-gate if (++mmu_p->dvma_free_rec_index == px_dvma_debug_rec) 16007c478bd9Sstevel@tonic-gate mmu_p->dvma_free_rec_index = 0; 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate ptr_save = mmu_p->dvma_active_list; 16037c478bd9Sstevel@tonic-gate for (ptr = ptr_save; ptr; ptr = ptr->next) { 16047c478bd9Sstevel@tonic-gate if ((ptr->dvma_addr == address) && (ptr->len = len)) 16057c478bd9Sstevel@tonic-gate break; 16067c478bd9Sstevel@tonic-gate ptr_save = ptr; 16077c478bd9Sstevel@tonic-gate } 16087c478bd9Sstevel@tonic-gate if (!ptr) { 16097c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x", 16109fc8611eSDaniel Ice (long)address, len); 16117c478bd9Sstevel@tonic-gate goto done; 16127c478bd9Sstevel@tonic-gate } 16137c478bd9Sstevel@tonic-gate if (ptr == mmu_p->dvma_active_list) 16147c478bd9Sstevel@tonic-gate mmu_p->dvma_active_list = ptr->next; 16157c478bd9Sstevel@tonic-gate else 16167c478bd9Sstevel@tonic-gate ptr_save->next = ptr->next; 16177c478bd9Sstevel@tonic-gate kmem_free(ptr, sizeof (struct px_dvma_rec)); 16187c478bd9Sstevel@tonic-gate mmu_p->dvma_active_count--; 16197c478bd9Sstevel@tonic-gate done: 16207c478bd9Sstevel@tonic-gate mutex_exit(&mmu_p->dvma_debug_lock); 16217c478bd9Sstevel@tonic-gate } 16227c478bd9Sstevel@tonic-gate 16237c478bd9Sstevel@tonic-gate #ifdef DEBUG 16247c478bd9Sstevel@tonic-gate void 16257c478bd9Sstevel@tonic-gate px_dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp) 16267c478bd9Sstevel@tonic-gate { 16277c478bd9Sstevel@tonic-gate DBG(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n", 16289fc8611eSDaniel Ice hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size); 16297c478bd9Sstevel@tonic-gate DBG(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n", 16309fc8611eSDaniel Ice hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags, 16319fc8611eSDaniel Ice hp->dmai_nwin); 16327c478bd9Sstevel@tonic-gate DBG(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n", 16339fc8611eSDaniel Ice hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0); 16347c478bd9Sstevel@tonic-gate DBG(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n", 16359fc8611eSDaniel Ice hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr, 16369fc8611eSDaniel Ice hp->dmai_cookie); 16377c478bd9Sstevel@tonic-gate } 16387c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 1639