/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2019 Joyent, Inc. */ /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ /* All Rights Reserved */ /* * University Copyright- Copyright (c) 1982, 1986, 1988 * The Regents of the University of California * All Rights Reserved * * University Acknowledgment- Portions of this document are derived from * software developed by the University of California, Berkeley, and its * contributors. */ /* * VM - segment of a mapped device. * * This segment driver is used when mapping character special devices. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if DEBUG int segdev_debug; #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; } #else #define DEBUGF(level, args) #endif /* Default timeout for devmap context management */ #define CTX_TIMEOUT_VALUE 0 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ { mutex_enter(&dhp->dh_lock); } #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ { mutex_exit(&dhp->dh_lock); } #define round_down_p2(a, s) ((a) & ~((s) - 1)) #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1)) /* * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize */ #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \ (((uvaddr | paddr) & (pgsize - 1)) == 0) #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \ (((uvaddr ^ paddr) & (pgsize - 1)) == 0) #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */ #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */ static struct devmap_ctx *devmapctx_list = NULL; static struct devmap_softlock *devmap_slist = NULL; /* * mutex, vnode and page for the page of zeros we use for the trash mappings. * One trash page is allocated on the first ddi_umem_setup call that uses it * XXX Eventually, we may want to combine this with what segnf does when all * hat layers implement HAT_NOFAULT. * * The trash page is used when the backing store for a userland mapping is * removed but the application semantics do not take kindly to a SIGBUS. * In that scenario, the applications pages are mapped to some dummy page * which returns garbage on read and writes go into a common place. * (Perfect for NO_FAULT semantics) * The device driver is responsible to communicating to the app with some * other mechanism that such remapping has happened and the app should take * corrective action. * We can also use an anonymous memory page as there is no requirement to * keep the page locked, however this complicates the fault code. RFE. */ static struct vnode trashvp; static struct page *trashpp; /* Non-pageable kernel memory is allocated from the umem_np_arena. */ static vmem_t *umem_np_arena; /* Set the cookie to a value we know will never be a valid umem_cookie */ #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1) /* * Macros to check if type of devmap handle */ #define cookie_is_devmem(c) \ ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE) #define cookie_is_pmem(c) \ ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE) #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\ ((c)->type == KMEM_PAGEABLE)) #define dhp_is_devmem(dhp) \ (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie))) #define dhp_is_pmem(dhp) \ (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie))) #define dhp_is_kpmem(dhp) \ (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie))) /* * Private seg op routines. */ static int segdev_dup(struct seg *, struct seg *); static int segdev_unmap(struct seg *, caddr_t, size_t); static void segdev_free(struct seg *); static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t, enum fault_type, enum seg_rw); static faultcode_t segdev_faulta(struct seg *, caddr_t); static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t); static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t); static void segdev_badop(void); static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t); static size_t segdev_incore(struct seg *, caddr_t, size_t, char *); static int segdev_lockop(struct seg *, caddr_t, size_t, int, int, ulong_t *, size_t); static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *); static u_offset_t segdev_getoffset(struct seg *, caddr_t); static int segdev_gettype(struct seg *, caddr_t); static int segdev_getvp(struct seg *, caddr_t, struct vnode **); static int segdev_advise(struct seg *, caddr_t, size_t, uint_t); static void segdev_dump(struct seg *); static int segdev_pagelock(struct seg *, caddr_t, size_t, struct page ***, enum lock_type, enum seg_rw); static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t); static int segdev_getmemid(struct seg *, caddr_t, memid_t *); static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t); static int segdev_capable(struct seg *, segcapability_t); /* * XXX this struct is used by rootnex_map_fault to identify * the segment it has been passed. So if you make it * "static" you'll need to fix rootnex_map_fault. */ struct seg_ops segdev_ops = { segdev_dup, segdev_unmap, segdev_free, segdev_fault, segdev_faulta, segdev_setprot, segdev_checkprot, (int (*)())segdev_badop, /* kluster */ (size_t (*)(struct seg *))NULL, /* swapout */ segdev_sync, /* sync */ segdev_incore, segdev_lockop, /* lockop */ segdev_getprot, segdev_getoffset, segdev_gettype, segdev_getvp, segdev_advise, segdev_dump, segdev_pagelock, segdev_setpagesize, segdev_getmemid, segdev_getpolicy, segdev_capable, seg_inherit_notsup }; /* * Private segdev support routines */ static struct segdev_data *sdp_alloc(void); static void segdev_softunlock(struct hat *, struct seg *, caddr_t, size_t, enum seg_rw); static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t, struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *); static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t, size_t, enum fault_type, enum seg_rw, devmap_handle_t *); static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t); static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t); static void devmap_softlock_rele(devmap_handle_t *); static void devmap_ctx_rele(devmap_handle_t *); static void devmap_ctxto(void *); static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr); static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len, ulong_t *opfn, ulong_t *pagesize); static void free_devmap_handle(devmap_handle_t *dhp); static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp, struct seg *newseg); static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp); static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len); static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr); static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr, offset_t off, size_t len, uint_t flags); static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr, size_t *llen, caddr_t *laddr); static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len); static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag); static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size); static void *devmap_umem_alloc_np(size_t size, size_t flags); static void devmap_umem_free_np(void *addr, size_t size); /* * routines to lock and unlock underlying segkp segment for * KMEM_PAGEABLE type cookies. */ static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t); static void release_kpmem_lock(struct ddi_umem_cookie *, size_t); /* * Routines to synchronize F_SOFTLOCK and F_INVAL faults for * drivers with devmap_access callbacks */ static int devmap_softlock_enter(struct devmap_softlock *, size_t, enum fault_type); static void devmap_softlock_exit(struct devmap_softlock *, size_t, enum fault_type); static kmutex_t devmapctx_lock; static kmutex_t devmap_slock; /* * Initialize the thread callbacks and thread private data. */ static struct devmap_ctx * devmap_ctxinit(dev_t dev, ulong_t id) { struct devmap_ctx *devctx; struct devmap_ctx *tmp; dev_info_t *dip; tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP); mutex_enter(&devmapctx_lock); dip = e_ddi_hold_devi_by_dev(dev, 0); ASSERT(dip != NULL); ddi_release_devi(dip); for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next) if ((devctx->dip == dip) && (devctx->id == id)) break; if (devctx == NULL) { devctx = tmp; devctx->dip = dip; devctx->id = id; mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL); devctx->next = devmapctx_list; devmapctx_list = devctx; } else kmem_free(tmp, sizeof (struct devmap_ctx)); mutex_enter(&devctx->lock); devctx->refcnt++; mutex_exit(&devctx->lock); mutex_exit(&devmapctx_lock); return (devctx); } /* * Timeout callback called if a CPU has not given up the device context * within dhp->dh_timeout_length ticks */ static void devmap_ctxto(void *data) { struct devmap_ctx *devctx = data; TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO, "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx); mutex_enter(&devctx->lock); /* * Set oncpu = 0 so the next mapping trying to get the device context * can. */ devctx->oncpu = 0; devctx->timeout = 0; cv_signal(&devctx->cv); mutex_exit(&devctx->lock); } /* * Create a device segment. */ int segdev_create(struct seg **segpp, void *argsp) { struct seg *seg = *segpp; struct segdev_data *sdp; struct segdev_crargs *a = (struct segdev_crargs *)argsp; devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data; int error; /* * Since the address space is "write" locked, we * don't need the segment lock to protect "segdev" data. */ ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP); sdp = sdp_alloc(); sdp->mapfunc = a->mapfunc; sdp->offset = a->offset; sdp->prot = a->prot; sdp->maxprot = a->maxprot; sdp->type = a->type; sdp->pageprot = 0; sdp->softlockcnt = 0; sdp->vpage = NULL; if (sdp->mapfunc == NULL) sdp->devmap_data = dhp; else sdp->devmap_data = dhp = NULL; sdp->hat_flags = a->hat_flags; sdp->hat_attr = a->hat_attr; /* * Currently, hat_flags supports only HAT_LOAD_NOCONSIST */ ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST)); /* * Hold shadow vnode -- segdev only deals with * character (VCHR) devices. We use the common * vp to hang pages on. */ sdp->vp = specfind(a->dev, VCHR); ASSERT(sdp->vp != NULL); seg->s_ops = &segdev_ops; seg->s_data = sdp; while (dhp != NULL) { dhp->dh_seg = seg; dhp = dhp->dh_next; } /* * Inform the vnode of the new mapping. */ /* * It is ok to use pass sdp->maxprot to ADDMAP rather than to use * dhp specific maxprot because spec_addmap does not use maxprot. */ error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset, seg->s_as, seg->s_base, seg->s_size, sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL); if (error != 0) { sdp->devmap_data = NULL; hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_UNLOAD_UNMAP); } else { /* * Mappings of /dev/null don't count towards the VSZ of a * process. Mappings of /dev/null have no mapping type. */ if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED | MAP_PRIVATE)) == 0) { seg->s_as->a_resvsize -= seg->s_size; } } return (error); } static struct segdev_data * sdp_alloc(void) { struct segdev_data *sdp; sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP); rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL); return (sdp); } /* * Duplicate seg and return new segment in newseg. */ static int segdev_dup(struct seg *seg, struct seg *newseg) { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; struct segdev_data *newsdp; devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; size_t npages; int ret; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP, "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg); DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n", (void *)dhp, (void *)seg)); /* * Since the address space is "write" locked, we * don't need the segment lock to protect "segdev" data. */ ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); newsdp = sdp_alloc(); newseg->s_ops = seg->s_ops; newseg->s_data = (void *)newsdp; VN_HOLD(sdp->vp); newsdp->vp = sdp->vp; newsdp->mapfunc = sdp->mapfunc; newsdp->offset = sdp->offset; newsdp->pageprot = sdp->pageprot; newsdp->prot = sdp->prot; newsdp->maxprot = sdp->maxprot; newsdp->type = sdp->type; newsdp->hat_attr = sdp->hat_attr; newsdp->hat_flags = sdp->hat_flags; newsdp->softlockcnt = 0; /* * Initialize per page data if the segment we are * dup'ing has per page information. */ npages = seg_pages(newseg); if (sdp->vpage != NULL) { size_t nbytes = vpgtob(npages); newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP); bcopy(sdp->vpage, newsdp->vpage, nbytes); } else newsdp->vpage = NULL; /* * duplicate devmap handles */ if (dhp != NULL) { ret = devmap_handle_dup(dhp, (devmap_handle_t **)&newsdp->devmap_data, newseg); if (ret != 0) { TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1, "segdev_dup:ret1 ret=%x, dhp=%p seg=%p", ret, (void *)dhp, (void *)seg); DEBUGF(1, (CE_CONT, "segdev_dup: ret %x dhp %p seg %p\n", ret, (void *)dhp, (void *)seg)); return (ret); } } /* * Inform the common vnode of the new mapping. */ return (VOP_ADDMAP(VTOCVP(newsdp->vp), newsdp->offset, newseg->s_as, newseg->s_base, newseg->s_size, newsdp->prot, newsdp->maxprot, sdp->type, CRED(), NULL)); } /* * duplicate devmap handles */ static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp, struct seg *newseg) { devmap_handle_t *newdhp_save = NULL; devmap_handle_t *newdhp = NULL; struct devmap_callback_ctl *callbackops; while (dhp != NULL) { newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP); /* Need to lock the original dhp while copying if REMAP */ HOLD_DHP_LOCK(dhp); bcopy(dhp, newdhp, sizeof (devmap_handle_t)); RELE_DHP_LOCK(dhp); newdhp->dh_seg = newseg; newdhp->dh_next = NULL; if (newdhp_save != NULL) newdhp_save->dh_next = newdhp; else *new_dhp = newdhp; newdhp_save = newdhp; callbackops = &newdhp->dh_callbackops; if (dhp->dh_softlock != NULL) newdhp->dh_softlock = devmap_softlock_init( newdhp->dh_dev, (ulong_t)callbackops->devmap_access); if (dhp->dh_ctx != NULL) newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev, (ulong_t)callbackops->devmap_access); /* * Initialize dh_lock if we want to do remap. */ if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) { mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); newdhp->dh_flags |= DEVMAP_LOCK_INITED; } if (callbackops->devmap_dup != NULL) { int ret; /* * Call the dup callback so that the driver can * duplicate its private data. */ ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp, (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp); if (ret != 0) { /* * We want to free up this segment as the driver * has indicated that we can't dup it. But we * don't want to call the drivers, devmap_unmap, * callback function as the driver does not * think this segment exists. The caller of * devmap_dup will call seg_free on newseg * as it was the caller that allocated the * segment. */ DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: " "newdhp %p dhp %p\n", (void *)newdhp, (void *)dhp)); callbackops->devmap_unmap = NULL; return (ret); } } dhp = dhp->dh_next; } return (0); } /* * Split a segment at addr for length len. */ /*ARGSUSED*/ static int segdev_unmap(struct seg *seg, caddr_t addr, size_t len) { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; register struct segdev_data *nsdp; register struct seg *nseg; register size_t opages; /* old segment size in pages */ register size_t npages; /* new segment size in pages */ register size_t dpages; /* pages being deleted (unmapped) */ register size_t nbytes; devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; devmap_handle_t *dhpp; devmap_handle_t *newdhp; struct devmap_callback_ctl *callbackops; caddr_t nbase; offset_t off; ulong_t nsize; size_t mlen, sz; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP, "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx", (void *)dhp, (void *)seg, (void *)addr, len); DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n", (void *)dhp, (void *)seg, (void *)addr, len)); /* * Since the address space is "write" locked, we * don't need the segment lock to protect "segdev" data. */ ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); if ((sz = sdp->softlockcnt) > 0) { /* * Fail the unmap if pages are SOFTLOCKed through this mapping. * softlockcnt is protected from change by the as write lock. */ TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1, "segdev_unmap:error softlockcnt = %ld", sz); DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz)); return (EAGAIN); } /* * Check for bad sizes */ if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size || (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) panic("segdev_unmap"); if (dhp != NULL) { devmap_handle_t *tdhp; /* * If large page size was used in hat_devload(), * the same page size must be used in hat_unload(). */ dhpp = tdhp = devmap_find_handle(dhp, addr); while (tdhp != NULL) { if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) { break; } tdhp = tdhp->dh_next; } if (tdhp != NULL) { /* found a dhp using large pages */ size_t slen = len; size_t mlen; size_t soff; soff = (ulong_t)(addr - dhpp->dh_uvaddr); while (slen != 0) { mlen = MIN(slen, (dhpp->dh_len - soff)); hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr, dhpp->dh_len, HAT_UNLOAD_UNMAP); dhpp = dhpp->dh_next; ASSERT(slen >= mlen); slen -= mlen; soff = 0; } } else hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP); } else { /* * Unload any hardware translations in the range * to be taken out. */ hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP); } /* * get the user offset which will used in the driver callbacks */ off = sdp->offset + (offset_t)(addr - seg->s_base); /* * Inform the vnode of the unmapping. */ ASSERT(sdp->vp != NULL); (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len, sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL); /* * Check for entire segment */ if (addr == seg->s_base && len == seg->s_size) { seg_free(seg); return (0); } opages = seg_pages(seg); dpages = btop(len); npages = opages - dpages; /* * Check for beginning of segment */ if (addr == seg->s_base) { if (sdp->vpage != NULL) { register struct vpage *ovpage; ovpage = sdp->vpage; /* keep pointer to vpage */ nbytes = vpgtob(npages); sdp->vpage = kmem_alloc(nbytes, KM_SLEEP); bcopy(&ovpage[dpages], sdp->vpage, nbytes); /* free up old vpage */ kmem_free(ovpage, vpgtob(opages)); } /* * free devmap handles from the beginning of the mapping. */ if (dhp != NULL) devmap_handle_unmap_head(dhp, len); sdp->offset += (offset_t)len; seg->s_base += len; seg->s_size -= len; return (0); } /* * Check for end of segment */ if (addr + len == seg->s_base + seg->s_size) { if (sdp->vpage != NULL) { register struct vpage *ovpage; ovpage = sdp->vpage; /* keep pointer to vpage */ nbytes = vpgtob(npages); sdp->vpage = kmem_alloc(nbytes, KM_SLEEP); bcopy(ovpage, sdp->vpage, nbytes); /* free up old vpage */ kmem_free(ovpage, vpgtob(opages)); } seg->s_size -= len; /* * free devmap handles from addr to the end of the mapping. */ if (dhp != NULL) devmap_handle_unmap_tail(dhp, addr); return (0); } /* * The section to go is in the middle of the segment, * have to make it into two segments. nseg is made for * the high end while seg is cut down at the low end. */ nbase = addr + len; /* new seg base */ nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */ seg->s_size = addr - seg->s_base; /* shrink old seg */ nseg = seg_alloc(seg->s_as, nbase, nsize); if (nseg == NULL) panic("segdev_unmap seg_alloc"); TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2, "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg); DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n", (void *)seg, (void *)nseg)); nsdp = sdp_alloc(); nseg->s_ops = seg->s_ops; nseg->s_data = (void *)nsdp; VN_HOLD(sdp->vp); nsdp->mapfunc = sdp->mapfunc; nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base); nsdp->vp = sdp->vp; nsdp->pageprot = sdp->pageprot; nsdp->prot = sdp->prot; nsdp->maxprot = sdp->maxprot; nsdp->type = sdp->type; nsdp->hat_attr = sdp->hat_attr; nsdp->hat_flags = sdp->hat_flags; nsdp->softlockcnt = 0; /* * Initialize per page data if the segment we are * dup'ing has per page information. */ if (sdp->vpage != NULL) { /* need to split vpage into two arrays */ register size_t nnbytes; register size_t nnpages; register struct vpage *ovpage; ovpage = sdp->vpage; /* keep pointer to vpage */ npages = seg_pages(seg); /* seg has shrunk */ nbytes = vpgtob(npages); nnpages = seg_pages(nseg); nnbytes = vpgtob(nnpages); sdp->vpage = kmem_alloc(nbytes, KM_SLEEP); bcopy(ovpage, sdp->vpage, nbytes); nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP); bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes); /* free up old vpage */ kmem_free(ovpage, vpgtob(opages)); } else nsdp->vpage = NULL; /* * unmap dhps. */ if (dhp == NULL) { nsdp->devmap_data = NULL; return (0); } while (dhp != NULL) { callbackops = &dhp->dh_callbackops; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3, "segdev_unmap: dhp=%p addr=%p", dhp, addr); DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n", (void *)dhp, (void *)addr, (void *)dhp->dh_uvaddr, dhp->dh_len)); if (addr == (dhp->dh_uvaddr + dhp->dh_len)) { dhpp = dhp->dh_next; dhp->dh_next = NULL; dhp = dhpp; } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) { dhp = dhp->dh_next; } else if (addr > dhp->dh_uvaddr && (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) { /* * is enclosed by dhp. * create a newdhp that begins at addr+len and * ends at dhp->dh_uvaddr+dhp->dh_len. */ newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP); HOLD_DHP_LOCK(dhp); bcopy(dhp, newdhp, sizeof (devmap_handle_t)); RELE_DHP_LOCK(dhp); newdhp->dh_seg = nseg; newdhp->dh_next = dhp->dh_next; if (dhp->dh_softlock != NULL) newdhp->dh_softlock = devmap_softlock_init( newdhp->dh_dev, (ulong_t)callbackops->devmap_access); if (dhp->dh_ctx != NULL) newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev, (ulong_t)callbackops->devmap_access); if (newdhp->dh_flags & DEVMAP_LOCK_INITED) { mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); } if (callbackops->devmap_unmap != NULL) (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, off, len, dhp, &dhp->dh_pvtp, newdhp, &newdhp->dh_pvtp); mlen = len + (addr - dhp->dh_uvaddr); devmap_handle_reduce_len(newdhp, mlen); nsdp->devmap_data = newdhp; /* XX Changing len should recalculate LARGE flag */ dhp->dh_len = addr - dhp->dh_uvaddr; dhpp = dhp->dh_next; dhp->dh_next = NULL; dhp = dhpp; } else if ((addr > dhp->dh_uvaddr) && ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) { mlen = dhp->dh_len + dhp->dh_uvaddr - addr; /* * spans over dhps. */ if (callbackops->devmap_unmap != NULL) (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, off, mlen, (devmap_cookie_t *)dhp, &dhp->dh_pvtp, NULL, NULL); /* XX Changing len should recalculate LARGE flag */ dhp->dh_len = addr - dhp->dh_uvaddr; dhpp = dhp->dh_next; dhp->dh_next = NULL; dhp = dhpp; nsdp->devmap_data = dhp; } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) { /* * dhp is enclosed by . */ dhp->dh_seg = nseg; nsdp->devmap_data = dhp; dhp = devmap_handle_unmap(dhp); nsdp->devmap_data = dhp; /* XX redundant? */ } else if (((addr + len) > dhp->dh_uvaddr) && ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) { mlen = addr + len - dhp->dh_uvaddr; if (callbackops->devmap_unmap != NULL) (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff, mlen, NULL, NULL, dhp, &dhp->dh_pvtp); devmap_handle_reduce_len(dhp, mlen); nsdp->devmap_data = dhp; dhp->dh_seg = nseg; dhp = dhp->dh_next; } else { dhp->dh_seg = nseg; dhp = dhp->dh_next; } } return (0); } /* * Utility function handles reducing the length of a devmap handle during unmap * Note that is only used for unmapping the front portion of the handler, * i.e., we are bumping up the offset/pfn etc up by len * Do not use if reducing length at the tail. */ static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len) { struct ddi_umem_cookie *cp; struct devmap_pmem_cookie *pcp; /* * adjust devmap handle fields */ ASSERT(len < dhp->dh_len); /* Make sure only page-aligned changes are done */ ASSERT((len & PAGEOFFSET) == 0); dhp->dh_len -= len; dhp->dh_uoff += (offset_t)len; dhp->dh_roff += (offset_t)len; dhp->dh_uvaddr += len; /* Need to grab dhp lock if REMAP */ HOLD_DHP_LOCK(dhp); cp = dhp->dh_cookie; if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) { if (cookie_is_devmem(cp)) { dhp->dh_pfn += btop(len); } else if (cookie_is_pmem(cp)) { pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && dhp->dh_roff < ptob(pcp->dp_npages)); } else { ASSERT(dhp->dh_roff < cp->size); ASSERT(dhp->dh_cvaddr >= cp->cvaddr && dhp->dh_cvaddr < (cp->cvaddr + cp->size)); ASSERT((dhp->dh_cvaddr + len) <= (cp->cvaddr + cp->size)); dhp->dh_cvaddr += len; } } /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */ RELE_DHP_LOCK(dhp); } /* * Free devmap handle, dhp. * Return the next devmap handle on the linked list. */ static devmap_handle_t * devmap_handle_unmap(devmap_handle_t *dhp) { struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops; struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data; devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data; ASSERT(dhp != NULL); /* * before we free up dhp, call the driver's devmap_unmap entry point * to free resources allocated for this dhp. */ if (callbackops->devmap_unmap != NULL) { (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff, dhp->dh_len, NULL, NULL, NULL, NULL); } if (dhpp == dhp) { /* releasing first dhp, change sdp data */ sdp->devmap_data = dhp->dh_next; } else { while (dhpp->dh_next != dhp) { dhpp = dhpp->dh_next; } dhpp->dh_next = dhp->dh_next; } dhpp = dhp->dh_next; /* return value is next dhp in chain */ if (dhp->dh_softlock != NULL) devmap_softlock_rele(dhp); if (dhp->dh_ctx != NULL) devmap_ctx_rele(dhp); if (dhp->dh_flags & DEVMAP_LOCK_INITED) { mutex_destroy(&dhp->dh_lock); } kmem_free(dhp, sizeof (devmap_handle_t)); return (dhpp); } /* * Free complete devmap handles from dhp for len bytes * dhp can be either the first handle or a subsequent handle */ static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len) { struct devmap_callback_ctl *callbackops; /* * free the devmap handles covered by len. */ while (len >= dhp->dh_len) { len -= dhp->dh_len; dhp = devmap_handle_unmap(dhp); } if (len != 0) { /* partial unmap at head of first remaining dhp */ callbackops = &dhp->dh_callbackops; /* * Call the unmap callback so the drivers can make * adjustment on its private data. */ if (callbackops->devmap_unmap != NULL) (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp); devmap_handle_reduce_len(dhp, len); } } /* * Free devmap handles to truncate the mapping after addr * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again) * Also could then use the routine in middle unmap case too */ static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr) { register struct seg *seg = dhp->dh_seg; register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data; struct devmap_callback_ctl *callbackops; register devmap_handle_t *dhpp; size_t maplen; ulong_t off; size_t len; maplen = (size_t)(addr - dhp->dh_uvaddr); dhph = devmap_find_handle(dhph, addr); while (dhph != NULL) { if (maplen == 0) { dhph = devmap_handle_unmap(dhph); } else { callbackops = &dhph->dh_callbackops; len = dhph->dh_len - maplen; off = (ulong_t)sdp->offset + (addr - seg->s_base); /* * Call the unmap callback so the driver * can make adjustments on its private data. */ if (callbackops->devmap_unmap != NULL) (*callbackops->devmap_unmap)(dhph, dhph->dh_pvtp, off, len, (devmap_cookie_t *)dhph, &dhph->dh_pvtp, NULL, NULL); /* XXX Reducing len needs to recalculate LARGE flag */ dhph->dh_len = maplen; maplen = 0; dhpp = dhph->dh_next; dhph->dh_next = NULL; dhph = dhpp; } } /* end while */ } /* * Free a segment. */ static void segdev_free(struct seg *seg) { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE, "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg); DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n", (void *)dhp, (void *)seg)); /* * Since the address space is "write" locked, we * don't need the segment lock to protect "segdev" data. */ ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); while (dhp != NULL) dhp = devmap_handle_unmap(dhp); VN_RELE(sdp->vp); if (sdp->vpage != NULL) kmem_free(sdp->vpage, vpgtob(seg_pages(seg))); rw_destroy(&sdp->lock); kmem_free(sdp, sizeof (*sdp)); } static void free_devmap_handle(devmap_handle_t *dhp) { register devmap_handle_t *dhpp; /* * free up devmap handle */ while (dhp != NULL) { dhpp = dhp->dh_next; if (dhp->dh_flags & DEVMAP_LOCK_INITED) { mutex_destroy(&dhp->dh_lock); } if (dhp->dh_softlock != NULL) devmap_softlock_rele(dhp); if (dhp->dh_ctx != NULL) devmap_ctx_rele(dhp); kmem_free(dhp, sizeof (devmap_handle_t)); dhp = dhpp; } } /* * routines to lock and unlock underlying segkp segment for * KMEM_PAGEABLE type cookies. * segkp only allows a single pending F_SOFTLOCK * we keep track of number of locks in the cookie so we can * have multiple pending faults and manage the calls to segkp. * RFE: if segkp supports either pagelock or can support multiple * calls to F_SOFTLOCK, then these routines can go away. * If pagelock, segdev_faultpage can fault on a page by page basis * and simplifies the code quite a bit. * if multiple calls allowed but not partial ranges, then need for * cookie->lock and locked count goes away, code can call as_fault directly */ static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages) { int err = 0; ASSERT(cookie_is_kpmem(cookie)); /* * Fault in pages in segkp with F_SOFTLOCK. * We want to hold the lock until all pages have been loaded. * segkp only allows single caller to hold SOFTLOCK, so cookie * holds a count so we dont call into segkp multiple times */ mutex_enter(&cookie->lock); /* * Check for overflow in locked field */ if ((UINT32_MAX - cookie->locked) < npages) { err = FC_MAKE_ERR(ENOMEM); } else if (cookie->locked == 0) { /* First time locking */ err = as_fault(kas.a_hat, &kas, cookie->cvaddr, cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE); } if (!err) { cookie->locked += npages; } mutex_exit(&cookie->lock); return (err); } static void release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages) { mutex_enter(&cookie->lock); ASSERT(cookie_is_kpmem(cookie)); ASSERT(cookie->locked >= npages); cookie->locked -= (uint_t)npages; if (cookie->locked == 0) { /* Last unlock */ if (as_fault(kas.a_hat, &kas, cookie->cvaddr, cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE)) panic("segdev releasing kpmem lock %p", (void *)cookie); } mutex_exit(&cookie->lock); } /* * Routines to synchronize F_SOFTLOCK and F_INVAL faults for * drivers with devmap_access callbacks * slock->softlocked basically works like a rw lock * -ve counts => F_SOFTLOCK in progress * +ve counts => F_INVAL/F_PROT in progress * We allow only one F_SOFTLOCK at a time * but can have multiple pending F_INVAL/F_PROT calls * * This routine waits using cv_wait_sig so killing processes is more graceful * Returns EINTR if coming out of this routine due to a signal, 0 otherwise */ static int devmap_softlock_enter( struct devmap_softlock *slock, size_t npages, enum fault_type type) { if (npages == 0) return (0); mutex_enter(&(slock->lock)); switch (type) { case F_SOFTLOCK : while (slock->softlocked) { if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) { /* signalled */ mutex_exit(&(slock->lock)); return (EINTR); } } slock->softlocked -= npages; /* -ve count => locked */ break; case F_INVAL : case F_PROT : while (slock->softlocked < 0) if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) { /* signalled */ mutex_exit(&(slock->lock)); return (EINTR); } slock->softlocked += npages; /* +ve count => f_invals */ break; default: ASSERT(0); } mutex_exit(&(slock->lock)); return (0); } static void devmap_softlock_exit( struct devmap_softlock *slock, size_t npages, enum fault_type type) { if (slock == NULL) return; mutex_enter(&(slock->lock)); switch (type) { case F_SOFTLOCK : ASSERT(-slock->softlocked >= npages); slock->softlocked += npages; /* -ve count is softlocked */ if (slock->softlocked == 0) cv_signal(&slock->cv); break; case F_INVAL : case F_PROT: ASSERT(slock->softlocked >= npages); slock->softlocked -= npages; if (slock->softlocked == 0) cv_signal(&slock->cv); break; default: ASSERT(0); } mutex_exit(&(slock->lock)); } /* * Do a F_SOFTUNLOCK call over the range requested. * The range must have already been F_SOFTLOCK'ed. * The segment lock should be held, (but not the segment private lock?) * The softunlock code below does not adjust for large page sizes * assumes the caller already did any addr/len adjustments for * pagesize mappings before calling. */ /*ARGSUSED*/ static void segdev_softunlock( struct hat *hat, /* the hat */ struct seg *seg, /* seg_dev of interest */ caddr_t addr, /* base address of range */ size_t len, /* number of bytes */ enum seg_rw rw) /* type of access at fault */ { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK, "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx", dhp_head, sdp, addr, len); DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx " "addr %p len %lx\n", (void *)dhp_head, sdp->softlockcnt, (void *)addr, len)); hat_unlock(hat, addr, len); if (dhp_head != NULL) { devmap_handle_t *dhp; size_t mlen; size_t tlen = len; ulong_t off; dhp = devmap_find_handle(dhp_head, addr); ASSERT(dhp != NULL); off = (ulong_t)(addr - dhp->dh_uvaddr); while (tlen != 0) { mlen = MIN(tlen, (dhp->dh_len - off)); /* * unlock segkp memory, locked during F_SOFTLOCK */ if (dhp_is_kpmem(dhp)) { release_kpmem_lock( (struct ddi_umem_cookie *)dhp->dh_cookie, btopr(mlen)); } /* * Do the softlock accounting for devmap_access */ if (dhp->dh_callbackops.devmap_access != NULL) { devmap_softlock_exit(dhp->dh_softlock, btopr(mlen), F_SOFTLOCK); } tlen -= mlen; dhp = dhp->dh_next; off = 0; } } mutex_enter(&freemem_lock); ASSERT(sdp->softlockcnt >= btopr(len)); sdp->softlockcnt -= btopr(len); mutex_exit(&freemem_lock); if (sdp->softlockcnt == 0) { /* * All SOFTLOCKS are gone. Wakeup any waiting * unmappers so they can try again to unmap. * Check for waiters first without the mutex * held so we don't always grab the mutex on * softunlocks. */ if (AS_ISUNMAPWAIT(seg->s_as)) { mutex_enter(&seg->s_as->a_contents); if (AS_ISUNMAPWAIT(seg->s_as)) { AS_CLRUNMAPWAIT(seg->s_as); cv_broadcast(&seg->s_as->a_cv); } mutex_exit(&seg->s_as->a_contents); } } } /* * Handle fault for a single page. * Done in a separate routine so we can handle errors more easily. * This routine is called only from segdev_faultpages() * when looping over the range of addresses requested. The segment lock is held. */ static faultcode_t segdev_faultpage( struct hat *hat, /* the hat */ struct seg *seg, /* seg_dev of interest */ caddr_t addr, /* address in as */ struct vpage *vpage, /* pointer to vpage for seg, addr */ enum fault_type type, /* type of fault */ enum seg_rw rw, /* type of access at fault */ devmap_handle_t *dhp) /* devmap handle if any for this page */ { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; uint_t prot; pfn_t pfnum = PFN_INVALID; u_offset_t offset; uint_t hat_flags; dev_info_t *dip; TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE, "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr); DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n", (void *)dhp, (void *)seg, (void *)addr)); /* * Initialize protection value for this page. * If we have per page protection values check it now. */ if (sdp->pageprot) { uint_t protchk; switch (rw) { case S_READ: protchk = PROT_READ; break; case S_WRITE: protchk = PROT_WRITE; break; case S_EXEC: protchk = PROT_EXEC; break; case S_OTHER: default: protchk = PROT_READ | PROT_WRITE | PROT_EXEC; break; } prot = VPP_PROT(vpage); if ((prot & protchk) == 0) return (FC_PROT); /* illegal access type */ } else { prot = sdp->prot; /* caller has already done segment level protection check */ } if (type == F_SOFTLOCK) { mutex_enter(&freemem_lock); sdp->softlockcnt++; mutex_exit(&freemem_lock); } hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD); offset = sdp->offset + (u_offset_t)(addr - seg->s_base); /* * In the devmap framework, sdp->mapfunc is set to NULL. we can get * pfnum from dhp->dh_pfn (at beginning of segment) and offset from * seg->s_base. */ if (dhp == NULL) { /* If segment has devmap_data, then dhp should be non-NULL */ ASSERT(sdp->devmap_data == NULL); pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev, (off_t)offset, prot); prot |= sdp->hat_attr; } else { ulong_t off; struct ddi_umem_cookie *cp; struct devmap_pmem_cookie *pcp; /* ensure the dhp passed in contains addr. */ ASSERT(dhp == devmap_find_handle( (devmap_handle_t *)sdp->devmap_data, addr)); off = addr - dhp->dh_uvaddr; /* * This routine assumes that the caller makes sure that the * fields in dhp used below are unchanged due to remap during * this call. Caller does HOLD_DHP_LOCK if neeed */ cp = dhp->dh_cookie; if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { pfnum = PFN_INVALID; } else if (cookie_is_devmem(cp)) { pfnum = dhp->dh_pfn + btop(off); } else if (cookie_is_pmem(cp)) { pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && dhp->dh_roff < ptob(pcp->dp_npages)); pfnum = page_pptonum( pcp->dp_pparray[btop(off + dhp->dh_roff)]); } else { ASSERT(dhp->dh_roff < cp->size); ASSERT(dhp->dh_cvaddr >= cp->cvaddr && dhp->dh_cvaddr < (cp->cvaddr + cp->size)); ASSERT((dhp->dh_cvaddr + off) <= (cp->cvaddr + cp->size)); ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <= (cp->cvaddr + cp->size)); switch (cp->type) { case UMEM_LOCKED : if (cp->pparray != NULL) { ASSERT((dhp->dh_roff & PAGEOFFSET) == 0); pfnum = page_pptonum( cp->pparray[btop(off + dhp->dh_roff)]); } else { pfnum = hat_getpfnum( ((proc_t *)cp->procp)->p_as->a_hat, cp->cvaddr + off); } break; case UMEM_TRASH : pfnum = page_pptonum(trashpp); /* * We should set hat_flags to HAT_NOFAULT also * However, not all hat layers implement this */ break; case KMEM_PAGEABLE: case KMEM_NON_PAGEABLE: pfnum = hat_getpfnum(kas.a_hat, dhp->dh_cvaddr + off); break; default : pfnum = PFN_INVALID; break; } } prot |= dhp->dh_hat_attr; } if (pfnum == PFN_INVALID) { return (FC_MAKE_ERR(EFAULT)); } /* prot should already be OR'ed in with hat_attributes if needed */ TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1, "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x", pfnum, pf_is_memory(pfnum), prot, hat_flags); DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x " "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags)); if (pf_is_memory(pfnum) || (dhp != NULL)) { /* * It's not _really_ required here to pass sdp->hat_flags * to hat_devload even though we do it. * This is because hat figures it out DEVMEM mappings * are non-consistent, anyway. */ hat_devload(hat, addr, PAGESIZE, pfnum, prot, hat_flags | sdp->hat_flags); return (0); } /* * Fall through to the case where devmap is not used and need to call * up the device tree to set up the mapping */ dip = VTOS(VTOCVP(sdp->vp))->s_dip; ASSERT(dip); /* * When calling ddi_map_fault, we do not OR in sdp->hat_attr * This is because this calls drivers which may not expect * prot to have any other values than PROT_ALL * The root nexus driver has a hack to peek into the segment * structure and then OR in sdp->hat_attr. * XX In case the bus_ops interfaces are ever revisited * we need to fix this. prot should include other hat attributes */ if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL, (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) { return (FC_MAKE_ERR(EFAULT)); } return (0); } static faultcode_t segdev_fault( struct hat *hat, /* the hat */ struct seg *seg, /* the seg_dev of interest */ caddr_t addr, /* the address of the fault */ size_t len, /* the length of the range */ enum fault_type type, /* type of fault */ enum seg_rw rw) /* type of access at fault */ { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data; devmap_handle_t *dhp; struct devmap_softlock *slock = NULL; ulong_t slpage = 0; ulong_t off; caddr_t maddr = addr; int err; int err_is_faultcode = 0; TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT, "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x", (void *)dhp_head, (void *)seg, (void *)addr, len, type); DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p " "addr %p len %lx type %x\n", (void *)dhp_head, (void *)seg, (void *)addr, len, type)); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); /* Handle non-devmap case */ if (dhp_head == NULL) return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL)); /* Find devmap handle */ if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) return (FC_NOMAP); /* * The seg_dev driver does not implement copy-on-write, * and always loads translations with maximal allowed permissions * but we got an fault trying to access the device. * Servicing the fault is not going to result in any better result * RFE: If we want devmap_access callbacks to be involved in F_PROT * faults, then the code below is written for that * Pending resolution of the following: * - determine if the F_INVAL/F_SOFTLOCK syncing * is needed for F_PROT also or not. The code below assumes it does * - If driver sees F_PROT and calls devmap_load with same type, * then segdev_faultpages will fail with FC_PROT anyway, need to * change that so calls from devmap_load to segdev_faultpages for * F_PROT type are retagged to F_INVAL. * RFE: Today we dont have drivers that use devmap and want to handle * F_PROT calls. The code in segdev_fault* is written to allow * this case but is not tested. A driver that needs this capability * should be able to remove the short-circuit case; resolve the * above issues and "should" work. */ if (type == F_PROT) { return (FC_PROT); } /* * Loop through dhp list calling devmap_access or segdev_faultpages for * each devmap handle. * drivers which implement devmap_access can interpose on faults and do * device-appropriate special actions before calling devmap_load. */ /* * Unfortunately, this simple loop has turned out to expose a variety * of complex problems which results in the following convoluted code. * * First, a desire to handle a serialization of F_SOFTLOCK calls * to the driver within the framework. * This results in a dh_softlock structure that is on a per device * (or device instance) basis and serializes devmap_access calls. * Ideally we would need to do this for underlying * memory/device regions that are being faulted on * but that is hard to identify and with REMAP, harder * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t. * to F_SOFTLOCK calls to the driver. * These serializations are to simplify the driver programmer model. * To support these two features, the code first goes through the * devmap handles and counts the pages (slpage) that are covered * by devmap_access callbacks. * This part ends with a devmap_softlock_enter call * which allows only one F_SOFTLOCK active on a device instance, * but multiple F_INVAL/F_PROTs can be active except when a * F_SOFTLOCK is active * * Next, we dont short-circuit the fault code upfront to call * segdev_softunlock for F_SOFTUNLOCK, because we must use * the same length when we softlock and softunlock. * * -Hat layers may not support softunlocking lengths less than the * original length when there is large page support. * -kpmem locking is dependent on keeping the lengths same. * -if drivers handled F_SOFTLOCK, they probably also expect to * see an F_SOFTUNLOCK of the same length * Hence, if extending lengths during softlock, * softunlock has to make the same adjustments and goes through * the same loop calling segdev_faultpages/segdev_softunlock * But some of the synchronization and error handling is different */ if (type != F_SOFTUNLOCK) { devmap_handle_t *dhpp = dhp; size_t slen = len; /* * Calculate count of pages that are : * a) within the (potentially extended) fault region * b) AND covered by devmap handle with devmap_access */ off = (ulong_t)(addr - dhpp->dh_uvaddr); while (slen != 0) { size_t mlen; /* * Softlocking on a region that allows remap is * unsupported due to unresolved locking issues * XXX: unclear what these are? * One potential is that if there is a pending * softlock, then a remap should not be allowed * until the unlock is done. This is easily * fixed by returning error in devmap*remap on * checking the dh->dh_softlock->softlocked value */ if ((type == F_SOFTLOCK) && (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) { return (FC_NOSUPPORT); } mlen = MIN(slen, (dhpp->dh_len - off)); if (dhpp->dh_callbackops.devmap_access) { size_t llen; caddr_t laddr; /* * use extended length for large page mappings */ HOLD_DHP_LOCK(dhpp); if ((sdp->pageprot == 0) && (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) { devmap_get_large_pgsize(dhpp, mlen, maddr, &llen, &laddr); } else { llen = mlen; } RELE_DHP_LOCK(dhpp); slpage += btopr(llen); slock = dhpp->dh_softlock; } maddr += mlen; ASSERT(slen >= mlen); slen -= mlen; dhpp = dhpp->dh_next; off = 0; } /* * synchonize with other faulting threads and wait till safe * devmap_softlock_enter might return due to signal in cv_wait * * devmap_softlock_enter has to be called outside of while loop * to prevent a deadlock if len spans over multiple dhps. * dh_softlock is based on device instance and if multiple dhps * use the same device instance, the second dhp's LOCK call * will hang waiting on the first to complete. * devmap_setup verifies that slocks in a dhp_chain are same. * RFE: this deadlock only hold true for F_SOFTLOCK. For * F_INVAL/F_PROT, since we now allow multiple in parallel, * we could have done the softlock_enter inside the loop * and supported multi-dhp mappings with dissimilar devices */ if (err = devmap_softlock_enter(slock, slpage, type)) return (FC_MAKE_ERR(err)); } /* reset 'maddr' to the start addr of the range of fault. */ maddr = addr; /* calculate the offset corresponds to 'addr' in the first dhp. */ off = (ulong_t)(addr - dhp->dh_uvaddr); /* * The fault length may span over multiple dhps. * Loop until the total length is satisfied. */ while (len != 0) { size_t llen; size_t mlen; caddr_t laddr; /* * mlen is the smaller of 'len' and the length * from addr to the end of mapping defined by dhp. */ mlen = MIN(len, (dhp->dh_len - off)); HOLD_DHP_LOCK(dhp); /* * Pass the extended length and address to devmap_access * if large pagesize is used for loading address translations. */ if ((sdp->pageprot == 0) && (dhp->dh_flags & DEVMAP_FLAG_LARGE)) { devmap_get_large_pgsize(dhp, mlen, maddr, &llen, &laddr); ASSERT(maddr == addr || laddr == maddr); } else { llen = mlen; laddr = maddr; } if (dhp->dh_callbackops.devmap_access != NULL) { offset_t aoff; aoff = sdp->offset + (offset_t)(laddr - seg->s_base); /* * call driver's devmap_access entry point which will * call devmap_load/contextmgmt to load the translations * * We drop the dhp_lock before calling access so * drivers can call devmap_*_remap within access */ RELE_DHP_LOCK(dhp); err = (*dhp->dh_callbackops.devmap_access)( dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw); } else { /* * If no devmap_access entry point, then load mappings * hold dhp_lock across faultpages if REMAP */ err = segdev_faultpages(hat, seg, laddr, llen, type, rw, dhp); err_is_faultcode = 1; RELE_DHP_LOCK(dhp); } if (err) { if ((type == F_SOFTLOCK) && (maddr > addr)) { /* * If not first dhp, use * segdev_fault(F_SOFTUNLOCK) for prior dhps * While this is recursion, it is incorrect to * call just segdev_softunlock * if we are using either large pages * or devmap_access. It will be more right * to go through the same loop as above * rather than call segdev_softunlock directly * It will use the right lenghths as well as * call into the driver devmap_access routines. */ size_t done = (size_t)(maddr - addr); (void) segdev_fault(hat, seg, addr, done, F_SOFTUNLOCK, S_OTHER); /* * reduce slpage by number of pages * released by segdev_softunlock */ ASSERT(slpage >= btopr(done)); devmap_softlock_exit(slock, slpage - btopr(done), type); } else { devmap_softlock_exit(slock, slpage, type); } /* * Segdev_faultpages() already returns a faultcode, * hence, result from segdev_faultpages() should be * returned directly. */ if (err_is_faultcode) return (err); return (FC_MAKE_ERR(err)); } maddr += mlen; ASSERT(len >= mlen); len -= mlen; dhp = dhp->dh_next; off = 0; ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr); } /* * release the softlock count at end of fault * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK */ if ((type == F_INVAL) || (type == F_PROT)) devmap_softlock_exit(slock, slpage, type); return (0); } /* * segdev_faultpages * * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load * This routine assumes that the callers makes sure that the fields * in dhp used below are not changed due to remap during this call. * Caller does HOLD_DHP_LOCK if neeed * This routine returns a faultcode_t as a return value for segdev_fault. */ static faultcode_t segdev_faultpages( struct hat *hat, /* the hat */ struct seg *seg, /* the seg_dev of interest */ caddr_t addr, /* the address of the fault */ size_t len, /* the length of the range */ enum fault_type type, /* type of fault */ enum seg_rw rw, /* type of access at fault */ devmap_handle_t *dhp) /* devmap handle */ { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; register caddr_t a; struct vpage *vpage; struct ddi_umem_cookie *kpmem_cookie = NULL; int err; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES, "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx", (void *)dhp, (void *)seg, (void *)addr, len); DEBUGF(5, (CE_CONT, "segdev_faultpages: " "dhp %p seg %p addr %p len %lx\n", (void *)dhp, (void *)seg, (void *)addr, len)); /* * The seg_dev driver does not implement copy-on-write, * and always loads translations with maximal allowed permissions * but we got an fault trying to access the device. * Servicing the fault is not going to result in any better result * XXX: If we want to allow devmap_access to handle F_PROT calls, * This code should be removed and let the normal fault handling * take care of finding the error */ if (type == F_PROT) { return (FC_PROT); } if (type == F_SOFTUNLOCK) { segdev_softunlock(hat, seg, addr, len, rw); return (0); } /* * For kernel pageable memory, fault/lock segkp pages * We hold this until the completion of this * fault (INVAL/PROT) or till unlock (SOFTLOCK). */ if ((dhp != NULL) && dhp_is_kpmem(dhp)) { kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie; if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len))) return (err); } /* * If we have the same protections for the entire segment, * insure that the access being attempted is legitimate. */ rw_enter(&sdp->lock, RW_READER); if (sdp->pageprot == 0) { uint_t protchk; switch (rw) { case S_READ: protchk = PROT_READ; break; case S_WRITE: protchk = PROT_WRITE; break; case S_EXEC: protchk = PROT_EXEC; break; case S_OTHER: default: protchk = PROT_READ | PROT_WRITE | PROT_EXEC; break; } if ((sdp->prot & protchk) == 0) { rw_exit(&sdp->lock); /* undo kpmem locking */ if (kpmem_cookie != NULL) { release_kpmem_lock(kpmem_cookie, btopr(len)); } return (FC_PROT); /* illegal access type */ } } /* * we do a single hat_devload for the range if * - devmap framework (dhp is not NULL), * - pageprot == 0, i.e., no per-page protection set and * - is device pages, irrespective of whether we are using large pages */ if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) { pfn_t pfnum; uint_t hat_flags; if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { rw_exit(&sdp->lock); return (FC_NOMAP); } if (type == F_SOFTLOCK) { mutex_enter(&freemem_lock); sdp->softlockcnt += btopr(len); mutex_exit(&freemem_lock); } hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD); pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr)); ASSERT(!pf_is_memory(pfnum)); hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr, hat_flags | sdp->hat_flags); rw_exit(&sdp->lock); return (0); } /* Handle cases where we have to loop through fault handling per-page */ if (sdp->vpage == NULL) vpage = NULL; else vpage = &sdp->vpage[seg_page(seg, addr)]; /* loop over the address range handling each fault */ for (a = addr; a < addr + len; a += PAGESIZE) { if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) { break; } if (vpage != NULL) vpage++; } rw_exit(&sdp->lock); if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */ size_t done = (size_t)(a - addr); /* pages fault successfully */ if (done > 0) { /* use softunlock for those pages */ segdev_softunlock(hat, seg, addr, done, S_OTHER); } if (kpmem_cookie != NULL) { /* release kpmem lock for rest of pages */ ASSERT(len >= done); release_kpmem_lock(kpmem_cookie, btopr(len - done)); } } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) { /* for non-SOFTLOCK cases, release kpmem */ release_kpmem_lock(kpmem_cookie, btopr(len)); } return (err); } /* * Asynchronous page fault. We simply do nothing since this * entry point is not supposed to load up the translation. */ /*ARGSUSED*/ static faultcode_t segdev_faulta(struct seg *seg, caddr_t addr) { TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA, "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); return (0); } static int segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; register devmap_handle_t *dhp; register struct vpage *vp, *evp; devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data; ulong_t off; size_t mlen, sz; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT, "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x", (void *)seg, (void *)addr, len, prot); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) { /* * Fail the setprot if pages are SOFTLOCKed through this * mapping. * Softlockcnt is protected from change by the as read lock. */ TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1, "segdev_setprot:error softlockcnt=%lx", sz); DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz)); return (EAGAIN); } if (dhp_head != NULL) { if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) return (EINVAL); /* * check if violate maxprot. */ off = (ulong_t)(addr - dhp->dh_uvaddr); mlen = len; while (dhp) { if ((dhp->dh_maxprot & prot) != prot) return (EACCES); /* violated maxprot */ if (mlen > (dhp->dh_len - off)) { mlen -= dhp->dh_len - off; dhp = dhp->dh_next; off = 0; } else break; } } else { if ((sdp->maxprot & prot) != prot) return (EACCES); } rw_enter(&sdp->lock, RW_WRITER); if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) { if (sdp->prot == prot) { rw_exit(&sdp->lock); return (0); /* all done */ } sdp->prot = (uchar_t)prot; } else { sdp->pageprot = 1; if (sdp->vpage == NULL) { /* * First time through setting per page permissions, * initialize all the vpage structures to prot */ sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)), KM_SLEEP); evp = &sdp->vpage[seg_pages(seg)]; for (vp = sdp->vpage; vp < evp; vp++) VPP_SETPROT(vp, sdp->prot); } /* * Now go change the needed vpages protections. */ evp = &sdp->vpage[seg_page(seg, addr + len)]; for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) VPP_SETPROT(vp, prot); } rw_exit(&sdp->lock); if (dhp_head != NULL) { devmap_handle_t *tdhp; /* * If large page size was used in hat_devload(), * the same page size must be used in hat_unload(). */ dhp = tdhp = devmap_find_handle(dhp_head, addr); while (tdhp != NULL) { if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) { break; } tdhp = tdhp->dh_next; } if (tdhp) { size_t slen = len; size_t mlen; size_t soff; soff = (ulong_t)(addr - dhp->dh_uvaddr); while (slen != 0) { mlen = MIN(slen, (dhp->dh_len - soff)); hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr, dhp->dh_len, HAT_UNLOAD); dhp = dhp->dh_next; ASSERT(slen >= mlen); slen -= mlen; soff = 0; } return (0); } } if ((prot & ~PROT_USER) == PROT_NONE) { hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD); } else { /* * RFE: the segment should keep track of all attributes * allowing us to remove the deprecated hat_chgprot * and use hat_chgattr. */ hat_chgprot(seg->s_as->a_hat, addr, len, prot); } return (0); } static int segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; struct vpage *vp, *evp; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT, "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x", (void *)seg, (void *)addr, len, prot); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); /* * If segment protection can be used, simply check against them */ rw_enter(&sdp->lock, RW_READER); if (sdp->pageprot == 0) { register int err; err = ((sdp->prot & prot) != prot) ? EACCES : 0; rw_exit(&sdp->lock); return (err); } /* * Have to check down to the vpage level */ evp = &sdp->vpage[seg_page(seg, addr + len)]; for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) { if ((VPP_PROT(vp) & prot) != prot) { rw_exit(&sdp->lock); return (EACCES); } } rw_exit(&sdp->lock); return (0); } static int segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; size_t pgno; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT, "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p", (void *)seg, (void *)addr, len, (void *)protv); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1; if (pgno != 0) { rw_enter(&sdp->lock, RW_READER); if (sdp->pageprot == 0) { do { protv[--pgno] = sdp->prot; } while (pgno != 0); } else { size_t pgoff = seg_page(seg, addr); do { pgno--; protv[pgno] = VPP_PROT(&sdp->vpage[pgno + pgoff]); } while (pgno != 0); } rw_exit(&sdp->lock); } return (0); } static u_offset_t segdev_getoffset(register struct seg *seg, caddr_t addr) { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET, "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); return ((u_offset_t)sdp->offset + (addr - seg->s_base)); } /*ARGSUSED*/ static int segdev_gettype(register struct seg *seg, caddr_t addr) { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE, "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); return (sdp->type); } /*ARGSUSED*/ static int segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp) { register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP, "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); /* * Note that this vp is the common_vp of the device, where the * pages are hung .. */ *vpp = VTOCVP(sdp->vp); return (0); } static void segdev_badop(void) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP, "segdev_badop:start"); panic("segdev_badop"); /*NOTREACHED*/ } /* * segdev pages are not in the cache, and thus can't really be controlled. * Hence, syncs are simply always successful. */ /*ARGSUSED*/ static int segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start"); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); return (0); } /* * segdev pages are always "in core". */ /*ARGSUSED*/ static size_t segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec) { size_t v = 0; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start"); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE, v += PAGESIZE) *vec++ = 1; return (v); } /* * segdev pages are not in the cache, and thus can't really be controlled. * Hence, locks are simply always successful. */ /*ARGSUSED*/ static int segdev_lockop(struct seg *seg, caddr_t addr, size_t len, int attr, int op, ulong_t *lockmap, size_t pos) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start"); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); return (0); } /* * segdev pages are not in the cache, and thus can't really be controlled. * Hence, advise is simply always successful. */ /*ARGSUSED*/ static int segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start"); ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); return (0); } /* * segdev pages are not dumped, so we just return */ static void segdev_dump(struct seg *seg __unused) { } /* * ddi_segmap_setup: Used by drivers who wish specify mapping attributes * for a segment. Called from a drivers segmap(9E) * routine. */ /*ARGSUSED*/ int ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp, off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred, ddi_device_acc_attr_t *accattrp, uint_t rnumber) { struct segdev_crargs dev_a; int (*mapfunc)(dev_t dev, off_t off, int prot); uint_t hat_attr; pfn_t pfn; int error, i; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP, "ddi_segmap_setup:start"); if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev) return (ENODEV); /* * Character devices that support the d_mmap * interface can only be mmap'ed shared. */ if ((flags & MAP_TYPE) != MAP_SHARED) return (EINVAL); /* * Check that this region is indeed mappable on this platform. * Use the mapping function. */ if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1) return (ENXIO); if (len == 0) return (ENXIO); /* * Check to ensure that the entire range is * legal and we are not trying to map in * more than the device will let us. */ /* * Save the pfn at offset here. This pfn will be * used later to get user address. */ pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset, maxprot); if (pfn == PFN_INVALID) return (ENXIO); for (i = PAGESIZE; i < len; i += PAGESIZE) { if (cdev_mmap(mapfunc, dev, offset + i, maxprot) == PFN_INVALID) return (ENXIO); } as_rangelock(as); /* Pick an address w/o worrying about any vac alignment constraints. */ error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags); if (error != 0) { as_rangeunlock(as); return (error); } dev_a.mapfunc = mapfunc; dev_a.dev = dev; dev_a.offset = (offset_t)offset; dev_a.type = flags & MAP_TYPE; dev_a.prot = (uchar_t)prot; dev_a.maxprot = (uchar_t)maxprot; dev_a.hat_attr = hat_attr; dev_a.hat_flags = 0; dev_a.devmap_data = NULL; error = as_map(as, *addrp, len, segdev_create, &dev_a); as_rangeunlock(as); return (error); } /*ARGSUSED*/ static int segdev_pagelock(struct seg *seg, caddr_t addr, size_t len, struct page ***ppp, enum lock_type type, enum seg_rw rw) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK, "segdev_pagelock:start"); return (ENOTSUP); } /*ARGSUSED*/ static int segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len, uint_t szc) { return (ENOTSUP); } /* * devmap_device: Used by devmap framework to establish mapping * called by devmap_seup(9F) during map setup time. */ /*ARGSUSED*/ static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr, offset_t off, size_t len, uint_t flags) { devmap_handle_t *rdhp, *maxdhp; struct segdev_crargs dev_a; int err; uint_t maxprot = PROT_ALL; offset_t offset = 0; pfn_t pfn; struct devmap_pmem_cookie *pcp; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE, "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx", (void *)dhp, (void *)addr, off, len); DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n", (void *)dhp, (void *)addr, off, len)); as_rangelock(as); if ((flags & MAP_FIXED) == 0) { offset_t aligned_off; rdhp = maxdhp = dhp; while (rdhp != NULL) { maxdhp = (maxdhp->dh_len > rdhp->dh_len) ? maxdhp : rdhp; rdhp = rdhp->dh_next; maxprot |= dhp->dh_maxprot; } offset = maxdhp->dh_uoff - dhp->dh_uoff; /* * Use the dhp that has the * largest len to get user address. */ /* * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr, * use 0 which is as good as any other. */ if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) { aligned_off = (offset_t)0; } else if (dhp_is_devmem(maxdhp)) { aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset; } else if (dhp_is_pmem(maxdhp)) { pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie; pfn = page_pptonum( pcp->dp_pparray[btop(maxdhp->dh_roff)]); aligned_off = (offset_t)ptob(pfn) - offset; } else { aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr - offset; } /* * Pick an address aligned to dh_cookie. * for kernel memory/user memory, cookie is cvaddr. * for device memory, cookie is physical address. */ map_addr(addr, len, aligned_off, 1, flags); if (*addr == NULL) { as_rangeunlock(as); return (ENOMEM); } } else { /* * User-specified address; blow away any previous mappings. */ (void) as_unmap(as, *addr, len); } dev_a.mapfunc = NULL; dev_a.dev = dhp->dh_dev; dev_a.type = flags & MAP_TYPE; dev_a.offset = off; /* * sdp->maxprot has the least restrict protection of all dhps. */ dev_a.maxprot = maxprot; dev_a.prot = dhp->dh_prot; /* * devmap uses dhp->dh_hat_attr for hat. */ dev_a.hat_flags = 0; dev_a.hat_attr = 0; dev_a.devmap_data = (void *)dhp; err = as_map(as, *addr, len, segdev_create, &dev_a); as_rangeunlock(as); return (err); } int devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len, uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t, size_t, uint_t, uint_t)) { register devmap_handle_t *dhp = (devmap_handle_t *)dhc; struct devmap_ctx *devctx; int do_timeout = 0; int ret; #ifdef lint pvtp = pvtp; #endif TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT, "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx", (void *)dhp, off, len); DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n", (void *)dhp, off, len)); if (ctxmgt == NULL) return (FC_HWERR); devctx = dhp->dh_ctx; /* * If we are on an MP system with more than one cpu running * and if a thread on some CPU already has the context, wait * for it to finish if there is a hysteresis timeout. * * We call cv_wait() instead of cv_wait_sig() because * it does not matter much if it returned due to a signal * or due to a cv_signal() or cv_broadcast(). In either event * we need to complete the mapping otherwise the processes * will die with a SEGV. */ if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) { TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1, "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p", devctx, dhp); do_timeout = 1; mutex_enter(&devctx->lock); while (devctx->oncpu) cv_wait(&devctx->cv, &devctx->lock); devctx->oncpu = 1; mutex_exit(&devctx->lock); } /* * Call the contextmgt callback so that the driver can handle * the fault. */ ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw); /* * If devmap_access() returned -1, then there was a hardware * error so we need to convert the return value to something * that trap() will understand. Otherwise, the return value * is already a fault code generated by devmap_unload() * or devmap_load(). */ if (ret) { TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2, "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p", ret, dhp, devctx); DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n", ret, (void *)dhp)); if (devctx->oncpu) { mutex_enter(&devctx->lock); devctx->oncpu = 0; cv_signal(&devctx->cv); mutex_exit(&devctx->lock); } return (FC_HWERR); } /* * Setup the timeout if we need to */ if (do_timeout) { mutex_enter(&devctx->lock); if (dhp->dh_timeout_length > 0) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3, "devmap_do_ctxmgt:timeout set"); devctx->timeout = timeout(devmap_ctxto, devctx, dhp->dh_timeout_length); } else { /* * We don't want to wait so set oncpu to * 0 and wake up anyone waiting. */ TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4, "devmap_do_ctxmgt:timeout not set"); devctx->oncpu = 0; cv_signal(&devctx->cv); } mutex_exit(&devctx->lock); } return (DDI_SUCCESS); } /* * end of mapping * poff fault_offset | * base | | | * | | | | * V V V V * +-----------+---------------+-------+---------+-------+ * ^ ^ ^ ^ * |<--- offset--->|<-len->| | * |<--- dh_len(size of mapping) --->| * |<-- pg -->| * -->|rlen|<-- */ static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len, ulong_t *opfn, ulong_t *pagesize) { register int level; ulong_t pg; ulong_t poff; ulong_t base; caddr_t uvaddr; long rlen; TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP, "devmap_roundup:start dhp=%p off=%lx len=%lx", (void *)dhp, offset, len); DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n", (void *)dhp, offset, len)); /* * get the max. pagesize that is aligned within the range * . * * The calculations below use physical address to ddetermine * the page size to use. The same calculations can use the * virtual address to determine the page size. */ pg = 0; poff = 0; base = (ulong_t)ptob(dhp->dh_pfn); for (level = dhp->dh_mmulevel; level >= 0; level--) { pg = page_get_pagesize(level); poff = ((base + offset) & ~(pg - 1)); uvaddr = dhp->dh_uvaddr + (poff - base); if ((poff >= base) && ((poff + pg) <= (base + dhp->dh_len)) && VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg)) break; } TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1, "devmap_roundup: base=%lx poff=%lx dhp=%p", base, poff, dhp); DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n", base, poff, dhp->dh_pfn)); ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg)); ASSERT(level >= 0); *pagesize = pg; *opfn = dhp->dh_pfn + btop(poff - base); rlen = len + offset - (poff - base + pg); ASSERT(rlen < (long)len); TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2, "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p", (void *)dhp, level, rlen, pagesize, opfn); DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p " "level %x rlen %lx psize %lx opfn %lx\n", (void *)dhp, level, rlen, *pagesize, *opfn)); return ((ulong_t)((rlen > 0) ? rlen : 0)); } /* * find the dhp that contains addr. */ static devmap_handle_t * devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr) { devmap_handle_t *dhp; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE, "devmap_find_handle:start"); dhp = dhp_head; while (dhp) { if (addr >= dhp->dh_uvaddr && addr < (dhp->dh_uvaddr + dhp->dh_len)) return (dhp); dhp = dhp->dh_next; } return ((devmap_handle_t *)NULL); } /* * devmap_unload: * Marks a segdev segment or pages if offset->offset+len * is not the entire segment as intercept and unloads the * pages in the range offset -> offset+len. */ int devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len) { register devmap_handle_t *dhp = (devmap_handle_t *)dhc; caddr_t addr; ulong_t size; ssize_t soff; TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD, "devmap_unload:start dhp=%p offset=%llx len=%lx", (void *)dhp, offset, len); DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n", (void *)dhp, offset, len)); soff = (ssize_t)(offset - dhp->dh_uoff); soff = round_down_p2(soff, PAGESIZE); if (soff < 0 || soff >= dhp->dh_len) return (FC_MAKE_ERR(EINVAL)); /* * Address and size must be page aligned. Len is set to the * number of bytes in the number of pages that are required to * support len. Offset is set to the byte offset of the first byte * of the page that contains offset. */ len = round_up_p2(len, PAGESIZE); /* * If len is == 0, then calculate the size by getting * the number of bytes from offset to the end of the segment. */ if (len == 0) size = dhp->dh_len - soff; else { size = len; if ((soff + size) > dhp->dh_len) return (FC_MAKE_ERR(EINVAL)); } /* * The address is offset bytes from the base address of * the dhp. */ addr = (caddr_t)(soff + dhp->dh_uvaddr); /* * If large page size was used in hat_devload(), * the same page size must be used in hat_unload(). */ if (dhp->dh_flags & DEVMAP_FLAG_LARGE) { hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); } else { hat_unload(dhp->dh_seg->s_as->a_hat, addr, size, HAT_UNLOAD|HAT_UNLOAD_OTHER); } return (0); } /* * calculates the optimal page size that will be used for hat_devload(). */ static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr, size_t *llen, caddr_t *laddr) { ulong_t off; ulong_t pfn; ulong_t pgsize; uint_t first = 1; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE, "devmap_get_large_pgsize:start"); /* * RFE - Code only supports large page mappings for devmem * This code could be changed in future if we want to support * large page mappings for kernel exported memory. */ ASSERT(dhp_is_devmem(dhp)); ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)); *llen = 0; off = (ulong_t)(addr - dhp->dh_uvaddr); while ((long)len > 0) { /* * get the optimal pfn to minimize address translations. * devmap_roundup() returns residue bytes for next round * calculations. */ len = devmap_roundup(dhp, off, len, &pfn, &pgsize); if (first) { *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn); first = 0; } *llen += pgsize; off = ptob(pfn - dhp->dh_pfn) + pgsize; } /* Large page mapping len/addr cover more range than original fault */ ASSERT(*llen >= len && *laddr <= addr); ASSERT((*laddr + *llen) >= (addr + len)); } /* * Initialize the devmap_softlock structure. */ static struct devmap_softlock * devmap_softlock_init(dev_t dev, ulong_t id) { struct devmap_softlock *slock; struct devmap_softlock *tmp; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT, "devmap_softlock_init:start"); tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP); mutex_enter(&devmap_slock); for (slock = devmap_slist; slock != NULL; slock = slock->next) if ((slock->dev == dev) && (slock->id == id)) break; if (slock == NULL) { slock = tmp; slock->dev = dev; slock->id = id; mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&slock->cv, NULL, CV_DEFAULT, NULL); slock->next = devmap_slist; devmap_slist = slock; } else kmem_free(tmp, sizeof (struct devmap_softlock)); mutex_enter(&slock->lock); slock->refcnt++; mutex_exit(&slock->lock); mutex_exit(&devmap_slock); return (slock); } /* * Wake up processes that sleep on softlocked. * Free dh_softlock if refcnt is 0. */ static void devmap_softlock_rele(devmap_handle_t *dhp) { struct devmap_softlock *slock = dhp->dh_softlock; struct devmap_softlock *tmp; struct devmap_softlock *parent; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE, "devmap_softlock_rele:start"); mutex_enter(&devmap_slock); mutex_enter(&slock->lock); ASSERT(slock->refcnt > 0); slock->refcnt--; /* * If no one is using the device, free up the slock data. */ if (slock->refcnt == 0) { slock->softlocked = 0; cv_signal(&slock->cv); if (devmap_slist == slock) devmap_slist = slock->next; else { parent = devmap_slist; for (tmp = devmap_slist->next; tmp != NULL; tmp = tmp->next) { if (tmp == slock) { parent->next = tmp->next; break; } parent = tmp; } } mutex_exit(&slock->lock); mutex_destroy(&slock->lock); cv_destroy(&slock->cv); kmem_free(slock, sizeof (struct devmap_softlock)); } else mutex_exit(&slock->lock); mutex_exit(&devmap_slock); } /* * Wake up processes that sleep on dh_ctx->locked. * Free dh_ctx if refcnt is 0. */ static void devmap_ctx_rele(devmap_handle_t *dhp) { struct devmap_ctx *devctx = dhp->dh_ctx; struct devmap_ctx *tmp; struct devmap_ctx *parent; timeout_id_t tid; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE, "devmap_ctx_rele:start"); mutex_enter(&devmapctx_lock); mutex_enter(&devctx->lock); ASSERT(devctx->refcnt > 0); devctx->refcnt--; /* * If no one is using the device, free up the devctx data. */ if (devctx->refcnt == 0) { /* * Untimeout any threads using this mapping as they are about * to go away. */ if (devctx->timeout != 0) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1, "devmap_ctx_rele:untimeout ctx->timeout"); tid = devctx->timeout; mutex_exit(&devctx->lock); (void) untimeout(tid); mutex_enter(&devctx->lock); } devctx->oncpu = 0; cv_signal(&devctx->cv); if (devmapctx_list == devctx) devmapctx_list = devctx->next; else { parent = devmapctx_list; for (tmp = devmapctx_list->next; tmp != NULL; tmp = tmp->next) { if (tmp == devctx) { parent->next = tmp->next; break; } parent = tmp; } } mutex_exit(&devctx->lock); mutex_destroy(&devctx->lock); cv_destroy(&devctx->cv); kmem_free(devctx, sizeof (struct devmap_ctx)); } else mutex_exit(&devctx->lock); mutex_exit(&devmapctx_lock); } /* * devmap_load: * Marks a segdev segment or pages if offset->offset+len * is not the entire segment as nointercept and faults in * the pages in the range offset -> offset+len. */ int devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type, uint_t rw) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; struct as *asp = dhp->dh_seg->s_as; caddr_t addr; ulong_t size; ssize_t soff; /* offset from the beginning of the segment */ int rc; TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD, "devmap_load:start dhp=%p offset=%llx len=%lx", (void *)dhp, offset, len); DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n", (void *)dhp, offset, len)); /* * Hat layer only supports devload to process' context for which * the as lock is held. Verify here and return error if drivers * inadvertently call devmap_load on a wrong devmap handle. */ if ((asp != &kas) && !AS_LOCK_HELD(asp)) return (FC_MAKE_ERR(EINVAL)); soff = (ssize_t)(offset - dhp->dh_uoff); soff = round_down_p2(soff, PAGESIZE); if (soff < 0 || soff >= dhp->dh_len) return (FC_MAKE_ERR(EINVAL)); /* * Address and size must be page aligned. Len is set to the * number of bytes in the number of pages that are required to * support len. Offset is set to the byte offset of the first byte * of the page that contains offset. */ len = round_up_p2(len, PAGESIZE); /* * If len == 0, then calculate the size by getting * the number of bytes from offset to the end of the segment. */ if (len == 0) size = dhp->dh_len - soff; else { size = len; if ((soff + size) > dhp->dh_len) return (FC_MAKE_ERR(EINVAL)); } /* * The address is offset bytes from the base address of * the segment. */ addr = (caddr_t)(soff + dhp->dh_uvaddr); HOLD_DHP_LOCK(dhp); rc = segdev_faultpages(asp->a_hat, dhp->dh_seg, addr, size, type, rw, dhp); RELE_DHP_LOCK(dhp); return (rc); } int devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp, size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred) { register devmap_handle_t *dhp; int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t, size_t *, uint_t); int (*mmap)(dev_t, off_t, int); struct devmap_callback_ctl *callbackops; devmap_handle_t *dhp_head = NULL; devmap_handle_t *dhp_prev = NULL; devmap_handle_t *dhp_curr; caddr_t addr; int map_flag; int ret; ulong_t total_len; size_t map_len; size_t resid_len = len; offset_t map_off = off; struct devmap_softlock *slock = NULL; #ifdef lint cred = cred; #endif TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP, "devmap_setup:start off=%llx len=%lx", off, len); DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n", off, len)); devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap; mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap; /* * driver must provide devmap(9E) entry point in cb_ops to use the * devmap framework. */ if (devmap == NULL || devmap == nulldev || devmap == nodev) return (EINVAL); /* * To protect from an inadvertent entry because the devmap entry point * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and * mmap is NULL. */ map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag; if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev)) return (EINVAL); /* * devmap allows mmap(2) to map multiple registers. * one devmap_handle is created for each register mapped. */ for (total_len = 0; total_len < len; total_len += map_len) { dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP); if (dhp_prev != NULL) dhp_prev->dh_next = dhp; else dhp_head = dhp; dhp_prev = dhp; dhp->dh_prot = prot; dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot; dhp->dh_dev = dev; dhp->dh_timeout_length = CTX_TIMEOUT_VALUE; dhp->dh_uoff = map_off; /* * Get mapping specific info from * the driver, such as rnumber, roff, len, callbackops, * accattrp and, if the mapping is for kernel memory, * ddi_umem_cookie. */ if ((ret = cdev_devmap(dev, dhp, map_off, resid_len, &map_len, get_udatamodel())) != 0) { free_devmap_handle(dhp_head); return (ENXIO); } if (map_len & PAGEOFFSET) { free_devmap_handle(dhp_head); return (EINVAL); } callbackops = &dhp->dh_callbackops; if ((callbackops->devmap_access == NULL) || (callbackops->devmap_access == nulldev) || (callbackops->devmap_access == nodev)) { /* * Normally devmap does not support MAP_PRIVATE unless * the drivers provide a valid devmap_access routine. */ if ((flags & MAP_PRIVATE) != 0) { free_devmap_handle(dhp_head); return (EINVAL); } } else { /* * Initialize dhp_softlock and dh_ctx if the drivers * provide devmap_access. */ dhp->dh_softlock = devmap_softlock_init(dev, (ulong_t)callbackops->devmap_access); dhp->dh_ctx = devmap_ctxinit(dev, (ulong_t)callbackops->devmap_access); /* * segdev_fault can only work when all * dh_softlock in a multi-dhp mapping * are same. see comments in segdev_fault * This code keeps track of the first * dh_softlock allocated in slock and * compares all later allocations and if * not similar, returns an error. */ if (slock == NULL) slock = dhp->dh_softlock; if (slock != dhp->dh_softlock) { free_devmap_handle(dhp_head); return (ENOTSUP); } } map_off += map_len; resid_len -= map_len; } /* * get the user virtual address and establish the mapping between * uvaddr and device physical address. */ if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags)) != 0) { /* * free devmap handles if error during the mapping. */ free_devmap_handle(dhp_head); return (ret); } /* * call the driver's devmap_map callback to do more after the mapping, * such as to allocate driver private data for context management. */ dhp = dhp_head; map_off = off; addr = *addrp; while (dhp != NULL) { callbackops = &dhp->dh_callbackops; dhp->dh_uvaddr = addr; dhp_curr = dhp; if (callbackops->devmap_map != NULL) { ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp, dev, flags, map_off, dhp->dh_len, &dhp->dh_pvtp); if (ret != 0) { struct segdev_data *sdp; /* * call driver's devmap_unmap entry point * to free driver resources. */ dhp = dhp_head; map_off = off; while (dhp != dhp_curr) { callbackops = &dhp->dh_callbackops; if (callbackops->devmap_unmap != NULL) { (*callbackops->devmap_unmap)( dhp, dhp->dh_pvtp, map_off, dhp->dh_len, NULL, NULL, NULL, NULL); } map_off += dhp->dh_len; dhp = dhp->dh_next; } sdp = dhp_head->dh_seg->s_data; sdp->devmap_data = NULL; free_devmap_handle(dhp_head); return (ENXIO); } } map_off += dhp->dh_len; addr += dhp->dh_len; dhp = dhp->dh_next; } return (0); } int ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp, off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred) { TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP, "devmap_segmap:start"); return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp, (size_t)len, prot, maxprot, flags, cred)); } /* * Called from devmap_devmem_setup/remap to see if can use large pages for * this device mapping. * Also calculate the max. page size for this mapping. * this page size will be used in fault routine for * optimal page size calculations. */ static void devmap_devmem_large_page_setup(devmap_handle_t *dhp) { ASSERT(dhp_is_devmem(dhp)); dhp->dh_mmulevel = 0; /* * use large page size only if: * 1. device memory. * 2. mmu supports multiple page sizes, * 3. Driver did not disallow it * 4. dhp length is at least as big as the large pagesize * 5. the uvaddr and pfn are large pagesize aligned */ if (page_num_pagesizes() > 1 && !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) { ulong_t base; int level; base = (ulong_t)ptob(dhp->dh_pfn); for (level = 1; level < page_num_pagesizes(); level++) { size_t pgsize = page_get_pagesize(level); if ((dhp->dh_len < pgsize) || (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr, base, pgsize))) { break; } } dhp->dh_mmulevel = level - 1; } if (dhp->dh_mmulevel > 0) { dhp->dh_flags |= DEVMAP_FLAG_LARGE; } else { dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; } } /* * Called by driver devmap routine to pass device specific info to * the framework. used for device memory mapping only. */ int devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip, struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff, size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; ddi_acc_handle_t handle; ddi_map_req_t mr; ddi_acc_hdl_t *hp; int err; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP, "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx", (void *)dhp, roff, rnumber, (uint_t)len); DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx " "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); /* * First to check if this function has been called for this dhp. */ if (dhp->dh_flags & DEVMAP_SETUP_DONE) return (DDI_FAILURE); if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) return (DDI_FAILURE); if (flags & DEVMAP_MAPPING_INVALID) { /* * Don't go up the tree to get pfn if the driver specifies * DEVMAP_MAPPING_INVALID in flags. * * If DEVMAP_MAPPING_INVALID is specified, we have to grant * remap permission. */ if (!(flags & DEVMAP_ALLOW_REMAP)) { return (DDI_FAILURE); } dhp->dh_pfn = PFN_INVALID; } else { handle = impl_acc_hdl_alloc(KM_SLEEP, NULL); if (handle == NULL) return (DDI_FAILURE); hp = impl_acc_hdl_get(handle); hp->ah_vers = VERS_ACCHDL; hp->ah_dip = dip; hp->ah_rnumber = rnumber; hp->ah_offset = roff; hp->ah_len = len; if (accattrp != NULL) hp->ah_acc = *accattrp; mr.map_op = DDI_MO_MAP_LOCKED; mr.map_type = DDI_MT_RNUMBER; mr.map_obj.rnumber = rnumber; mr.map_prot = maxprot & dhp->dh_orig_maxprot; mr.map_flags = DDI_MF_DEVICE_MAPPING; mr.map_handlep = hp; mr.map_vers = DDI_MAP_VERSION; /* * up the device tree to get pfn. * The rootnex_map_regspec() routine in nexus drivers has been * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING. */ err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn); dhp->dh_hat_attr = hp->ah_hat_flags; impl_acc_hdl_free(handle); if (err) return (DDI_FAILURE); } /* Should not be using devmem setup for memory pages */ ASSERT(!pf_is_memory(dhp->dh_pfn)); /* Only some of the flags bits are settable by the driver */ dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); dhp->dh_len = ptob(btopr(len)); dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; dhp->dh_roff = ptob(btop(roff)); /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */ devmap_devmem_large_page_setup(dhp); dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); if (callbackops != NULL) { bcopy(callbackops, &dhp->dh_callbackops, sizeof (struct devmap_callback_ctl)); } /* * Initialize dh_lock if we want to do remap. */ if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); dhp->dh_flags |= DEVMAP_LOCK_INITED; } dhp->dh_flags |= DEVMAP_SETUP_DONE; return (DDI_SUCCESS); } int devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip, uint_t rnumber, offset_t roff, size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; ddi_acc_handle_t handle; ddi_map_req_t mr; ddi_acc_hdl_t *hp; pfn_t pfn; uint_t hat_flags; int err; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP, "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx", (void *)dhp, roff, rnumber, (uint_t)len); DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx " "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); /* * Return failure if setup has not been done or no remap permission * has been granted during the setup. */ if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) return (DDI_FAILURE); /* Only DEVMAP_MAPPING_INVALID flag supported for remap */ if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID)) return (DDI_FAILURE); if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) return (DDI_FAILURE); if (!(flags & DEVMAP_MAPPING_INVALID)) { handle = impl_acc_hdl_alloc(KM_SLEEP, NULL); if (handle == NULL) return (DDI_FAILURE); } HOLD_DHP_LOCK(dhp); /* * Unload the old mapping, so next fault will setup the new mappings * Do this while holding the dhp lock so other faults dont reestablish * the mappings */ hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); if (flags & DEVMAP_MAPPING_INVALID) { dhp->dh_flags |= DEVMAP_MAPPING_INVALID; dhp->dh_pfn = PFN_INVALID; } else { /* clear any prior DEVMAP_MAPPING_INVALID flag */ dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID; hp = impl_acc_hdl_get(handle); hp->ah_vers = VERS_ACCHDL; hp->ah_dip = dip; hp->ah_rnumber = rnumber; hp->ah_offset = roff; hp->ah_len = len; if (accattrp != NULL) hp->ah_acc = *accattrp; mr.map_op = DDI_MO_MAP_LOCKED; mr.map_type = DDI_MT_RNUMBER; mr.map_obj.rnumber = rnumber; mr.map_prot = maxprot & dhp->dh_orig_maxprot; mr.map_flags = DDI_MF_DEVICE_MAPPING; mr.map_handlep = hp; mr.map_vers = DDI_MAP_VERSION; /* * up the device tree to get pfn. * The rootnex_map_regspec() routine in nexus drivers has been * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING. */ err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn); hat_flags = hp->ah_hat_flags; impl_acc_hdl_free(handle); if (err) { RELE_DHP_LOCK(dhp); return (DDI_FAILURE); } /* * Store result of ddi_map first in local variables, as we do * not want to overwrite the existing dhp with wrong data. */ dhp->dh_pfn = pfn; dhp->dh_hat_attr = hat_flags; } /* clear the large page size flag */ dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; dhp->dh_roff = ptob(btop(roff)); /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */ devmap_devmem_large_page_setup(dhp); dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); RELE_DHP_LOCK(dhp); return (DDI_SUCCESS); } /* * called by driver devmap routine to pass kernel virtual address mapping * info to the framework. used only for kernel memory * allocated from ddi_umem_alloc(). */ int devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip, struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie; #ifdef lint dip = dip; #endif TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP, "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx", (void *)dhp, off, cookie, len); DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx " "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); if (cookie == NULL) return (DDI_FAILURE); /* For UMEM_TRASH, this restriction is not needed */ if ((off + len) > cp->size) return (DDI_FAILURE); /* check if the cache attributes are supported */ if (i_ddi_check_cache_attr(flags) == B_FALSE) return (DDI_FAILURE); /* * First to check if this function has been called for this dhp. */ if (dhp->dh_flags & DEVMAP_SETUP_DONE) return (DDI_FAILURE); if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) return (DDI_FAILURE); if (flags & DEVMAP_MAPPING_INVALID) { /* * If DEVMAP_MAPPING_INVALID is specified, we have to grant * remap permission. */ if (!(flags & DEVMAP_ALLOW_REMAP)) { return (DDI_FAILURE); } } else { dhp->dh_cookie = cookie; dhp->dh_roff = ptob(btop(off)); dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; /* set HAT cache attributes */ i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); /* set HAT endianess attributes */ i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); } /* * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload(); * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to * create consistent mappings but our intention was to create * non-consistent mappings. * * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent * mappings. * * kernel exported memory: hat figures it out it's memory and always * creates consistent mappings. * * /dev/mem: non-consistent mappings. See comments in common/io/mem.c * * /dev/kmem: consistent mappings are created unless they are * MAP_FIXED. We _explicitly_ tell hat to create non-consistent * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED * mappings of /dev/kmem. See common/io/mem.c */ /* Only some of the flags bits are settable by the driver */ dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); dhp->dh_len = ptob(btopr(len)); dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); if (callbackops != NULL) { bcopy(callbackops, &dhp->dh_callbackops, sizeof (struct devmap_callback_ctl)); } /* * Initialize dh_lock if we want to do remap. */ if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); dhp->dh_flags |= DEVMAP_LOCK_INITED; } dhp->dh_flags |= DEVMAP_SETUP_DONE; return (DDI_SUCCESS); } int devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip, ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie; TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP, "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx", (void *)dhp, off, cookie, len); DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx " "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); #ifdef lint dip = dip; accattrp = accattrp; #endif /* * Reture failure if setup has not been done or no remap permission * has been granted during the setup. */ if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) return (DDI_FAILURE); /* No flags supported for remap yet */ if (flags != 0) return (DDI_FAILURE); /* check if the cache attributes are supported */ if (i_ddi_check_cache_attr(flags) == B_FALSE) return (DDI_FAILURE); if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) return (DDI_FAILURE); /* For UMEM_TRASH, this restriction is not needed */ if ((off + len) > cp->size) return (DDI_FAILURE); HOLD_DHP_LOCK(dhp); /* * Unload the old mapping, so next fault will setup the new mappings * Do this while holding the dhp lock so other faults dont reestablish * the mappings */ hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); dhp->dh_cookie = cookie; dhp->dh_roff = ptob(btop(off)); dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; /* set HAT cache attributes */ i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); /* set HAT endianess attributes */ i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); /* clear the large page size flag */ dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); RELE_DHP_LOCK(dhp); return (DDI_SUCCESS); } /* * to set timeout value for the driver's context management callback, e.g. * devmap_access(). */ void devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT, "devmap_set_ctx_timeout:start dhp=%p ticks=%x", (void *)dhp, ticks); dhp->dh_timeout_length = ticks; } int devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off, size_t len, uint_t type, uint_t rw) { #ifdef lint pvtp = pvtp; #endif TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS, "devmap_default_access:start"); return (devmap_load(dhp, off, len, type, rw)); } /* * segkmem_alloc() wrapper to allocate memory which is both * non-relocatable (for DR) and sharelocked, since the rest * of this segment driver requires it. */ static void * devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag) { ASSERT(vmp != NULL); ASSERT(kvseg.s_base != NULL); vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED); return (segkmem_alloc(vmp, size, vmflag)); } /* * This is where things are a bit incestuous with seg_kmem: unlike * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so * we need to do a bit of a dance around that to prevent duplication of * code until we decide to bite the bullet and implement a new kernel * segment for driver-allocated memory that is exported to user space. */ static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size) { page_t *pp; caddr_t addr = inaddr; caddr_t eaddr; pgcnt_t npages = btopr(size); ASSERT(vmp != NULL); ASSERT(kvseg.s_base != NULL); ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { /* * Use page_find() instead of page_lookup() to find the page * since we know that it is hashed and has a shared lock. */ pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr); if (pp == NULL) panic("devmap_free_pages: page not found"); if (!page_tryupgrade(pp)) { page_unlock(pp); pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL); if (pp == NULL) panic("devmap_free_pages: page already freed"); } /* Clear p_lckcnt so page_destroy() doesn't update availrmem */ pp->p_lckcnt = 0; page_destroy(pp, 0); } page_unresv(npages); if (vmp != NULL) vmem_free(vmp, inaddr, size); } /* * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for * allocating non-pageable kmem in response to a ddi_umem_alloc() * default request. For now we allocate our own pages and we keep * them long-term sharelocked, since: A) the fault routines expect the * memory to already be locked; B) pageable umem is already long-term * locked; C) it's a lot of work to make it otherwise, particularly * since the nexus layer expects the pages to never fault. An RFE is to * not keep the pages long-term locked, but instead to be able to * take faults on them and simply look them up in kvp in case we * fault on them. Even then, we must take care not to let pageout * steal them from us since the data must remain resident; if we * do this we must come up with some way to pin the pages to prevent * faults while a driver is doing DMA to/from them. */ static void * devmap_umem_alloc_np(size_t size, size_t flags) { void *buf; int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP; buf = vmem_alloc(umem_np_arena, size, vmflags); if (buf != NULL) bzero(buf, size); return (buf); } static void devmap_umem_free_np(void *addr, size_t size) { vmem_free(umem_np_arena, addr, size); } /* * allocate page aligned kernel memory for exporting to user land. * The devmap framework will use the cookie allocated by ddi_umem_alloc() * to find a user virtual address that is in same color as the address * allocated here. */ void * ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie) { register size_t len = ptob(btopr(size)); void *buf = NULL; struct ddi_umem_cookie *cp; int iflags = 0; *cookie = NULL; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC, "devmap_umem_alloc:start"); if (len == 0) return ((void *)NULL); /* * allocate cookie */ if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie), flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) { ASSERT(flags & DDI_UMEM_NOSLEEP); return ((void *)NULL); } if (flags & DDI_UMEM_PAGEABLE) { /* Only one of the flags is allowed */ ASSERT(!(flags & DDI_UMEM_TRASH)); /* initialize resource with 0 */ iflags = KPD_ZERO; /* * to allocate unlocked pageable memory, use segkp_get() to * create a segkp segment. Since segkp can only service kas, * other segment drivers such as segdev have to do * as_fault(segkp, SOFTLOCK) in its fault routine, */ if (flags & DDI_UMEM_NOSLEEP) iflags |= KPD_NOWAIT; if ((buf = segkp_get(segkp, len, iflags)) == NULL) { kmem_free(cp, sizeof (struct ddi_umem_cookie)); return ((void *)NULL); } cp->type = KMEM_PAGEABLE; mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL); cp->locked = 0; } else if (flags & DDI_UMEM_TRASH) { /* Only one of the flags is allowed */ ASSERT(!(flags & DDI_UMEM_PAGEABLE)); cp->type = UMEM_TRASH; buf = NULL; } else { if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) { kmem_free(cp, sizeof (struct ddi_umem_cookie)); return ((void *)NULL); } cp->type = KMEM_NON_PAGEABLE; } /* * need to save size here. size will be used when * we do kmem_free. */ cp->size = len; cp->cvaddr = (caddr_t)buf; *cookie = (void *)cp; return (buf); } void ddi_umem_free(ddi_umem_cookie_t cookie) { struct ddi_umem_cookie *cp; TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE, "devmap_umem_free:start"); /* * if cookie is NULL, no effects on the system */ if (cookie == NULL) return; cp = (struct ddi_umem_cookie *)cookie; switch (cp->type) { case KMEM_PAGEABLE : ASSERT(cp->cvaddr != NULL && cp->size != 0); /* * Check if there are still any pending faults on the cookie * while the driver is deleting it, * XXX - could change to an ASSERT but wont catch errant drivers */ mutex_enter(&cp->lock); if (cp->locked) { mutex_exit(&cp->lock); panic("ddi_umem_free for cookie with pending faults %p", (void *)cp); } segkp_release(segkp, cp->cvaddr); /* * release mutex associated with this cookie. */ mutex_destroy(&cp->lock); break; case KMEM_NON_PAGEABLE : ASSERT(cp->cvaddr != NULL && cp->size != 0); devmap_umem_free_np(cp->cvaddr, cp->size); break; case UMEM_TRASH : break; case UMEM_LOCKED : /* Callers should use ddi_umem_unlock for this type */ ddi_umem_unlock(cookie); /* Frees the cookie too */ return; default: /* panic so we can diagnose the underlying cause */ panic("ddi_umem_free: illegal cookie type 0x%x\n", cp->type); } kmem_free(cookie, sizeof (struct ddi_umem_cookie)); } static int segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; /* * It looks as if it is always mapped shared */ TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID, "segdev_getmemid:start"); memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp); memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base); return (0); } /*ARGSUSED*/ static lgrp_mem_policy_info_t * segdev_getpolicy(struct seg *seg, caddr_t addr) { return (NULL); } /*ARGSUSED*/ static int segdev_capable(struct seg *seg, segcapability_t capability) { return (0); } /* * ddi_umem_alloc() non-pageable quantum cache max size. * This is just a SWAG. */ #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE) /* * Initialize seg_dev from boot. This routine sets up the trash page * and creates the umem_np_arena used to back non-pageable memory * requests. */ void segdev_init(void) { struct seg kseg; umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE, devmap_alloc_pages, devmap_free_pages, heap_arena, DEVMAP_UMEM_QUANTUM, VM_SLEEP); kseg.s_as = &kas; trashpp = page_create_va(&trashvp, 0, PAGESIZE, PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL); if (trashpp == NULL) panic("segdev_init: failed to create trash page"); pagezero(trashpp, 0, PAGESIZE); page_downgrade(trashpp); } /* * Invoke platform-dependent support routines so that /proc can have * the platform code deal with curious hardware. */ int segdev_copyfrom(struct seg *seg, caddr_t uaddr, const void *devaddr, void *kaddr, size_t len) { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; struct snode *sp = VTOS(VTOCVP(sdp->vp)); return (e_ddi_copyfromdev(sp->s_dip, (off_t)(uaddr - seg->s_base), devaddr, kaddr, len)); } int segdev_copyto(struct seg *seg, caddr_t uaddr, const void *kaddr, void *devaddr, size_t len) { struct segdev_data *sdp = (struct segdev_data *)seg->s_data; struct snode *sp = VTOS(VTOCVP(sdp->vp)); return (e_ddi_copytodev(sp->s_dip, (off_t)(uaddr - seg->s_base), kaddr, devaddr, len)); }