17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57b93957cSeota * Common Development and Distribution License (the "License"). 67b93957cSeota * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217b93957cSeota 227c478bd9Sstevel@tonic-gate /* 23*17965fd8SKrishnendu Sadhukhan - Sun Microsystems * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 287c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate /* 317c478bd9Sstevel@tonic-gate * University Copyright- Copyright (c) 1982, 1986, 1988 327c478bd9Sstevel@tonic-gate * The Regents of the University of California 337c478bd9Sstevel@tonic-gate * All Rights Reserved 347c478bd9Sstevel@tonic-gate * 357c478bd9Sstevel@tonic-gate * University Acknowledgment- Portions of this document are derived from 367c478bd9Sstevel@tonic-gate * software developed by the University of California, Berkeley, and its 377c478bd9Sstevel@tonic-gate * contributors. 387c478bd9Sstevel@tonic-gate */ 397c478bd9Sstevel@tonic-gate 407c478bd9Sstevel@tonic-gate /* 417c478bd9Sstevel@tonic-gate * VM - segment of a mapped device. 427c478bd9Sstevel@tonic-gate * 437c478bd9Sstevel@tonic-gate * This segment driver is used when mapping character special devices. 447c478bd9Sstevel@tonic-gate */ 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate #include <sys/types.h> 477c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 487c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 497c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 507c478bd9Sstevel@tonic-gate #include <sys/systm.h> 517c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 527c478bd9Sstevel@tonic-gate #include <sys/mman.h> 537c478bd9Sstevel@tonic-gate #include <sys/errno.h> 547c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 567c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 577c478bd9Sstevel@tonic-gate #include <sys/proc.h> 587c478bd9Sstevel@tonic-gate #include <sys/conf.h> 597c478bd9Sstevel@tonic-gate #include <sys/debug.h> 607c478bd9Sstevel@tonic-gate #include <sys/ddidevmap.h> 617b93957cSeota #include <sys/ddi_implfuncs.h> 627c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 637c478bd9Sstevel@tonic-gate 647c478bd9Sstevel@tonic-gate #include <vm/page.h> 657c478bd9Sstevel@tonic-gate #include <vm/hat.h> 667c478bd9Sstevel@tonic-gate #include <vm/as.h> 677c478bd9Sstevel@tonic-gate #include <vm/seg.h> 687c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h> 697c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h> 707c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 717c478bd9Sstevel@tonic-gate #include <vm/vpage.h> 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 747c478bd9Sstevel@tonic-gate #include <sys/esunddi.h> 757c478bd9Sstevel@tonic-gate #include <sys/fs/snode.h> 767c478bd9Sstevel@tonic-gate 777b93957cSeota 787c478bd9Sstevel@tonic-gate #if DEBUG 797c478bd9Sstevel@tonic-gate int segdev_debug; 807c478bd9Sstevel@tonic-gate #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; } 817c478bd9Sstevel@tonic-gate #else 827c478bd9Sstevel@tonic-gate #define DEBUGF(level, args) 837c478bd9Sstevel@tonic-gate #endif 847c478bd9Sstevel@tonic-gate 857c478bd9Sstevel@tonic-gate /* Default timeout for devmap context management */ 867c478bd9Sstevel@tonic-gate #define CTX_TIMEOUT_VALUE 0 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ 897c478bd9Sstevel@tonic-gate { mutex_enter(&dhp->dh_lock); } 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ 927c478bd9Sstevel@tonic-gate { mutex_exit(&dhp->dh_lock); } 937c478bd9Sstevel@tonic-gate 947c478bd9Sstevel@tonic-gate #define round_down_p2(a, s) ((a) & ~((s) - 1)) 957c478bd9Sstevel@tonic-gate #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1)) 967c478bd9Sstevel@tonic-gate 977c478bd9Sstevel@tonic-gate /* 987c478bd9Sstevel@tonic-gate * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary 997c478bd9Sstevel@tonic-gate * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize 1007c478bd9Sstevel@tonic-gate */ 1017c478bd9Sstevel@tonic-gate #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \ 1027c478bd9Sstevel@tonic-gate (((uvaddr | paddr) & (pgsize - 1)) == 0) 1037c478bd9Sstevel@tonic-gate #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \ 1047c478bd9Sstevel@tonic-gate (((uvaddr ^ paddr) & (pgsize - 1)) == 0) 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */ 1077c478bd9Sstevel@tonic-gate 1087c478bd9Sstevel@tonic-gate #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */ 1097c478bd9Sstevel@tonic-gate 1107c478bd9Sstevel@tonic-gate static struct devmap_ctx *devmapctx_list = NULL; 1117c478bd9Sstevel@tonic-gate static struct devmap_softlock *devmap_slist = NULL; 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate /* 1147c478bd9Sstevel@tonic-gate * mutex, vnode and page for the page of zeros we use for the trash mappings. 1157c478bd9Sstevel@tonic-gate * One trash page is allocated on the first ddi_umem_setup call that uses it 1167c478bd9Sstevel@tonic-gate * XXX Eventually, we may want to combine this with what segnf does when all 1177c478bd9Sstevel@tonic-gate * hat layers implement HAT_NOFAULT. 1187c478bd9Sstevel@tonic-gate * 1197c478bd9Sstevel@tonic-gate * The trash page is used when the backing store for a userland mapping is 1207c478bd9Sstevel@tonic-gate * removed but the application semantics do not take kindly to a SIGBUS. 1217c478bd9Sstevel@tonic-gate * In that scenario, the applications pages are mapped to some dummy page 1227c478bd9Sstevel@tonic-gate * which returns garbage on read and writes go into a common place. 1237c478bd9Sstevel@tonic-gate * (Perfect for NO_FAULT semantics) 1247c478bd9Sstevel@tonic-gate * The device driver is responsible to communicating to the app with some 1257c478bd9Sstevel@tonic-gate * other mechanism that such remapping has happened and the app should take 1267c478bd9Sstevel@tonic-gate * corrective action. 1277c478bd9Sstevel@tonic-gate * We can also use an anonymous memory page as there is no requirement to 1287c478bd9Sstevel@tonic-gate * keep the page locked, however this complicates the fault code. RFE. 1297c478bd9Sstevel@tonic-gate */ 1307c478bd9Sstevel@tonic-gate static struct vnode trashvp; 1317c478bd9Sstevel@tonic-gate static struct page *trashpp; 1327c478bd9Sstevel@tonic-gate 1337c478bd9Sstevel@tonic-gate /* Non-pageable kernel memory is allocated from the umem_np_arena. */ 1347c478bd9Sstevel@tonic-gate static vmem_t *umem_np_arena; 1357c478bd9Sstevel@tonic-gate 1367c478bd9Sstevel@tonic-gate /* Set the cookie to a value we know will never be a valid umem_cookie */ 1377c478bd9Sstevel@tonic-gate #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1) 1387c478bd9Sstevel@tonic-gate 1397c478bd9Sstevel@tonic-gate /* 1407c478bd9Sstevel@tonic-gate * Macros to check if type of devmap handle 1417c478bd9Sstevel@tonic-gate */ 1427c478bd9Sstevel@tonic-gate #define cookie_is_devmem(c) \ 1437c478bd9Sstevel@tonic-gate ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE) 1447c478bd9Sstevel@tonic-gate 1457c478bd9Sstevel@tonic-gate #define cookie_is_pmem(c) \ 1467c478bd9Sstevel@tonic-gate ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE) 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\ 1497c478bd9Sstevel@tonic-gate ((c)->type == KMEM_PAGEABLE)) 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate #define dhp_is_devmem(dhp) \ 1527c478bd9Sstevel@tonic-gate (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie))) 1537c478bd9Sstevel@tonic-gate 1547c478bd9Sstevel@tonic-gate #define dhp_is_pmem(dhp) \ 1557c478bd9Sstevel@tonic-gate (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie))) 1567c478bd9Sstevel@tonic-gate 1577c478bd9Sstevel@tonic-gate #define dhp_is_kpmem(dhp) \ 1587c478bd9Sstevel@tonic-gate (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie))) 1597c478bd9Sstevel@tonic-gate 1607c478bd9Sstevel@tonic-gate /* 1617c478bd9Sstevel@tonic-gate * Private seg op routines. 1627c478bd9Sstevel@tonic-gate */ 1637c478bd9Sstevel@tonic-gate static int segdev_dup(struct seg *, struct seg *); 1647c478bd9Sstevel@tonic-gate static int segdev_unmap(struct seg *, caddr_t, size_t); 1657c478bd9Sstevel@tonic-gate static void segdev_free(struct seg *); 1667c478bd9Sstevel@tonic-gate static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t, 1677c478bd9Sstevel@tonic-gate enum fault_type, enum seg_rw); 1687c478bd9Sstevel@tonic-gate static faultcode_t segdev_faulta(struct seg *, caddr_t); 1697c478bd9Sstevel@tonic-gate static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t); 1707c478bd9Sstevel@tonic-gate static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t); 1717c478bd9Sstevel@tonic-gate static void segdev_badop(void); 1727c478bd9Sstevel@tonic-gate static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t); 1737c478bd9Sstevel@tonic-gate static size_t segdev_incore(struct seg *, caddr_t, size_t, char *); 1747c478bd9Sstevel@tonic-gate static int segdev_lockop(struct seg *, caddr_t, size_t, int, int, 1757c478bd9Sstevel@tonic-gate ulong_t *, size_t); 1767c478bd9Sstevel@tonic-gate static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *); 1777c478bd9Sstevel@tonic-gate static u_offset_t segdev_getoffset(struct seg *, caddr_t); 1787c478bd9Sstevel@tonic-gate static int segdev_gettype(struct seg *, caddr_t); 1797c478bd9Sstevel@tonic-gate static int segdev_getvp(struct seg *, caddr_t, struct vnode **); 1807c478bd9Sstevel@tonic-gate static int segdev_advise(struct seg *, caddr_t, size_t, uint_t); 1817c478bd9Sstevel@tonic-gate static void segdev_dump(struct seg *); 1827c478bd9Sstevel@tonic-gate static int segdev_pagelock(struct seg *, caddr_t, size_t, 1837c478bd9Sstevel@tonic-gate struct page ***, enum lock_type, enum seg_rw); 1847c478bd9Sstevel@tonic-gate static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t); 1857c478bd9Sstevel@tonic-gate static int segdev_getmemid(struct seg *, caddr_t, memid_t *); 1867c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t); 1871bd5c35fSelowe static int segdev_capable(struct seg *, segcapability_t); 1887c478bd9Sstevel@tonic-gate 1897c478bd9Sstevel@tonic-gate /* 1907c478bd9Sstevel@tonic-gate * XXX this struct is used by rootnex_map_fault to identify 1917c478bd9Sstevel@tonic-gate * the segment it has been passed. So if you make it 1927c478bd9Sstevel@tonic-gate * "static" you'll need to fix rootnex_map_fault. 1937c478bd9Sstevel@tonic-gate */ 1947c478bd9Sstevel@tonic-gate struct seg_ops segdev_ops = { 1957c478bd9Sstevel@tonic-gate segdev_dup, 1967c478bd9Sstevel@tonic-gate segdev_unmap, 1977c478bd9Sstevel@tonic-gate segdev_free, 1987c478bd9Sstevel@tonic-gate segdev_fault, 1997c478bd9Sstevel@tonic-gate segdev_faulta, 2007c478bd9Sstevel@tonic-gate segdev_setprot, 2017c478bd9Sstevel@tonic-gate segdev_checkprot, 2027c478bd9Sstevel@tonic-gate (int (*)())segdev_badop, /* kluster */ 2037c478bd9Sstevel@tonic-gate (size_t (*)(struct seg *))NULL, /* swapout */ 2047c478bd9Sstevel@tonic-gate segdev_sync, /* sync */ 2057c478bd9Sstevel@tonic-gate segdev_incore, 2067c478bd9Sstevel@tonic-gate segdev_lockop, /* lockop */ 2077c478bd9Sstevel@tonic-gate segdev_getprot, 2087c478bd9Sstevel@tonic-gate segdev_getoffset, 2097c478bd9Sstevel@tonic-gate segdev_gettype, 2107c478bd9Sstevel@tonic-gate segdev_getvp, 2117c478bd9Sstevel@tonic-gate segdev_advise, 2127c478bd9Sstevel@tonic-gate segdev_dump, 2137c478bd9Sstevel@tonic-gate segdev_pagelock, 2147c478bd9Sstevel@tonic-gate segdev_setpagesize, 2157c478bd9Sstevel@tonic-gate segdev_getmemid, 2167c478bd9Sstevel@tonic-gate segdev_getpolicy, 2171bd5c35fSelowe segdev_capable, 2187c478bd9Sstevel@tonic-gate }; 2197c478bd9Sstevel@tonic-gate 2207c478bd9Sstevel@tonic-gate /* 2217c478bd9Sstevel@tonic-gate * Private segdev support routines 2227c478bd9Sstevel@tonic-gate */ 2237c478bd9Sstevel@tonic-gate static struct segdev_data *sdp_alloc(void); 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate static void segdev_softunlock(struct hat *, struct seg *, caddr_t, 2267c478bd9Sstevel@tonic-gate size_t, enum seg_rw); 2277c478bd9Sstevel@tonic-gate 2287c478bd9Sstevel@tonic-gate static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t, 2297c478bd9Sstevel@tonic-gate struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *); 2307c478bd9Sstevel@tonic-gate 2317c478bd9Sstevel@tonic-gate static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t, 2327c478bd9Sstevel@tonic-gate size_t, enum fault_type, enum seg_rw, devmap_handle_t *); 2337c478bd9Sstevel@tonic-gate 2347c478bd9Sstevel@tonic-gate static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t); 2357c478bd9Sstevel@tonic-gate static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t); 2367c478bd9Sstevel@tonic-gate static void devmap_softlock_rele(devmap_handle_t *); 2377c478bd9Sstevel@tonic-gate static void devmap_ctx_rele(devmap_handle_t *); 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate static void devmap_ctxto(void *); 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head, 2427c478bd9Sstevel@tonic-gate caddr_t addr); 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len, 2457c478bd9Sstevel@tonic-gate ulong_t *opfn, ulong_t *pagesize); 2467c478bd9Sstevel@tonic-gate 2477c478bd9Sstevel@tonic-gate static void free_devmap_handle(devmap_handle_t *dhp); 2487c478bd9Sstevel@tonic-gate 2497c478bd9Sstevel@tonic-gate static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp, 2507c478bd9Sstevel@tonic-gate struct seg *newseg); 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp); 2537c478bd9Sstevel@tonic-gate 2547c478bd9Sstevel@tonic-gate static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len); 2557c478bd9Sstevel@tonic-gate 2567c478bd9Sstevel@tonic-gate static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr); 2577c478bd9Sstevel@tonic-gate 2587c478bd9Sstevel@tonic-gate static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr, 2597c478bd9Sstevel@tonic-gate offset_t off, size_t len, uint_t flags); 2607c478bd9Sstevel@tonic-gate 2617c478bd9Sstevel@tonic-gate static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, 2627c478bd9Sstevel@tonic-gate caddr_t addr, size_t *llen, caddr_t *laddr); 2637c478bd9Sstevel@tonic-gate 2647c478bd9Sstevel@tonic-gate static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len); 2657c478bd9Sstevel@tonic-gate 2667c478bd9Sstevel@tonic-gate static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag); 2677c478bd9Sstevel@tonic-gate static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size); 2687c478bd9Sstevel@tonic-gate 2697c478bd9Sstevel@tonic-gate static void *devmap_umem_alloc_np(size_t size, size_t flags); 2707c478bd9Sstevel@tonic-gate static void devmap_umem_free_np(void *addr, size_t size); 2717c478bd9Sstevel@tonic-gate 2727c478bd9Sstevel@tonic-gate /* 2737c478bd9Sstevel@tonic-gate * routines to lock and unlock underlying segkp segment for 2747c478bd9Sstevel@tonic-gate * KMEM_PAGEABLE type cookies. 2757c478bd9Sstevel@tonic-gate */ 2767c478bd9Sstevel@tonic-gate static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t); 2777c478bd9Sstevel@tonic-gate static void release_kpmem_lock(struct ddi_umem_cookie *, size_t); 2787c478bd9Sstevel@tonic-gate 2797c478bd9Sstevel@tonic-gate /* 2807c478bd9Sstevel@tonic-gate * Routines to synchronize F_SOFTLOCK and F_INVAL faults for 2817c478bd9Sstevel@tonic-gate * drivers with devmap_access callbacks 2827c478bd9Sstevel@tonic-gate */ 2837c478bd9Sstevel@tonic-gate static int devmap_softlock_enter(struct devmap_softlock *, size_t, 2847c478bd9Sstevel@tonic-gate enum fault_type); 2857c478bd9Sstevel@tonic-gate static void devmap_softlock_exit(struct devmap_softlock *, size_t, 2867c478bd9Sstevel@tonic-gate enum fault_type); 2877c478bd9Sstevel@tonic-gate 2887c478bd9Sstevel@tonic-gate static kmutex_t devmapctx_lock; 2897c478bd9Sstevel@tonic-gate 2907c478bd9Sstevel@tonic-gate static kmutex_t devmap_slock; 2917c478bd9Sstevel@tonic-gate 2927c478bd9Sstevel@tonic-gate /* 2937c478bd9Sstevel@tonic-gate * Initialize the thread callbacks and thread private data. 2947c478bd9Sstevel@tonic-gate */ 2957c478bd9Sstevel@tonic-gate static struct devmap_ctx * 2967c478bd9Sstevel@tonic-gate devmap_ctxinit(dev_t dev, ulong_t id) 2977c478bd9Sstevel@tonic-gate { 2987c478bd9Sstevel@tonic-gate struct devmap_ctx *devctx; 2997c478bd9Sstevel@tonic-gate struct devmap_ctx *tmp; 3007c478bd9Sstevel@tonic-gate dev_info_t *dip; 3017c478bd9Sstevel@tonic-gate 3027c478bd9Sstevel@tonic-gate tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP); 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate mutex_enter(&devmapctx_lock); 3057c478bd9Sstevel@tonic-gate 3067c478bd9Sstevel@tonic-gate dip = e_ddi_hold_devi_by_dev(dev, 0); 3077c478bd9Sstevel@tonic-gate ASSERT(dip != NULL); 3087c478bd9Sstevel@tonic-gate ddi_release_devi(dip); 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next) 3117c478bd9Sstevel@tonic-gate if ((devctx->dip == dip) && (devctx->id == id)) 3127c478bd9Sstevel@tonic-gate break; 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate if (devctx == NULL) { 3157c478bd9Sstevel@tonic-gate devctx = tmp; 3167c478bd9Sstevel@tonic-gate devctx->dip = dip; 3177c478bd9Sstevel@tonic-gate devctx->id = id; 3187c478bd9Sstevel@tonic-gate mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL); 3197c478bd9Sstevel@tonic-gate cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL); 3207c478bd9Sstevel@tonic-gate devctx->next = devmapctx_list; 3217c478bd9Sstevel@tonic-gate devmapctx_list = devctx; 3227c478bd9Sstevel@tonic-gate } else 3237c478bd9Sstevel@tonic-gate kmem_free(tmp, sizeof (struct devmap_ctx)); 3247c478bd9Sstevel@tonic-gate 3257c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 3267c478bd9Sstevel@tonic-gate devctx->refcnt++; 3277c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 3287c478bd9Sstevel@tonic-gate mutex_exit(&devmapctx_lock); 3297c478bd9Sstevel@tonic-gate 3307c478bd9Sstevel@tonic-gate return (devctx); 3317c478bd9Sstevel@tonic-gate } 3327c478bd9Sstevel@tonic-gate 3337c478bd9Sstevel@tonic-gate /* 3347c478bd9Sstevel@tonic-gate * Timeout callback called if a CPU has not given up the device context 3357c478bd9Sstevel@tonic-gate * within dhp->dh_timeout_length ticks 3367c478bd9Sstevel@tonic-gate */ 3377c478bd9Sstevel@tonic-gate static void 3387c478bd9Sstevel@tonic-gate devmap_ctxto(void *data) 3397c478bd9Sstevel@tonic-gate { 3407c478bd9Sstevel@tonic-gate struct devmap_ctx *devctx = data; 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO, 3437c478bd9Sstevel@tonic-gate "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx); 3447c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 3457c478bd9Sstevel@tonic-gate /* 3467c478bd9Sstevel@tonic-gate * Set oncpu = 0 so the next mapping trying to get the device context 3477c478bd9Sstevel@tonic-gate * can. 3487c478bd9Sstevel@tonic-gate */ 3497c478bd9Sstevel@tonic-gate devctx->oncpu = 0; 3507c478bd9Sstevel@tonic-gate devctx->timeout = 0; 3517c478bd9Sstevel@tonic-gate cv_signal(&devctx->cv); 3527c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 3537c478bd9Sstevel@tonic-gate } 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate /* 3567c478bd9Sstevel@tonic-gate * Create a device segment. 3577c478bd9Sstevel@tonic-gate */ 3587c478bd9Sstevel@tonic-gate int 3597c478bd9Sstevel@tonic-gate segdev_create(struct seg *seg, void *argsp) 3607c478bd9Sstevel@tonic-gate { 3617c478bd9Sstevel@tonic-gate struct segdev_data *sdp; 3627c478bd9Sstevel@tonic-gate struct segdev_crargs *a = (struct segdev_crargs *)argsp; 3637c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data; 3647c478bd9Sstevel@tonic-gate int error; 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate /* 3677c478bd9Sstevel@tonic-gate * Since the address space is "write" locked, we 3687c478bd9Sstevel@tonic-gate * don't need the segment lock to protect "segdev" data. 3697c478bd9Sstevel@tonic-gate */ 3707c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP); 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate sdp = sdp_alloc(); 3757c478bd9Sstevel@tonic-gate 3767c478bd9Sstevel@tonic-gate sdp->mapfunc = a->mapfunc; 3777c478bd9Sstevel@tonic-gate sdp->offset = a->offset; 3787c478bd9Sstevel@tonic-gate sdp->prot = a->prot; 3797c478bd9Sstevel@tonic-gate sdp->maxprot = a->maxprot; 3807c478bd9Sstevel@tonic-gate sdp->type = a->type; 3817c478bd9Sstevel@tonic-gate sdp->pageprot = 0; 3827c478bd9Sstevel@tonic-gate sdp->softlockcnt = 0; 3837c478bd9Sstevel@tonic-gate sdp->vpage = NULL; 3847c478bd9Sstevel@tonic-gate 3857c478bd9Sstevel@tonic-gate if (sdp->mapfunc == NULL) 3867c478bd9Sstevel@tonic-gate sdp->devmap_data = dhp; 3877c478bd9Sstevel@tonic-gate else 3887c478bd9Sstevel@tonic-gate sdp->devmap_data = dhp = NULL; 3897c478bd9Sstevel@tonic-gate 3907c478bd9Sstevel@tonic-gate sdp->hat_flags = a->hat_flags; 3917c478bd9Sstevel@tonic-gate sdp->hat_attr = a->hat_attr; 3927c478bd9Sstevel@tonic-gate 3937c478bd9Sstevel@tonic-gate /* 3947c478bd9Sstevel@tonic-gate * Currently, hat_flags supports only HAT_LOAD_NOCONSIST 3957c478bd9Sstevel@tonic-gate */ 3967c478bd9Sstevel@tonic-gate ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST)); 3977c478bd9Sstevel@tonic-gate 3987c478bd9Sstevel@tonic-gate /* 3997c478bd9Sstevel@tonic-gate * Hold shadow vnode -- segdev only deals with 4007c478bd9Sstevel@tonic-gate * character (VCHR) devices. We use the common 4017c478bd9Sstevel@tonic-gate * vp to hang pages on. 4027c478bd9Sstevel@tonic-gate */ 4037c478bd9Sstevel@tonic-gate sdp->vp = specfind(a->dev, VCHR); 4047c478bd9Sstevel@tonic-gate ASSERT(sdp->vp != NULL); 4057c478bd9Sstevel@tonic-gate 4067c478bd9Sstevel@tonic-gate seg->s_ops = &segdev_ops; 4077c478bd9Sstevel@tonic-gate seg->s_data = sdp; 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate while (dhp != NULL) { 4107c478bd9Sstevel@tonic-gate dhp->dh_seg = seg; 4117c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 4127c478bd9Sstevel@tonic-gate } 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate /* 4157c478bd9Sstevel@tonic-gate * Inform the vnode of the new mapping. 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate /* 4187c478bd9Sstevel@tonic-gate * It is ok to use pass sdp->maxprot to ADDMAP rather than to use 4197c478bd9Sstevel@tonic-gate * dhp specific maxprot because spec_addmap does not use maxprot. 4207c478bd9Sstevel@tonic-gate */ 4217c478bd9Sstevel@tonic-gate error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset, 4227c478bd9Sstevel@tonic-gate seg->s_as, seg->s_base, seg->s_size, 423da6c28aaSamw sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL); 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate if (error != 0) { 4267c478bd9Sstevel@tonic-gate sdp->devmap_data = NULL; 4277c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size, 4287c478bd9Sstevel@tonic-gate HAT_UNLOAD_UNMAP); 429*17965fd8SKrishnendu Sadhukhan - Sun Microsystems } else { 430*17965fd8SKrishnendu Sadhukhan - Sun Microsystems /* 431*17965fd8SKrishnendu Sadhukhan - Sun Microsystems * Mappings of /dev/null don't count towards the VSZ of a 432*17965fd8SKrishnendu Sadhukhan - Sun Microsystems * process. Mappings of /dev/null have no mapping type. 433*17965fd8SKrishnendu Sadhukhan - Sun Microsystems */ 434*17965fd8SKrishnendu Sadhukhan - Sun Microsystems if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED | 435*17965fd8SKrishnendu Sadhukhan - Sun Microsystems MAP_PRIVATE)) == 0) { 436*17965fd8SKrishnendu Sadhukhan - Sun Microsystems seg->s_as->a_resvsize -= seg->s_size; 437*17965fd8SKrishnendu Sadhukhan - Sun Microsystems } 4387c478bd9Sstevel@tonic-gate } 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate return (error); 4417c478bd9Sstevel@tonic-gate } 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate static struct segdev_data * 4447c478bd9Sstevel@tonic-gate sdp_alloc(void) 4457c478bd9Sstevel@tonic-gate { 4467c478bd9Sstevel@tonic-gate struct segdev_data *sdp; 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP); 44944374aaeSsvemuri rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL); 4507c478bd9Sstevel@tonic-gate 4517c478bd9Sstevel@tonic-gate return (sdp); 4527c478bd9Sstevel@tonic-gate } 4537c478bd9Sstevel@tonic-gate 4547c478bd9Sstevel@tonic-gate /* 4557c478bd9Sstevel@tonic-gate * Duplicate seg and return new segment in newseg. 4567c478bd9Sstevel@tonic-gate */ 4577c478bd9Sstevel@tonic-gate static int 4587c478bd9Sstevel@tonic-gate segdev_dup(struct seg *seg, struct seg *newseg) 4597c478bd9Sstevel@tonic-gate { 4607c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 4617c478bd9Sstevel@tonic-gate struct segdev_data *newsdp; 4627c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; 4637c478bd9Sstevel@tonic-gate size_t npages; 4647c478bd9Sstevel@tonic-gate int ret; 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP, 4677c478bd9Sstevel@tonic-gate "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg); 4687c478bd9Sstevel@tonic-gate 4697c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n", 4707c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg)); 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate /* 4737c478bd9Sstevel@tonic-gate * Since the address space is "write" locked, we 4747c478bd9Sstevel@tonic-gate * don't need the segment lock to protect "segdev" data. 4757c478bd9Sstevel@tonic-gate */ 4767c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 4777c478bd9Sstevel@tonic-gate 4787c478bd9Sstevel@tonic-gate newsdp = sdp_alloc(); 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate newseg->s_ops = seg->s_ops; 4817c478bd9Sstevel@tonic-gate newseg->s_data = (void *)newsdp; 4827c478bd9Sstevel@tonic-gate 4837c478bd9Sstevel@tonic-gate VN_HOLD(sdp->vp); 4847c478bd9Sstevel@tonic-gate newsdp->vp = sdp->vp; 4857c478bd9Sstevel@tonic-gate newsdp->mapfunc = sdp->mapfunc; 4867c478bd9Sstevel@tonic-gate newsdp->offset = sdp->offset; 4877c478bd9Sstevel@tonic-gate newsdp->pageprot = sdp->pageprot; 4887c478bd9Sstevel@tonic-gate newsdp->prot = sdp->prot; 4897c478bd9Sstevel@tonic-gate newsdp->maxprot = sdp->maxprot; 4907c478bd9Sstevel@tonic-gate newsdp->type = sdp->type; 4917c478bd9Sstevel@tonic-gate newsdp->hat_attr = sdp->hat_attr; 4927c478bd9Sstevel@tonic-gate newsdp->hat_flags = sdp->hat_flags; 4937c478bd9Sstevel@tonic-gate newsdp->softlockcnt = 0; 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate /* 4967c478bd9Sstevel@tonic-gate * Initialize per page data if the segment we are 4977c478bd9Sstevel@tonic-gate * dup'ing has per page information. 4987c478bd9Sstevel@tonic-gate */ 4997c478bd9Sstevel@tonic-gate npages = seg_pages(newseg); 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate if (sdp->vpage != NULL) { 5027c478bd9Sstevel@tonic-gate size_t nbytes = vpgtob(npages); 5037c478bd9Sstevel@tonic-gate 5047c478bd9Sstevel@tonic-gate newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP); 5057c478bd9Sstevel@tonic-gate bcopy(sdp->vpage, newsdp->vpage, nbytes); 5067c478bd9Sstevel@tonic-gate } else 5077c478bd9Sstevel@tonic-gate newsdp->vpage = NULL; 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate /* 5107c478bd9Sstevel@tonic-gate * duplicate devmap handles 5117c478bd9Sstevel@tonic-gate */ 5127c478bd9Sstevel@tonic-gate if (dhp != NULL) { 5137c478bd9Sstevel@tonic-gate ret = devmap_handle_dup(dhp, 51444374aaeSsvemuri (devmap_handle_t **)&newsdp->devmap_data, newseg); 5157c478bd9Sstevel@tonic-gate if (ret != 0) { 5167c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1, 5177c478bd9Sstevel@tonic-gate "segdev_dup:ret1 ret=%x, dhp=%p seg=%p", 5187c478bd9Sstevel@tonic-gate ret, (void *)dhp, (void *)seg); 5197c478bd9Sstevel@tonic-gate DEBUGF(1, (CE_CONT, 5207c478bd9Sstevel@tonic-gate "segdev_dup: ret %x dhp %p seg %p\n", 5217c478bd9Sstevel@tonic-gate ret, (void *)dhp, (void *)seg)); 5227c478bd9Sstevel@tonic-gate return (ret); 5237c478bd9Sstevel@tonic-gate } 5247c478bd9Sstevel@tonic-gate } 5257c478bd9Sstevel@tonic-gate 5267c478bd9Sstevel@tonic-gate /* 5277c478bd9Sstevel@tonic-gate * Inform the common vnode of the new mapping. 5287c478bd9Sstevel@tonic-gate */ 5297c478bd9Sstevel@tonic-gate return (VOP_ADDMAP(VTOCVP(newsdp->vp), 53044374aaeSsvemuri newsdp->offset, newseg->s_as, 53144374aaeSsvemuri newseg->s_base, newseg->s_size, newsdp->prot, 53244374aaeSsvemuri newsdp->maxprot, sdp->type, CRED(), NULL)); 5337c478bd9Sstevel@tonic-gate } 5347c478bd9Sstevel@tonic-gate 5357c478bd9Sstevel@tonic-gate /* 5367c478bd9Sstevel@tonic-gate * duplicate devmap handles 5377c478bd9Sstevel@tonic-gate */ 5387c478bd9Sstevel@tonic-gate static int 5397c478bd9Sstevel@tonic-gate devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp, 5407c478bd9Sstevel@tonic-gate struct seg *newseg) 5417c478bd9Sstevel@tonic-gate { 5427c478bd9Sstevel@tonic-gate devmap_handle_t *newdhp_save = NULL; 5437c478bd9Sstevel@tonic-gate devmap_handle_t *newdhp = NULL; 5447c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops; 5457c478bd9Sstevel@tonic-gate 5467c478bd9Sstevel@tonic-gate while (dhp != NULL) { 5477c478bd9Sstevel@tonic-gate newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP); 5487c478bd9Sstevel@tonic-gate 5497c478bd9Sstevel@tonic-gate /* Need to lock the original dhp while copying if REMAP */ 5507c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 5517c478bd9Sstevel@tonic-gate bcopy(dhp, newdhp, sizeof (devmap_handle_t)); 5527c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 5537c478bd9Sstevel@tonic-gate newdhp->dh_seg = newseg; 5547c478bd9Sstevel@tonic-gate newdhp->dh_next = NULL; 5557c478bd9Sstevel@tonic-gate if (newdhp_save != NULL) 5567c478bd9Sstevel@tonic-gate newdhp_save->dh_next = newdhp; 5577c478bd9Sstevel@tonic-gate else 5587c478bd9Sstevel@tonic-gate *new_dhp = newdhp; 5597c478bd9Sstevel@tonic-gate newdhp_save = newdhp; 5607c478bd9Sstevel@tonic-gate 5617c478bd9Sstevel@tonic-gate callbackops = &newdhp->dh_callbackops; 5627c478bd9Sstevel@tonic-gate 5637c478bd9Sstevel@tonic-gate if (dhp->dh_softlock != NULL) 5647c478bd9Sstevel@tonic-gate newdhp->dh_softlock = devmap_softlock_init( 5657c478bd9Sstevel@tonic-gate newdhp->dh_dev, 5667c478bd9Sstevel@tonic-gate (ulong_t)callbackops->devmap_access); 5677c478bd9Sstevel@tonic-gate if (dhp->dh_ctx != NULL) 5687c478bd9Sstevel@tonic-gate newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev, 5697c478bd9Sstevel@tonic-gate (ulong_t)callbackops->devmap_access); 5707c478bd9Sstevel@tonic-gate 5717c478bd9Sstevel@tonic-gate /* 5727c478bd9Sstevel@tonic-gate * Initialize dh_lock if we want to do remap. 5737c478bd9Sstevel@tonic-gate */ 5747c478bd9Sstevel@tonic-gate if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) { 5757c478bd9Sstevel@tonic-gate mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); 5767c478bd9Sstevel@tonic-gate newdhp->dh_flags |= DEVMAP_LOCK_INITED; 5777c478bd9Sstevel@tonic-gate } 5787c478bd9Sstevel@tonic-gate 5797c478bd9Sstevel@tonic-gate if (callbackops->devmap_dup != NULL) { 5807c478bd9Sstevel@tonic-gate int ret; 5817c478bd9Sstevel@tonic-gate 5827c478bd9Sstevel@tonic-gate /* 5837c478bd9Sstevel@tonic-gate * Call the dup callback so that the driver can 5847c478bd9Sstevel@tonic-gate * duplicate its private data. 5857c478bd9Sstevel@tonic-gate */ 5867c478bd9Sstevel@tonic-gate ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp, 58744374aaeSsvemuri (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp); 5887c478bd9Sstevel@tonic-gate 5897c478bd9Sstevel@tonic-gate if (ret != 0) { 5907c478bd9Sstevel@tonic-gate /* 5917c478bd9Sstevel@tonic-gate * We want to free up this segment as the driver 5927c478bd9Sstevel@tonic-gate * has indicated that we can't dup it. But we 5937c478bd9Sstevel@tonic-gate * don't want to call the drivers, devmap_unmap, 5947c478bd9Sstevel@tonic-gate * callback function as the driver does not 5957c478bd9Sstevel@tonic-gate * think this segment exists. The caller of 5967c478bd9Sstevel@tonic-gate * devmap_dup will call seg_free on newseg 5977c478bd9Sstevel@tonic-gate * as it was the caller that allocated the 5987c478bd9Sstevel@tonic-gate * segment. 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: " 6017c478bd9Sstevel@tonic-gate "newdhp %p dhp %p\n", (void *)newdhp, 6027c478bd9Sstevel@tonic-gate (void *)dhp)); 6037c478bd9Sstevel@tonic-gate callbackops->devmap_unmap = NULL; 6047c478bd9Sstevel@tonic-gate return (ret); 6057c478bd9Sstevel@tonic-gate } 6067c478bd9Sstevel@tonic-gate } 6077c478bd9Sstevel@tonic-gate 6087c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 6097c478bd9Sstevel@tonic-gate } 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate return (0); 6127c478bd9Sstevel@tonic-gate } 6137c478bd9Sstevel@tonic-gate 6147c478bd9Sstevel@tonic-gate /* 6157c478bd9Sstevel@tonic-gate * Split a segment at addr for length len. 6167c478bd9Sstevel@tonic-gate */ 6177c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6187c478bd9Sstevel@tonic-gate static int 6197c478bd9Sstevel@tonic-gate segdev_unmap(struct seg *seg, caddr_t addr, size_t len) 6207c478bd9Sstevel@tonic-gate { 6217c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 6227c478bd9Sstevel@tonic-gate register struct segdev_data *nsdp; 6237c478bd9Sstevel@tonic-gate register struct seg *nseg; 6247c478bd9Sstevel@tonic-gate register size_t opages; /* old segment size in pages */ 6257c478bd9Sstevel@tonic-gate register size_t npages; /* new segment size in pages */ 6267c478bd9Sstevel@tonic-gate register size_t dpages; /* pages being deleted (unmapped) */ 6277c478bd9Sstevel@tonic-gate register size_t nbytes; 6287c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; 6297c478bd9Sstevel@tonic-gate devmap_handle_t *dhpp; 6307c478bd9Sstevel@tonic-gate devmap_handle_t *newdhp; 6317c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops; 6327c478bd9Sstevel@tonic-gate caddr_t nbase; 6337c478bd9Sstevel@tonic-gate offset_t off; 6347c478bd9Sstevel@tonic-gate ulong_t nsize; 6357c478bd9Sstevel@tonic-gate size_t mlen, sz; 6367c478bd9Sstevel@tonic-gate 6377c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP, 6387c478bd9Sstevel@tonic-gate "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx", 6397c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg, (void *)addr, len); 6407c478bd9Sstevel@tonic-gate 6417c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n", 6427c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg, (void *)addr, len)); 6437c478bd9Sstevel@tonic-gate 6447c478bd9Sstevel@tonic-gate /* 6457c478bd9Sstevel@tonic-gate * Since the address space is "write" locked, we 6467c478bd9Sstevel@tonic-gate * don't need the segment lock to protect "segdev" data. 6477c478bd9Sstevel@tonic-gate */ 6487c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate if ((sz = sdp->softlockcnt) > 0) { 6517c478bd9Sstevel@tonic-gate /* 6527c478bd9Sstevel@tonic-gate * Fail the unmap if pages are SOFTLOCKed through this mapping. 6537c478bd9Sstevel@tonic-gate * softlockcnt is protected from change by the as write lock. 6547c478bd9Sstevel@tonic-gate */ 6557c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1, 6567c478bd9Sstevel@tonic-gate "segdev_unmap:error softlockcnt = %ld", sz); 6577c478bd9Sstevel@tonic-gate DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz)); 6587c478bd9Sstevel@tonic-gate return (EAGAIN); 6597c478bd9Sstevel@tonic-gate } 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate /* 6627c478bd9Sstevel@tonic-gate * Check for bad sizes 6637c478bd9Sstevel@tonic-gate */ 6647c478bd9Sstevel@tonic-gate if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size || 6657c478bd9Sstevel@tonic-gate (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET)) 6667c478bd9Sstevel@tonic-gate panic("segdev_unmap"); 6677c478bd9Sstevel@tonic-gate 6687c478bd9Sstevel@tonic-gate if (dhp != NULL) { 6697c478bd9Sstevel@tonic-gate devmap_handle_t *tdhp; 6707c478bd9Sstevel@tonic-gate /* 6717c478bd9Sstevel@tonic-gate * If large page size was used in hat_devload(), 6727c478bd9Sstevel@tonic-gate * the same page size must be used in hat_unload(). 6737c478bd9Sstevel@tonic-gate */ 6747c478bd9Sstevel@tonic-gate dhpp = tdhp = devmap_find_handle(dhp, addr); 6757c478bd9Sstevel@tonic-gate while (tdhp != NULL) { 6767c478bd9Sstevel@tonic-gate if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) { 6777c478bd9Sstevel@tonic-gate break; 6787c478bd9Sstevel@tonic-gate } 6797c478bd9Sstevel@tonic-gate tdhp = tdhp->dh_next; 6807c478bd9Sstevel@tonic-gate } 6817c478bd9Sstevel@tonic-gate if (tdhp != NULL) { /* found a dhp using large pages */ 6827c478bd9Sstevel@tonic-gate size_t slen = len; 6837c478bd9Sstevel@tonic-gate size_t mlen; 6847c478bd9Sstevel@tonic-gate size_t soff; 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate soff = (ulong_t)(addr - dhpp->dh_uvaddr); 6877c478bd9Sstevel@tonic-gate while (slen != 0) { 6887c478bd9Sstevel@tonic-gate mlen = MIN(slen, (dhpp->dh_len - soff)); 6897c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr, 69044374aaeSsvemuri dhpp->dh_len, HAT_UNLOAD_UNMAP); 6917c478bd9Sstevel@tonic-gate dhpp = dhpp->dh_next; 6927c478bd9Sstevel@tonic-gate ASSERT(slen >= mlen); 6937c478bd9Sstevel@tonic-gate slen -= mlen; 6947c478bd9Sstevel@tonic-gate soff = 0; 6957c478bd9Sstevel@tonic-gate } 6967c478bd9Sstevel@tonic-gate } else 6977c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, 69844374aaeSsvemuri HAT_UNLOAD_UNMAP); 6997c478bd9Sstevel@tonic-gate } else { 7007c478bd9Sstevel@tonic-gate /* 7017c478bd9Sstevel@tonic-gate * Unload any hardware translations in the range 7027c478bd9Sstevel@tonic-gate * to be taken out. 7037c478bd9Sstevel@tonic-gate */ 7047c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP); 7057c478bd9Sstevel@tonic-gate } 7067c478bd9Sstevel@tonic-gate 7077c478bd9Sstevel@tonic-gate /* 7087c478bd9Sstevel@tonic-gate * get the user offset which will used in the driver callbacks 7097c478bd9Sstevel@tonic-gate */ 7107c478bd9Sstevel@tonic-gate off = sdp->offset + (offset_t)(addr - seg->s_base); 7117c478bd9Sstevel@tonic-gate 7127c478bd9Sstevel@tonic-gate /* 7137c478bd9Sstevel@tonic-gate * Inform the vnode of the unmapping. 7147c478bd9Sstevel@tonic-gate */ 7157c478bd9Sstevel@tonic-gate ASSERT(sdp->vp != NULL); 7167c478bd9Sstevel@tonic-gate (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len, 71744374aaeSsvemuri sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL); 7187c478bd9Sstevel@tonic-gate 7197c478bd9Sstevel@tonic-gate /* 7207c478bd9Sstevel@tonic-gate * Check for entire segment 7217c478bd9Sstevel@tonic-gate */ 7227c478bd9Sstevel@tonic-gate if (addr == seg->s_base && len == seg->s_size) { 7237c478bd9Sstevel@tonic-gate seg_free(seg); 7247c478bd9Sstevel@tonic-gate return (0); 7257c478bd9Sstevel@tonic-gate } 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate opages = seg_pages(seg); 7287c478bd9Sstevel@tonic-gate dpages = btop(len); 7297c478bd9Sstevel@tonic-gate npages = opages - dpages; 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate /* 7327c478bd9Sstevel@tonic-gate * Check for beginning of segment 7337c478bd9Sstevel@tonic-gate */ 7347c478bd9Sstevel@tonic-gate if (addr == seg->s_base) { 7357c478bd9Sstevel@tonic-gate if (sdp->vpage != NULL) { 7367c478bd9Sstevel@tonic-gate register struct vpage *ovpage; 7377c478bd9Sstevel@tonic-gate 7387c478bd9Sstevel@tonic-gate ovpage = sdp->vpage; /* keep pointer to vpage */ 7397c478bd9Sstevel@tonic-gate 7407c478bd9Sstevel@tonic-gate nbytes = vpgtob(npages); 7417c478bd9Sstevel@tonic-gate sdp->vpage = kmem_alloc(nbytes, KM_SLEEP); 7427c478bd9Sstevel@tonic-gate bcopy(&ovpage[dpages], sdp->vpage, nbytes); 7437c478bd9Sstevel@tonic-gate 7447c478bd9Sstevel@tonic-gate /* free up old vpage */ 7457c478bd9Sstevel@tonic-gate kmem_free(ovpage, vpgtob(opages)); 7467c478bd9Sstevel@tonic-gate } 7477c478bd9Sstevel@tonic-gate 7487c478bd9Sstevel@tonic-gate /* 7497c478bd9Sstevel@tonic-gate * free devmap handles from the beginning of the mapping. 7507c478bd9Sstevel@tonic-gate */ 7517c478bd9Sstevel@tonic-gate if (dhp != NULL) 7527c478bd9Sstevel@tonic-gate devmap_handle_unmap_head(dhp, len); 7537c478bd9Sstevel@tonic-gate 7547c478bd9Sstevel@tonic-gate sdp->offset += (offset_t)len; 7557c478bd9Sstevel@tonic-gate 7567c478bd9Sstevel@tonic-gate seg->s_base += len; 7577c478bd9Sstevel@tonic-gate seg->s_size -= len; 7587c478bd9Sstevel@tonic-gate 7597c478bd9Sstevel@tonic-gate return (0); 7607c478bd9Sstevel@tonic-gate } 7617c478bd9Sstevel@tonic-gate 7627c478bd9Sstevel@tonic-gate /* 7637c478bd9Sstevel@tonic-gate * Check for end of segment 7647c478bd9Sstevel@tonic-gate */ 7657c478bd9Sstevel@tonic-gate if (addr + len == seg->s_base + seg->s_size) { 7667c478bd9Sstevel@tonic-gate if (sdp->vpage != NULL) { 7677c478bd9Sstevel@tonic-gate register struct vpage *ovpage; 7687c478bd9Sstevel@tonic-gate 7697c478bd9Sstevel@tonic-gate ovpage = sdp->vpage; /* keep pointer to vpage */ 7707c478bd9Sstevel@tonic-gate 7717c478bd9Sstevel@tonic-gate nbytes = vpgtob(npages); 7727c478bd9Sstevel@tonic-gate sdp->vpage = kmem_alloc(nbytes, KM_SLEEP); 7737c478bd9Sstevel@tonic-gate bcopy(ovpage, sdp->vpage, nbytes); 7747c478bd9Sstevel@tonic-gate 7757c478bd9Sstevel@tonic-gate /* free up old vpage */ 7767c478bd9Sstevel@tonic-gate kmem_free(ovpage, vpgtob(opages)); 7777c478bd9Sstevel@tonic-gate } 7787c478bd9Sstevel@tonic-gate seg->s_size -= len; 7797c478bd9Sstevel@tonic-gate 7807c478bd9Sstevel@tonic-gate /* 7817c478bd9Sstevel@tonic-gate * free devmap handles from addr to the end of the mapping. 7827c478bd9Sstevel@tonic-gate */ 7837c478bd9Sstevel@tonic-gate if (dhp != NULL) 7847c478bd9Sstevel@tonic-gate devmap_handle_unmap_tail(dhp, addr); 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate return (0); 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate 7897c478bd9Sstevel@tonic-gate /* 7907c478bd9Sstevel@tonic-gate * The section to go is in the middle of the segment, 7917c478bd9Sstevel@tonic-gate * have to make it into two segments. nseg is made for 7927c478bd9Sstevel@tonic-gate * the high end while seg is cut down at the low end. 7937c478bd9Sstevel@tonic-gate */ 7947c478bd9Sstevel@tonic-gate nbase = addr + len; /* new seg base */ 7957c478bd9Sstevel@tonic-gate nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */ 7967c478bd9Sstevel@tonic-gate seg->s_size = addr - seg->s_base; /* shrink old seg */ 7977c478bd9Sstevel@tonic-gate nseg = seg_alloc(seg->s_as, nbase, nsize); 7987c478bd9Sstevel@tonic-gate if (nseg == NULL) 7997c478bd9Sstevel@tonic-gate panic("segdev_unmap seg_alloc"); 8007c478bd9Sstevel@tonic-gate 8017c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2, 8027c478bd9Sstevel@tonic-gate "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg); 8037c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n", 8047c478bd9Sstevel@tonic-gate (void *)seg, (void *)nseg)); 8057c478bd9Sstevel@tonic-gate nsdp = sdp_alloc(); 8067c478bd9Sstevel@tonic-gate 8077c478bd9Sstevel@tonic-gate nseg->s_ops = seg->s_ops; 8087c478bd9Sstevel@tonic-gate nseg->s_data = (void *)nsdp; 8097c478bd9Sstevel@tonic-gate 8107c478bd9Sstevel@tonic-gate VN_HOLD(sdp->vp); 8117c478bd9Sstevel@tonic-gate nsdp->mapfunc = sdp->mapfunc; 8127c478bd9Sstevel@tonic-gate nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base); 8137c478bd9Sstevel@tonic-gate nsdp->vp = sdp->vp; 8147c478bd9Sstevel@tonic-gate nsdp->pageprot = sdp->pageprot; 8157c478bd9Sstevel@tonic-gate nsdp->prot = sdp->prot; 8167c478bd9Sstevel@tonic-gate nsdp->maxprot = sdp->maxprot; 8177c478bd9Sstevel@tonic-gate nsdp->type = sdp->type; 8187c478bd9Sstevel@tonic-gate nsdp->hat_attr = sdp->hat_attr; 8197c478bd9Sstevel@tonic-gate nsdp->hat_flags = sdp->hat_flags; 8207c478bd9Sstevel@tonic-gate nsdp->softlockcnt = 0; 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate /* 8237c478bd9Sstevel@tonic-gate * Initialize per page data if the segment we are 8247c478bd9Sstevel@tonic-gate * dup'ing has per page information. 8257c478bd9Sstevel@tonic-gate */ 8267c478bd9Sstevel@tonic-gate if (sdp->vpage != NULL) { 8277c478bd9Sstevel@tonic-gate /* need to split vpage into two arrays */ 8287c478bd9Sstevel@tonic-gate register size_t nnbytes; 8297c478bd9Sstevel@tonic-gate register size_t nnpages; 8307c478bd9Sstevel@tonic-gate register struct vpage *ovpage; 8317c478bd9Sstevel@tonic-gate 8327c478bd9Sstevel@tonic-gate ovpage = sdp->vpage; /* keep pointer to vpage */ 8337c478bd9Sstevel@tonic-gate 8347c478bd9Sstevel@tonic-gate npages = seg_pages(seg); /* seg has shrunk */ 8357c478bd9Sstevel@tonic-gate nbytes = vpgtob(npages); 8367c478bd9Sstevel@tonic-gate nnpages = seg_pages(nseg); 8377c478bd9Sstevel@tonic-gate nnbytes = vpgtob(nnpages); 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate sdp->vpage = kmem_alloc(nbytes, KM_SLEEP); 8407c478bd9Sstevel@tonic-gate bcopy(ovpage, sdp->vpage, nbytes); 8417c478bd9Sstevel@tonic-gate 8427c478bd9Sstevel@tonic-gate nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP); 8437c478bd9Sstevel@tonic-gate bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes); 8447c478bd9Sstevel@tonic-gate 8457c478bd9Sstevel@tonic-gate /* free up old vpage */ 8467c478bd9Sstevel@tonic-gate kmem_free(ovpage, vpgtob(opages)); 8477c478bd9Sstevel@tonic-gate } else 8487c478bd9Sstevel@tonic-gate nsdp->vpage = NULL; 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate /* 8517c478bd9Sstevel@tonic-gate * unmap dhps. 8527c478bd9Sstevel@tonic-gate */ 8537c478bd9Sstevel@tonic-gate if (dhp == NULL) { 8547c478bd9Sstevel@tonic-gate nsdp->devmap_data = NULL; 8557c478bd9Sstevel@tonic-gate return (0); 8567c478bd9Sstevel@tonic-gate } 8577c478bd9Sstevel@tonic-gate while (dhp != NULL) { 8587c478bd9Sstevel@tonic-gate callbackops = &dhp->dh_callbackops; 8597c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3, 8607c478bd9Sstevel@tonic-gate "segdev_unmap: dhp=%p addr=%p", dhp, addr); 8617c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n", 8627c478bd9Sstevel@tonic-gate (void *)dhp, (void *)addr, 8637c478bd9Sstevel@tonic-gate (void *)dhp->dh_uvaddr, dhp->dh_len)); 8647c478bd9Sstevel@tonic-gate 8657c478bd9Sstevel@tonic-gate if (addr == (dhp->dh_uvaddr + dhp->dh_len)) { 8667c478bd9Sstevel@tonic-gate dhpp = dhp->dh_next; 8677c478bd9Sstevel@tonic-gate dhp->dh_next = NULL; 8687c478bd9Sstevel@tonic-gate dhp = dhpp; 8697c478bd9Sstevel@tonic-gate } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) { 8707c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 8717c478bd9Sstevel@tonic-gate } else if (addr > dhp->dh_uvaddr && 87244374aaeSsvemuri (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) { 8737c478bd9Sstevel@tonic-gate /* 8747c478bd9Sstevel@tonic-gate * <addr, addr+len> is enclosed by dhp. 8757c478bd9Sstevel@tonic-gate * create a newdhp that begins at addr+len and 8767c478bd9Sstevel@tonic-gate * ends at dhp->dh_uvaddr+dhp->dh_len. 8777c478bd9Sstevel@tonic-gate */ 8787c478bd9Sstevel@tonic-gate newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP); 8797c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 8807c478bd9Sstevel@tonic-gate bcopy(dhp, newdhp, sizeof (devmap_handle_t)); 8817c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 8827c478bd9Sstevel@tonic-gate newdhp->dh_seg = nseg; 8837c478bd9Sstevel@tonic-gate newdhp->dh_next = dhp->dh_next; 8847c478bd9Sstevel@tonic-gate if (dhp->dh_softlock != NULL) 8857c478bd9Sstevel@tonic-gate newdhp->dh_softlock = devmap_softlock_init( 88644374aaeSsvemuri newdhp->dh_dev, 88744374aaeSsvemuri (ulong_t)callbackops->devmap_access); 8887c478bd9Sstevel@tonic-gate if (dhp->dh_ctx != NULL) 8897c478bd9Sstevel@tonic-gate newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev, 89044374aaeSsvemuri (ulong_t)callbackops->devmap_access); 8917c478bd9Sstevel@tonic-gate if (newdhp->dh_flags & DEVMAP_LOCK_INITED) { 8927c478bd9Sstevel@tonic-gate mutex_init(&newdhp->dh_lock, 8937c478bd9Sstevel@tonic-gate NULL, MUTEX_DEFAULT, NULL); 8947c478bd9Sstevel@tonic-gate } 8957c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) 8967c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, 89744374aaeSsvemuri off, len, dhp, &dhp->dh_pvtp, 89844374aaeSsvemuri newdhp, &newdhp->dh_pvtp); 8997c478bd9Sstevel@tonic-gate mlen = len + (addr - dhp->dh_uvaddr); 9007c478bd9Sstevel@tonic-gate devmap_handle_reduce_len(newdhp, mlen); 9017c478bd9Sstevel@tonic-gate nsdp->devmap_data = newdhp; 9027c478bd9Sstevel@tonic-gate /* XX Changing len should recalculate LARGE flag */ 9037c478bd9Sstevel@tonic-gate dhp->dh_len = addr - dhp->dh_uvaddr; 9047c478bd9Sstevel@tonic-gate dhpp = dhp->dh_next; 9057c478bd9Sstevel@tonic-gate dhp->dh_next = NULL; 9067c478bd9Sstevel@tonic-gate dhp = dhpp; 9077c478bd9Sstevel@tonic-gate } else if ((addr > dhp->dh_uvaddr) && 90844374aaeSsvemuri ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) { 9097c478bd9Sstevel@tonic-gate mlen = dhp->dh_len + dhp->dh_uvaddr - addr; 9107c478bd9Sstevel@tonic-gate /* 9117c478bd9Sstevel@tonic-gate * <addr, addr+len> spans over dhps. 9127c478bd9Sstevel@tonic-gate */ 9137c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) 9147c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, 91544374aaeSsvemuri off, mlen, (devmap_cookie_t *)dhp, 91644374aaeSsvemuri &dhp->dh_pvtp, NULL, NULL); 9177c478bd9Sstevel@tonic-gate /* XX Changing len should recalculate LARGE flag */ 9187c478bd9Sstevel@tonic-gate dhp->dh_len = addr - dhp->dh_uvaddr; 9197c478bd9Sstevel@tonic-gate dhpp = dhp->dh_next; 9207c478bd9Sstevel@tonic-gate dhp->dh_next = NULL; 9217c478bd9Sstevel@tonic-gate dhp = dhpp; 9227c478bd9Sstevel@tonic-gate nsdp->devmap_data = dhp; 9237c478bd9Sstevel@tonic-gate } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) { 9247c478bd9Sstevel@tonic-gate /* 9257c478bd9Sstevel@tonic-gate * dhp is enclosed by <addr, addr+len>. 9267c478bd9Sstevel@tonic-gate */ 9277c478bd9Sstevel@tonic-gate dhp->dh_seg = nseg; 9287c478bd9Sstevel@tonic-gate nsdp->devmap_data = dhp; 9297c478bd9Sstevel@tonic-gate dhp = devmap_handle_unmap(dhp); 9307c478bd9Sstevel@tonic-gate nsdp->devmap_data = dhp; /* XX redundant? */ 9317c478bd9Sstevel@tonic-gate } else if (((addr + len) > dhp->dh_uvaddr) && 93244374aaeSsvemuri ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) { 9337c478bd9Sstevel@tonic-gate mlen = addr + len - dhp->dh_uvaddr; 9347c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) 9357c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, 93644374aaeSsvemuri dhp->dh_uoff, mlen, NULL, 93744374aaeSsvemuri NULL, dhp, &dhp->dh_pvtp); 9387c478bd9Sstevel@tonic-gate devmap_handle_reduce_len(dhp, mlen); 9397c478bd9Sstevel@tonic-gate nsdp->devmap_data = dhp; 9407c478bd9Sstevel@tonic-gate dhp->dh_seg = nseg; 9417c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 9427c478bd9Sstevel@tonic-gate } else { 9437c478bd9Sstevel@tonic-gate dhp->dh_seg = nseg; 9447c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 9457c478bd9Sstevel@tonic-gate } 9467c478bd9Sstevel@tonic-gate } 9477c478bd9Sstevel@tonic-gate return (0); 9487c478bd9Sstevel@tonic-gate } 9497c478bd9Sstevel@tonic-gate 9507c478bd9Sstevel@tonic-gate /* 9517c478bd9Sstevel@tonic-gate * Utility function handles reducing the length of a devmap handle during unmap 9527c478bd9Sstevel@tonic-gate * Note that is only used for unmapping the front portion of the handler, 9537c478bd9Sstevel@tonic-gate * i.e., we are bumping up the offset/pfn etc up by len 9547c478bd9Sstevel@tonic-gate * Do not use if reducing length at the tail. 9557c478bd9Sstevel@tonic-gate */ 9567c478bd9Sstevel@tonic-gate static void 9577c478bd9Sstevel@tonic-gate devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len) 9587c478bd9Sstevel@tonic-gate { 9597c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *cp; 9607c478bd9Sstevel@tonic-gate struct devmap_pmem_cookie *pcp; 9617c478bd9Sstevel@tonic-gate /* 9627c478bd9Sstevel@tonic-gate * adjust devmap handle fields 9637c478bd9Sstevel@tonic-gate */ 9647c478bd9Sstevel@tonic-gate ASSERT(len < dhp->dh_len); 9657c478bd9Sstevel@tonic-gate 9667c478bd9Sstevel@tonic-gate /* Make sure only page-aligned changes are done */ 9677c478bd9Sstevel@tonic-gate ASSERT((len & PAGEOFFSET) == 0); 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate dhp->dh_len -= len; 9707c478bd9Sstevel@tonic-gate dhp->dh_uoff += (offset_t)len; 9717c478bd9Sstevel@tonic-gate dhp->dh_roff += (offset_t)len; 9727c478bd9Sstevel@tonic-gate dhp->dh_uvaddr += len; 9737c478bd9Sstevel@tonic-gate /* Need to grab dhp lock if REMAP */ 9747c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 9757c478bd9Sstevel@tonic-gate cp = dhp->dh_cookie; 9767c478bd9Sstevel@tonic-gate if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) { 9777c478bd9Sstevel@tonic-gate if (cookie_is_devmem(cp)) { 9787c478bd9Sstevel@tonic-gate dhp->dh_pfn += btop(len); 9797c478bd9Sstevel@tonic-gate } else if (cookie_is_pmem(cp)) { 9807c478bd9Sstevel@tonic-gate pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; 9817c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && 98244374aaeSsvemuri dhp->dh_roff < ptob(pcp->dp_npages)); 9837c478bd9Sstevel@tonic-gate } else { 9847c478bd9Sstevel@tonic-gate ASSERT(dhp->dh_roff < cp->size); 9857c478bd9Sstevel@tonic-gate ASSERT(dhp->dh_cvaddr >= cp->cvaddr && 98644374aaeSsvemuri dhp->dh_cvaddr < (cp->cvaddr + cp->size)); 9877c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_cvaddr + len) <= 98844374aaeSsvemuri (cp->cvaddr + cp->size)); 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate dhp->dh_cvaddr += len; 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate } 9937c478bd9Sstevel@tonic-gate /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */ 9947c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 9957c478bd9Sstevel@tonic-gate } 9967c478bd9Sstevel@tonic-gate 9977c478bd9Sstevel@tonic-gate /* 9987c478bd9Sstevel@tonic-gate * Free devmap handle, dhp. 9997c478bd9Sstevel@tonic-gate * Return the next devmap handle on the linked list. 10007c478bd9Sstevel@tonic-gate */ 10017c478bd9Sstevel@tonic-gate static devmap_handle_t * 10027c478bd9Sstevel@tonic-gate devmap_handle_unmap(devmap_handle_t *dhp) 10037c478bd9Sstevel@tonic-gate { 10047c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops; 10057c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data; 10067c478bd9Sstevel@tonic-gate devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data; 10077c478bd9Sstevel@tonic-gate 10087c478bd9Sstevel@tonic-gate ASSERT(dhp != NULL); 10097c478bd9Sstevel@tonic-gate 10107c478bd9Sstevel@tonic-gate /* 10117c478bd9Sstevel@tonic-gate * before we free up dhp, call the driver's devmap_unmap entry point 10127c478bd9Sstevel@tonic-gate * to free resources allocated for this dhp. 10137c478bd9Sstevel@tonic-gate */ 10147c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) { 10157c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff, 101644374aaeSsvemuri dhp->dh_len, NULL, NULL, NULL, NULL); 10177c478bd9Sstevel@tonic-gate } 10187c478bd9Sstevel@tonic-gate 10197c478bd9Sstevel@tonic-gate if (dhpp == dhp) { /* releasing first dhp, change sdp data */ 10207c478bd9Sstevel@tonic-gate sdp->devmap_data = dhp->dh_next; 10217c478bd9Sstevel@tonic-gate } else { 10227c478bd9Sstevel@tonic-gate while (dhpp->dh_next != dhp) { 10237c478bd9Sstevel@tonic-gate dhpp = dhpp->dh_next; 10247c478bd9Sstevel@tonic-gate } 10257c478bd9Sstevel@tonic-gate dhpp->dh_next = dhp->dh_next; 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate dhpp = dhp->dh_next; /* return value is next dhp in chain */ 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate if (dhp->dh_softlock != NULL) 10307c478bd9Sstevel@tonic-gate devmap_softlock_rele(dhp); 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate if (dhp->dh_ctx != NULL) 10337c478bd9Sstevel@tonic-gate devmap_ctx_rele(dhp); 10347c478bd9Sstevel@tonic-gate 10357c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_LOCK_INITED) { 10367c478bd9Sstevel@tonic-gate mutex_destroy(&dhp->dh_lock); 10377c478bd9Sstevel@tonic-gate } 10387c478bd9Sstevel@tonic-gate kmem_free(dhp, sizeof (devmap_handle_t)); 10397c478bd9Sstevel@tonic-gate 10407c478bd9Sstevel@tonic-gate return (dhpp); 10417c478bd9Sstevel@tonic-gate } 10427c478bd9Sstevel@tonic-gate 10437c478bd9Sstevel@tonic-gate /* 10447c478bd9Sstevel@tonic-gate * Free complete devmap handles from dhp for len bytes 10457c478bd9Sstevel@tonic-gate * dhp can be either the first handle or a subsequent handle 10467c478bd9Sstevel@tonic-gate */ 10477c478bd9Sstevel@tonic-gate static void 10487c478bd9Sstevel@tonic-gate devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len) 10497c478bd9Sstevel@tonic-gate { 10507c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops; 10517c478bd9Sstevel@tonic-gate 10527c478bd9Sstevel@tonic-gate /* 10537c478bd9Sstevel@tonic-gate * free the devmap handles covered by len. 10547c478bd9Sstevel@tonic-gate */ 10557c478bd9Sstevel@tonic-gate while (len >= dhp->dh_len) { 10567c478bd9Sstevel@tonic-gate len -= dhp->dh_len; 10577c478bd9Sstevel@tonic-gate dhp = devmap_handle_unmap(dhp); 10587c478bd9Sstevel@tonic-gate } 10597c478bd9Sstevel@tonic-gate if (len != 0) { /* partial unmap at head of first remaining dhp */ 10607c478bd9Sstevel@tonic-gate callbackops = &dhp->dh_callbackops; 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate /* 10637c478bd9Sstevel@tonic-gate * Call the unmap callback so the drivers can make 10647c478bd9Sstevel@tonic-gate * adjustment on its private data. 10657c478bd9Sstevel@tonic-gate */ 10667c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) 10677c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, 10687c478bd9Sstevel@tonic-gate dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp); 10697c478bd9Sstevel@tonic-gate devmap_handle_reduce_len(dhp, len); 10707c478bd9Sstevel@tonic-gate } 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate 10737c478bd9Sstevel@tonic-gate /* 10747c478bd9Sstevel@tonic-gate * Free devmap handles to truncate the mapping after addr 10757c478bd9Sstevel@tonic-gate * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again) 10767c478bd9Sstevel@tonic-gate * Also could then use the routine in middle unmap case too 10777c478bd9Sstevel@tonic-gate */ 10787c478bd9Sstevel@tonic-gate static void 10797c478bd9Sstevel@tonic-gate devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr) 10807c478bd9Sstevel@tonic-gate { 10817c478bd9Sstevel@tonic-gate register struct seg *seg = dhp->dh_seg; 10827c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 10837c478bd9Sstevel@tonic-gate register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data; 10847c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops; 10857c478bd9Sstevel@tonic-gate register devmap_handle_t *dhpp; 10867c478bd9Sstevel@tonic-gate size_t maplen; 10877c478bd9Sstevel@tonic-gate ulong_t off; 10887c478bd9Sstevel@tonic-gate size_t len; 10897c478bd9Sstevel@tonic-gate 10907c478bd9Sstevel@tonic-gate maplen = (size_t)(addr - dhp->dh_uvaddr); 10917c478bd9Sstevel@tonic-gate dhph = devmap_find_handle(dhph, addr); 10927c478bd9Sstevel@tonic-gate 10937c478bd9Sstevel@tonic-gate while (dhph != NULL) { 10947c478bd9Sstevel@tonic-gate if (maplen == 0) { 10957c478bd9Sstevel@tonic-gate dhph = devmap_handle_unmap(dhph); 10967c478bd9Sstevel@tonic-gate } else { 10977c478bd9Sstevel@tonic-gate callbackops = &dhph->dh_callbackops; 10987c478bd9Sstevel@tonic-gate len = dhph->dh_len - maplen; 10997c478bd9Sstevel@tonic-gate off = (ulong_t)sdp->offset + (addr - seg->s_base); 11007c478bd9Sstevel@tonic-gate /* 11017c478bd9Sstevel@tonic-gate * Call the unmap callback so the driver 11027c478bd9Sstevel@tonic-gate * can make adjustments on its private data. 11037c478bd9Sstevel@tonic-gate */ 11047c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) 11057c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)(dhph, 110644374aaeSsvemuri dhph->dh_pvtp, off, len, 110744374aaeSsvemuri (devmap_cookie_t *)dhph, 110844374aaeSsvemuri &dhph->dh_pvtp, NULL, NULL); 11097c478bd9Sstevel@tonic-gate /* XXX Reducing len needs to recalculate LARGE flag */ 11107c478bd9Sstevel@tonic-gate dhph->dh_len = maplen; 11117c478bd9Sstevel@tonic-gate maplen = 0; 11127c478bd9Sstevel@tonic-gate dhpp = dhph->dh_next; 11137c478bd9Sstevel@tonic-gate dhph->dh_next = NULL; 11147c478bd9Sstevel@tonic-gate dhph = dhpp; 11157c478bd9Sstevel@tonic-gate } 11167c478bd9Sstevel@tonic-gate } /* end while */ 11177c478bd9Sstevel@tonic-gate } 11187c478bd9Sstevel@tonic-gate 11197c478bd9Sstevel@tonic-gate /* 11207c478bd9Sstevel@tonic-gate * Free a segment. 11217c478bd9Sstevel@tonic-gate */ 11227c478bd9Sstevel@tonic-gate static void 11237c478bd9Sstevel@tonic-gate segdev_free(struct seg *seg) 11247c478bd9Sstevel@tonic-gate { 11257c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 11267c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data; 11277c478bd9Sstevel@tonic-gate 11287c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE, 11297c478bd9Sstevel@tonic-gate "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg); 11307c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n", 11317c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg)); 11327c478bd9Sstevel@tonic-gate 11337c478bd9Sstevel@tonic-gate /* 11347c478bd9Sstevel@tonic-gate * Since the address space is "write" locked, we 11357c478bd9Sstevel@tonic-gate * don't need the segment lock to protect "segdev" data. 11367c478bd9Sstevel@tonic-gate */ 11377c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock)); 11387c478bd9Sstevel@tonic-gate 11397c478bd9Sstevel@tonic-gate while (dhp != NULL) 11407c478bd9Sstevel@tonic-gate dhp = devmap_handle_unmap(dhp); 11417c478bd9Sstevel@tonic-gate 11427c478bd9Sstevel@tonic-gate VN_RELE(sdp->vp); 11437c478bd9Sstevel@tonic-gate if (sdp->vpage != NULL) 11447c478bd9Sstevel@tonic-gate kmem_free(sdp->vpage, vpgtob(seg_pages(seg))); 11457c478bd9Sstevel@tonic-gate 114644374aaeSsvemuri rw_destroy(&sdp->lock); 11477c478bd9Sstevel@tonic-gate kmem_free(sdp, sizeof (*sdp)); 11487c478bd9Sstevel@tonic-gate } 11497c478bd9Sstevel@tonic-gate 11507c478bd9Sstevel@tonic-gate static void 11517c478bd9Sstevel@tonic-gate free_devmap_handle(devmap_handle_t *dhp) 11527c478bd9Sstevel@tonic-gate { 11537c478bd9Sstevel@tonic-gate register devmap_handle_t *dhpp; 11547c478bd9Sstevel@tonic-gate 11557c478bd9Sstevel@tonic-gate /* 11567c478bd9Sstevel@tonic-gate * free up devmap handle 11577c478bd9Sstevel@tonic-gate */ 11587c478bd9Sstevel@tonic-gate while (dhp != NULL) { 11597c478bd9Sstevel@tonic-gate dhpp = dhp->dh_next; 11607c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_LOCK_INITED) { 11617c478bd9Sstevel@tonic-gate mutex_destroy(&dhp->dh_lock); 11627c478bd9Sstevel@tonic-gate } 11637c478bd9Sstevel@tonic-gate 11647c478bd9Sstevel@tonic-gate if (dhp->dh_softlock != NULL) 11657c478bd9Sstevel@tonic-gate devmap_softlock_rele(dhp); 11667c478bd9Sstevel@tonic-gate 11677c478bd9Sstevel@tonic-gate if (dhp->dh_ctx != NULL) 11687c478bd9Sstevel@tonic-gate devmap_ctx_rele(dhp); 11697c478bd9Sstevel@tonic-gate 11707c478bd9Sstevel@tonic-gate kmem_free(dhp, sizeof (devmap_handle_t)); 11717c478bd9Sstevel@tonic-gate dhp = dhpp; 11727c478bd9Sstevel@tonic-gate } 11737c478bd9Sstevel@tonic-gate } 11747c478bd9Sstevel@tonic-gate 11757c478bd9Sstevel@tonic-gate /* 11767c478bd9Sstevel@tonic-gate * routines to lock and unlock underlying segkp segment for 11777c478bd9Sstevel@tonic-gate * KMEM_PAGEABLE type cookies. 11787c478bd9Sstevel@tonic-gate * segkp only allows a single pending F_SOFTLOCK 11797c478bd9Sstevel@tonic-gate * we keep track of number of locks in the cookie so we can 11807c478bd9Sstevel@tonic-gate * have multiple pending faults and manage the calls to segkp. 11817c478bd9Sstevel@tonic-gate * RFE: if segkp supports either pagelock or can support multiple 11827c478bd9Sstevel@tonic-gate * calls to F_SOFTLOCK, then these routines can go away. 11837c478bd9Sstevel@tonic-gate * If pagelock, segdev_faultpage can fault on a page by page basis 11847c478bd9Sstevel@tonic-gate * and simplifies the code quite a bit. 11857c478bd9Sstevel@tonic-gate * if multiple calls allowed but not partial ranges, then need for 11867c478bd9Sstevel@tonic-gate * cookie->lock and locked count goes away, code can call as_fault directly 11877c478bd9Sstevel@tonic-gate */ 11887c478bd9Sstevel@tonic-gate static faultcode_t 11897c478bd9Sstevel@tonic-gate acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages) 11907c478bd9Sstevel@tonic-gate { 11917c478bd9Sstevel@tonic-gate int err = 0; 11927c478bd9Sstevel@tonic-gate ASSERT(cookie_is_kpmem(cookie)); 11937c478bd9Sstevel@tonic-gate /* 11947c478bd9Sstevel@tonic-gate * Fault in pages in segkp with F_SOFTLOCK. 11957c478bd9Sstevel@tonic-gate * We want to hold the lock until all pages have been loaded. 11967c478bd9Sstevel@tonic-gate * segkp only allows single caller to hold SOFTLOCK, so cookie 11977c478bd9Sstevel@tonic-gate * holds a count so we dont call into segkp multiple times 11987c478bd9Sstevel@tonic-gate */ 11997c478bd9Sstevel@tonic-gate mutex_enter(&cookie->lock); 12007c478bd9Sstevel@tonic-gate 12017c478bd9Sstevel@tonic-gate /* 12027c478bd9Sstevel@tonic-gate * Check for overflow in locked field 12037c478bd9Sstevel@tonic-gate */ 12047c478bd9Sstevel@tonic-gate if ((UINT32_MAX - cookie->locked) < npages) { 12057c478bd9Sstevel@tonic-gate err = FC_MAKE_ERR(ENOMEM); 12067c478bd9Sstevel@tonic-gate } else if (cookie->locked == 0) { 12077c478bd9Sstevel@tonic-gate /* First time locking */ 12087c478bd9Sstevel@tonic-gate err = as_fault(kas.a_hat, &kas, cookie->cvaddr, 12097c478bd9Sstevel@tonic-gate cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE); 12107c478bd9Sstevel@tonic-gate } 12117c478bd9Sstevel@tonic-gate if (!err) { 12127c478bd9Sstevel@tonic-gate cookie->locked += npages; 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate mutex_exit(&cookie->lock); 12157c478bd9Sstevel@tonic-gate return (err); 12167c478bd9Sstevel@tonic-gate } 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate static void 12197c478bd9Sstevel@tonic-gate release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages) 12207c478bd9Sstevel@tonic-gate { 12217c478bd9Sstevel@tonic-gate mutex_enter(&cookie->lock); 12227c478bd9Sstevel@tonic-gate ASSERT(cookie_is_kpmem(cookie)); 12237c478bd9Sstevel@tonic-gate ASSERT(cookie->locked >= npages); 12247c478bd9Sstevel@tonic-gate cookie->locked -= (uint_t)npages; 12257c478bd9Sstevel@tonic-gate if (cookie->locked == 0) { 12267c478bd9Sstevel@tonic-gate /* Last unlock */ 12277c478bd9Sstevel@tonic-gate if (as_fault(kas.a_hat, &kas, cookie->cvaddr, 12287c478bd9Sstevel@tonic-gate cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE)) 12297c478bd9Sstevel@tonic-gate panic("segdev releasing kpmem lock %p", (void *)cookie); 12307c478bd9Sstevel@tonic-gate } 12317c478bd9Sstevel@tonic-gate mutex_exit(&cookie->lock); 12327c478bd9Sstevel@tonic-gate } 12337c478bd9Sstevel@tonic-gate 12347c478bd9Sstevel@tonic-gate /* 12357c478bd9Sstevel@tonic-gate * Routines to synchronize F_SOFTLOCK and F_INVAL faults for 12367c478bd9Sstevel@tonic-gate * drivers with devmap_access callbacks 12377c478bd9Sstevel@tonic-gate * slock->softlocked basically works like a rw lock 12387c478bd9Sstevel@tonic-gate * -ve counts => F_SOFTLOCK in progress 12397c478bd9Sstevel@tonic-gate * +ve counts => F_INVAL/F_PROT in progress 12407c478bd9Sstevel@tonic-gate * We allow only one F_SOFTLOCK at a time 12417c478bd9Sstevel@tonic-gate * but can have multiple pending F_INVAL/F_PROT calls 12427c478bd9Sstevel@tonic-gate * 12437c478bd9Sstevel@tonic-gate * This routine waits using cv_wait_sig so killing processes is more graceful 12447c478bd9Sstevel@tonic-gate * Returns EINTR if coming out of this routine due to a signal, 0 otherwise 12457c478bd9Sstevel@tonic-gate */ 12467c478bd9Sstevel@tonic-gate static int devmap_softlock_enter( 12477c478bd9Sstevel@tonic-gate struct devmap_softlock *slock, 12487c478bd9Sstevel@tonic-gate size_t npages, 12497c478bd9Sstevel@tonic-gate enum fault_type type) 12507c478bd9Sstevel@tonic-gate { 12517c478bd9Sstevel@tonic-gate if (npages == 0) 12527c478bd9Sstevel@tonic-gate return (0); 12537c478bd9Sstevel@tonic-gate mutex_enter(&(slock->lock)); 12547c478bd9Sstevel@tonic-gate switch (type) { 12557c478bd9Sstevel@tonic-gate case F_SOFTLOCK : 12567c478bd9Sstevel@tonic-gate while (slock->softlocked) { 12577c478bd9Sstevel@tonic-gate if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) { 12587c478bd9Sstevel@tonic-gate /* signalled */ 12597c478bd9Sstevel@tonic-gate mutex_exit(&(slock->lock)); 12607c478bd9Sstevel@tonic-gate return (EINTR); 12617c478bd9Sstevel@tonic-gate } 12627c478bd9Sstevel@tonic-gate } 12637c478bd9Sstevel@tonic-gate slock->softlocked -= npages; /* -ve count => locked */ 12647c478bd9Sstevel@tonic-gate break; 12657c478bd9Sstevel@tonic-gate case F_INVAL : 12667c478bd9Sstevel@tonic-gate case F_PROT : 12677c478bd9Sstevel@tonic-gate while (slock->softlocked < 0) 12687c478bd9Sstevel@tonic-gate if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) { 12697c478bd9Sstevel@tonic-gate /* signalled */ 12707c478bd9Sstevel@tonic-gate mutex_exit(&(slock->lock)); 12717c478bd9Sstevel@tonic-gate return (EINTR); 12727c478bd9Sstevel@tonic-gate } 12737c478bd9Sstevel@tonic-gate slock->softlocked += npages; /* +ve count => f_invals */ 12747c478bd9Sstevel@tonic-gate break; 12757c478bd9Sstevel@tonic-gate default: 12767c478bd9Sstevel@tonic-gate ASSERT(0); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate mutex_exit(&(slock->lock)); 12797c478bd9Sstevel@tonic-gate return (0); 12807c478bd9Sstevel@tonic-gate } 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate static void devmap_softlock_exit( 12837c478bd9Sstevel@tonic-gate struct devmap_softlock *slock, 12847c478bd9Sstevel@tonic-gate size_t npages, 12857c478bd9Sstevel@tonic-gate enum fault_type type) 12867c478bd9Sstevel@tonic-gate { 12877c478bd9Sstevel@tonic-gate if (slock == NULL) 12887c478bd9Sstevel@tonic-gate return; 12897c478bd9Sstevel@tonic-gate mutex_enter(&(slock->lock)); 12907c478bd9Sstevel@tonic-gate switch (type) { 12917c478bd9Sstevel@tonic-gate case F_SOFTLOCK : 12927c478bd9Sstevel@tonic-gate ASSERT(-slock->softlocked >= npages); 12937c478bd9Sstevel@tonic-gate slock->softlocked += npages; /* -ve count is softlocked */ 12947c478bd9Sstevel@tonic-gate if (slock->softlocked == 0) 12957c478bd9Sstevel@tonic-gate cv_signal(&slock->cv); 12967c478bd9Sstevel@tonic-gate break; 12977c478bd9Sstevel@tonic-gate case F_INVAL : 12987c478bd9Sstevel@tonic-gate case F_PROT: 12997c478bd9Sstevel@tonic-gate ASSERT(slock->softlocked >= npages); 13007c478bd9Sstevel@tonic-gate slock->softlocked -= npages; 13017c478bd9Sstevel@tonic-gate if (slock->softlocked == 0) 13027c478bd9Sstevel@tonic-gate cv_signal(&slock->cv); 13037c478bd9Sstevel@tonic-gate break; 13047c478bd9Sstevel@tonic-gate default: 13057c478bd9Sstevel@tonic-gate ASSERT(0); 13067c478bd9Sstevel@tonic-gate } 13077c478bd9Sstevel@tonic-gate mutex_exit(&(slock->lock)); 13087c478bd9Sstevel@tonic-gate } 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate /* 13117c478bd9Sstevel@tonic-gate * Do a F_SOFTUNLOCK call over the range requested. 13127c478bd9Sstevel@tonic-gate * The range must have already been F_SOFTLOCK'ed. 13137c478bd9Sstevel@tonic-gate * The segment lock should be held, (but not the segment private lock?) 13147c478bd9Sstevel@tonic-gate * The softunlock code below does not adjust for large page sizes 13157c478bd9Sstevel@tonic-gate * assumes the caller already did any addr/len adjustments for 13167c478bd9Sstevel@tonic-gate * pagesize mappings before calling. 13177c478bd9Sstevel@tonic-gate */ 13187c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13197c478bd9Sstevel@tonic-gate static void 13207c478bd9Sstevel@tonic-gate segdev_softunlock( 13217c478bd9Sstevel@tonic-gate struct hat *hat, /* the hat */ 13227c478bd9Sstevel@tonic-gate struct seg *seg, /* seg_dev of interest */ 13237c478bd9Sstevel@tonic-gate caddr_t addr, /* base address of range */ 13247c478bd9Sstevel@tonic-gate size_t len, /* number of bytes */ 13257c478bd9Sstevel@tonic-gate enum seg_rw rw) /* type of access at fault */ 13267c478bd9Sstevel@tonic-gate { 13277c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 13287c478bd9Sstevel@tonic-gate devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data; 13297c478bd9Sstevel@tonic-gate 13307c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK, 13317c478bd9Sstevel@tonic-gate "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx", 13327c478bd9Sstevel@tonic-gate dhp_head, sdp, addr, len); 13337c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx " 13347c478bd9Sstevel@tonic-gate "addr %p len %lx\n", 13357c478bd9Sstevel@tonic-gate (void *)dhp_head, sdp->softlockcnt, (void *)addr, len)); 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate hat_unlock(hat, addr, len); 13387c478bd9Sstevel@tonic-gate 13397c478bd9Sstevel@tonic-gate if (dhp_head != NULL) { 13407c478bd9Sstevel@tonic-gate devmap_handle_t *dhp; 13417c478bd9Sstevel@tonic-gate size_t mlen; 1342d5f2339fSmec size_t tlen = len; 13437c478bd9Sstevel@tonic-gate ulong_t off; 13447c478bd9Sstevel@tonic-gate 13457c478bd9Sstevel@tonic-gate dhp = devmap_find_handle(dhp_head, addr); 13467c478bd9Sstevel@tonic-gate ASSERT(dhp != NULL); 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate off = (ulong_t)(addr - dhp->dh_uvaddr); 1349d5f2339fSmec while (tlen != 0) { 1350d5f2339fSmec mlen = MIN(tlen, (dhp->dh_len - off)); 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate /* 13537c478bd9Sstevel@tonic-gate * unlock segkp memory, locked during F_SOFTLOCK 13547c478bd9Sstevel@tonic-gate */ 13557c478bd9Sstevel@tonic-gate if (dhp_is_kpmem(dhp)) { 13567c478bd9Sstevel@tonic-gate release_kpmem_lock( 13577c478bd9Sstevel@tonic-gate (struct ddi_umem_cookie *)dhp->dh_cookie, 13587c478bd9Sstevel@tonic-gate btopr(mlen)); 13597c478bd9Sstevel@tonic-gate } 13607c478bd9Sstevel@tonic-gate 13617c478bd9Sstevel@tonic-gate /* 13627c478bd9Sstevel@tonic-gate * Do the softlock accounting for devmap_access 13637c478bd9Sstevel@tonic-gate */ 13647c478bd9Sstevel@tonic-gate if (dhp->dh_callbackops.devmap_access != NULL) { 13657c478bd9Sstevel@tonic-gate devmap_softlock_exit(dhp->dh_softlock, 136644374aaeSsvemuri btopr(mlen), F_SOFTLOCK); 13677c478bd9Sstevel@tonic-gate } 13687c478bd9Sstevel@tonic-gate 1369d5f2339fSmec tlen -= mlen; 13707c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 13717c478bd9Sstevel@tonic-gate off = 0; 13727c478bd9Sstevel@tonic-gate } 13737c478bd9Sstevel@tonic-gate } 13747c478bd9Sstevel@tonic-gate 13757c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 13767c478bd9Sstevel@tonic-gate ASSERT(sdp->softlockcnt >= btopr(len)); 13777c478bd9Sstevel@tonic-gate sdp->softlockcnt -= btopr(len); 13787c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 13797c478bd9Sstevel@tonic-gate if (sdp->softlockcnt == 0) { 13807c478bd9Sstevel@tonic-gate /* 13817c478bd9Sstevel@tonic-gate * All SOFTLOCKS are gone. Wakeup any waiting 13827c478bd9Sstevel@tonic-gate * unmappers so they can try again to unmap. 13837c478bd9Sstevel@tonic-gate * Check for waiters first without the mutex 13847c478bd9Sstevel@tonic-gate * held so we don't always grab the mutex on 13857c478bd9Sstevel@tonic-gate * softunlocks. 13867c478bd9Sstevel@tonic-gate */ 13877c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 13887c478bd9Sstevel@tonic-gate mutex_enter(&seg->s_as->a_contents); 13897c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(seg->s_as)) { 13907c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(seg->s_as); 13917c478bd9Sstevel@tonic-gate cv_broadcast(&seg->s_as->a_cv); 13927c478bd9Sstevel@tonic-gate } 13937c478bd9Sstevel@tonic-gate mutex_exit(&seg->s_as->a_contents); 13947c478bd9Sstevel@tonic-gate } 13957c478bd9Sstevel@tonic-gate } 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate } 13987c478bd9Sstevel@tonic-gate 13997c478bd9Sstevel@tonic-gate /* 14007c478bd9Sstevel@tonic-gate * Handle fault for a single page. 14017c478bd9Sstevel@tonic-gate * Done in a separate routine so we can handle errors more easily. 14027c478bd9Sstevel@tonic-gate * This routine is called only from segdev_faultpages() 14037c478bd9Sstevel@tonic-gate * when looping over the range of addresses requested. The segment lock is held. 14047c478bd9Sstevel@tonic-gate */ 14057c478bd9Sstevel@tonic-gate static faultcode_t 14067c478bd9Sstevel@tonic-gate segdev_faultpage( 14077c478bd9Sstevel@tonic-gate struct hat *hat, /* the hat */ 14087c478bd9Sstevel@tonic-gate struct seg *seg, /* seg_dev of interest */ 14097c478bd9Sstevel@tonic-gate caddr_t addr, /* address in as */ 14107c478bd9Sstevel@tonic-gate struct vpage *vpage, /* pointer to vpage for seg, addr */ 14117c478bd9Sstevel@tonic-gate enum fault_type type, /* type of fault */ 14127c478bd9Sstevel@tonic-gate enum seg_rw rw, /* type of access at fault */ 14137c478bd9Sstevel@tonic-gate devmap_handle_t *dhp) /* devmap handle if any for this page */ 14147c478bd9Sstevel@tonic-gate { 14157c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 14167c478bd9Sstevel@tonic-gate uint_t prot; 14177c478bd9Sstevel@tonic-gate pfn_t pfnum = PFN_INVALID; 14187c478bd9Sstevel@tonic-gate u_offset_t offset; 14197c478bd9Sstevel@tonic-gate uint_t hat_flags; 14207c478bd9Sstevel@tonic-gate dev_info_t *dip; 14217c478bd9Sstevel@tonic-gate 14227c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE, 14237c478bd9Sstevel@tonic-gate "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr); 14247c478bd9Sstevel@tonic-gate DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n", 14257c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg, (void *)addr)); 14267c478bd9Sstevel@tonic-gate 14277c478bd9Sstevel@tonic-gate /* 14287c478bd9Sstevel@tonic-gate * Initialize protection value for this page. 14297c478bd9Sstevel@tonic-gate * If we have per page protection values check it now. 14307c478bd9Sstevel@tonic-gate */ 14317c478bd9Sstevel@tonic-gate if (sdp->pageprot) { 14327c478bd9Sstevel@tonic-gate uint_t protchk; 14337c478bd9Sstevel@tonic-gate 14347c478bd9Sstevel@tonic-gate switch (rw) { 14357c478bd9Sstevel@tonic-gate case S_READ: 14367c478bd9Sstevel@tonic-gate protchk = PROT_READ; 14377c478bd9Sstevel@tonic-gate break; 14387c478bd9Sstevel@tonic-gate case S_WRITE: 14397c478bd9Sstevel@tonic-gate protchk = PROT_WRITE; 14407c478bd9Sstevel@tonic-gate break; 14417c478bd9Sstevel@tonic-gate case S_EXEC: 14427c478bd9Sstevel@tonic-gate protchk = PROT_EXEC; 14437c478bd9Sstevel@tonic-gate break; 14447c478bd9Sstevel@tonic-gate case S_OTHER: 14457c478bd9Sstevel@tonic-gate default: 14467c478bd9Sstevel@tonic-gate protchk = PROT_READ | PROT_WRITE | PROT_EXEC; 14477c478bd9Sstevel@tonic-gate break; 14487c478bd9Sstevel@tonic-gate } 14497c478bd9Sstevel@tonic-gate 14507c478bd9Sstevel@tonic-gate prot = VPP_PROT(vpage); 14517c478bd9Sstevel@tonic-gate if ((prot & protchk) == 0) 14527c478bd9Sstevel@tonic-gate return (FC_PROT); /* illegal access type */ 14537c478bd9Sstevel@tonic-gate } else { 14547c478bd9Sstevel@tonic-gate prot = sdp->prot; 14557c478bd9Sstevel@tonic-gate /* caller has already done segment level protection check */ 14567c478bd9Sstevel@tonic-gate } 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 14597c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 14607c478bd9Sstevel@tonic-gate sdp->softlockcnt++; 14617c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 14627c478bd9Sstevel@tonic-gate } 14637c478bd9Sstevel@tonic-gate 14647c478bd9Sstevel@tonic-gate hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD); 14657c478bd9Sstevel@tonic-gate offset = sdp->offset + (u_offset_t)(addr - seg->s_base); 14667c478bd9Sstevel@tonic-gate /* 14677c478bd9Sstevel@tonic-gate * In the devmap framework, sdp->mapfunc is set to NULL. we can get 14687c478bd9Sstevel@tonic-gate * pfnum from dhp->dh_pfn (at beginning of segment) and offset from 14697c478bd9Sstevel@tonic-gate * seg->s_base. 14707c478bd9Sstevel@tonic-gate */ 14717c478bd9Sstevel@tonic-gate if (dhp == NULL) { 14727c478bd9Sstevel@tonic-gate /* If segment has devmap_data, then dhp should be non-NULL */ 14737c478bd9Sstevel@tonic-gate ASSERT(sdp->devmap_data == NULL); 14747c478bd9Sstevel@tonic-gate pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev, 147544374aaeSsvemuri (off_t)offset, prot); 14767c478bd9Sstevel@tonic-gate prot |= sdp->hat_attr; 14777c478bd9Sstevel@tonic-gate } else { 14787c478bd9Sstevel@tonic-gate ulong_t off; 14797c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *cp; 14807c478bd9Sstevel@tonic-gate struct devmap_pmem_cookie *pcp; 14817c478bd9Sstevel@tonic-gate 14827c478bd9Sstevel@tonic-gate /* ensure the dhp passed in contains addr. */ 14837c478bd9Sstevel@tonic-gate ASSERT(dhp == devmap_find_handle( 148444374aaeSsvemuri (devmap_handle_t *)sdp->devmap_data, addr)); 14857c478bd9Sstevel@tonic-gate 14867c478bd9Sstevel@tonic-gate off = addr - dhp->dh_uvaddr; 14877c478bd9Sstevel@tonic-gate 14887c478bd9Sstevel@tonic-gate /* 14897c478bd9Sstevel@tonic-gate * This routine assumes that the caller makes sure that the 14907c478bd9Sstevel@tonic-gate * fields in dhp used below are unchanged due to remap during 14917c478bd9Sstevel@tonic-gate * this call. Caller does HOLD_DHP_LOCK if neeed 14927c478bd9Sstevel@tonic-gate */ 14937c478bd9Sstevel@tonic-gate cp = dhp->dh_cookie; 14947c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { 14957c478bd9Sstevel@tonic-gate pfnum = PFN_INVALID; 14967c478bd9Sstevel@tonic-gate } else if (cookie_is_devmem(cp)) { 14977c478bd9Sstevel@tonic-gate pfnum = dhp->dh_pfn + btop(off); 14987c478bd9Sstevel@tonic-gate } else if (cookie_is_pmem(cp)) { 14997c478bd9Sstevel@tonic-gate pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie; 15007c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 && 150144374aaeSsvemuri dhp->dh_roff < ptob(pcp->dp_npages)); 15027c478bd9Sstevel@tonic-gate pfnum = page_pptonum( 15037c478bd9Sstevel@tonic-gate pcp->dp_pparray[btop(off + dhp->dh_roff)]); 15047c478bd9Sstevel@tonic-gate } else { 15057c478bd9Sstevel@tonic-gate ASSERT(dhp->dh_roff < cp->size); 15067c478bd9Sstevel@tonic-gate ASSERT(dhp->dh_cvaddr >= cp->cvaddr && 150744374aaeSsvemuri dhp->dh_cvaddr < (cp->cvaddr + cp->size)); 15087c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_cvaddr + off) <= 150944374aaeSsvemuri (cp->cvaddr + cp->size)); 15107c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <= 151144374aaeSsvemuri (cp->cvaddr + cp->size)); 15127c478bd9Sstevel@tonic-gate 15137c478bd9Sstevel@tonic-gate switch (cp->type) { 15147c478bd9Sstevel@tonic-gate case UMEM_LOCKED : 151544374aaeSsvemuri if (cp->pparray != NULL) { 151644374aaeSsvemuri ASSERT((dhp->dh_roff & 151744374aaeSsvemuri PAGEOFFSET) == 0); 151844374aaeSsvemuri pfnum = page_pptonum( 151944374aaeSsvemuri cp->pparray[btop(off + 152044374aaeSsvemuri dhp->dh_roff)]); 152144374aaeSsvemuri } else { 152244374aaeSsvemuri pfnum = hat_getpfnum( 152344374aaeSsvemuri ((proc_t *)cp->procp)->p_as->a_hat, 152444374aaeSsvemuri cp->cvaddr + off); 152544374aaeSsvemuri } 152644374aaeSsvemuri break; 15277c478bd9Sstevel@tonic-gate case UMEM_TRASH : 152844374aaeSsvemuri pfnum = page_pptonum(trashpp); 152944374aaeSsvemuri /* 153044374aaeSsvemuri * We should set hat_flags to HAT_NOFAULT also 153144374aaeSsvemuri * However, not all hat layers implement this 153244374aaeSsvemuri */ 153344374aaeSsvemuri break; 15347c478bd9Sstevel@tonic-gate case KMEM_PAGEABLE: 15357c478bd9Sstevel@tonic-gate case KMEM_NON_PAGEABLE: 153644374aaeSsvemuri pfnum = hat_getpfnum(kas.a_hat, 153744374aaeSsvemuri dhp->dh_cvaddr + off); 153844374aaeSsvemuri break; 15397c478bd9Sstevel@tonic-gate default : 154044374aaeSsvemuri pfnum = PFN_INVALID; 154144374aaeSsvemuri break; 15427c478bd9Sstevel@tonic-gate } 15437c478bd9Sstevel@tonic-gate } 15447c478bd9Sstevel@tonic-gate prot |= dhp->dh_hat_attr; 15457c478bd9Sstevel@tonic-gate } 15467c478bd9Sstevel@tonic-gate if (pfnum == PFN_INVALID) { 15477c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EFAULT)); 15487c478bd9Sstevel@tonic-gate } 15497c478bd9Sstevel@tonic-gate /* prot should already be OR'ed in with hat_attributes if needed */ 15507c478bd9Sstevel@tonic-gate 15517c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1, 15527c478bd9Sstevel@tonic-gate "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x", 15537c478bd9Sstevel@tonic-gate pfnum, pf_is_memory(pfnum), prot, hat_flags); 15547c478bd9Sstevel@tonic-gate DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x " 15557c478bd9Sstevel@tonic-gate "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags)); 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate if (pf_is_memory(pfnum) || (dhp != NULL)) { 15587c478bd9Sstevel@tonic-gate /* 15597c478bd9Sstevel@tonic-gate * It's not _really_ required here to pass sdp->hat_flags 15607c478bd9Sstevel@tonic-gate * to hat_devload even though we do it. 15617c478bd9Sstevel@tonic-gate * This is because hat figures it out DEVMEM mappings 15627c478bd9Sstevel@tonic-gate * are non-consistent, anyway. 15637c478bd9Sstevel@tonic-gate */ 15647c478bd9Sstevel@tonic-gate hat_devload(hat, addr, PAGESIZE, pfnum, 156544374aaeSsvemuri prot, hat_flags | sdp->hat_flags); 15667c478bd9Sstevel@tonic-gate return (0); 15677c478bd9Sstevel@tonic-gate } 15687c478bd9Sstevel@tonic-gate 15697c478bd9Sstevel@tonic-gate /* 15707c478bd9Sstevel@tonic-gate * Fall through to the case where devmap is not used and need to call 15717c478bd9Sstevel@tonic-gate * up the device tree to set up the mapping 15727c478bd9Sstevel@tonic-gate */ 15737c478bd9Sstevel@tonic-gate 15747c478bd9Sstevel@tonic-gate dip = VTOS(VTOCVP(sdp->vp))->s_dip; 15757c478bd9Sstevel@tonic-gate ASSERT(dip); 15767c478bd9Sstevel@tonic-gate 15777c478bd9Sstevel@tonic-gate /* 15787c478bd9Sstevel@tonic-gate * When calling ddi_map_fault, we do not OR in sdp->hat_attr 15797c478bd9Sstevel@tonic-gate * This is because this calls drivers which may not expect 15807c478bd9Sstevel@tonic-gate * prot to have any other values than PROT_ALL 15817c478bd9Sstevel@tonic-gate * The root nexus driver has a hack to peek into the segment 15827c478bd9Sstevel@tonic-gate * structure and then OR in sdp->hat_attr. 15837c478bd9Sstevel@tonic-gate * XX In case the bus_ops interfaces are ever revisited 15847c478bd9Sstevel@tonic-gate * we need to fix this. prot should include other hat attributes 15857c478bd9Sstevel@tonic-gate */ 15867c478bd9Sstevel@tonic-gate if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL, 15877c478bd9Sstevel@tonic-gate (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) { 15887c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EFAULT)); 15897c478bd9Sstevel@tonic-gate } 15907c478bd9Sstevel@tonic-gate return (0); 15917c478bd9Sstevel@tonic-gate } 15927c478bd9Sstevel@tonic-gate 15937c478bd9Sstevel@tonic-gate static faultcode_t 15947c478bd9Sstevel@tonic-gate segdev_fault( 15957c478bd9Sstevel@tonic-gate struct hat *hat, /* the hat */ 15967c478bd9Sstevel@tonic-gate struct seg *seg, /* the seg_dev of interest */ 15977c478bd9Sstevel@tonic-gate caddr_t addr, /* the address of the fault */ 15987c478bd9Sstevel@tonic-gate size_t len, /* the length of the range */ 15997c478bd9Sstevel@tonic-gate enum fault_type type, /* type of fault */ 16007c478bd9Sstevel@tonic-gate enum seg_rw rw) /* type of access at fault */ 16017c478bd9Sstevel@tonic-gate { 16027c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 16037c478bd9Sstevel@tonic-gate devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data; 16047c478bd9Sstevel@tonic-gate devmap_handle_t *dhp; 16057c478bd9Sstevel@tonic-gate struct devmap_softlock *slock = NULL; 16067c478bd9Sstevel@tonic-gate ulong_t slpage = 0; 16077c478bd9Sstevel@tonic-gate ulong_t off; 16087c478bd9Sstevel@tonic-gate caddr_t maddr = addr; 16097c478bd9Sstevel@tonic-gate int err; 16107c478bd9Sstevel@tonic-gate int err_is_faultcode = 0; 16117c478bd9Sstevel@tonic-gate 16127c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT, 16137c478bd9Sstevel@tonic-gate "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x", 16147c478bd9Sstevel@tonic-gate (void *)dhp_head, (void *)seg, (void *)addr, len, type); 16157c478bd9Sstevel@tonic-gate DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p " 16167c478bd9Sstevel@tonic-gate "addr %p len %lx type %x\n", 16177c478bd9Sstevel@tonic-gate (void *)dhp_head, (void *)seg, (void *)addr, len, type)); 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 16207c478bd9Sstevel@tonic-gate 16217c478bd9Sstevel@tonic-gate /* Handle non-devmap case */ 16227c478bd9Sstevel@tonic-gate if (dhp_head == NULL) 16237c478bd9Sstevel@tonic-gate return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL)); 16247c478bd9Sstevel@tonic-gate 16257c478bd9Sstevel@tonic-gate /* Find devmap handle */ 16267c478bd9Sstevel@tonic-gate if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) 16277c478bd9Sstevel@tonic-gate return (FC_NOMAP); 16287c478bd9Sstevel@tonic-gate 16297c478bd9Sstevel@tonic-gate /* 16307c478bd9Sstevel@tonic-gate * The seg_dev driver does not implement copy-on-write, 16317c478bd9Sstevel@tonic-gate * and always loads translations with maximal allowed permissions 16327c478bd9Sstevel@tonic-gate * but we got an fault trying to access the device. 16337c478bd9Sstevel@tonic-gate * Servicing the fault is not going to result in any better result 16347c478bd9Sstevel@tonic-gate * RFE: If we want devmap_access callbacks to be involved in F_PROT 16357c478bd9Sstevel@tonic-gate * faults, then the code below is written for that 16367c478bd9Sstevel@tonic-gate * Pending resolution of the following: 16377c478bd9Sstevel@tonic-gate * - determine if the F_INVAL/F_SOFTLOCK syncing 16387c478bd9Sstevel@tonic-gate * is needed for F_PROT also or not. The code below assumes it does 16397c478bd9Sstevel@tonic-gate * - If driver sees F_PROT and calls devmap_load with same type, 16407c478bd9Sstevel@tonic-gate * then segdev_faultpages will fail with FC_PROT anyway, need to 16417c478bd9Sstevel@tonic-gate * change that so calls from devmap_load to segdev_faultpages for 16427c478bd9Sstevel@tonic-gate * F_PROT type are retagged to F_INVAL. 16437c478bd9Sstevel@tonic-gate * RFE: Today we dont have drivers that use devmap and want to handle 16447c478bd9Sstevel@tonic-gate * F_PROT calls. The code in segdev_fault* is written to allow 16457c478bd9Sstevel@tonic-gate * this case but is not tested. A driver that needs this capability 16467c478bd9Sstevel@tonic-gate * should be able to remove the short-circuit case; resolve the 16477c478bd9Sstevel@tonic-gate * above issues and "should" work. 16487c478bd9Sstevel@tonic-gate */ 16497c478bd9Sstevel@tonic-gate if (type == F_PROT) { 16507c478bd9Sstevel@tonic-gate return (FC_PROT); 16517c478bd9Sstevel@tonic-gate } 16527c478bd9Sstevel@tonic-gate 16537c478bd9Sstevel@tonic-gate /* 16547c478bd9Sstevel@tonic-gate * Loop through dhp list calling devmap_access or segdev_faultpages for 16557c478bd9Sstevel@tonic-gate * each devmap handle. 16567c478bd9Sstevel@tonic-gate * drivers which implement devmap_access can interpose on faults and do 16577c478bd9Sstevel@tonic-gate * device-appropriate special actions before calling devmap_load. 16587c478bd9Sstevel@tonic-gate */ 16597c478bd9Sstevel@tonic-gate 16607c478bd9Sstevel@tonic-gate /* 16617c478bd9Sstevel@tonic-gate * Unfortunately, this simple loop has turned out to expose a variety 16627c478bd9Sstevel@tonic-gate * of complex problems which results in the following convoluted code. 16637c478bd9Sstevel@tonic-gate * 16647c478bd9Sstevel@tonic-gate * First, a desire to handle a serialization of F_SOFTLOCK calls 16657c478bd9Sstevel@tonic-gate * to the driver within the framework. 16667c478bd9Sstevel@tonic-gate * This results in a dh_softlock structure that is on a per device 16677c478bd9Sstevel@tonic-gate * (or device instance) basis and serializes devmap_access calls. 16687c478bd9Sstevel@tonic-gate * Ideally we would need to do this for underlying 16697c478bd9Sstevel@tonic-gate * memory/device regions that are being faulted on 16707c478bd9Sstevel@tonic-gate * but that is hard to identify and with REMAP, harder 16717c478bd9Sstevel@tonic-gate * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t. 16727c478bd9Sstevel@tonic-gate * to F_SOFTLOCK calls to the driver. 16737c478bd9Sstevel@tonic-gate * These serializations are to simplify the driver programmer model. 16747c478bd9Sstevel@tonic-gate * To support these two features, the code first goes through the 16757c478bd9Sstevel@tonic-gate * devmap handles and counts the pages (slpage) that are covered 16767c478bd9Sstevel@tonic-gate * by devmap_access callbacks. 16777c478bd9Sstevel@tonic-gate * This part ends with a devmap_softlock_enter call 16787c478bd9Sstevel@tonic-gate * which allows only one F_SOFTLOCK active on a device instance, 16797c478bd9Sstevel@tonic-gate * but multiple F_INVAL/F_PROTs can be active except when a 16807c478bd9Sstevel@tonic-gate * F_SOFTLOCK is active 16817c478bd9Sstevel@tonic-gate * 16827c478bd9Sstevel@tonic-gate * Next, we dont short-circuit the fault code upfront to call 16837c478bd9Sstevel@tonic-gate * segdev_softunlock for F_SOFTUNLOCK, because we must use 16847c478bd9Sstevel@tonic-gate * the same length when we softlock and softunlock. 16857c478bd9Sstevel@tonic-gate * 16867c478bd9Sstevel@tonic-gate * -Hat layers may not support softunlocking lengths less than the 16877c478bd9Sstevel@tonic-gate * original length when there is large page support. 16887c478bd9Sstevel@tonic-gate * -kpmem locking is dependent on keeping the lengths same. 16897c478bd9Sstevel@tonic-gate * -if drivers handled F_SOFTLOCK, they probably also expect to 16907c478bd9Sstevel@tonic-gate * see an F_SOFTUNLOCK of the same length 16917c478bd9Sstevel@tonic-gate * Hence, if extending lengths during softlock, 16927c478bd9Sstevel@tonic-gate * softunlock has to make the same adjustments and goes through 16937c478bd9Sstevel@tonic-gate * the same loop calling segdev_faultpages/segdev_softunlock 16947c478bd9Sstevel@tonic-gate * But some of the synchronization and error handling is different 16957c478bd9Sstevel@tonic-gate */ 16967c478bd9Sstevel@tonic-gate 16977c478bd9Sstevel@tonic-gate if (type != F_SOFTUNLOCK) { 16987c478bd9Sstevel@tonic-gate devmap_handle_t *dhpp = dhp; 16997c478bd9Sstevel@tonic-gate size_t slen = len; 17007c478bd9Sstevel@tonic-gate 17017c478bd9Sstevel@tonic-gate /* 17027c478bd9Sstevel@tonic-gate * Calculate count of pages that are : 17037c478bd9Sstevel@tonic-gate * a) within the (potentially extended) fault region 17047c478bd9Sstevel@tonic-gate * b) AND covered by devmap handle with devmap_access 17057c478bd9Sstevel@tonic-gate */ 17067c478bd9Sstevel@tonic-gate off = (ulong_t)(addr - dhpp->dh_uvaddr); 17077c478bd9Sstevel@tonic-gate while (slen != 0) { 17087c478bd9Sstevel@tonic-gate size_t mlen; 17097c478bd9Sstevel@tonic-gate 17107c478bd9Sstevel@tonic-gate /* 17117c478bd9Sstevel@tonic-gate * Softlocking on a region that allows remap is 17127c478bd9Sstevel@tonic-gate * unsupported due to unresolved locking issues 17137c478bd9Sstevel@tonic-gate * XXX: unclear what these are? 17147c478bd9Sstevel@tonic-gate * One potential is that if there is a pending 17157c478bd9Sstevel@tonic-gate * softlock, then a remap should not be allowed 17167c478bd9Sstevel@tonic-gate * until the unlock is done. This is easily 17177c478bd9Sstevel@tonic-gate * fixed by returning error in devmap*remap on 17187c478bd9Sstevel@tonic-gate * checking the dh->dh_softlock->softlocked value 17197c478bd9Sstevel@tonic-gate */ 17207c478bd9Sstevel@tonic-gate if ((type == F_SOFTLOCK) && 17217c478bd9Sstevel@tonic-gate (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) { 17227c478bd9Sstevel@tonic-gate return (FC_NOSUPPORT); 17237c478bd9Sstevel@tonic-gate } 17247c478bd9Sstevel@tonic-gate 17257c478bd9Sstevel@tonic-gate mlen = MIN(slen, (dhpp->dh_len - off)); 17267c478bd9Sstevel@tonic-gate if (dhpp->dh_callbackops.devmap_access) { 17277c478bd9Sstevel@tonic-gate size_t llen; 17287c478bd9Sstevel@tonic-gate caddr_t laddr; 17297c478bd9Sstevel@tonic-gate /* 17307c478bd9Sstevel@tonic-gate * use extended length for large page mappings 17317c478bd9Sstevel@tonic-gate */ 17327c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhpp); 17337c478bd9Sstevel@tonic-gate if ((sdp->pageprot == 0) && 17347c478bd9Sstevel@tonic-gate (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) { 17357c478bd9Sstevel@tonic-gate devmap_get_large_pgsize(dhpp, 17367c478bd9Sstevel@tonic-gate mlen, maddr, &llen, &laddr); 17377c478bd9Sstevel@tonic-gate } else { 17387c478bd9Sstevel@tonic-gate llen = mlen; 17397c478bd9Sstevel@tonic-gate } 17407c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhpp); 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate slpage += btopr(llen); 17437c478bd9Sstevel@tonic-gate slock = dhpp->dh_softlock; 17447c478bd9Sstevel@tonic-gate } 17457c478bd9Sstevel@tonic-gate maddr += mlen; 17467c478bd9Sstevel@tonic-gate ASSERT(slen >= mlen); 17477c478bd9Sstevel@tonic-gate slen -= mlen; 17487c478bd9Sstevel@tonic-gate dhpp = dhpp->dh_next; 17497c478bd9Sstevel@tonic-gate off = 0; 17507c478bd9Sstevel@tonic-gate } 17517c478bd9Sstevel@tonic-gate /* 17527c478bd9Sstevel@tonic-gate * synchonize with other faulting threads and wait till safe 17537c478bd9Sstevel@tonic-gate * devmap_softlock_enter might return due to signal in cv_wait 17547c478bd9Sstevel@tonic-gate * 17557c478bd9Sstevel@tonic-gate * devmap_softlock_enter has to be called outside of while loop 17567c478bd9Sstevel@tonic-gate * to prevent a deadlock if len spans over multiple dhps. 17577c478bd9Sstevel@tonic-gate * dh_softlock is based on device instance and if multiple dhps 17587c478bd9Sstevel@tonic-gate * use the same device instance, the second dhp's LOCK call 17597c478bd9Sstevel@tonic-gate * will hang waiting on the first to complete. 17607c478bd9Sstevel@tonic-gate * devmap_setup verifies that slocks in a dhp_chain are same. 17617c478bd9Sstevel@tonic-gate * RFE: this deadlock only hold true for F_SOFTLOCK. For 17627c478bd9Sstevel@tonic-gate * F_INVAL/F_PROT, since we now allow multiple in parallel, 17637c478bd9Sstevel@tonic-gate * we could have done the softlock_enter inside the loop 17647c478bd9Sstevel@tonic-gate * and supported multi-dhp mappings with dissimilar devices 17657c478bd9Sstevel@tonic-gate */ 17667c478bd9Sstevel@tonic-gate if (err = devmap_softlock_enter(slock, slpage, type)) 17677c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(err)); 17687c478bd9Sstevel@tonic-gate } 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate /* reset 'maddr' to the start addr of the range of fault. */ 17717c478bd9Sstevel@tonic-gate maddr = addr; 17727c478bd9Sstevel@tonic-gate 17737c478bd9Sstevel@tonic-gate /* calculate the offset corresponds to 'addr' in the first dhp. */ 17747c478bd9Sstevel@tonic-gate off = (ulong_t)(addr - dhp->dh_uvaddr); 17757c478bd9Sstevel@tonic-gate 17767c478bd9Sstevel@tonic-gate /* 17777c478bd9Sstevel@tonic-gate * The fault length may span over multiple dhps. 17787c478bd9Sstevel@tonic-gate * Loop until the total length is satisfied. 17797c478bd9Sstevel@tonic-gate */ 17807c478bd9Sstevel@tonic-gate while (len != 0) { 17817c478bd9Sstevel@tonic-gate size_t llen; 17827c478bd9Sstevel@tonic-gate size_t mlen; 17837c478bd9Sstevel@tonic-gate caddr_t laddr; 17847c478bd9Sstevel@tonic-gate 17857c478bd9Sstevel@tonic-gate /* 17867c478bd9Sstevel@tonic-gate * mlen is the smaller of 'len' and the length 17877c478bd9Sstevel@tonic-gate * from addr to the end of mapping defined by dhp. 17887c478bd9Sstevel@tonic-gate */ 17897c478bd9Sstevel@tonic-gate mlen = MIN(len, (dhp->dh_len - off)); 17907c478bd9Sstevel@tonic-gate 17917c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 17927c478bd9Sstevel@tonic-gate /* 17937c478bd9Sstevel@tonic-gate * Pass the extended length and address to devmap_access 17947c478bd9Sstevel@tonic-gate * if large pagesize is used for loading address translations. 17957c478bd9Sstevel@tonic-gate */ 17967c478bd9Sstevel@tonic-gate if ((sdp->pageprot == 0) && 17977c478bd9Sstevel@tonic-gate (dhp->dh_flags & DEVMAP_FLAG_LARGE)) { 17987c478bd9Sstevel@tonic-gate devmap_get_large_pgsize(dhp, mlen, maddr, 179944374aaeSsvemuri &llen, &laddr); 18007c478bd9Sstevel@tonic-gate ASSERT(maddr == addr || laddr == maddr); 18017c478bd9Sstevel@tonic-gate } else { 18027c478bd9Sstevel@tonic-gate llen = mlen; 18037c478bd9Sstevel@tonic-gate laddr = maddr; 18047c478bd9Sstevel@tonic-gate } 18057c478bd9Sstevel@tonic-gate 18067c478bd9Sstevel@tonic-gate if (dhp->dh_callbackops.devmap_access != NULL) { 18077c478bd9Sstevel@tonic-gate offset_t aoff; 18087c478bd9Sstevel@tonic-gate 18097c478bd9Sstevel@tonic-gate aoff = sdp->offset + (offset_t)(laddr - seg->s_base); 18107c478bd9Sstevel@tonic-gate 18117c478bd9Sstevel@tonic-gate /* 18127c478bd9Sstevel@tonic-gate * call driver's devmap_access entry point which will 18137c478bd9Sstevel@tonic-gate * call devmap_load/contextmgmt to load the translations 18147c478bd9Sstevel@tonic-gate * 18157c478bd9Sstevel@tonic-gate * We drop the dhp_lock before calling access so 18167c478bd9Sstevel@tonic-gate * drivers can call devmap_*_remap within access 18177c478bd9Sstevel@tonic-gate */ 18187c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 18197c478bd9Sstevel@tonic-gate 18207c478bd9Sstevel@tonic-gate err = (*dhp->dh_callbackops.devmap_access)( 18217c478bd9Sstevel@tonic-gate dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw); 18227c478bd9Sstevel@tonic-gate } else { 18237c478bd9Sstevel@tonic-gate /* 18247c478bd9Sstevel@tonic-gate * If no devmap_access entry point, then load mappings 18257c478bd9Sstevel@tonic-gate * hold dhp_lock across faultpages if REMAP 18267c478bd9Sstevel@tonic-gate */ 18277c478bd9Sstevel@tonic-gate err = segdev_faultpages(hat, seg, laddr, llen, 18287c478bd9Sstevel@tonic-gate type, rw, dhp); 18297c478bd9Sstevel@tonic-gate err_is_faultcode = 1; 18307c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 18317c478bd9Sstevel@tonic-gate } 18327c478bd9Sstevel@tonic-gate 18337c478bd9Sstevel@tonic-gate if (err) { 18347c478bd9Sstevel@tonic-gate if ((type == F_SOFTLOCK) && (maddr > addr)) { 18357c478bd9Sstevel@tonic-gate /* 18367c478bd9Sstevel@tonic-gate * If not first dhp, use 18377c478bd9Sstevel@tonic-gate * segdev_fault(F_SOFTUNLOCK) for prior dhps 18387c478bd9Sstevel@tonic-gate * While this is recursion, it is incorrect to 18397c478bd9Sstevel@tonic-gate * call just segdev_softunlock 18407c478bd9Sstevel@tonic-gate * if we are using either large pages 18417c478bd9Sstevel@tonic-gate * or devmap_access. It will be more right 18427c478bd9Sstevel@tonic-gate * to go through the same loop as above 18437c478bd9Sstevel@tonic-gate * rather than call segdev_softunlock directly 18447c478bd9Sstevel@tonic-gate * It will use the right lenghths as well as 18457c478bd9Sstevel@tonic-gate * call into the driver devmap_access routines. 18467c478bd9Sstevel@tonic-gate */ 18477c478bd9Sstevel@tonic-gate size_t done = (size_t)(maddr - addr); 18487c478bd9Sstevel@tonic-gate (void) segdev_fault(hat, seg, addr, done, 184944374aaeSsvemuri F_SOFTUNLOCK, S_OTHER); 18507c478bd9Sstevel@tonic-gate /* 18517c478bd9Sstevel@tonic-gate * reduce slpage by number of pages 18527c478bd9Sstevel@tonic-gate * released by segdev_softunlock 18537c478bd9Sstevel@tonic-gate */ 18547c478bd9Sstevel@tonic-gate ASSERT(slpage >= btopr(done)); 18557c478bd9Sstevel@tonic-gate devmap_softlock_exit(slock, 185644374aaeSsvemuri slpage - btopr(done), type); 18577c478bd9Sstevel@tonic-gate } else { 18587c478bd9Sstevel@tonic-gate devmap_softlock_exit(slock, slpage, type); 18597c478bd9Sstevel@tonic-gate } 18607c478bd9Sstevel@tonic-gate 18617c478bd9Sstevel@tonic-gate 18627c478bd9Sstevel@tonic-gate /* 18637c478bd9Sstevel@tonic-gate * Segdev_faultpages() already returns a faultcode, 18647c478bd9Sstevel@tonic-gate * hence, result from segdev_faultpages() should be 18657c478bd9Sstevel@tonic-gate * returned directly. 18667c478bd9Sstevel@tonic-gate */ 18677c478bd9Sstevel@tonic-gate if (err_is_faultcode) 18687c478bd9Sstevel@tonic-gate return (err); 18697c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(err)); 18707c478bd9Sstevel@tonic-gate } 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate maddr += mlen; 18737c478bd9Sstevel@tonic-gate ASSERT(len >= mlen); 18747c478bd9Sstevel@tonic-gate len -= mlen; 18757c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 18767c478bd9Sstevel@tonic-gate off = 0; 18777c478bd9Sstevel@tonic-gate 18787c478bd9Sstevel@tonic-gate ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr); 18797c478bd9Sstevel@tonic-gate } 18807c478bd9Sstevel@tonic-gate /* 18817c478bd9Sstevel@tonic-gate * release the softlock count at end of fault 18827c478bd9Sstevel@tonic-gate * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK 18837c478bd9Sstevel@tonic-gate */ 18847c478bd9Sstevel@tonic-gate if ((type == F_INVAL) || (type == F_PROT)) 18857c478bd9Sstevel@tonic-gate devmap_softlock_exit(slock, slpage, type); 18867c478bd9Sstevel@tonic-gate return (0); 18877c478bd9Sstevel@tonic-gate } 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate /* 18907c478bd9Sstevel@tonic-gate * segdev_faultpages 18917c478bd9Sstevel@tonic-gate * 18927c478bd9Sstevel@tonic-gate * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load 18937c478bd9Sstevel@tonic-gate * This routine assumes that the callers makes sure that the fields 18947c478bd9Sstevel@tonic-gate * in dhp used below are not changed due to remap during this call. 18957c478bd9Sstevel@tonic-gate * Caller does HOLD_DHP_LOCK if neeed 18967c478bd9Sstevel@tonic-gate * This routine returns a faultcode_t as a return value for segdev_fault. 18977c478bd9Sstevel@tonic-gate */ 18987c478bd9Sstevel@tonic-gate static faultcode_t 18997c478bd9Sstevel@tonic-gate segdev_faultpages( 19007c478bd9Sstevel@tonic-gate struct hat *hat, /* the hat */ 19017c478bd9Sstevel@tonic-gate struct seg *seg, /* the seg_dev of interest */ 19027c478bd9Sstevel@tonic-gate caddr_t addr, /* the address of the fault */ 19037c478bd9Sstevel@tonic-gate size_t len, /* the length of the range */ 19047c478bd9Sstevel@tonic-gate enum fault_type type, /* type of fault */ 19057c478bd9Sstevel@tonic-gate enum seg_rw rw, /* type of access at fault */ 19067c478bd9Sstevel@tonic-gate devmap_handle_t *dhp) /* devmap handle */ 19077c478bd9Sstevel@tonic-gate { 19087c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 19097c478bd9Sstevel@tonic-gate register caddr_t a; 19107c478bd9Sstevel@tonic-gate struct vpage *vpage; 19117c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *kpmem_cookie = NULL; 19127c478bd9Sstevel@tonic-gate int err; 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES, 19157c478bd9Sstevel@tonic-gate "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx", 19167c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg, (void *)addr, len); 19177c478bd9Sstevel@tonic-gate DEBUGF(5, (CE_CONT, "segdev_faultpages: " 19187c478bd9Sstevel@tonic-gate "dhp %p seg %p addr %p len %lx\n", 19197c478bd9Sstevel@tonic-gate (void *)dhp, (void *)seg, (void *)addr, len)); 19207c478bd9Sstevel@tonic-gate 19217c478bd9Sstevel@tonic-gate /* 19227c478bd9Sstevel@tonic-gate * The seg_dev driver does not implement copy-on-write, 19237c478bd9Sstevel@tonic-gate * and always loads translations with maximal allowed permissions 19247c478bd9Sstevel@tonic-gate * but we got an fault trying to access the device. 19257c478bd9Sstevel@tonic-gate * Servicing the fault is not going to result in any better result 19267c478bd9Sstevel@tonic-gate * XXX: If we want to allow devmap_access to handle F_PROT calls, 19277c478bd9Sstevel@tonic-gate * This code should be removed and let the normal fault handling 19287c478bd9Sstevel@tonic-gate * take care of finding the error 19297c478bd9Sstevel@tonic-gate */ 19307c478bd9Sstevel@tonic-gate if (type == F_PROT) { 19317c478bd9Sstevel@tonic-gate return (FC_PROT); 19327c478bd9Sstevel@tonic-gate } 19337c478bd9Sstevel@tonic-gate 19347c478bd9Sstevel@tonic-gate if (type == F_SOFTUNLOCK) { 19357c478bd9Sstevel@tonic-gate segdev_softunlock(hat, seg, addr, len, rw); 19367c478bd9Sstevel@tonic-gate return (0); 19377c478bd9Sstevel@tonic-gate } 19387c478bd9Sstevel@tonic-gate 19397c478bd9Sstevel@tonic-gate /* 19407c478bd9Sstevel@tonic-gate * For kernel pageable memory, fault/lock segkp pages 19417c478bd9Sstevel@tonic-gate * We hold this until the completion of this 19427c478bd9Sstevel@tonic-gate * fault (INVAL/PROT) or till unlock (SOFTLOCK). 19437c478bd9Sstevel@tonic-gate */ 19447c478bd9Sstevel@tonic-gate if ((dhp != NULL) && dhp_is_kpmem(dhp)) { 19457c478bd9Sstevel@tonic-gate kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie; 19467c478bd9Sstevel@tonic-gate if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len))) 19477c478bd9Sstevel@tonic-gate return (err); 19487c478bd9Sstevel@tonic-gate } 19497c478bd9Sstevel@tonic-gate 19507c478bd9Sstevel@tonic-gate /* 19517c478bd9Sstevel@tonic-gate * If we have the same protections for the entire segment, 19527c478bd9Sstevel@tonic-gate * insure that the access being attempted is legitimate. 19537c478bd9Sstevel@tonic-gate */ 195444374aaeSsvemuri rw_enter(&sdp->lock, RW_READER); 19557c478bd9Sstevel@tonic-gate if (sdp->pageprot == 0) { 19567c478bd9Sstevel@tonic-gate uint_t protchk; 19577c478bd9Sstevel@tonic-gate 19587c478bd9Sstevel@tonic-gate switch (rw) { 19597c478bd9Sstevel@tonic-gate case S_READ: 19607c478bd9Sstevel@tonic-gate protchk = PROT_READ; 19617c478bd9Sstevel@tonic-gate break; 19627c478bd9Sstevel@tonic-gate case S_WRITE: 19637c478bd9Sstevel@tonic-gate protchk = PROT_WRITE; 19647c478bd9Sstevel@tonic-gate break; 19657c478bd9Sstevel@tonic-gate case S_EXEC: 19667c478bd9Sstevel@tonic-gate protchk = PROT_EXEC; 19677c478bd9Sstevel@tonic-gate break; 19687c478bd9Sstevel@tonic-gate case S_OTHER: 19697c478bd9Sstevel@tonic-gate default: 19707c478bd9Sstevel@tonic-gate protchk = PROT_READ | PROT_WRITE | PROT_EXEC; 19717c478bd9Sstevel@tonic-gate break; 19727c478bd9Sstevel@tonic-gate } 19737c478bd9Sstevel@tonic-gate 19747c478bd9Sstevel@tonic-gate if ((sdp->prot & protchk) == 0) { 197544374aaeSsvemuri rw_exit(&sdp->lock); 19767c478bd9Sstevel@tonic-gate /* undo kpmem locking */ 19777c478bd9Sstevel@tonic-gate if (kpmem_cookie != NULL) { 19787c478bd9Sstevel@tonic-gate release_kpmem_lock(kpmem_cookie, btopr(len)); 19797c478bd9Sstevel@tonic-gate } 19807c478bd9Sstevel@tonic-gate return (FC_PROT); /* illegal access type */ 19817c478bd9Sstevel@tonic-gate } 19827c478bd9Sstevel@tonic-gate } 19837c478bd9Sstevel@tonic-gate 19847c478bd9Sstevel@tonic-gate /* 19857c478bd9Sstevel@tonic-gate * we do a single hat_devload for the range if 19867c478bd9Sstevel@tonic-gate * - devmap framework (dhp is not NULL), 19877c478bd9Sstevel@tonic-gate * - pageprot == 0, i.e., no per-page protection set and 19887c478bd9Sstevel@tonic-gate * - is device pages, irrespective of whether we are using large pages 19897c478bd9Sstevel@tonic-gate */ 19907c478bd9Sstevel@tonic-gate if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) { 19917c478bd9Sstevel@tonic-gate pfn_t pfnum; 19927c478bd9Sstevel@tonic-gate uint_t hat_flags; 19937c478bd9Sstevel@tonic-gate 19947c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) { 199544374aaeSsvemuri rw_exit(&sdp->lock); 19967c478bd9Sstevel@tonic-gate return (FC_NOMAP); 19977c478bd9Sstevel@tonic-gate } 19987c478bd9Sstevel@tonic-gate 19997c478bd9Sstevel@tonic-gate if (type == F_SOFTLOCK) { 20007c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 20017c478bd9Sstevel@tonic-gate sdp->softlockcnt += btopr(len); 20027c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 20037c478bd9Sstevel@tonic-gate } 20047c478bd9Sstevel@tonic-gate 20057c478bd9Sstevel@tonic-gate hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD); 20067c478bd9Sstevel@tonic-gate pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr)); 20077c478bd9Sstevel@tonic-gate ASSERT(!pf_is_memory(pfnum)); 20087c478bd9Sstevel@tonic-gate 20097c478bd9Sstevel@tonic-gate hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr, 201044374aaeSsvemuri hat_flags | sdp->hat_flags); 201144374aaeSsvemuri rw_exit(&sdp->lock); 20127c478bd9Sstevel@tonic-gate return (0); 20137c478bd9Sstevel@tonic-gate } 20147c478bd9Sstevel@tonic-gate 20157c478bd9Sstevel@tonic-gate /* Handle cases where we have to loop through fault handling per-page */ 20167c478bd9Sstevel@tonic-gate 20177c478bd9Sstevel@tonic-gate if (sdp->vpage == NULL) 20187c478bd9Sstevel@tonic-gate vpage = NULL; 20197c478bd9Sstevel@tonic-gate else 20207c478bd9Sstevel@tonic-gate vpage = &sdp->vpage[seg_page(seg, addr)]; 20217c478bd9Sstevel@tonic-gate 20227c478bd9Sstevel@tonic-gate /* loop over the address range handling each fault */ 20237c478bd9Sstevel@tonic-gate for (a = addr; a < addr + len; a += PAGESIZE) { 20247c478bd9Sstevel@tonic-gate if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) { 20257c478bd9Sstevel@tonic-gate break; 20267c478bd9Sstevel@tonic-gate } 20277c478bd9Sstevel@tonic-gate if (vpage != NULL) 20287c478bd9Sstevel@tonic-gate vpage++; 20297c478bd9Sstevel@tonic-gate } 203044374aaeSsvemuri rw_exit(&sdp->lock); 20317c478bd9Sstevel@tonic-gate if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */ 20327c478bd9Sstevel@tonic-gate size_t done = (size_t)(a - addr); /* pages fault successfully */ 20337c478bd9Sstevel@tonic-gate if (done > 0) { 20347c478bd9Sstevel@tonic-gate /* use softunlock for those pages */ 20357c478bd9Sstevel@tonic-gate segdev_softunlock(hat, seg, addr, done, S_OTHER); 20367c478bd9Sstevel@tonic-gate } 20377c478bd9Sstevel@tonic-gate if (kpmem_cookie != NULL) { 20387c478bd9Sstevel@tonic-gate /* release kpmem lock for rest of pages */ 20397c478bd9Sstevel@tonic-gate ASSERT(len >= done); 20407c478bd9Sstevel@tonic-gate release_kpmem_lock(kpmem_cookie, btopr(len - done)); 20417c478bd9Sstevel@tonic-gate } 20427c478bd9Sstevel@tonic-gate } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) { 20437c478bd9Sstevel@tonic-gate /* for non-SOFTLOCK cases, release kpmem */ 20447c478bd9Sstevel@tonic-gate release_kpmem_lock(kpmem_cookie, btopr(len)); 20457c478bd9Sstevel@tonic-gate } 20467c478bd9Sstevel@tonic-gate return (err); 20477c478bd9Sstevel@tonic-gate } 20487c478bd9Sstevel@tonic-gate 20497c478bd9Sstevel@tonic-gate /* 20507c478bd9Sstevel@tonic-gate * Asynchronous page fault. We simply do nothing since this 20517c478bd9Sstevel@tonic-gate * entry point is not supposed to load up the translation. 20527c478bd9Sstevel@tonic-gate */ 20537c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20547c478bd9Sstevel@tonic-gate static faultcode_t 20557c478bd9Sstevel@tonic-gate segdev_faulta(struct seg *seg, caddr_t addr) 20567c478bd9Sstevel@tonic-gate { 20577c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA, 20587c478bd9Sstevel@tonic-gate "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr); 20597c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 20607c478bd9Sstevel@tonic-gate 20617c478bd9Sstevel@tonic-gate return (0); 20627c478bd9Sstevel@tonic-gate } 20637c478bd9Sstevel@tonic-gate 20647c478bd9Sstevel@tonic-gate static int 20657c478bd9Sstevel@tonic-gate segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 20667c478bd9Sstevel@tonic-gate { 20677c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 20687c478bd9Sstevel@tonic-gate register devmap_handle_t *dhp; 20697c478bd9Sstevel@tonic-gate register struct vpage *vp, *evp; 20707c478bd9Sstevel@tonic-gate devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data; 20717c478bd9Sstevel@tonic-gate ulong_t off; 20727c478bd9Sstevel@tonic-gate size_t mlen, sz; 20737c478bd9Sstevel@tonic-gate 20747c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT, 20757c478bd9Sstevel@tonic-gate "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x", 20767c478bd9Sstevel@tonic-gate (void *)seg, (void *)addr, len, prot); 20777c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 20787c478bd9Sstevel@tonic-gate 20797c478bd9Sstevel@tonic-gate if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) { 20807c478bd9Sstevel@tonic-gate /* 20817c478bd9Sstevel@tonic-gate * Fail the setprot if pages are SOFTLOCKed through this 20827c478bd9Sstevel@tonic-gate * mapping. 20837c478bd9Sstevel@tonic-gate * Softlockcnt is protected from change by the as read lock. 20847c478bd9Sstevel@tonic-gate */ 20857c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1, 20867c478bd9Sstevel@tonic-gate "segdev_setprot:error softlockcnt=%lx", sz); 20877c478bd9Sstevel@tonic-gate DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz)); 20887c478bd9Sstevel@tonic-gate return (EAGAIN); 20897c478bd9Sstevel@tonic-gate } 20907c478bd9Sstevel@tonic-gate 20917c478bd9Sstevel@tonic-gate if (dhp_head != NULL) { 20927c478bd9Sstevel@tonic-gate if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL) 20937c478bd9Sstevel@tonic-gate return (EINVAL); 20947c478bd9Sstevel@tonic-gate 20957c478bd9Sstevel@tonic-gate /* 20967c478bd9Sstevel@tonic-gate * check if violate maxprot. 20977c478bd9Sstevel@tonic-gate */ 20987c478bd9Sstevel@tonic-gate off = (ulong_t)(addr - dhp->dh_uvaddr); 20997c478bd9Sstevel@tonic-gate mlen = len; 21007c478bd9Sstevel@tonic-gate while (dhp) { 21017c478bd9Sstevel@tonic-gate if ((dhp->dh_maxprot & prot) != prot) 21027c478bd9Sstevel@tonic-gate return (EACCES); /* violated maxprot */ 21037c478bd9Sstevel@tonic-gate 21047c478bd9Sstevel@tonic-gate if (mlen > (dhp->dh_len - off)) { 21057c478bd9Sstevel@tonic-gate mlen -= dhp->dh_len - off; 21067c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 21077c478bd9Sstevel@tonic-gate off = 0; 21087c478bd9Sstevel@tonic-gate } else 21097c478bd9Sstevel@tonic-gate break; 21107c478bd9Sstevel@tonic-gate } 21117c478bd9Sstevel@tonic-gate } else { 21127c478bd9Sstevel@tonic-gate if ((sdp->maxprot & prot) != prot) 21137c478bd9Sstevel@tonic-gate return (EACCES); 21147c478bd9Sstevel@tonic-gate } 21157c478bd9Sstevel@tonic-gate 211644374aaeSsvemuri rw_enter(&sdp->lock, RW_WRITER); 21177c478bd9Sstevel@tonic-gate if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) { 21187c478bd9Sstevel@tonic-gate if (sdp->prot == prot) { 211944374aaeSsvemuri rw_exit(&sdp->lock); 21207c478bd9Sstevel@tonic-gate return (0); /* all done */ 21217c478bd9Sstevel@tonic-gate } 21227c478bd9Sstevel@tonic-gate sdp->prot = (uchar_t)prot; 21237c478bd9Sstevel@tonic-gate } else { 21247c478bd9Sstevel@tonic-gate sdp->pageprot = 1; 21257c478bd9Sstevel@tonic-gate if (sdp->vpage == NULL) { 21267c478bd9Sstevel@tonic-gate /* 21277c478bd9Sstevel@tonic-gate * First time through setting per page permissions, 21287c478bd9Sstevel@tonic-gate * initialize all the vpage structures to prot 21297c478bd9Sstevel@tonic-gate */ 21307c478bd9Sstevel@tonic-gate sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)), 21317c478bd9Sstevel@tonic-gate KM_SLEEP); 21327c478bd9Sstevel@tonic-gate evp = &sdp->vpage[seg_pages(seg)]; 21337c478bd9Sstevel@tonic-gate for (vp = sdp->vpage; vp < evp; vp++) 21347c478bd9Sstevel@tonic-gate VPP_SETPROT(vp, sdp->prot); 21357c478bd9Sstevel@tonic-gate } 21367c478bd9Sstevel@tonic-gate /* 21377c478bd9Sstevel@tonic-gate * Now go change the needed vpages protections. 21387c478bd9Sstevel@tonic-gate */ 21397c478bd9Sstevel@tonic-gate evp = &sdp->vpage[seg_page(seg, addr + len)]; 21407c478bd9Sstevel@tonic-gate for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) 21417c478bd9Sstevel@tonic-gate VPP_SETPROT(vp, prot); 21427c478bd9Sstevel@tonic-gate } 214344374aaeSsvemuri rw_exit(&sdp->lock); 21447c478bd9Sstevel@tonic-gate 21457c478bd9Sstevel@tonic-gate if (dhp_head != NULL) { 21467c478bd9Sstevel@tonic-gate devmap_handle_t *tdhp; 21477c478bd9Sstevel@tonic-gate /* 21487c478bd9Sstevel@tonic-gate * If large page size was used in hat_devload(), 21497c478bd9Sstevel@tonic-gate * the same page size must be used in hat_unload(). 21507c478bd9Sstevel@tonic-gate */ 21517c478bd9Sstevel@tonic-gate dhp = tdhp = devmap_find_handle(dhp_head, addr); 21527c478bd9Sstevel@tonic-gate while (tdhp != NULL) { 21537c478bd9Sstevel@tonic-gate if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) { 21547c478bd9Sstevel@tonic-gate break; 21557c478bd9Sstevel@tonic-gate } 21567c478bd9Sstevel@tonic-gate tdhp = tdhp->dh_next; 21577c478bd9Sstevel@tonic-gate } 21587c478bd9Sstevel@tonic-gate if (tdhp) { 21597c478bd9Sstevel@tonic-gate size_t slen = len; 21607c478bd9Sstevel@tonic-gate size_t mlen; 21617c478bd9Sstevel@tonic-gate size_t soff; 21627c478bd9Sstevel@tonic-gate 21637c478bd9Sstevel@tonic-gate soff = (ulong_t)(addr - dhp->dh_uvaddr); 21647c478bd9Sstevel@tonic-gate while (slen != 0) { 21657c478bd9Sstevel@tonic-gate mlen = MIN(slen, (dhp->dh_len - soff)); 21667c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr, 216744374aaeSsvemuri dhp->dh_len, HAT_UNLOAD); 21687c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 21697c478bd9Sstevel@tonic-gate ASSERT(slen >= mlen); 21707c478bd9Sstevel@tonic-gate slen -= mlen; 21717c478bd9Sstevel@tonic-gate soff = 0; 21727c478bd9Sstevel@tonic-gate } 21737c478bd9Sstevel@tonic-gate return (0); 21747c478bd9Sstevel@tonic-gate } 21757c478bd9Sstevel@tonic-gate } 21767c478bd9Sstevel@tonic-gate 21777c478bd9Sstevel@tonic-gate if ((prot & ~PROT_USER) == PROT_NONE) { 21787c478bd9Sstevel@tonic-gate hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD); 21797c478bd9Sstevel@tonic-gate } else { 21807c478bd9Sstevel@tonic-gate /* 21817c478bd9Sstevel@tonic-gate * RFE: the segment should keep track of all attributes 21827c478bd9Sstevel@tonic-gate * allowing us to remove the deprecated hat_chgprot 21837c478bd9Sstevel@tonic-gate * and use hat_chgattr. 21847c478bd9Sstevel@tonic-gate */ 21857c478bd9Sstevel@tonic-gate hat_chgprot(seg->s_as->a_hat, addr, len, prot); 21867c478bd9Sstevel@tonic-gate } 21877c478bd9Sstevel@tonic-gate 21887c478bd9Sstevel@tonic-gate return (0); 21897c478bd9Sstevel@tonic-gate } 21907c478bd9Sstevel@tonic-gate 21917c478bd9Sstevel@tonic-gate static int 21927c478bd9Sstevel@tonic-gate segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 21937c478bd9Sstevel@tonic-gate { 21947c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 21957c478bd9Sstevel@tonic-gate struct vpage *vp, *evp; 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT, 21987c478bd9Sstevel@tonic-gate "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x", 21997c478bd9Sstevel@tonic-gate (void *)seg, (void *)addr, len, prot); 22007c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 22017c478bd9Sstevel@tonic-gate 22027c478bd9Sstevel@tonic-gate /* 22037c478bd9Sstevel@tonic-gate * If segment protection can be used, simply check against them 22047c478bd9Sstevel@tonic-gate */ 220544374aaeSsvemuri rw_enter(&sdp->lock, RW_READER); 22067c478bd9Sstevel@tonic-gate if (sdp->pageprot == 0) { 22077c478bd9Sstevel@tonic-gate register int err; 22087c478bd9Sstevel@tonic-gate 22097c478bd9Sstevel@tonic-gate err = ((sdp->prot & prot) != prot) ? EACCES : 0; 221044374aaeSsvemuri rw_exit(&sdp->lock); 22117c478bd9Sstevel@tonic-gate return (err); 22127c478bd9Sstevel@tonic-gate } 22137c478bd9Sstevel@tonic-gate 22147c478bd9Sstevel@tonic-gate /* 22157c478bd9Sstevel@tonic-gate * Have to check down to the vpage level 22167c478bd9Sstevel@tonic-gate */ 22177c478bd9Sstevel@tonic-gate evp = &sdp->vpage[seg_page(seg, addr + len)]; 22187c478bd9Sstevel@tonic-gate for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) { 22197c478bd9Sstevel@tonic-gate if ((VPP_PROT(vp) & prot) != prot) { 222044374aaeSsvemuri rw_exit(&sdp->lock); 22217c478bd9Sstevel@tonic-gate return (EACCES); 22227c478bd9Sstevel@tonic-gate } 22237c478bd9Sstevel@tonic-gate } 222444374aaeSsvemuri rw_exit(&sdp->lock); 22257c478bd9Sstevel@tonic-gate return (0); 22267c478bd9Sstevel@tonic-gate } 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate static int 22297c478bd9Sstevel@tonic-gate segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 22307c478bd9Sstevel@tonic-gate { 22317c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 22327c478bd9Sstevel@tonic-gate size_t pgno; 22337c478bd9Sstevel@tonic-gate 22347c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT, 22357c478bd9Sstevel@tonic-gate "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p", 22367c478bd9Sstevel@tonic-gate (void *)seg, (void *)addr, len, (void *)protv); 22377c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 22387c478bd9Sstevel@tonic-gate 22397c478bd9Sstevel@tonic-gate pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1; 22407c478bd9Sstevel@tonic-gate if (pgno != 0) { 224144374aaeSsvemuri rw_enter(&sdp->lock, RW_READER); 22427c478bd9Sstevel@tonic-gate if (sdp->pageprot == 0) { 224344374aaeSsvemuri do { 22447c478bd9Sstevel@tonic-gate protv[--pgno] = sdp->prot; 224544374aaeSsvemuri } while (pgno != 0); 22467c478bd9Sstevel@tonic-gate } else { 22477c478bd9Sstevel@tonic-gate size_t pgoff = seg_page(seg, addr); 22487c478bd9Sstevel@tonic-gate 22497c478bd9Sstevel@tonic-gate do { 22507c478bd9Sstevel@tonic-gate pgno--; 22517c478bd9Sstevel@tonic-gate protv[pgno] = 225244374aaeSsvemuri VPP_PROT(&sdp->vpage[pgno + pgoff]); 22537c478bd9Sstevel@tonic-gate } while (pgno != 0); 22547c478bd9Sstevel@tonic-gate } 225544374aaeSsvemuri rw_exit(&sdp->lock); 22567c478bd9Sstevel@tonic-gate } 22577c478bd9Sstevel@tonic-gate return (0); 22587c478bd9Sstevel@tonic-gate } 22597c478bd9Sstevel@tonic-gate 22607c478bd9Sstevel@tonic-gate static u_offset_t 22617c478bd9Sstevel@tonic-gate segdev_getoffset(register struct seg *seg, caddr_t addr) 22627c478bd9Sstevel@tonic-gate { 22637c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 22647c478bd9Sstevel@tonic-gate 22657c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET, 22667c478bd9Sstevel@tonic-gate "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr); 22677c478bd9Sstevel@tonic-gate 22687c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 22697c478bd9Sstevel@tonic-gate 22707c478bd9Sstevel@tonic-gate return ((u_offset_t)sdp->offset + (addr - seg->s_base)); 22717c478bd9Sstevel@tonic-gate } 22727c478bd9Sstevel@tonic-gate 22737c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 22747c478bd9Sstevel@tonic-gate static int 22757c478bd9Sstevel@tonic-gate segdev_gettype(register struct seg *seg, caddr_t addr) 22767c478bd9Sstevel@tonic-gate { 22777c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 22787c478bd9Sstevel@tonic-gate 22797c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE, 22807c478bd9Sstevel@tonic-gate "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr); 22817c478bd9Sstevel@tonic-gate 22827c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 22837c478bd9Sstevel@tonic-gate 22847c478bd9Sstevel@tonic-gate return (sdp->type); 22857c478bd9Sstevel@tonic-gate } 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate 22887c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 22897c478bd9Sstevel@tonic-gate static int 22907c478bd9Sstevel@tonic-gate segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp) 22917c478bd9Sstevel@tonic-gate { 22927c478bd9Sstevel@tonic-gate register struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 22937c478bd9Sstevel@tonic-gate 22947c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP, 22957c478bd9Sstevel@tonic-gate "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr); 22967c478bd9Sstevel@tonic-gate 22977c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 22987c478bd9Sstevel@tonic-gate 22997c478bd9Sstevel@tonic-gate /* 23007c478bd9Sstevel@tonic-gate * Note that this vp is the common_vp of the device, where the 23017c478bd9Sstevel@tonic-gate * pages are hung .. 23027c478bd9Sstevel@tonic-gate */ 23037c478bd9Sstevel@tonic-gate *vpp = VTOCVP(sdp->vp); 23047c478bd9Sstevel@tonic-gate 23057c478bd9Sstevel@tonic-gate return (0); 23067c478bd9Sstevel@tonic-gate } 23077c478bd9Sstevel@tonic-gate 23087c478bd9Sstevel@tonic-gate static void 23097c478bd9Sstevel@tonic-gate segdev_badop(void) 23107c478bd9Sstevel@tonic-gate { 23117c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP, 231244374aaeSsvemuri "segdev_badop:start"); 23137c478bd9Sstevel@tonic-gate panic("segdev_badop"); 23147c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 23157c478bd9Sstevel@tonic-gate } 23167c478bd9Sstevel@tonic-gate 23177c478bd9Sstevel@tonic-gate /* 23187c478bd9Sstevel@tonic-gate * segdev pages are not in the cache, and thus can't really be controlled. 23197c478bd9Sstevel@tonic-gate * Hence, syncs are simply always successful. 23207c478bd9Sstevel@tonic-gate */ 23217c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23227c478bd9Sstevel@tonic-gate static int 23237c478bd9Sstevel@tonic-gate segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags) 23247c478bd9Sstevel@tonic-gate { 23257c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start"); 23267c478bd9Sstevel@tonic-gate 23277c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23287c478bd9Sstevel@tonic-gate 23297c478bd9Sstevel@tonic-gate return (0); 23307c478bd9Sstevel@tonic-gate } 23317c478bd9Sstevel@tonic-gate 23327c478bd9Sstevel@tonic-gate /* 23337c478bd9Sstevel@tonic-gate * segdev pages are always "in core". 23347c478bd9Sstevel@tonic-gate */ 23357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23367c478bd9Sstevel@tonic-gate static size_t 23377c478bd9Sstevel@tonic-gate segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec) 23387c478bd9Sstevel@tonic-gate { 23397c478bd9Sstevel@tonic-gate size_t v = 0; 23407c478bd9Sstevel@tonic-gate 23417c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start"); 23427c478bd9Sstevel@tonic-gate 23437c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE, 23467c478bd9Sstevel@tonic-gate v += PAGESIZE) 23477c478bd9Sstevel@tonic-gate *vec++ = 1; 23487c478bd9Sstevel@tonic-gate return (v); 23497c478bd9Sstevel@tonic-gate } 23507c478bd9Sstevel@tonic-gate 23517c478bd9Sstevel@tonic-gate /* 23527c478bd9Sstevel@tonic-gate * segdev pages are not in the cache, and thus can't really be controlled. 23537c478bd9Sstevel@tonic-gate * Hence, locks are simply always successful. 23547c478bd9Sstevel@tonic-gate */ 23557c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23567c478bd9Sstevel@tonic-gate static int 23577c478bd9Sstevel@tonic-gate segdev_lockop(struct seg *seg, caddr_t addr, 23587c478bd9Sstevel@tonic-gate size_t len, int attr, int op, ulong_t *lockmap, size_t pos) 23597c478bd9Sstevel@tonic-gate { 23607c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start"); 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23637c478bd9Sstevel@tonic-gate 23647c478bd9Sstevel@tonic-gate return (0); 23657c478bd9Sstevel@tonic-gate } 23667c478bd9Sstevel@tonic-gate 23677c478bd9Sstevel@tonic-gate /* 23687c478bd9Sstevel@tonic-gate * segdev pages are not in the cache, and thus can't really be controlled. 23697c478bd9Sstevel@tonic-gate * Hence, advise is simply always successful. 23707c478bd9Sstevel@tonic-gate */ 23717c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23727c478bd9Sstevel@tonic-gate static int 23737c478bd9Sstevel@tonic-gate segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 23747c478bd9Sstevel@tonic-gate { 23757c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start"); 23767c478bd9Sstevel@tonic-gate 23777c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 23787c478bd9Sstevel@tonic-gate 23797c478bd9Sstevel@tonic-gate return (0); 23807c478bd9Sstevel@tonic-gate } 23817c478bd9Sstevel@tonic-gate 23827c478bd9Sstevel@tonic-gate /* 23837c478bd9Sstevel@tonic-gate * segdev pages are not dumped, so we just return 23847c478bd9Sstevel@tonic-gate */ 23857c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23867c478bd9Sstevel@tonic-gate static void 23877c478bd9Sstevel@tonic-gate segdev_dump(struct seg *seg) 23887c478bd9Sstevel@tonic-gate {} 23897c478bd9Sstevel@tonic-gate 23907c478bd9Sstevel@tonic-gate /* 23917c478bd9Sstevel@tonic-gate * ddi_segmap_setup: Used by drivers who wish specify mapping attributes 23927c478bd9Sstevel@tonic-gate * for a segment. Called from a drivers segmap(9E) 23937c478bd9Sstevel@tonic-gate * routine. 23947c478bd9Sstevel@tonic-gate */ 23957c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23967c478bd9Sstevel@tonic-gate int 23977c478bd9Sstevel@tonic-gate ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp, 23987c478bd9Sstevel@tonic-gate off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred, 23997c478bd9Sstevel@tonic-gate ddi_device_acc_attr_t *accattrp, uint_t rnumber) 24007c478bd9Sstevel@tonic-gate { 24017c478bd9Sstevel@tonic-gate struct segdev_crargs dev_a; 24027c478bd9Sstevel@tonic-gate int (*mapfunc)(dev_t dev, off_t off, int prot); 24037c478bd9Sstevel@tonic-gate uint_t hat_attr; 24047c478bd9Sstevel@tonic-gate pfn_t pfn; 24057c478bd9Sstevel@tonic-gate int error, i; 24067c478bd9Sstevel@tonic-gate 24077c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP, 24087c478bd9Sstevel@tonic-gate "ddi_segmap_setup:start"); 24097c478bd9Sstevel@tonic-gate 24107c478bd9Sstevel@tonic-gate if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev) 24117c478bd9Sstevel@tonic-gate return (ENODEV); 24127c478bd9Sstevel@tonic-gate 24137c478bd9Sstevel@tonic-gate /* 24147c478bd9Sstevel@tonic-gate * Character devices that support the d_mmap 24157c478bd9Sstevel@tonic-gate * interface can only be mmap'ed shared. 24167c478bd9Sstevel@tonic-gate */ 24177c478bd9Sstevel@tonic-gate if ((flags & MAP_TYPE) != MAP_SHARED) 24187c478bd9Sstevel@tonic-gate return (EINVAL); 24197c478bd9Sstevel@tonic-gate 24207c478bd9Sstevel@tonic-gate /* 24217c478bd9Sstevel@tonic-gate * Check that this region is indeed mappable on this platform. 24227c478bd9Sstevel@tonic-gate * Use the mapping function. 24237c478bd9Sstevel@tonic-gate */ 24247c478bd9Sstevel@tonic-gate if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1) 24257c478bd9Sstevel@tonic-gate return (ENXIO); 24267c478bd9Sstevel@tonic-gate 24277c478bd9Sstevel@tonic-gate /* 24287c478bd9Sstevel@tonic-gate * Check to ensure that the entire range is 24297c478bd9Sstevel@tonic-gate * legal and we are not trying to map in 24307c478bd9Sstevel@tonic-gate * more than the device will let us. 24317c478bd9Sstevel@tonic-gate */ 24327c478bd9Sstevel@tonic-gate for (i = 0; i < len; i += PAGESIZE) { 24337c478bd9Sstevel@tonic-gate if (i == 0) { 24347c478bd9Sstevel@tonic-gate /* 24357c478bd9Sstevel@tonic-gate * Save the pfn at offset here. This pfn will be 24367c478bd9Sstevel@tonic-gate * used later to get user address. 24377c478bd9Sstevel@tonic-gate */ 24387c478bd9Sstevel@tonic-gate if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset, 243944374aaeSsvemuri maxprot)) == PFN_INVALID) 24407c478bd9Sstevel@tonic-gate return (ENXIO); 24417c478bd9Sstevel@tonic-gate } else { 24427c478bd9Sstevel@tonic-gate if (cdev_mmap(mapfunc, dev, offset + i, maxprot) == 244344374aaeSsvemuri PFN_INVALID) 24447c478bd9Sstevel@tonic-gate return (ENXIO); 24457c478bd9Sstevel@tonic-gate } 24467c478bd9Sstevel@tonic-gate } 24477c478bd9Sstevel@tonic-gate 24487c478bd9Sstevel@tonic-gate as_rangelock(as); 244960946fe0Smec /* Pick an address w/o worrying about any vac alignment constraints. */ 245060946fe0Smec error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags); 245160946fe0Smec if (error != 0) { 245260946fe0Smec as_rangeunlock(as); 245360946fe0Smec return (error); 24547c478bd9Sstevel@tonic-gate } 24557c478bd9Sstevel@tonic-gate 24567c478bd9Sstevel@tonic-gate dev_a.mapfunc = mapfunc; 24577c478bd9Sstevel@tonic-gate dev_a.dev = dev; 24587c478bd9Sstevel@tonic-gate dev_a.offset = (offset_t)offset; 24597c478bd9Sstevel@tonic-gate dev_a.type = flags & MAP_TYPE; 24607c478bd9Sstevel@tonic-gate dev_a.prot = (uchar_t)prot; 24617c478bd9Sstevel@tonic-gate dev_a.maxprot = (uchar_t)maxprot; 24627c478bd9Sstevel@tonic-gate dev_a.hat_attr = hat_attr; 24637c478bd9Sstevel@tonic-gate dev_a.hat_flags = 0; 24647c478bd9Sstevel@tonic-gate dev_a.devmap_data = NULL; 24657c478bd9Sstevel@tonic-gate 24667c478bd9Sstevel@tonic-gate error = as_map(as, *addrp, len, segdev_create, &dev_a); 24677c478bd9Sstevel@tonic-gate as_rangeunlock(as); 24687c478bd9Sstevel@tonic-gate return (error); 24697c478bd9Sstevel@tonic-gate 24707c478bd9Sstevel@tonic-gate } 24717c478bd9Sstevel@tonic-gate 24727c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 24737c478bd9Sstevel@tonic-gate static int 24747c478bd9Sstevel@tonic-gate segdev_pagelock(struct seg *seg, caddr_t addr, size_t len, 24757c478bd9Sstevel@tonic-gate struct page ***ppp, enum lock_type type, enum seg_rw rw) 24767c478bd9Sstevel@tonic-gate { 24777c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK, 24787c478bd9Sstevel@tonic-gate "segdev_pagelock:start"); 24797c478bd9Sstevel@tonic-gate return (ENOTSUP); 24807c478bd9Sstevel@tonic-gate } 24817c478bd9Sstevel@tonic-gate 24827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 24837c478bd9Sstevel@tonic-gate static int 24847c478bd9Sstevel@tonic-gate segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len, 24857c478bd9Sstevel@tonic-gate uint_t szc) 24867c478bd9Sstevel@tonic-gate { 24877c478bd9Sstevel@tonic-gate return (ENOTSUP); 24887c478bd9Sstevel@tonic-gate } 24897c478bd9Sstevel@tonic-gate 24907c478bd9Sstevel@tonic-gate /* 24917c478bd9Sstevel@tonic-gate * devmap_device: Used by devmap framework to establish mapping 24927c478bd9Sstevel@tonic-gate * called by devmap_seup(9F) during map setup time. 24937c478bd9Sstevel@tonic-gate */ 24947c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 24957c478bd9Sstevel@tonic-gate static int 24967c478bd9Sstevel@tonic-gate devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr, 24977c478bd9Sstevel@tonic-gate offset_t off, size_t len, uint_t flags) 24987c478bd9Sstevel@tonic-gate { 24997c478bd9Sstevel@tonic-gate devmap_handle_t *rdhp, *maxdhp; 25007c478bd9Sstevel@tonic-gate struct segdev_crargs dev_a; 25017c478bd9Sstevel@tonic-gate int err; 25027c478bd9Sstevel@tonic-gate uint_t maxprot = PROT_ALL; 25037c478bd9Sstevel@tonic-gate offset_t offset = 0; 25047c478bd9Sstevel@tonic-gate pfn_t pfn; 25057c478bd9Sstevel@tonic-gate struct devmap_pmem_cookie *pcp; 25067c478bd9Sstevel@tonic-gate 25077c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE, 25087c478bd9Sstevel@tonic-gate "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx", 25097c478bd9Sstevel@tonic-gate (void *)dhp, (void *)addr, off, len); 25107c478bd9Sstevel@tonic-gate 25117c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n", 25127c478bd9Sstevel@tonic-gate (void *)dhp, (void *)addr, off, len)); 25137c478bd9Sstevel@tonic-gate 25147c478bd9Sstevel@tonic-gate as_rangelock(as); 25157c478bd9Sstevel@tonic-gate if ((flags & MAP_FIXED) == 0) { 25167c478bd9Sstevel@tonic-gate offset_t aligned_off; 25177c478bd9Sstevel@tonic-gate 25187c478bd9Sstevel@tonic-gate rdhp = maxdhp = dhp; 25197c478bd9Sstevel@tonic-gate while (rdhp != NULL) { 25207c478bd9Sstevel@tonic-gate maxdhp = (maxdhp->dh_len > rdhp->dh_len) ? 252144374aaeSsvemuri maxdhp : rdhp; 25227c478bd9Sstevel@tonic-gate rdhp = rdhp->dh_next; 25237c478bd9Sstevel@tonic-gate maxprot |= dhp->dh_maxprot; 25247c478bd9Sstevel@tonic-gate } 25257c478bd9Sstevel@tonic-gate offset = maxdhp->dh_uoff - dhp->dh_uoff; 25267c478bd9Sstevel@tonic-gate 25277c478bd9Sstevel@tonic-gate /* 25287c478bd9Sstevel@tonic-gate * Use the dhp that has the 25297c478bd9Sstevel@tonic-gate * largest len to get user address. 25307c478bd9Sstevel@tonic-gate */ 25317c478bd9Sstevel@tonic-gate /* 25327c478bd9Sstevel@tonic-gate * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr, 25337c478bd9Sstevel@tonic-gate * use 0 which is as good as any other. 25347c478bd9Sstevel@tonic-gate */ 25357c478bd9Sstevel@tonic-gate if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) { 25367c478bd9Sstevel@tonic-gate aligned_off = (offset_t)0; 25377c478bd9Sstevel@tonic-gate } else if (dhp_is_devmem(maxdhp)) { 25387c478bd9Sstevel@tonic-gate aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset; 25397c478bd9Sstevel@tonic-gate } else if (dhp_is_pmem(maxdhp)) { 25407c478bd9Sstevel@tonic-gate pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie; 25417c478bd9Sstevel@tonic-gate pfn = page_pptonum( 25427c478bd9Sstevel@tonic-gate pcp->dp_pparray[btop(maxdhp->dh_roff)]); 25437c478bd9Sstevel@tonic-gate aligned_off = (offset_t)ptob(pfn) - offset; 25447c478bd9Sstevel@tonic-gate } else { 25457c478bd9Sstevel@tonic-gate aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr - 25467c478bd9Sstevel@tonic-gate offset; 25477c478bd9Sstevel@tonic-gate } 25487c478bd9Sstevel@tonic-gate 25497c478bd9Sstevel@tonic-gate /* 25507c478bd9Sstevel@tonic-gate * Pick an address aligned to dh_cookie. 25517c478bd9Sstevel@tonic-gate * for kernel memory/user memory, cookie is cvaddr. 25527c478bd9Sstevel@tonic-gate * for device memory, cookie is physical address. 25537c478bd9Sstevel@tonic-gate */ 25547c478bd9Sstevel@tonic-gate map_addr(addr, len, aligned_off, 1, flags); 25557c478bd9Sstevel@tonic-gate if (*addr == NULL) { 25567c478bd9Sstevel@tonic-gate as_rangeunlock(as); 25577c478bd9Sstevel@tonic-gate return (ENOMEM); 25587c478bd9Sstevel@tonic-gate } 25597c478bd9Sstevel@tonic-gate } else { 25607c478bd9Sstevel@tonic-gate /* 25617c478bd9Sstevel@tonic-gate * User-specified address; blow away any previous mappings. 25627c478bd9Sstevel@tonic-gate */ 25637c478bd9Sstevel@tonic-gate (void) as_unmap(as, *addr, len); 25647c478bd9Sstevel@tonic-gate } 25657c478bd9Sstevel@tonic-gate 25667c478bd9Sstevel@tonic-gate dev_a.mapfunc = NULL; 25677c478bd9Sstevel@tonic-gate dev_a.dev = dhp->dh_dev; 25687c478bd9Sstevel@tonic-gate dev_a.type = flags & MAP_TYPE; 25697c478bd9Sstevel@tonic-gate dev_a.offset = off; 25707c478bd9Sstevel@tonic-gate /* 25717c478bd9Sstevel@tonic-gate * sdp->maxprot has the least restrict protection of all dhps. 25727c478bd9Sstevel@tonic-gate */ 25737c478bd9Sstevel@tonic-gate dev_a.maxprot = maxprot; 25747c478bd9Sstevel@tonic-gate dev_a.prot = dhp->dh_prot; 25757c478bd9Sstevel@tonic-gate /* 25767c478bd9Sstevel@tonic-gate * devmap uses dhp->dh_hat_attr for hat. 25777c478bd9Sstevel@tonic-gate */ 25787c478bd9Sstevel@tonic-gate dev_a.hat_flags = 0; 25797c478bd9Sstevel@tonic-gate dev_a.hat_attr = 0; 25807c478bd9Sstevel@tonic-gate dev_a.devmap_data = (void *)dhp; 25817c478bd9Sstevel@tonic-gate 25827c478bd9Sstevel@tonic-gate err = as_map(as, *addr, len, segdev_create, &dev_a); 25837c478bd9Sstevel@tonic-gate as_rangeunlock(as); 25847c478bd9Sstevel@tonic-gate return (err); 25857c478bd9Sstevel@tonic-gate } 25867c478bd9Sstevel@tonic-gate 25877c478bd9Sstevel@tonic-gate int 25887c478bd9Sstevel@tonic-gate devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len, 25897c478bd9Sstevel@tonic-gate uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t, 25907c478bd9Sstevel@tonic-gate size_t, uint_t, uint_t)) 25917c478bd9Sstevel@tonic-gate { 25927c478bd9Sstevel@tonic-gate register devmap_handle_t *dhp = (devmap_handle_t *)dhc; 25937c478bd9Sstevel@tonic-gate struct devmap_ctx *devctx; 25947c478bd9Sstevel@tonic-gate int do_timeout = 0; 25957c478bd9Sstevel@tonic-gate int ret; 25967c478bd9Sstevel@tonic-gate 25977c478bd9Sstevel@tonic-gate #ifdef lint 25987c478bd9Sstevel@tonic-gate pvtp = pvtp; 25997c478bd9Sstevel@tonic-gate #endif 26007c478bd9Sstevel@tonic-gate 26017c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT, 26027c478bd9Sstevel@tonic-gate "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx", 26037c478bd9Sstevel@tonic-gate (void *)dhp, off, len); 26047c478bd9Sstevel@tonic-gate DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n", 26057c478bd9Sstevel@tonic-gate (void *)dhp, off, len)); 26067c478bd9Sstevel@tonic-gate 26077c478bd9Sstevel@tonic-gate if (ctxmgt == NULL) 26087c478bd9Sstevel@tonic-gate return (FC_HWERR); 26097c478bd9Sstevel@tonic-gate 26107c478bd9Sstevel@tonic-gate devctx = dhp->dh_ctx; 26117c478bd9Sstevel@tonic-gate 26127c478bd9Sstevel@tonic-gate /* 26137c478bd9Sstevel@tonic-gate * If we are on an MP system with more than one cpu running 26147c478bd9Sstevel@tonic-gate * and if a thread on some CPU already has the context, wait 26157c478bd9Sstevel@tonic-gate * for it to finish if there is a hysteresis timeout. 26167c478bd9Sstevel@tonic-gate * 26177c478bd9Sstevel@tonic-gate * We call cv_wait() instead of cv_wait_sig() because 26187c478bd9Sstevel@tonic-gate * it does not matter much if it returned due to a signal 26197c478bd9Sstevel@tonic-gate * or due to a cv_signal() or cv_broadcast(). In either event 26207c478bd9Sstevel@tonic-gate * we need to complete the mapping otherwise the processes 26217c478bd9Sstevel@tonic-gate * will die with a SEGV. 26227c478bd9Sstevel@tonic-gate */ 26237c478bd9Sstevel@tonic-gate if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) { 26247c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1, 26257c478bd9Sstevel@tonic-gate "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p", 26267c478bd9Sstevel@tonic-gate devctx, dhp); 26277c478bd9Sstevel@tonic-gate do_timeout = 1; 26287c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 26297c478bd9Sstevel@tonic-gate while (devctx->oncpu) 26307c478bd9Sstevel@tonic-gate cv_wait(&devctx->cv, &devctx->lock); 26317c478bd9Sstevel@tonic-gate devctx->oncpu = 1; 26327c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 26337c478bd9Sstevel@tonic-gate } 26347c478bd9Sstevel@tonic-gate 26357c478bd9Sstevel@tonic-gate /* 26367c478bd9Sstevel@tonic-gate * Call the contextmgt callback so that the driver can handle 26377c478bd9Sstevel@tonic-gate * the fault. 26387c478bd9Sstevel@tonic-gate */ 26397c478bd9Sstevel@tonic-gate ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw); 26407c478bd9Sstevel@tonic-gate 26417c478bd9Sstevel@tonic-gate /* 26427c478bd9Sstevel@tonic-gate * If devmap_access() returned -1, then there was a hardware 26437c478bd9Sstevel@tonic-gate * error so we need to convert the return value to something 26447c478bd9Sstevel@tonic-gate * that trap() will understand. Otherwise, the return value 26457c478bd9Sstevel@tonic-gate * is already a fault code generated by devmap_unload() 26467c478bd9Sstevel@tonic-gate * or devmap_load(). 26477c478bd9Sstevel@tonic-gate */ 26487c478bd9Sstevel@tonic-gate if (ret) { 26497c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2, 26507c478bd9Sstevel@tonic-gate "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p", 26517c478bd9Sstevel@tonic-gate ret, dhp, devctx); 26527c478bd9Sstevel@tonic-gate DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n", 26537c478bd9Sstevel@tonic-gate ret, (void *)dhp)); 26547c478bd9Sstevel@tonic-gate if (devctx->oncpu) { 26557c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 26567c478bd9Sstevel@tonic-gate devctx->oncpu = 0; 26577c478bd9Sstevel@tonic-gate cv_signal(&devctx->cv); 26587c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 26597c478bd9Sstevel@tonic-gate } 26607c478bd9Sstevel@tonic-gate return (FC_HWERR); 26617c478bd9Sstevel@tonic-gate } 26627c478bd9Sstevel@tonic-gate 26637c478bd9Sstevel@tonic-gate /* 26647c478bd9Sstevel@tonic-gate * Setup the timeout if we need to 26657c478bd9Sstevel@tonic-gate */ 26667c478bd9Sstevel@tonic-gate if (do_timeout) { 26677c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 26687c478bd9Sstevel@tonic-gate if (dhp->dh_timeout_length > 0) { 26697c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3, 26707c478bd9Sstevel@tonic-gate "devmap_do_ctxmgt:timeout set"); 26717c478bd9Sstevel@tonic-gate devctx->timeout = timeout(devmap_ctxto, 26727c478bd9Sstevel@tonic-gate devctx, dhp->dh_timeout_length); 26737c478bd9Sstevel@tonic-gate } else { 26747c478bd9Sstevel@tonic-gate /* 26757c478bd9Sstevel@tonic-gate * We don't want to wait so set oncpu to 26767c478bd9Sstevel@tonic-gate * 0 and wake up anyone waiting. 26777c478bd9Sstevel@tonic-gate */ 26787c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4, 26797c478bd9Sstevel@tonic-gate "devmap_do_ctxmgt:timeout not set"); 26807c478bd9Sstevel@tonic-gate devctx->oncpu = 0; 26817c478bd9Sstevel@tonic-gate cv_signal(&devctx->cv); 26827c478bd9Sstevel@tonic-gate } 26837c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 26847c478bd9Sstevel@tonic-gate } 26857c478bd9Sstevel@tonic-gate 26867c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 26877c478bd9Sstevel@tonic-gate } 26887c478bd9Sstevel@tonic-gate 26897c478bd9Sstevel@tonic-gate /* 26907c478bd9Sstevel@tonic-gate * end of mapping 26917c478bd9Sstevel@tonic-gate * poff fault_offset | 26927c478bd9Sstevel@tonic-gate * base | | | 26937c478bd9Sstevel@tonic-gate * | | | | 26947c478bd9Sstevel@tonic-gate * V V V V 26957c478bd9Sstevel@tonic-gate * +-----------+---------------+-------+---------+-------+ 26967c478bd9Sstevel@tonic-gate * ^ ^ ^ ^ 26977c478bd9Sstevel@tonic-gate * |<--- offset--->|<-len->| | 26987c478bd9Sstevel@tonic-gate * |<--- dh_len(size of mapping) --->| 26997c478bd9Sstevel@tonic-gate * |<-- pg -->| 27007c478bd9Sstevel@tonic-gate * -->|rlen|<-- 27017c478bd9Sstevel@tonic-gate */ 27027c478bd9Sstevel@tonic-gate static ulong_t 27037c478bd9Sstevel@tonic-gate devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len, 27047c478bd9Sstevel@tonic-gate ulong_t *opfn, ulong_t *pagesize) 27057c478bd9Sstevel@tonic-gate { 27067c478bd9Sstevel@tonic-gate register int level; 27077c478bd9Sstevel@tonic-gate ulong_t pg; 27087c478bd9Sstevel@tonic-gate ulong_t poff; 27097c478bd9Sstevel@tonic-gate ulong_t base; 27107c478bd9Sstevel@tonic-gate caddr_t uvaddr; 27117c478bd9Sstevel@tonic-gate long rlen; 27127c478bd9Sstevel@tonic-gate 27137c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP, 27147c478bd9Sstevel@tonic-gate "devmap_roundup:start dhp=%p off=%lx len=%lx", 27157c478bd9Sstevel@tonic-gate (void *)dhp, offset, len); 27167c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n", 27177c478bd9Sstevel@tonic-gate (void *)dhp, offset, len)); 27187c478bd9Sstevel@tonic-gate 27197c478bd9Sstevel@tonic-gate /* 27207c478bd9Sstevel@tonic-gate * get the max. pagesize that is aligned within the range 27217c478bd9Sstevel@tonic-gate * <dh_pfn, dh_pfn+offset>. 27227c478bd9Sstevel@tonic-gate * 27237c478bd9Sstevel@tonic-gate * The calculations below use physical address to ddetermine 27247c478bd9Sstevel@tonic-gate * the page size to use. The same calculations can use the 27257c478bd9Sstevel@tonic-gate * virtual address to determine the page size. 27267c478bd9Sstevel@tonic-gate */ 27277c478bd9Sstevel@tonic-gate base = (ulong_t)ptob(dhp->dh_pfn); 27287c478bd9Sstevel@tonic-gate for (level = dhp->dh_mmulevel; level >= 0; level--) { 27297c478bd9Sstevel@tonic-gate pg = page_get_pagesize(level); 27307c478bd9Sstevel@tonic-gate poff = ((base + offset) & ~(pg - 1)); 27317c478bd9Sstevel@tonic-gate uvaddr = dhp->dh_uvaddr + (poff - base); 27327c478bd9Sstevel@tonic-gate if ((poff >= base) && 27337c478bd9Sstevel@tonic-gate ((poff + pg) <= (base + dhp->dh_len)) && 27347c478bd9Sstevel@tonic-gate VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg)) 27357c478bd9Sstevel@tonic-gate break; 27367c478bd9Sstevel@tonic-gate } 27377c478bd9Sstevel@tonic-gate 27387c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1, 27397c478bd9Sstevel@tonic-gate "devmap_roundup: base=%lx poff=%lx dhp=%p", 27407c478bd9Sstevel@tonic-gate base, poff, dhp); 27417c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n", 27427c478bd9Sstevel@tonic-gate base, poff, dhp->dh_pfn)); 27437c478bd9Sstevel@tonic-gate 27447c478bd9Sstevel@tonic-gate ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg)); 27457c478bd9Sstevel@tonic-gate ASSERT(level >= 0); 27467c478bd9Sstevel@tonic-gate 27477c478bd9Sstevel@tonic-gate *pagesize = pg; 27487c478bd9Sstevel@tonic-gate *opfn = dhp->dh_pfn + btop(poff - base); 27497c478bd9Sstevel@tonic-gate 27507c478bd9Sstevel@tonic-gate rlen = len + offset - (poff - base + pg); 27517c478bd9Sstevel@tonic-gate 27527c478bd9Sstevel@tonic-gate ASSERT(rlen < (long)len); 27537c478bd9Sstevel@tonic-gate 27547c478bd9Sstevel@tonic-gate TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2, 27557c478bd9Sstevel@tonic-gate "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p", 27567c478bd9Sstevel@tonic-gate (void *)dhp, level, rlen, pagesize, opfn); 27577c478bd9Sstevel@tonic-gate DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p " 27587c478bd9Sstevel@tonic-gate "level %x rlen %lx psize %lx opfn %lx\n", 27597c478bd9Sstevel@tonic-gate (void *)dhp, level, rlen, *pagesize, *opfn)); 27607c478bd9Sstevel@tonic-gate 27617c478bd9Sstevel@tonic-gate return ((ulong_t)((rlen > 0) ? rlen : 0)); 27627c478bd9Sstevel@tonic-gate } 27637c478bd9Sstevel@tonic-gate 27647c478bd9Sstevel@tonic-gate /* 27657c478bd9Sstevel@tonic-gate * find the dhp that contains addr. 27667c478bd9Sstevel@tonic-gate */ 27677c478bd9Sstevel@tonic-gate static devmap_handle_t * 27687c478bd9Sstevel@tonic-gate devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr) 27697c478bd9Sstevel@tonic-gate { 27707c478bd9Sstevel@tonic-gate devmap_handle_t *dhp; 27717c478bd9Sstevel@tonic-gate 27727c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE, 27737c478bd9Sstevel@tonic-gate "devmap_find_handle:start"); 27747c478bd9Sstevel@tonic-gate 27757c478bd9Sstevel@tonic-gate dhp = dhp_head; 27767c478bd9Sstevel@tonic-gate while (dhp) { 27777c478bd9Sstevel@tonic-gate if (addr >= dhp->dh_uvaddr && 27787c478bd9Sstevel@tonic-gate addr < (dhp->dh_uvaddr + dhp->dh_len)) 27797c478bd9Sstevel@tonic-gate return (dhp); 27807c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 27817c478bd9Sstevel@tonic-gate } 27827c478bd9Sstevel@tonic-gate 27837c478bd9Sstevel@tonic-gate return ((devmap_handle_t *)NULL); 27847c478bd9Sstevel@tonic-gate } 27857c478bd9Sstevel@tonic-gate 27867c478bd9Sstevel@tonic-gate /* 27877c478bd9Sstevel@tonic-gate * devmap_unload: 27887c478bd9Sstevel@tonic-gate * Marks a segdev segment or pages if offset->offset+len 27897c478bd9Sstevel@tonic-gate * is not the entire segment as intercept and unloads the 27907c478bd9Sstevel@tonic-gate * pages in the range offset -> offset+len. 27917c478bd9Sstevel@tonic-gate */ 27927c478bd9Sstevel@tonic-gate int 27937c478bd9Sstevel@tonic-gate devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len) 27947c478bd9Sstevel@tonic-gate { 27957c478bd9Sstevel@tonic-gate register devmap_handle_t *dhp = (devmap_handle_t *)dhc; 27967c478bd9Sstevel@tonic-gate caddr_t addr; 27977c478bd9Sstevel@tonic-gate ulong_t size; 27987c478bd9Sstevel@tonic-gate ssize_t soff; 27997c478bd9Sstevel@tonic-gate 28007c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD, 28017c478bd9Sstevel@tonic-gate "devmap_unload:start dhp=%p offset=%llx len=%lx", 28027c478bd9Sstevel@tonic-gate (void *)dhp, offset, len); 28037c478bd9Sstevel@tonic-gate DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n", 28047c478bd9Sstevel@tonic-gate (void *)dhp, offset, len)); 28057c478bd9Sstevel@tonic-gate 28067c478bd9Sstevel@tonic-gate soff = (ssize_t)(offset - dhp->dh_uoff); 28077c478bd9Sstevel@tonic-gate soff = round_down_p2(soff, PAGESIZE); 28087c478bd9Sstevel@tonic-gate if (soff < 0 || soff >= dhp->dh_len) 28097c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EINVAL)); 28107c478bd9Sstevel@tonic-gate 28117c478bd9Sstevel@tonic-gate /* 28127c478bd9Sstevel@tonic-gate * Address and size must be page aligned. Len is set to the 28137c478bd9Sstevel@tonic-gate * number of bytes in the number of pages that are required to 28147c478bd9Sstevel@tonic-gate * support len. Offset is set to the byte offset of the first byte 28157c478bd9Sstevel@tonic-gate * of the page that contains offset. 28167c478bd9Sstevel@tonic-gate */ 28177c478bd9Sstevel@tonic-gate len = round_up_p2(len, PAGESIZE); 28187c478bd9Sstevel@tonic-gate 28197c478bd9Sstevel@tonic-gate /* 28207c478bd9Sstevel@tonic-gate * If len is == 0, then calculate the size by getting 28217c478bd9Sstevel@tonic-gate * the number of bytes from offset to the end of the segment. 28227c478bd9Sstevel@tonic-gate */ 28237c478bd9Sstevel@tonic-gate if (len == 0) 28247c478bd9Sstevel@tonic-gate size = dhp->dh_len - soff; 28257c478bd9Sstevel@tonic-gate else { 28267c478bd9Sstevel@tonic-gate size = len; 28277c478bd9Sstevel@tonic-gate if ((soff + size) > dhp->dh_len) 28287c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EINVAL)); 28297c478bd9Sstevel@tonic-gate } 28307c478bd9Sstevel@tonic-gate 28317c478bd9Sstevel@tonic-gate /* 28327c478bd9Sstevel@tonic-gate * The address is offset bytes from the base address of 28337c478bd9Sstevel@tonic-gate * the dhp. 28347c478bd9Sstevel@tonic-gate */ 28357c478bd9Sstevel@tonic-gate addr = (caddr_t)(soff + dhp->dh_uvaddr); 28367c478bd9Sstevel@tonic-gate 28377c478bd9Sstevel@tonic-gate /* 28387c478bd9Sstevel@tonic-gate * If large page size was used in hat_devload(), 28397c478bd9Sstevel@tonic-gate * the same page size must be used in hat_unload(). 28407c478bd9Sstevel@tonic-gate */ 28417c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_FLAG_LARGE) { 28427c478bd9Sstevel@tonic-gate hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, 284344374aaeSsvemuri dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); 28447c478bd9Sstevel@tonic-gate } else { 28457c478bd9Sstevel@tonic-gate hat_unload(dhp->dh_seg->s_as->a_hat, addr, size, 284644374aaeSsvemuri HAT_UNLOAD|HAT_UNLOAD_OTHER); 28477c478bd9Sstevel@tonic-gate } 28487c478bd9Sstevel@tonic-gate 28497c478bd9Sstevel@tonic-gate return (0); 28507c478bd9Sstevel@tonic-gate } 28517c478bd9Sstevel@tonic-gate 28527c478bd9Sstevel@tonic-gate /* 28537c478bd9Sstevel@tonic-gate * calculates the optimal page size that will be used for hat_devload(). 28547c478bd9Sstevel@tonic-gate */ 28557c478bd9Sstevel@tonic-gate static void 28567c478bd9Sstevel@tonic-gate devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr, 28577c478bd9Sstevel@tonic-gate size_t *llen, caddr_t *laddr) 28587c478bd9Sstevel@tonic-gate { 28597c478bd9Sstevel@tonic-gate ulong_t off; 28607c478bd9Sstevel@tonic-gate ulong_t pfn; 28617c478bd9Sstevel@tonic-gate ulong_t pgsize; 28627c478bd9Sstevel@tonic-gate uint_t first = 1; 28637c478bd9Sstevel@tonic-gate 28647c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE, 28657c478bd9Sstevel@tonic-gate "devmap_get_large_pgsize:start"); 28667c478bd9Sstevel@tonic-gate 28677c478bd9Sstevel@tonic-gate /* 28687c478bd9Sstevel@tonic-gate * RFE - Code only supports large page mappings for devmem 28697c478bd9Sstevel@tonic-gate * This code could be changed in future if we want to support 28707c478bd9Sstevel@tonic-gate * large page mappings for kernel exported memory. 28717c478bd9Sstevel@tonic-gate */ 28727c478bd9Sstevel@tonic-gate ASSERT(dhp_is_devmem(dhp)); 28737c478bd9Sstevel@tonic-gate ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)); 28747c478bd9Sstevel@tonic-gate 28757c478bd9Sstevel@tonic-gate *llen = 0; 28767c478bd9Sstevel@tonic-gate off = (ulong_t)(addr - dhp->dh_uvaddr); 28777c478bd9Sstevel@tonic-gate while ((long)len > 0) { 28787c478bd9Sstevel@tonic-gate /* 28797c478bd9Sstevel@tonic-gate * get the optimal pfn to minimize address translations. 28807c478bd9Sstevel@tonic-gate * devmap_roundup() returns residue bytes for next round 28817c478bd9Sstevel@tonic-gate * calculations. 28827c478bd9Sstevel@tonic-gate */ 28837c478bd9Sstevel@tonic-gate len = devmap_roundup(dhp, off, len, &pfn, &pgsize); 28847c478bd9Sstevel@tonic-gate 28857c478bd9Sstevel@tonic-gate if (first) { 28867c478bd9Sstevel@tonic-gate *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn); 28877c478bd9Sstevel@tonic-gate first = 0; 28887c478bd9Sstevel@tonic-gate } 28897c478bd9Sstevel@tonic-gate 28907c478bd9Sstevel@tonic-gate *llen += pgsize; 28917c478bd9Sstevel@tonic-gate off = ptob(pfn - dhp->dh_pfn) + pgsize; 28927c478bd9Sstevel@tonic-gate } 2893da6c28aaSamw /* Large page mapping len/addr cover more range than original fault */ 28947c478bd9Sstevel@tonic-gate ASSERT(*llen >= len && *laddr <= addr); 28957c478bd9Sstevel@tonic-gate ASSERT((*laddr + *llen) >= (addr + len)); 28967c478bd9Sstevel@tonic-gate } 28977c478bd9Sstevel@tonic-gate 28987c478bd9Sstevel@tonic-gate /* 28997c478bd9Sstevel@tonic-gate * Initialize the devmap_softlock structure. 29007c478bd9Sstevel@tonic-gate */ 29017c478bd9Sstevel@tonic-gate static struct devmap_softlock * 29027c478bd9Sstevel@tonic-gate devmap_softlock_init(dev_t dev, ulong_t id) 29037c478bd9Sstevel@tonic-gate { 29047c478bd9Sstevel@tonic-gate struct devmap_softlock *slock; 29057c478bd9Sstevel@tonic-gate struct devmap_softlock *tmp; 29067c478bd9Sstevel@tonic-gate 29077c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT, 29087c478bd9Sstevel@tonic-gate "devmap_softlock_init:start"); 29097c478bd9Sstevel@tonic-gate 29107c478bd9Sstevel@tonic-gate tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP); 29117c478bd9Sstevel@tonic-gate mutex_enter(&devmap_slock); 29127c478bd9Sstevel@tonic-gate 29137c478bd9Sstevel@tonic-gate for (slock = devmap_slist; slock != NULL; slock = slock->next) 29147c478bd9Sstevel@tonic-gate if ((slock->dev == dev) && (slock->id == id)) 29157c478bd9Sstevel@tonic-gate break; 29167c478bd9Sstevel@tonic-gate 29177c478bd9Sstevel@tonic-gate if (slock == NULL) { 29187c478bd9Sstevel@tonic-gate slock = tmp; 29197c478bd9Sstevel@tonic-gate slock->dev = dev; 29207c478bd9Sstevel@tonic-gate slock->id = id; 29217c478bd9Sstevel@tonic-gate mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL); 29227c478bd9Sstevel@tonic-gate cv_init(&slock->cv, NULL, CV_DEFAULT, NULL); 29237c478bd9Sstevel@tonic-gate slock->next = devmap_slist; 29247c478bd9Sstevel@tonic-gate devmap_slist = slock; 29257c478bd9Sstevel@tonic-gate } else 29267c478bd9Sstevel@tonic-gate kmem_free(tmp, sizeof (struct devmap_softlock)); 29277c478bd9Sstevel@tonic-gate 29287c478bd9Sstevel@tonic-gate mutex_enter(&slock->lock); 29297c478bd9Sstevel@tonic-gate slock->refcnt++; 29307c478bd9Sstevel@tonic-gate mutex_exit(&slock->lock); 29317c478bd9Sstevel@tonic-gate mutex_exit(&devmap_slock); 29327c478bd9Sstevel@tonic-gate 29337c478bd9Sstevel@tonic-gate return (slock); 29347c478bd9Sstevel@tonic-gate } 29357c478bd9Sstevel@tonic-gate 29367c478bd9Sstevel@tonic-gate /* 29377c478bd9Sstevel@tonic-gate * Wake up processes that sleep on softlocked. 29387c478bd9Sstevel@tonic-gate * Free dh_softlock if refcnt is 0. 29397c478bd9Sstevel@tonic-gate */ 29407c478bd9Sstevel@tonic-gate static void 29417c478bd9Sstevel@tonic-gate devmap_softlock_rele(devmap_handle_t *dhp) 29427c478bd9Sstevel@tonic-gate { 29437c478bd9Sstevel@tonic-gate struct devmap_softlock *slock = dhp->dh_softlock; 29447c478bd9Sstevel@tonic-gate struct devmap_softlock *tmp; 29457c478bd9Sstevel@tonic-gate struct devmap_softlock *parent; 29467c478bd9Sstevel@tonic-gate 29477c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE, 29487c478bd9Sstevel@tonic-gate "devmap_softlock_rele:start"); 29497c478bd9Sstevel@tonic-gate 29507c478bd9Sstevel@tonic-gate mutex_enter(&devmap_slock); 29517c478bd9Sstevel@tonic-gate mutex_enter(&slock->lock); 29527c478bd9Sstevel@tonic-gate 29537c478bd9Sstevel@tonic-gate ASSERT(slock->refcnt > 0); 29547c478bd9Sstevel@tonic-gate 29557c478bd9Sstevel@tonic-gate slock->refcnt--; 29567c478bd9Sstevel@tonic-gate 29577c478bd9Sstevel@tonic-gate /* 29587c478bd9Sstevel@tonic-gate * If no one is using the device, free up the slock data. 29597c478bd9Sstevel@tonic-gate */ 29607c478bd9Sstevel@tonic-gate if (slock->refcnt == 0) { 29617c478bd9Sstevel@tonic-gate slock->softlocked = 0; 29627c478bd9Sstevel@tonic-gate cv_signal(&slock->cv); 29637c478bd9Sstevel@tonic-gate 29647c478bd9Sstevel@tonic-gate if (devmap_slist == slock) 29657c478bd9Sstevel@tonic-gate devmap_slist = slock->next; 29667c478bd9Sstevel@tonic-gate else { 29677c478bd9Sstevel@tonic-gate parent = devmap_slist; 29687c478bd9Sstevel@tonic-gate for (tmp = devmap_slist->next; tmp != NULL; 296944374aaeSsvemuri tmp = tmp->next) { 29707c478bd9Sstevel@tonic-gate if (tmp == slock) { 29717c478bd9Sstevel@tonic-gate parent->next = tmp->next; 29727c478bd9Sstevel@tonic-gate break; 29737c478bd9Sstevel@tonic-gate } 29747c478bd9Sstevel@tonic-gate parent = tmp; 29757c478bd9Sstevel@tonic-gate } 29767c478bd9Sstevel@tonic-gate } 29777c478bd9Sstevel@tonic-gate mutex_exit(&slock->lock); 29787c478bd9Sstevel@tonic-gate mutex_destroy(&slock->lock); 29797c478bd9Sstevel@tonic-gate cv_destroy(&slock->cv); 29807c478bd9Sstevel@tonic-gate kmem_free(slock, sizeof (struct devmap_softlock)); 29817c478bd9Sstevel@tonic-gate } else 29827c478bd9Sstevel@tonic-gate mutex_exit(&slock->lock); 29837c478bd9Sstevel@tonic-gate 29847c478bd9Sstevel@tonic-gate mutex_exit(&devmap_slock); 29857c478bd9Sstevel@tonic-gate } 29867c478bd9Sstevel@tonic-gate 29877c478bd9Sstevel@tonic-gate /* 29887c478bd9Sstevel@tonic-gate * Wake up processes that sleep on dh_ctx->locked. 29897c478bd9Sstevel@tonic-gate * Free dh_ctx if refcnt is 0. 29907c478bd9Sstevel@tonic-gate */ 29917c478bd9Sstevel@tonic-gate static void 29927c478bd9Sstevel@tonic-gate devmap_ctx_rele(devmap_handle_t *dhp) 29937c478bd9Sstevel@tonic-gate { 29947c478bd9Sstevel@tonic-gate struct devmap_ctx *devctx = dhp->dh_ctx; 29957c478bd9Sstevel@tonic-gate struct devmap_ctx *tmp; 29967c478bd9Sstevel@tonic-gate struct devmap_ctx *parent; 29977c478bd9Sstevel@tonic-gate timeout_id_t tid; 29987c478bd9Sstevel@tonic-gate 29997c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE, 30007c478bd9Sstevel@tonic-gate "devmap_ctx_rele:start"); 30017c478bd9Sstevel@tonic-gate 30027c478bd9Sstevel@tonic-gate mutex_enter(&devmapctx_lock); 30037c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 30047c478bd9Sstevel@tonic-gate 30057c478bd9Sstevel@tonic-gate ASSERT(devctx->refcnt > 0); 30067c478bd9Sstevel@tonic-gate 30077c478bd9Sstevel@tonic-gate devctx->refcnt--; 30087c478bd9Sstevel@tonic-gate 30097c478bd9Sstevel@tonic-gate /* 30107c478bd9Sstevel@tonic-gate * If no one is using the device, free up the devctx data. 30117c478bd9Sstevel@tonic-gate */ 30127c478bd9Sstevel@tonic-gate if (devctx->refcnt == 0) { 30137c478bd9Sstevel@tonic-gate /* 30147c478bd9Sstevel@tonic-gate * Untimeout any threads using this mapping as they are about 30157c478bd9Sstevel@tonic-gate * to go away. 30167c478bd9Sstevel@tonic-gate */ 30177c478bd9Sstevel@tonic-gate if (devctx->timeout != 0) { 30187c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1, 30197c478bd9Sstevel@tonic-gate "devmap_ctx_rele:untimeout ctx->timeout"); 30207c478bd9Sstevel@tonic-gate 30217c478bd9Sstevel@tonic-gate tid = devctx->timeout; 30227c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 30237c478bd9Sstevel@tonic-gate (void) untimeout(tid); 30247c478bd9Sstevel@tonic-gate mutex_enter(&devctx->lock); 30257c478bd9Sstevel@tonic-gate } 30267c478bd9Sstevel@tonic-gate 30277c478bd9Sstevel@tonic-gate devctx->oncpu = 0; 30287c478bd9Sstevel@tonic-gate cv_signal(&devctx->cv); 30297c478bd9Sstevel@tonic-gate 30307c478bd9Sstevel@tonic-gate if (devmapctx_list == devctx) 30317c478bd9Sstevel@tonic-gate devmapctx_list = devctx->next; 30327c478bd9Sstevel@tonic-gate else { 30337c478bd9Sstevel@tonic-gate parent = devmapctx_list; 30347c478bd9Sstevel@tonic-gate for (tmp = devmapctx_list->next; tmp != NULL; 303544374aaeSsvemuri tmp = tmp->next) { 30367c478bd9Sstevel@tonic-gate if (tmp == devctx) { 30377c478bd9Sstevel@tonic-gate parent->next = tmp->next; 30387c478bd9Sstevel@tonic-gate break; 30397c478bd9Sstevel@tonic-gate } 30407c478bd9Sstevel@tonic-gate parent = tmp; 30417c478bd9Sstevel@tonic-gate } 30427c478bd9Sstevel@tonic-gate } 30437c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 30447c478bd9Sstevel@tonic-gate mutex_destroy(&devctx->lock); 30457c478bd9Sstevel@tonic-gate cv_destroy(&devctx->cv); 30467c478bd9Sstevel@tonic-gate kmem_free(devctx, sizeof (struct devmap_ctx)); 30477c478bd9Sstevel@tonic-gate } else 30487c478bd9Sstevel@tonic-gate mutex_exit(&devctx->lock); 30497c478bd9Sstevel@tonic-gate 30507c478bd9Sstevel@tonic-gate mutex_exit(&devmapctx_lock); 30517c478bd9Sstevel@tonic-gate } 30527c478bd9Sstevel@tonic-gate 30537c478bd9Sstevel@tonic-gate /* 30547c478bd9Sstevel@tonic-gate * devmap_load: 30557c478bd9Sstevel@tonic-gate * Marks a segdev segment or pages if offset->offset+len 30567c478bd9Sstevel@tonic-gate * is not the entire segment as nointercept and faults in 30577c478bd9Sstevel@tonic-gate * the pages in the range offset -> offset+len. 30587c478bd9Sstevel@tonic-gate */ 30597c478bd9Sstevel@tonic-gate int 30607c478bd9Sstevel@tonic-gate devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type, 30617c478bd9Sstevel@tonic-gate uint_t rw) 30627c478bd9Sstevel@tonic-gate { 30637c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)dhc; 30647c478bd9Sstevel@tonic-gate struct as *asp = dhp->dh_seg->s_as; 30657c478bd9Sstevel@tonic-gate caddr_t addr; 30667c478bd9Sstevel@tonic-gate ulong_t size; 30677c478bd9Sstevel@tonic-gate ssize_t soff; /* offset from the beginning of the segment */ 30687c478bd9Sstevel@tonic-gate int rc; 30697c478bd9Sstevel@tonic-gate 30707c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD, 30717c478bd9Sstevel@tonic-gate "devmap_load:start dhp=%p offset=%llx len=%lx", 307244374aaeSsvemuri (void *)dhp, offset, len); 30737c478bd9Sstevel@tonic-gate 30747c478bd9Sstevel@tonic-gate DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n", 30757c478bd9Sstevel@tonic-gate (void *)dhp, offset, len)); 30767c478bd9Sstevel@tonic-gate 30777c478bd9Sstevel@tonic-gate /* 30787c478bd9Sstevel@tonic-gate * Hat layer only supports devload to process' context for which 30797c478bd9Sstevel@tonic-gate * the as lock is held. Verify here and return error if drivers 30807c478bd9Sstevel@tonic-gate * inadvertently call devmap_load on a wrong devmap handle. 30817c478bd9Sstevel@tonic-gate */ 30827c478bd9Sstevel@tonic-gate if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock)) 30837c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EINVAL)); 30847c478bd9Sstevel@tonic-gate 30857c478bd9Sstevel@tonic-gate soff = (ssize_t)(offset - dhp->dh_uoff); 30867c478bd9Sstevel@tonic-gate soff = round_down_p2(soff, PAGESIZE); 30877c478bd9Sstevel@tonic-gate if (soff < 0 || soff >= dhp->dh_len) 30887c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EINVAL)); 30897c478bd9Sstevel@tonic-gate 30907c478bd9Sstevel@tonic-gate /* 30917c478bd9Sstevel@tonic-gate * Address and size must be page aligned. Len is set to the 30927c478bd9Sstevel@tonic-gate * number of bytes in the number of pages that are required to 30937c478bd9Sstevel@tonic-gate * support len. Offset is set to the byte offset of the first byte 30947c478bd9Sstevel@tonic-gate * of the page that contains offset. 30957c478bd9Sstevel@tonic-gate */ 30967c478bd9Sstevel@tonic-gate len = round_up_p2(len, PAGESIZE); 30977c478bd9Sstevel@tonic-gate 30987c478bd9Sstevel@tonic-gate /* 30997c478bd9Sstevel@tonic-gate * If len == 0, then calculate the size by getting 31007c478bd9Sstevel@tonic-gate * the number of bytes from offset to the end of the segment. 31017c478bd9Sstevel@tonic-gate */ 31027c478bd9Sstevel@tonic-gate if (len == 0) 31037c478bd9Sstevel@tonic-gate size = dhp->dh_len - soff; 31047c478bd9Sstevel@tonic-gate else { 31057c478bd9Sstevel@tonic-gate size = len; 31067c478bd9Sstevel@tonic-gate if ((soff + size) > dhp->dh_len) 31077c478bd9Sstevel@tonic-gate return (FC_MAKE_ERR(EINVAL)); 31087c478bd9Sstevel@tonic-gate } 31097c478bd9Sstevel@tonic-gate 31107c478bd9Sstevel@tonic-gate /* 31117c478bd9Sstevel@tonic-gate * The address is offset bytes from the base address of 31127c478bd9Sstevel@tonic-gate * the segment. 31137c478bd9Sstevel@tonic-gate */ 31147c478bd9Sstevel@tonic-gate addr = (caddr_t)(soff + dhp->dh_uvaddr); 31157c478bd9Sstevel@tonic-gate 31167c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 31177c478bd9Sstevel@tonic-gate rc = segdev_faultpages(asp->a_hat, 311844374aaeSsvemuri dhp->dh_seg, addr, size, type, rw, dhp); 31197c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 31207c478bd9Sstevel@tonic-gate return (rc); 31217c478bd9Sstevel@tonic-gate } 31227c478bd9Sstevel@tonic-gate 31237c478bd9Sstevel@tonic-gate int 31247c478bd9Sstevel@tonic-gate devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp, 31257c478bd9Sstevel@tonic-gate size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred) 31267c478bd9Sstevel@tonic-gate { 31277c478bd9Sstevel@tonic-gate register devmap_handle_t *dhp; 31287c478bd9Sstevel@tonic-gate int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t, 312944374aaeSsvemuri size_t *, uint_t); 31307c478bd9Sstevel@tonic-gate int (*mmap)(dev_t, off_t, int); 31317c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops; 31327c478bd9Sstevel@tonic-gate devmap_handle_t *dhp_head = NULL; 31337c478bd9Sstevel@tonic-gate devmap_handle_t *dhp_prev = NULL; 31347c478bd9Sstevel@tonic-gate devmap_handle_t *dhp_curr; 31357c478bd9Sstevel@tonic-gate caddr_t addr; 31367c478bd9Sstevel@tonic-gate int map_flag; 31377c478bd9Sstevel@tonic-gate int ret; 31387c478bd9Sstevel@tonic-gate ulong_t total_len; 31397c478bd9Sstevel@tonic-gate size_t map_len; 31407c478bd9Sstevel@tonic-gate size_t resid_len = len; 31417c478bd9Sstevel@tonic-gate offset_t map_off = off; 31427c478bd9Sstevel@tonic-gate struct devmap_softlock *slock = NULL; 31437c478bd9Sstevel@tonic-gate 31447c478bd9Sstevel@tonic-gate #ifdef lint 31457c478bd9Sstevel@tonic-gate cred = cred; 31467c478bd9Sstevel@tonic-gate #endif 31477c478bd9Sstevel@tonic-gate 31487c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP, 31497c478bd9Sstevel@tonic-gate "devmap_setup:start off=%llx len=%lx", off, len); 31507c478bd9Sstevel@tonic-gate DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n", 31517c478bd9Sstevel@tonic-gate off, len)); 31527c478bd9Sstevel@tonic-gate 31537c478bd9Sstevel@tonic-gate devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap; 31547c478bd9Sstevel@tonic-gate mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap; 31557c478bd9Sstevel@tonic-gate 31567c478bd9Sstevel@tonic-gate /* 31577c478bd9Sstevel@tonic-gate * driver must provide devmap(9E) entry point in cb_ops to use the 31587c478bd9Sstevel@tonic-gate * devmap framework. 31597c478bd9Sstevel@tonic-gate */ 31607c478bd9Sstevel@tonic-gate if (devmap == NULL || devmap == nulldev || devmap == nodev) 31617c478bd9Sstevel@tonic-gate return (EINVAL); 31627c478bd9Sstevel@tonic-gate 31637c478bd9Sstevel@tonic-gate /* 31647c478bd9Sstevel@tonic-gate * To protect from an inadvertent entry because the devmap entry point 31657c478bd9Sstevel@tonic-gate * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and 31667c478bd9Sstevel@tonic-gate * mmap is NULL. 31677c478bd9Sstevel@tonic-gate */ 31687c478bd9Sstevel@tonic-gate map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag; 31697c478bd9Sstevel@tonic-gate if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev)) 31707c478bd9Sstevel@tonic-gate return (EINVAL); 31717c478bd9Sstevel@tonic-gate 31727c478bd9Sstevel@tonic-gate /* 31737c478bd9Sstevel@tonic-gate * devmap allows mmap(2) to map multiple registers. 31747c478bd9Sstevel@tonic-gate * one devmap_handle is created for each register mapped. 31757c478bd9Sstevel@tonic-gate */ 31767c478bd9Sstevel@tonic-gate for (total_len = 0; total_len < len; total_len += map_len) { 31777c478bd9Sstevel@tonic-gate dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP); 31787c478bd9Sstevel@tonic-gate 31797c478bd9Sstevel@tonic-gate if (dhp_prev != NULL) 31807c478bd9Sstevel@tonic-gate dhp_prev->dh_next = dhp; 31817c478bd9Sstevel@tonic-gate else 31827c478bd9Sstevel@tonic-gate dhp_head = dhp; 31837c478bd9Sstevel@tonic-gate dhp_prev = dhp; 31847c478bd9Sstevel@tonic-gate 31857c478bd9Sstevel@tonic-gate dhp->dh_prot = prot; 31867c478bd9Sstevel@tonic-gate dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot; 31877c478bd9Sstevel@tonic-gate dhp->dh_dev = dev; 31887c478bd9Sstevel@tonic-gate dhp->dh_timeout_length = CTX_TIMEOUT_VALUE; 31897c478bd9Sstevel@tonic-gate dhp->dh_uoff = map_off; 31907c478bd9Sstevel@tonic-gate 31917c478bd9Sstevel@tonic-gate /* 31927c478bd9Sstevel@tonic-gate * Get mapping specific info from 31937c478bd9Sstevel@tonic-gate * the driver, such as rnumber, roff, len, callbackops, 31947c478bd9Sstevel@tonic-gate * accattrp and, if the mapping is for kernel memory, 31957c478bd9Sstevel@tonic-gate * ddi_umem_cookie. 31967c478bd9Sstevel@tonic-gate */ 31977c478bd9Sstevel@tonic-gate if ((ret = cdev_devmap(dev, dhp, map_off, 31987c478bd9Sstevel@tonic-gate resid_len, &map_len, get_udatamodel())) != 0) { 31997c478bd9Sstevel@tonic-gate free_devmap_handle(dhp_head); 32007c478bd9Sstevel@tonic-gate return (ENXIO); 32017c478bd9Sstevel@tonic-gate } 32027c478bd9Sstevel@tonic-gate 32037c478bd9Sstevel@tonic-gate if (map_len & PAGEOFFSET) { 32047c478bd9Sstevel@tonic-gate free_devmap_handle(dhp_head); 32057c478bd9Sstevel@tonic-gate return (EINVAL); 32067c478bd9Sstevel@tonic-gate } 32077c478bd9Sstevel@tonic-gate 32087c478bd9Sstevel@tonic-gate callbackops = &dhp->dh_callbackops; 32097c478bd9Sstevel@tonic-gate 32107c478bd9Sstevel@tonic-gate if ((callbackops->devmap_access == NULL) || 321144374aaeSsvemuri (callbackops->devmap_access == nulldev) || 321244374aaeSsvemuri (callbackops->devmap_access == nodev)) { 32137c478bd9Sstevel@tonic-gate /* 32147c478bd9Sstevel@tonic-gate * Normally devmap does not support MAP_PRIVATE unless 32157c478bd9Sstevel@tonic-gate * the drivers provide a valid devmap_access routine. 32167c478bd9Sstevel@tonic-gate */ 32177c478bd9Sstevel@tonic-gate if ((flags & MAP_PRIVATE) != 0) { 32187c478bd9Sstevel@tonic-gate free_devmap_handle(dhp_head); 32197c478bd9Sstevel@tonic-gate return (EINVAL); 32207c478bd9Sstevel@tonic-gate } 32217c478bd9Sstevel@tonic-gate } else { 32227c478bd9Sstevel@tonic-gate /* 32237c478bd9Sstevel@tonic-gate * Initialize dhp_softlock and dh_ctx if the drivers 32247c478bd9Sstevel@tonic-gate * provide devmap_access. 32257c478bd9Sstevel@tonic-gate */ 32267c478bd9Sstevel@tonic-gate dhp->dh_softlock = devmap_softlock_init(dev, 322744374aaeSsvemuri (ulong_t)callbackops->devmap_access); 32287c478bd9Sstevel@tonic-gate dhp->dh_ctx = devmap_ctxinit(dev, 322944374aaeSsvemuri (ulong_t)callbackops->devmap_access); 32307c478bd9Sstevel@tonic-gate 32317c478bd9Sstevel@tonic-gate /* 32327c478bd9Sstevel@tonic-gate * segdev_fault can only work when all 32337c478bd9Sstevel@tonic-gate * dh_softlock in a multi-dhp mapping 32347c478bd9Sstevel@tonic-gate * are same. see comments in segdev_fault 32357c478bd9Sstevel@tonic-gate * This code keeps track of the first 32367c478bd9Sstevel@tonic-gate * dh_softlock allocated in slock and 32377c478bd9Sstevel@tonic-gate * compares all later allocations and if 32387c478bd9Sstevel@tonic-gate * not similar, returns an error. 32397c478bd9Sstevel@tonic-gate */ 32407c478bd9Sstevel@tonic-gate if (slock == NULL) 32417c478bd9Sstevel@tonic-gate slock = dhp->dh_softlock; 32427c478bd9Sstevel@tonic-gate if (slock != dhp->dh_softlock) { 32437c478bd9Sstevel@tonic-gate free_devmap_handle(dhp_head); 32447c478bd9Sstevel@tonic-gate return (ENOTSUP); 32457c478bd9Sstevel@tonic-gate } 32467c478bd9Sstevel@tonic-gate } 32477c478bd9Sstevel@tonic-gate 32487c478bd9Sstevel@tonic-gate map_off += map_len; 32497c478bd9Sstevel@tonic-gate resid_len -= map_len; 32507c478bd9Sstevel@tonic-gate } 32517c478bd9Sstevel@tonic-gate 32527c478bd9Sstevel@tonic-gate /* 32537c478bd9Sstevel@tonic-gate * get the user virtual address and establish the mapping between 32547c478bd9Sstevel@tonic-gate * uvaddr and device physical address. 32557c478bd9Sstevel@tonic-gate */ 32567c478bd9Sstevel@tonic-gate if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags)) 325744374aaeSsvemuri != 0) { 32587c478bd9Sstevel@tonic-gate /* 32597c478bd9Sstevel@tonic-gate * free devmap handles if error during the mapping. 32607c478bd9Sstevel@tonic-gate */ 32617c478bd9Sstevel@tonic-gate free_devmap_handle(dhp_head); 32627c478bd9Sstevel@tonic-gate 32637c478bd9Sstevel@tonic-gate return (ret); 32647c478bd9Sstevel@tonic-gate } 32657c478bd9Sstevel@tonic-gate 32667c478bd9Sstevel@tonic-gate /* 32677c478bd9Sstevel@tonic-gate * call the driver's devmap_map callback to do more after the mapping, 32687c478bd9Sstevel@tonic-gate * such as to allocate driver private data for context management. 32697c478bd9Sstevel@tonic-gate */ 32707c478bd9Sstevel@tonic-gate dhp = dhp_head; 32717c478bd9Sstevel@tonic-gate map_off = off; 32727c478bd9Sstevel@tonic-gate addr = *addrp; 32737c478bd9Sstevel@tonic-gate while (dhp != NULL) { 32747c478bd9Sstevel@tonic-gate callbackops = &dhp->dh_callbackops; 32757c478bd9Sstevel@tonic-gate dhp->dh_uvaddr = addr; 32767c478bd9Sstevel@tonic-gate dhp_curr = dhp; 32777c478bd9Sstevel@tonic-gate if (callbackops->devmap_map != NULL) { 32787c478bd9Sstevel@tonic-gate ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp, 327944374aaeSsvemuri dev, flags, map_off, 328044374aaeSsvemuri dhp->dh_len, &dhp->dh_pvtp); 32817c478bd9Sstevel@tonic-gate if (ret != 0) { 32827c478bd9Sstevel@tonic-gate struct segdev_data *sdp; 32837c478bd9Sstevel@tonic-gate 32847c478bd9Sstevel@tonic-gate /* 32857c478bd9Sstevel@tonic-gate * call driver's devmap_unmap entry point 32867c478bd9Sstevel@tonic-gate * to free driver resources. 32877c478bd9Sstevel@tonic-gate */ 32887c478bd9Sstevel@tonic-gate dhp = dhp_head; 32897c478bd9Sstevel@tonic-gate map_off = off; 32907c478bd9Sstevel@tonic-gate while (dhp != dhp_curr) { 32917c478bd9Sstevel@tonic-gate callbackops = &dhp->dh_callbackops; 32927c478bd9Sstevel@tonic-gate if (callbackops->devmap_unmap != NULL) { 32937c478bd9Sstevel@tonic-gate (*callbackops->devmap_unmap)( 329444374aaeSsvemuri dhp, dhp->dh_pvtp, 329544374aaeSsvemuri map_off, dhp->dh_len, 329644374aaeSsvemuri NULL, NULL, NULL, NULL); 32977c478bd9Sstevel@tonic-gate } 32987c478bd9Sstevel@tonic-gate map_off += dhp->dh_len; 32997c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 33007c478bd9Sstevel@tonic-gate } 33017c478bd9Sstevel@tonic-gate sdp = dhp_head->dh_seg->s_data; 33027c478bd9Sstevel@tonic-gate sdp->devmap_data = NULL; 33037c478bd9Sstevel@tonic-gate free_devmap_handle(dhp_head); 33047c478bd9Sstevel@tonic-gate return (ENXIO); 33057c478bd9Sstevel@tonic-gate } 33067c478bd9Sstevel@tonic-gate } 33077c478bd9Sstevel@tonic-gate map_off += dhp->dh_len; 33087c478bd9Sstevel@tonic-gate addr += dhp->dh_len; 33097c478bd9Sstevel@tonic-gate dhp = dhp->dh_next; 33107c478bd9Sstevel@tonic-gate } 33117c478bd9Sstevel@tonic-gate 33127c478bd9Sstevel@tonic-gate return (0); 33137c478bd9Sstevel@tonic-gate } 33147c478bd9Sstevel@tonic-gate 33157c478bd9Sstevel@tonic-gate int 33167c478bd9Sstevel@tonic-gate ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp, 33177c478bd9Sstevel@tonic-gate off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred) 33187c478bd9Sstevel@tonic-gate { 33197c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP, 33207c478bd9Sstevel@tonic-gate "devmap_segmap:start"); 33217c478bd9Sstevel@tonic-gate return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp, 33227c478bd9Sstevel@tonic-gate (size_t)len, prot, maxprot, flags, cred)); 33237c478bd9Sstevel@tonic-gate } 33247c478bd9Sstevel@tonic-gate 33257c478bd9Sstevel@tonic-gate /* 33267c478bd9Sstevel@tonic-gate * Called from devmap_devmem_setup/remap to see if can use large pages for 33277c478bd9Sstevel@tonic-gate * this device mapping. 33287c478bd9Sstevel@tonic-gate * Also calculate the max. page size for this mapping. 33297c478bd9Sstevel@tonic-gate * this page size will be used in fault routine for 33307c478bd9Sstevel@tonic-gate * optimal page size calculations. 33317c478bd9Sstevel@tonic-gate */ 33327c478bd9Sstevel@tonic-gate static void 33337c478bd9Sstevel@tonic-gate devmap_devmem_large_page_setup(devmap_handle_t *dhp) 33347c478bd9Sstevel@tonic-gate { 33357c478bd9Sstevel@tonic-gate ASSERT(dhp_is_devmem(dhp)); 33367c478bd9Sstevel@tonic-gate dhp->dh_mmulevel = 0; 33377c478bd9Sstevel@tonic-gate 33387c478bd9Sstevel@tonic-gate /* 33397c478bd9Sstevel@tonic-gate * use large page size only if: 33407c478bd9Sstevel@tonic-gate * 1. device memory. 33417c478bd9Sstevel@tonic-gate * 2. mmu supports multiple page sizes, 33427c478bd9Sstevel@tonic-gate * 3. Driver did not disallow it 33437c478bd9Sstevel@tonic-gate * 4. dhp length is at least as big as the large pagesize 33447c478bd9Sstevel@tonic-gate * 5. the uvaddr and pfn are large pagesize aligned 33457c478bd9Sstevel@tonic-gate */ 33467c478bd9Sstevel@tonic-gate if (page_num_pagesizes() > 1 && 33477c478bd9Sstevel@tonic-gate !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) { 33487c478bd9Sstevel@tonic-gate ulong_t base; 33497c478bd9Sstevel@tonic-gate int level; 33507c478bd9Sstevel@tonic-gate 33517c478bd9Sstevel@tonic-gate base = (ulong_t)ptob(dhp->dh_pfn); 33527c478bd9Sstevel@tonic-gate for (level = 1; level < page_num_pagesizes(); level++) { 33537c478bd9Sstevel@tonic-gate size_t pgsize = page_get_pagesize(level); 33547c478bd9Sstevel@tonic-gate if ((dhp->dh_len < pgsize) || 33557c478bd9Sstevel@tonic-gate (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr, 335644374aaeSsvemuri base, pgsize))) { 33577c478bd9Sstevel@tonic-gate break; 33587c478bd9Sstevel@tonic-gate } 33597c478bd9Sstevel@tonic-gate } 33607c478bd9Sstevel@tonic-gate dhp->dh_mmulevel = level - 1; 33617c478bd9Sstevel@tonic-gate } 33627c478bd9Sstevel@tonic-gate if (dhp->dh_mmulevel > 0) { 33637c478bd9Sstevel@tonic-gate dhp->dh_flags |= DEVMAP_FLAG_LARGE; 33647c478bd9Sstevel@tonic-gate } else { 33657c478bd9Sstevel@tonic-gate dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; 33667c478bd9Sstevel@tonic-gate } 33677c478bd9Sstevel@tonic-gate } 33687c478bd9Sstevel@tonic-gate 33697c478bd9Sstevel@tonic-gate /* 33707c478bd9Sstevel@tonic-gate * Called by driver devmap routine to pass device specific info to 33717c478bd9Sstevel@tonic-gate * the framework. used for device memory mapping only. 33727c478bd9Sstevel@tonic-gate */ 33737c478bd9Sstevel@tonic-gate int 33747c478bd9Sstevel@tonic-gate devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip, 33757c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff, 33767c478bd9Sstevel@tonic-gate size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp) 33777c478bd9Sstevel@tonic-gate { 33787c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)dhc; 33797c478bd9Sstevel@tonic-gate ddi_acc_handle_t handle; 33807c478bd9Sstevel@tonic-gate ddi_map_req_t mr; 33817c478bd9Sstevel@tonic-gate ddi_acc_hdl_t *hp; 33827c478bd9Sstevel@tonic-gate int err; 33837c478bd9Sstevel@tonic-gate 33847c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP, 33857c478bd9Sstevel@tonic-gate "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx", 33867c478bd9Sstevel@tonic-gate (void *)dhp, roff, rnumber, (uint_t)len); 33877c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx " 33887c478bd9Sstevel@tonic-gate "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); 33897c478bd9Sstevel@tonic-gate 33907c478bd9Sstevel@tonic-gate /* 33917c478bd9Sstevel@tonic-gate * First to check if this function has been called for this dhp. 33927c478bd9Sstevel@tonic-gate */ 33937c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_SETUP_DONE) 33947c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 33957c478bd9Sstevel@tonic-gate 33967c478bd9Sstevel@tonic-gate if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) 33977c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 33987c478bd9Sstevel@tonic-gate 33997c478bd9Sstevel@tonic-gate if (flags & DEVMAP_MAPPING_INVALID) { 34007c478bd9Sstevel@tonic-gate /* 34017c478bd9Sstevel@tonic-gate * Don't go up the tree to get pfn if the driver specifies 34027c478bd9Sstevel@tonic-gate * DEVMAP_MAPPING_INVALID in flags. 34037c478bd9Sstevel@tonic-gate * 34047c478bd9Sstevel@tonic-gate * If DEVMAP_MAPPING_INVALID is specified, we have to grant 34057c478bd9Sstevel@tonic-gate * remap permission. 34067c478bd9Sstevel@tonic-gate */ 34077c478bd9Sstevel@tonic-gate if (!(flags & DEVMAP_ALLOW_REMAP)) { 34087c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 34097c478bd9Sstevel@tonic-gate } 34107c478bd9Sstevel@tonic-gate dhp->dh_pfn = PFN_INVALID; 34117c478bd9Sstevel@tonic-gate } else { 34127c478bd9Sstevel@tonic-gate handle = impl_acc_hdl_alloc(KM_SLEEP, NULL); 34137c478bd9Sstevel@tonic-gate if (handle == NULL) 34147c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 34157c478bd9Sstevel@tonic-gate 34167c478bd9Sstevel@tonic-gate hp = impl_acc_hdl_get(handle); 34177c478bd9Sstevel@tonic-gate hp->ah_vers = VERS_ACCHDL; 34187c478bd9Sstevel@tonic-gate hp->ah_dip = dip; 34197c478bd9Sstevel@tonic-gate hp->ah_rnumber = rnumber; 34207c478bd9Sstevel@tonic-gate hp->ah_offset = roff; 34217c478bd9Sstevel@tonic-gate hp->ah_len = len; 34227c478bd9Sstevel@tonic-gate if (accattrp != NULL) 34237c478bd9Sstevel@tonic-gate hp->ah_acc = *accattrp; 34247c478bd9Sstevel@tonic-gate 34257c478bd9Sstevel@tonic-gate mr.map_op = DDI_MO_MAP_LOCKED; 34267c478bd9Sstevel@tonic-gate mr.map_type = DDI_MT_RNUMBER; 34277c478bd9Sstevel@tonic-gate mr.map_obj.rnumber = rnumber; 34287c478bd9Sstevel@tonic-gate mr.map_prot = maxprot & dhp->dh_orig_maxprot; 34297c478bd9Sstevel@tonic-gate mr.map_flags = DDI_MF_DEVICE_MAPPING; 34307c478bd9Sstevel@tonic-gate mr.map_handlep = hp; 34317c478bd9Sstevel@tonic-gate mr.map_vers = DDI_MAP_VERSION; 34327c478bd9Sstevel@tonic-gate 34337c478bd9Sstevel@tonic-gate /* 34347c478bd9Sstevel@tonic-gate * up the device tree to get pfn. 34357c478bd9Sstevel@tonic-gate * The rootnex_map_regspec() routine in nexus drivers has been 34367c478bd9Sstevel@tonic-gate * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING. 34377c478bd9Sstevel@tonic-gate */ 34387c478bd9Sstevel@tonic-gate err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn); 34397c478bd9Sstevel@tonic-gate dhp->dh_hat_attr = hp->ah_hat_flags; 34407c478bd9Sstevel@tonic-gate impl_acc_hdl_free(handle); 34417c478bd9Sstevel@tonic-gate 34427c478bd9Sstevel@tonic-gate if (err) 34437c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 34447c478bd9Sstevel@tonic-gate } 34457c478bd9Sstevel@tonic-gate /* Should not be using devmem setup for memory pages */ 34467c478bd9Sstevel@tonic-gate ASSERT(!pf_is_memory(dhp->dh_pfn)); 34477c478bd9Sstevel@tonic-gate 34487c478bd9Sstevel@tonic-gate /* Only some of the flags bits are settable by the driver */ 34497c478bd9Sstevel@tonic-gate dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); 34507c478bd9Sstevel@tonic-gate dhp->dh_len = ptob(btopr(len)); 34517c478bd9Sstevel@tonic-gate 34527c478bd9Sstevel@tonic-gate dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; 34537c478bd9Sstevel@tonic-gate dhp->dh_roff = ptob(btop(roff)); 34547c478bd9Sstevel@tonic-gate 34557c478bd9Sstevel@tonic-gate /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */ 34567c478bd9Sstevel@tonic-gate devmap_devmem_large_page_setup(dhp); 34577c478bd9Sstevel@tonic-gate dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; 34587c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); 34597c478bd9Sstevel@tonic-gate 34607c478bd9Sstevel@tonic-gate 34617c478bd9Sstevel@tonic-gate if (callbackops != NULL) { 34627c478bd9Sstevel@tonic-gate bcopy(callbackops, &dhp->dh_callbackops, 34637c478bd9Sstevel@tonic-gate sizeof (struct devmap_callback_ctl)); 34647c478bd9Sstevel@tonic-gate } 34657c478bd9Sstevel@tonic-gate 34667c478bd9Sstevel@tonic-gate /* 34677c478bd9Sstevel@tonic-gate * Initialize dh_lock if we want to do remap. 34687c478bd9Sstevel@tonic-gate */ 34697c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { 34707c478bd9Sstevel@tonic-gate mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); 34717c478bd9Sstevel@tonic-gate dhp->dh_flags |= DEVMAP_LOCK_INITED; 34727c478bd9Sstevel@tonic-gate } 34737c478bd9Sstevel@tonic-gate 34747c478bd9Sstevel@tonic-gate dhp->dh_flags |= DEVMAP_SETUP_DONE; 34757c478bd9Sstevel@tonic-gate 34767c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 34777c478bd9Sstevel@tonic-gate } 34787c478bd9Sstevel@tonic-gate 34797c478bd9Sstevel@tonic-gate int 34807c478bd9Sstevel@tonic-gate devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip, 34817c478bd9Sstevel@tonic-gate uint_t rnumber, offset_t roff, size_t len, uint_t maxprot, 34827c478bd9Sstevel@tonic-gate uint_t flags, ddi_device_acc_attr_t *accattrp) 34837c478bd9Sstevel@tonic-gate { 34847c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)dhc; 34857c478bd9Sstevel@tonic-gate ddi_acc_handle_t handle; 34867c478bd9Sstevel@tonic-gate ddi_map_req_t mr; 34877c478bd9Sstevel@tonic-gate ddi_acc_hdl_t *hp; 34887c478bd9Sstevel@tonic-gate pfn_t pfn; 34897c478bd9Sstevel@tonic-gate uint_t hat_flags; 34907c478bd9Sstevel@tonic-gate int err; 34917c478bd9Sstevel@tonic-gate 34927c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP, 34937c478bd9Sstevel@tonic-gate "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx", 34947c478bd9Sstevel@tonic-gate (void *)dhp, roff, rnumber, (uint_t)len); 34957c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx " 34967c478bd9Sstevel@tonic-gate "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len)); 34977c478bd9Sstevel@tonic-gate 34987c478bd9Sstevel@tonic-gate /* 34997c478bd9Sstevel@tonic-gate * Return failure if setup has not been done or no remap permission 35007c478bd9Sstevel@tonic-gate * has been granted during the setup. 35017c478bd9Sstevel@tonic-gate */ 35027c478bd9Sstevel@tonic-gate if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || 35037c478bd9Sstevel@tonic-gate (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) 35047c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 35057c478bd9Sstevel@tonic-gate 35067c478bd9Sstevel@tonic-gate /* Only DEVMAP_MAPPING_INVALID flag supported for remap */ 35077c478bd9Sstevel@tonic-gate if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID)) 35087c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 35097c478bd9Sstevel@tonic-gate 35107c478bd9Sstevel@tonic-gate if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) 35117c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 35127c478bd9Sstevel@tonic-gate 35137c478bd9Sstevel@tonic-gate if (!(flags & DEVMAP_MAPPING_INVALID)) { 35147c478bd9Sstevel@tonic-gate handle = impl_acc_hdl_alloc(KM_SLEEP, NULL); 35157c478bd9Sstevel@tonic-gate if (handle == NULL) 35167c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 35177c478bd9Sstevel@tonic-gate } 35187c478bd9Sstevel@tonic-gate 35197c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 35207c478bd9Sstevel@tonic-gate 35217c478bd9Sstevel@tonic-gate /* 35227c478bd9Sstevel@tonic-gate * Unload the old mapping, so next fault will setup the new mappings 35237c478bd9Sstevel@tonic-gate * Do this while holding the dhp lock so other faults dont reestablish 35247c478bd9Sstevel@tonic-gate * the mappings 35257c478bd9Sstevel@tonic-gate */ 35267c478bd9Sstevel@tonic-gate hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, 352744374aaeSsvemuri dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); 35287c478bd9Sstevel@tonic-gate 35297c478bd9Sstevel@tonic-gate if (flags & DEVMAP_MAPPING_INVALID) { 35307c478bd9Sstevel@tonic-gate dhp->dh_flags |= DEVMAP_MAPPING_INVALID; 35317c478bd9Sstevel@tonic-gate dhp->dh_pfn = PFN_INVALID; 35327c478bd9Sstevel@tonic-gate } else { 35337c478bd9Sstevel@tonic-gate /* clear any prior DEVMAP_MAPPING_INVALID flag */ 35347c478bd9Sstevel@tonic-gate dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID; 35357c478bd9Sstevel@tonic-gate hp = impl_acc_hdl_get(handle); 35367c478bd9Sstevel@tonic-gate hp->ah_vers = VERS_ACCHDL; 35377c478bd9Sstevel@tonic-gate hp->ah_dip = dip; 35387c478bd9Sstevel@tonic-gate hp->ah_rnumber = rnumber; 35397c478bd9Sstevel@tonic-gate hp->ah_offset = roff; 35407c478bd9Sstevel@tonic-gate hp->ah_len = len; 35417c478bd9Sstevel@tonic-gate if (accattrp != NULL) 35427c478bd9Sstevel@tonic-gate hp->ah_acc = *accattrp; 35437c478bd9Sstevel@tonic-gate 35447c478bd9Sstevel@tonic-gate mr.map_op = DDI_MO_MAP_LOCKED; 35457c478bd9Sstevel@tonic-gate mr.map_type = DDI_MT_RNUMBER; 35467c478bd9Sstevel@tonic-gate mr.map_obj.rnumber = rnumber; 35477c478bd9Sstevel@tonic-gate mr.map_prot = maxprot & dhp->dh_orig_maxprot; 35487c478bd9Sstevel@tonic-gate mr.map_flags = DDI_MF_DEVICE_MAPPING; 35497c478bd9Sstevel@tonic-gate mr.map_handlep = hp; 35507c478bd9Sstevel@tonic-gate mr.map_vers = DDI_MAP_VERSION; 35517c478bd9Sstevel@tonic-gate 35527c478bd9Sstevel@tonic-gate /* 35537c478bd9Sstevel@tonic-gate * up the device tree to get pfn. 35547c478bd9Sstevel@tonic-gate * The rootnex_map_regspec() routine in nexus drivers has been 35557c478bd9Sstevel@tonic-gate * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING. 35567c478bd9Sstevel@tonic-gate */ 35577c478bd9Sstevel@tonic-gate err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn); 35587c478bd9Sstevel@tonic-gate hat_flags = hp->ah_hat_flags; 35597c478bd9Sstevel@tonic-gate impl_acc_hdl_free(handle); 35607c478bd9Sstevel@tonic-gate if (err) { 35617c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 35627c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 35637c478bd9Sstevel@tonic-gate } 35647c478bd9Sstevel@tonic-gate /* 35657c478bd9Sstevel@tonic-gate * Store result of ddi_map first in local variables, as we do 35667c478bd9Sstevel@tonic-gate * not want to overwrite the existing dhp with wrong data. 35677c478bd9Sstevel@tonic-gate */ 35687c478bd9Sstevel@tonic-gate dhp->dh_pfn = pfn; 35697c478bd9Sstevel@tonic-gate dhp->dh_hat_attr = hat_flags; 35707c478bd9Sstevel@tonic-gate } 35717c478bd9Sstevel@tonic-gate 35727c478bd9Sstevel@tonic-gate /* clear the large page size flag */ 35737c478bd9Sstevel@tonic-gate dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; 35747c478bd9Sstevel@tonic-gate 35757c478bd9Sstevel@tonic-gate dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE; 35767c478bd9Sstevel@tonic-gate dhp->dh_roff = ptob(btop(roff)); 35777c478bd9Sstevel@tonic-gate 35787c478bd9Sstevel@tonic-gate /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */ 35797c478bd9Sstevel@tonic-gate devmap_devmem_large_page_setup(dhp); 35807c478bd9Sstevel@tonic-gate dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; 35817c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); 35827c478bd9Sstevel@tonic-gate 35837c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 35847c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 35857c478bd9Sstevel@tonic-gate } 35867c478bd9Sstevel@tonic-gate 35877c478bd9Sstevel@tonic-gate /* 35887c478bd9Sstevel@tonic-gate * called by driver devmap routine to pass kernel virtual address mapping 35897c478bd9Sstevel@tonic-gate * info to the framework. used only for kernel memory 35907c478bd9Sstevel@tonic-gate * allocated from ddi_umem_alloc(). 35917c478bd9Sstevel@tonic-gate */ 35927c478bd9Sstevel@tonic-gate int 35937c478bd9Sstevel@tonic-gate devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip, 35947c478bd9Sstevel@tonic-gate struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie, 35957c478bd9Sstevel@tonic-gate offset_t off, size_t len, uint_t maxprot, uint_t flags, 35967c478bd9Sstevel@tonic-gate ddi_device_acc_attr_t *accattrp) 35977c478bd9Sstevel@tonic-gate { 35987c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)dhc; 35997c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie; 36007c478bd9Sstevel@tonic-gate 36017c478bd9Sstevel@tonic-gate #ifdef lint 36027c478bd9Sstevel@tonic-gate dip = dip; 36037c478bd9Sstevel@tonic-gate #endif 36047c478bd9Sstevel@tonic-gate 36057c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP, 36067c478bd9Sstevel@tonic-gate "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx", 36077c478bd9Sstevel@tonic-gate (void *)dhp, off, cookie, len); 36087c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx " 36097c478bd9Sstevel@tonic-gate "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); 36107c478bd9Sstevel@tonic-gate 36117c478bd9Sstevel@tonic-gate if (cookie == NULL) 36127c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 36137c478bd9Sstevel@tonic-gate 36147c478bd9Sstevel@tonic-gate /* For UMEM_TRASH, this restriction is not needed */ 36157c478bd9Sstevel@tonic-gate if ((off + len) > cp->size) 36167c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 36177c478bd9Sstevel@tonic-gate 361883220025Seota /* check if the cache attributes are supported */ 361983220025Seota if (i_ddi_check_cache_attr(flags) == B_FALSE) 36207b93957cSeota return (DDI_FAILURE); 36217b93957cSeota 36227c478bd9Sstevel@tonic-gate /* 36237c478bd9Sstevel@tonic-gate * First to check if this function has been called for this dhp. 36247c478bd9Sstevel@tonic-gate */ 36257c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_SETUP_DONE) 36267c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 36277c478bd9Sstevel@tonic-gate 36287c478bd9Sstevel@tonic-gate if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) 36297c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 36307c478bd9Sstevel@tonic-gate 36317c478bd9Sstevel@tonic-gate if (flags & DEVMAP_MAPPING_INVALID) { 36327c478bd9Sstevel@tonic-gate /* 36337c478bd9Sstevel@tonic-gate * If DEVMAP_MAPPING_INVALID is specified, we have to grant 36347c478bd9Sstevel@tonic-gate * remap permission. 36357c478bd9Sstevel@tonic-gate */ 36367c478bd9Sstevel@tonic-gate if (!(flags & DEVMAP_ALLOW_REMAP)) { 36377c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 36387c478bd9Sstevel@tonic-gate } 36397c478bd9Sstevel@tonic-gate } else { 36407c478bd9Sstevel@tonic-gate dhp->dh_cookie = cookie; 36417c478bd9Sstevel@tonic-gate dhp->dh_roff = ptob(btop(off)); 36427c478bd9Sstevel@tonic-gate dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; 36437b93957cSeota /* set HAT cache attributes */ 36447b93957cSeota i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); 36457b93957cSeota /* set HAT endianess attributes */ 36467b93957cSeota i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); 36477c478bd9Sstevel@tonic-gate } 36487c478bd9Sstevel@tonic-gate 36497c478bd9Sstevel@tonic-gate /* 36507c478bd9Sstevel@tonic-gate * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload(); 36517c478bd9Sstevel@tonic-gate * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to 36527c478bd9Sstevel@tonic-gate * create consistent mappings but our intention was to create 36537c478bd9Sstevel@tonic-gate * non-consistent mappings. 36547c478bd9Sstevel@tonic-gate * 36557c478bd9Sstevel@tonic-gate * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent 36567c478bd9Sstevel@tonic-gate * mappings. 36577c478bd9Sstevel@tonic-gate * 36587c478bd9Sstevel@tonic-gate * kernel exported memory: hat figures it out it's memory and always 36597c478bd9Sstevel@tonic-gate * creates consistent mappings. 36607c478bd9Sstevel@tonic-gate * 36617c478bd9Sstevel@tonic-gate * /dev/mem: non-consistent mappings. See comments in common/io/mem.c 36627c478bd9Sstevel@tonic-gate * 36637c478bd9Sstevel@tonic-gate * /dev/kmem: consistent mappings are created unless they are 36647c478bd9Sstevel@tonic-gate * MAP_FIXED. We _explicitly_ tell hat to create non-consistent 36657c478bd9Sstevel@tonic-gate * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED 36667c478bd9Sstevel@tonic-gate * mappings of /dev/kmem. See common/io/mem.c 36677c478bd9Sstevel@tonic-gate */ 36687c478bd9Sstevel@tonic-gate 36697c478bd9Sstevel@tonic-gate /* Only some of the flags bits are settable by the driver */ 36707c478bd9Sstevel@tonic-gate dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); 36717c478bd9Sstevel@tonic-gate 36727c478bd9Sstevel@tonic-gate dhp->dh_len = ptob(btopr(len)); 36737c478bd9Sstevel@tonic-gate dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; 36747c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); 36757c478bd9Sstevel@tonic-gate 36767c478bd9Sstevel@tonic-gate if (callbackops != NULL) { 36777c478bd9Sstevel@tonic-gate bcopy(callbackops, &dhp->dh_callbackops, 36787c478bd9Sstevel@tonic-gate sizeof (struct devmap_callback_ctl)); 36797c478bd9Sstevel@tonic-gate } 36807c478bd9Sstevel@tonic-gate /* 36817c478bd9Sstevel@tonic-gate * Initialize dh_lock if we want to do remap. 36827c478bd9Sstevel@tonic-gate */ 36837c478bd9Sstevel@tonic-gate if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { 36847c478bd9Sstevel@tonic-gate mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); 36857c478bd9Sstevel@tonic-gate dhp->dh_flags |= DEVMAP_LOCK_INITED; 36867c478bd9Sstevel@tonic-gate } 36877c478bd9Sstevel@tonic-gate 36887c478bd9Sstevel@tonic-gate dhp->dh_flags |= DEVMAP_SETUP_DONE; 36897c478bd9Sstevel@tonic-gate 36907c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 36917c478bd9Sstevel@tonic-gate } 36927c478bd9Sstevel@tonic-gate 36937c478bd9Sstevel@tonic-gate int 36947c478bd9Sstevel@tonic-gate devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip, 36957c478bd9Sstevel@tonic-gate ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot, 36967c478bd9Sstevel@tonic-gate uint_t flags, ddi_device_acc_attr_t *accattrp) 36977c478bd9Sstevel@tonic-gate { 36987c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)dhc; 36997c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie; 37007c478bd9Sstevel@tonic-gate 37017c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP, 37027c478bd9Sstevel@tonic-gate "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx", 37037c478bd9Sstevel@tonic-gate (void *)dhp, off, cookie, len); 37047c478bd9Sstevel@tonic-gate DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx " 37057c478bd9Sstevel@tonic-gate "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len)); 37067c478bd9Sstevel@tonic-gate 37077c478bd9Sstevel@tonic-gate #ifdef lint 37087c478bd9Sstevel@tonic-gate dip = dip; 37097c478bd9Sstevel@tonic-gate accattrp = accattrp; 37107c478bd9Sstevel@tonic-gate #endif 37117c478bd9Sstevel@tonic-gate /* 37127c478bd9Sstevel@tonic-gate * Reture failure if setup has not been done or no remap permission 37137c478bd9Sstevel@tonic-gate * has been granted during the setup. 37147c478bd9Sstevel@tonic-gate */ 37157c478bd9Sstevel@tonic-gate if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || 371644374aaeSsvemuri (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) 37177c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 37187c478bd9Sstevel@tonic-gate 37197c478bd9Sstevel@tonic-gate /* No flags supported for remap yet */ 37207c478bd9Sstevel@tonic-gate if (flags != 0) 37217c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 37227c478bd9Sstevel@tonic-gate 372383220025Seota /* check if the cache attributes are supported */ 372483220025Seota if (i_ddi_check_cache_attr(flags) == B_FALSE) 37257b93957cSeota return (DDI_FAILURE); 37267b93957cSeota 37277c478bd9Sstevel@tonic-gate if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) 37287c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 37297c478bd9Sstevel@tonic-gate 37307c478bd9Sstevel@tonic-gate /* For UMEM_TRASH, this restriction is not needed */ 37317c478bd9Sstevel@tonic-gate if ((off + len) > cp->size) 37327c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 37337c478bd9Sstevel@tonic-gate 37347c478bd9Sstevel@tonic-gate HOLD_DHP_LOCK(dhp); 37357c478bd9Sstevel@tonic-gate /* 37367c478bd9Sstevel@tonic-gate * Unload the old mapping, so next fault will setup the new mappings 37377c478bd9Sstevel@tonic-gate * Do this while holding the dhp lock so other faults dont reestablish 37387c478bd9Sstevel@tonic-gate * the mappings 37397c478bd9Sstevel@tonic-gate */ 37407c478bd9Sstevel@tonic-gate hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, 374144374aaeSsvemuri dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); 37427c478bd9Sstevel@tonic-gate 37437c478bd9Sstevel@tonic-gate dhp->dh_cookie = cookie; 37447c478bd9Sstevel@tonic-gate dhp->dh_roff = ptob(btop(off)); 37457c478bd9Sstevel@tonic-gate dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff; 37467b93957cSeota /* set HAT cache attributes */ 37477b93957cSeota i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr); 37487b93957cSeota /* set HAT endianess attributes */ 37497b93957cSeota i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr); 37507c478bd9Sstevel@tonic-gate 37517c478bd9Sstevel@tonic-gate /* clear the large page size flag */ 37527c478bd9Sstevel@tonic-gate dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; 37537c478bd9Sstevel@tonic-gate 37547c478bd9Sstevel@tonic-gate dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; 37557c478bd9Sstevel@tonic-gate ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); 37567c478bd9Sstevel@tonic-gate RELE_DHP_LOCK(dhp); 37577c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 37587c478bd9Sstevel@tonic-gate } 37597c478bd9Sstevel@tonic-gate 37607c478bd9Sstevel@tonic-gate /* 37617c478bd9Sstevel@tonic-gate * to set timeout value for the driver's context management callback, e.g. 37627c478bd9Sstevel@tonic-gate * devmap_access(). 37637c478bd9Sstevel@tonic-gate */ 37647c478bd9Sstevel@tonic-gate void 37657c478bd9Sstevel@tonic-gate devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks) 37667c478bd9Sstevel@tonic-gate { 37677c478bd9Sstevel@tonic-gate devmap_handle_t *dhp = (devmap_handle_t *)dhc; 37687c478bd9Sstevel@tonic-gate 37697c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT, 37707c478bd9Sstevel@tonic-gate "devmap_set_ctx_timeout:start dhp=%p ticks=%x", 37717c478bd9Sstevel@tonic-gate (void *)dhp, ticks); 37727c478bd9Sstevel@tonic-gate dhp->dh_timeout_length = ticks; 37737c478bd9Sstevel@tonic-gate } 37747c478bd9Sstevel@tonic-gate 37757c478bd9Sstevel@tonic-gate int 37767c478bd9Sstevel@tonic-gate devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off, 37777c478bd9Sstevel@tonic-gate size_t len, uint_t type, uint_t rw) 37787c478bd9Sstevel@tonic-gate { 37797c478bd9Sstevel@tonic-gate #ifdef lint 37807c478bd9Sstevel@tonic-gate pvtp = pvtp; 37817c478bd9Sstevel@tonic-gate #endif 37827c478bd9Sstevel@tonic-gate 37837c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS, 37847c478bd9Sstevel@tonic-gate "devmap_default_access:start"); 37857c478bd9Sstevel@tonic-gate return (devmap_load(dhp, off, len, type, rw)); 37867c478bd9Sstevel@tonic-gate } 37877c478bd9Sstevel@tonic-gate 37887c478bd9Sstevel@tonic-gate /* 37897c478bd9Sstevel@tonic-gate * segkmem_alloc() wrapper to allocate memory which is both 37907c478bd9Sstevel@tonic-gate * non-relocatable (for DR) and sharelocked, since the rest 37917c478bd9Sstevel@tonic-gate * of this segment driver requires it. 37927c478bd9Sstevel@tonic-gate */ 37937c478bd9Sstevel@tonic-gate static void * 37947c478bd9Sstevel@tonic-gate devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag) 37957c478bd9Sstevel@tonic-gate { 37967c478bd9Sstevel@tonic-gate ASSERT(vmp != NULL); 37977c478bd9Sstevel@tonic-gate ASSERT(kvseg.s_base != NULL); 37987c478bd9Sstevel@tonic-gate vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED); 37997c478bd9Sstevel@tonic-gate return (segkmem_alloc(vmp, size, vmflag)); 38007c478bd9Sstevel@tonic-gate } 38017c478bd9Sstevel@tonic-gate 38027c478bd9Sstevel@tonic-gate /* 3803da6c28aaSamw * This is where things are a bit incestuous with seg_kmem: unlike 38047c478bd9Sstevel@tonic-gate * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so 38057c478bd9Sstevel@tonic-gate * we need to do a bit of a dance around that to prevent duplication of 38067c478bd9Sstevel@tonic-gate * code until we decide to bite the bullet and implement a new kernel 38077c478bd9Sstevel@tonic-gate * segment for driver-allocated memory that is exported to user space. 38087c478bd9Sstevel@tonic-gate */ 38097c478bd9Sstevel@tonic-gate static void 38107c478bd9Sstevel@tonic-gate devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size) 38117c478bd9Sstevel@tonic-gate { 38127c478bd9Sstevel@tonic-gate page_t *pp; 38137c478bd9Sstevel@tonic-gate caddr_t addr = inaddr; 38147c478bd9Sstevel@tonic-gate caddr_t eaddr; 38157c478bd9Sstevel@tonic-gate pgcnt_t npages = btopr(size); 38167c478bd9Sstevel@tonic-gate 38177c478bd9Sstevel@tonic-gate ASSERT(vmp != NULL); 38187c478bd9Sstevel@tonic-gate ASSERT(kvseg.s_base != NULL); 38197c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0); 38207c478bd9Sstevel@tonic-gate 38217c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK); 38227c478bd9Sstevel@tonic-gate 38237c478bd9Sstevel@tonic-gate for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) { 38247c478bd9Sstevel@tonic-gate /* 38257c478bd9Sstevel@tonic-gate * Use page_find() instead of page_lookup() to find the page 38267c478bd9Sstevel@tonic-gate * since we know that it is hashed and has a shared lock. 38277c478bd9Sstevel@tonic-gate */ 38287c478bd9Sstevel@tonic-gate pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr); 38297c478bd9Sstevel@tonic-gate 38307c478bd9Sstevel@tonic-gate if (pp == NULL) 38317c478bd9Sstevel@tonic-gate panic("devmap_free_pages: page not found"); 38327c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 38337c478bd9Sstevel@tonic-gate page_unlock(pp); 38347c478bd9Sstevel@tonic-gate pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, 38357c478bd9Sstevel@tonic-gate SE_EXCL); 38367c478bd9Sstevel@tonic-gate if (pp == NULL) 38377c478bd9Sstevel@tonic-gate panic("devmap_free_pages: page already freed"); 38387c478bd9Sstevel@tonic-gate } 38397c478bd9Sstevel@tonic-gate /* Clear p_lckcnt so page_destroy() doesn't update availrmem */ 38407c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 38417c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 38427c478bd9Sstevel@tonic-gate } 38437c478bd9Sstevel@tonic-gate page_unresv(npages); 38447c478bd9Sstevel@tonic-gate 38457c478bd9Sstevel@tonic-gate if (vmp != NULL) 38467c478bd9Sstevel@tonic-gate vmem_free(vmp, inaddr, size); 38477c478bd9Sstevel@tonic-gate } 38487c478bd9Sstevel@tonic-gate 38497c478bd9Sstevel@tonic-gate /* 38507c478bd9Sstevel@tonic-gate * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for 38517c478bd9Sstevel@tonic-gate * allocating non-pageable kmem in response to a ddi_umem_alloc() 38527c478bd9Sstevel@tonic-gate * default request. For now we allocate our own pages and we keep 38537c478bd9Sstevel@tonic-gate * them long-term sharelocked, since: A) the fault routines expect the 38547c478bd9Sstevel@tonic-gate * memory to already be locked; B) pageable umem is already long-term 3855da6c28aaSamw * locked; C) it's a lot of work to make it otherwise, particularly 38567c478bd9Sstevel@tonic-gate * since the nexus layer expects the pages to never fault. An RFE is to 38577c478bd9Sstevel@tonic-gate * not keep the pages long-term locked, but instead to be able to 38587c478bd9Sstevel@tonic-gate * take faults on them and simply look them up in kvp in case we 38597c478bd9Sstevel@tonic-gate * fault on them. Even then, we must take care not to let pageout 38607c478bd9Sstevel@tonic-gate * steal them from us since the data must remain resident; if we 38617c478bd9Sstevel@tonic-gate * do this we must come up with some way to pin the pages to prevent 38627c478bd9Sstevel@tonic-gate * faults while a driver is doing DMA to/from them. 38637c478bd9Sstevel@tonic-gate */ 38647c478bd9Sstevel@tonic-gate static void * 38657c478bd9Sstevel@tonic-gate devmap_umem_alloc_np(size_t size, size_t flags) 38667c478bd9Sstevel@tonic-gate { 38677c478bd9Sstevel@tonic-gate void *buf; 38687c478bd9Sstevel@tonic-gate int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP; 38697c478bd9Sstevel@tonic-gate 38707c478bd9Sstevel@tonic-gate buf = vmem_alloc(umem_np_arena, size, vmflags); 38717c478bd9Sstevel@tonic-gate if (buf != NULL) 38727c478bd9Sstevel@tonic-gate bzero(buf, size); 38737c478bd9Sstevel@tonic-gate return (buf); 38747c478bd9Sstevel@tonic-gate } 38757c478bd9Sstevel@tonic-gate 38767c478bd9Sstevel@tonic-gate static void 38777c478bd9Sstevel@tonic-gate devmap_umem_free_np(void *addr, size_t size) 38787c478bd9Sstevel@tonic-gate { 38797c478bd9Sstevel@tonic-gate vmem_free(umem_np_arena, addr, size); 38807c478bd9Sstevel@tonic-gate } 38817c478bd9Sstevel@tonic-gate 38827c478bd9Sstevel@tonic-gate /* 38837c478bd9Sstevel@tonic-gate * allocate page aligned kernel memory for exporting to user land. 38847c478bd9Sstevel@tonic-gate * The devmap framework will use the cookie allocated by ddi_umem_alloc() 38857c478bd9Sstevel@tonic-gate * to find a user virtual address that is in same color as the address 38867c478bd9Sstevel@tonic-gate * allocated here. 38877c478bd9Sstevel@tonic-gate */ 38887c478bd9Sstevel@tonic-gate void * 38897c478bd9Sstevel@tonic-gate ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie) 38907c478bd9Sstevel@tonic-gate { 38917c478bd9Sstevel@tonic-gate register size_t len = ptob(btopr(size)); 38927c478bd9Sstevel@tonic-gate void *buf = NULL; 38937c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *cp; 38947c478bd9Sstevel@tonic-gate int iflags = 0; 38957c478bd9Sstevel@tonic-gate 38967c478bd9Sstevel@tonic-gate *cookie = NULL; 38977c478bd9Sstevel@tonic-gate 38987c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC, 38997c478bd9Sstevel@tonic-gate "devmap_umem_alloc:start"); 39007c478bd9Sstevel@tonic-gate if (len == 0) 39017c478bd9Sstevel@tonic-gate return ((void *)NULL); 39027c478bd9Sstevel@tonic-gate 39037c478bd9Sstevel@tonic-gate /* 39047c478bd9Sstevel@tonic-gate * allocate cookie 39057c478bd9Sstevel@tonic-gate */ 39067c478bd9Sstevel@tonic-gate if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie), 390744374aaeSsvemuri flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) { 39087c478bd9Sstevel@tonic-gate ASSERT(flags & DDI_UMEM_NOSLEEP); 39097c478bd9Sstevel@tonic-gate return ((void *)NULL); 39107c478bd9Sstevel@tonic-gate } 39117c478bd9Sstevel@tonic-gate 39127c478bd9Sstevel@tonic-gate if (flags & DDI_UMEM_PAGEABLE) { 39137c478bd9Sstevel@tonic-gate /* Only one of the flags is allowed */ 39147c478bd9Sstevel@tonic-gate ASSERT(!(flags & DDI_UMEM_TRASH)); 39157c478bd9Sstevel@tonic-gate /* initialize resource with 0 */ 39167c478bd9Sstevel@tonic-gate iflags = KPD_ZERO; 39177c478bd9Sstevel@tonic-gate 39187c478bd9Sstevel@tonic-gate /* 39197c478bd9Sstevel@tonic-gate * to allocate unlocked pageable memory, use segkp_get() to 39207c478bd9Sstevel@tonic-gate * create a segkp segment. Since segkp can only service kas, 39217c478bd9Sstevel@tonic-gate * other segment drivers such as segdev have to do 39227c478bd9Sstevel@tonic-gate * as_fault(segkp, SOFTLOCK) in its fault routine, 39237c478bd9Sstevel@tonic-gate */ 39247c478bd9Sstevel@tonic-gate if (flags & DDI_UMEM_NOSLEEP) 39257c478bd9Sstevel@tonic-gate iflags |= KPD_NOWAIT; 39267c478bd9Sstevel@tonic-gate 39277c478bd9Sstevel@tonic-gate if ((buf = segkp_get(segkp, len, iflags)) == NULL) { 39287c478bd9Sstevel@tonic-gate kmem_free(cp, sizeof (struct ddi_umem_cookie)); 39297c478bd9Sstevel@tonic-gate return ((void *)NULL); 39307c478bd9Sstevel@tonic-gate } 39317c478bd9Sstevel@tonic-gate cp->type = KMEM_PAGEABLE; 39327c478bd9Sstevel@tonic-gate mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL); 39337c478bd9Sstevel@tonic-gate cp->locked = 0; 39347c478bd9Sstevel@tonic-gate } else if (flags & DDI_UMEM_TRASH) { 39357c478bd9Sstevel@tonic-gate /* Only one of the flags is allowed */ 39367c478bd9Sstevel@tonic-gate ASSERT(!(flags & DDI_UMEM_PAGEABLE)); 39377c478bd9Sstevel@tonic-gate cp->type = UMEM_TRASH; 39387c478bd9Sstevel@tonic-gate buf = NULL; 39397c478bd9Sstevel@tonic-gate } else { 39407c478bd9Sstevel@tonic-gate if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) { 39417c478bd9Sstevel@tonic-gate kmem_free(cp, sizeof (struct ddi_umem_cookie)); 39427c478bd9Sstevel@tonic-gate return ((void *)NULL); 39437c478bd9Sstevel@tonic-gate } 39447c478bd9Sstevel@tonic-gate 39457c478bd9Sstevel@tonic-gate cp->type = KMEM_NON_PAGEABLE; 39467c478bd9Sstevel@tonic-gate } 39477c478bd9Sstevel@tonic-gate 39487c478bd9Sstevel@tonic-gate /* 39497c478bd9Sstevel@tonic-gate * need to save size here. size will be used when 39507c478bd9Sstevel@tonic-gate * we do kmem_free. 39517c478bd9Sstevel@tonic-gate */ 39527c478bd9Sstevel@tonic-gate cp->size = len; 39537c478bd9Sstevel@tonic-gate cp->cvaddr = (caddr_t)buf; 39547c478bd9Sstevel@tonic-gate 39557c478bd9Sstevel@tonic-gate *cookie = (void *)cp; 39567c478bd9Sstevel@tonic-gate return (buf); 39577c478bd9Sstevel@tonic-gate } 39587c478bd9Sstevel@tonic-gate 39597c478bd9Sstevel@tonic-gate void 39607c478bd9Sstevel@tonic-gate ddi_umem_free(ddi_umem_cookie_t cookie) 39617c478bd9Sstevel@tonic-gate { 39627c478bd9Sstevel@tonic-gate struct ddi_umem_cookie *cp; 39637c478bd9Sstevel@tonic-gate 39647c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE, 39657c478bd9Sstevel@tonic-gate "devmap_umem_free:start"); 39667c478bd9Sstevel@tonic-gate 39677c478bd9Sstevel@tonic-gate /* 39687c478bd9Sstevel@tonic-gate * if cookie is NULL, no effects on the system 39697c478bd9Sstevel@tonic-gate */ 39707c478bd9Sstevel@tonic-gate if (cookie == NULL) 39717c478bd9Sstevel@tonic-gate return; 39727c478bd9Sstevel@tonic-gate 39737c478bd9Sstevel@tonic-gate cp = (struct ddi_umem_cookie *)cookie; 39747c478bd9Sstevel@tonic-gate 39757c478bd9Sstevel@tonic-gate switch (cp->type) { 39767c478bd9Sstevel@tonic-gate case KMEM_PAGEABLE : 39777c478bd9Sstevel@tonic-gate ASSERT(cp->cvaddr != NULL && cp->size != 0); 39787c478bd9Sstevel@tonic-gate /* 39797c478bd9Sstevel@tonic-gate * Check if there are still any pending faults on the cookie 39807c478bd9Sstevel@tonic-gate * while the driver is deleting it, 39817c478bd9Sstevel@tonic-gate * XXX - could change to an ASSERT but wont catch errant drivers 39827c478bd9Sstevel@tonic-gate */ 39837c478bd9Sstevel@tonic-gate mutex_enter(&cp->lock); 39847c478bd9Sstevel@tonic-gate if (cp->locked) { 39857c478bd9Sstevel@tonic-gate mutex_exit(&cp->lock); 39867c478bd9Sstevel@tonic-gate panic("ddi_umem_free for cookie with pending faults %p", 39877c478bd9Sstevel@tonic-gate (void *)cp); 39887c478bd9Sstevel@tonic-gate return; 39897c478bd9Sstevel@tonic-gate } 39907c478bd9Sstevel@tonic-gate 39917c478bd9Sstevel@tonic-gate segkp_release(segkp, cp->cvaddr); 39927c478bd9Sstevel@tonic-gate 39937c478bd9Sstevel@tonic-gate /* 39947c478bd9Sstevel@tonic-gate * release mutex associated with this cookie. 39957c478bd9Sstevel@tonic-gate */ 39967c478bd9Sstevel@tonic-gate mutex_destroy(&cp->lock); 39977c478bd9Sstevel@tonic-gate break; 39987c478bd9Sstevel@tonic-gate case KMEM_NON_PAGEABLE : 39997c478bd9Sstevel@tonic-gate ASSERT(cp->cvaddr != NULL && cp->size != 0); 40007c478bd9Sstevel@tonic-gate devmap_umem_free_np(cp->cvaddr, cp->size); 40017c478bd9Sstevel@tonic-gate break; 40027c478bd9Sstevel@tonic-gate case UMEM_TRASH : 40037c478bd9Sstevel@tonic-gate break; 40047c478bd9Sstevel@tonic-gate case UMEM_LOCKED : 40057c478bd9Sstevel@tonic-gate /* Callers should use ddi_umem_unlock for this type */ 40067c478bd9Sstevel@tonic-gate ddi_umem_unlock(cookie); 40077c478bd9Sstevel@tonic-gate /* Frees the cookie too */ 40087c478bd9Sstevel@tonic-gate return; 40097c478bd9Sstevel@tonic-gate default: 40107c478bd9Sstevel@tonic-gate /* panic so we can diagnose the underlying cause */ 40117c478bd9Sstevel@tonic-gate panic("ddi_umem_free: illegal cookie type 0x%x\n", 40127c478bd9Sstevel@tonic-gate cp->type); 40137c478bd9Sstevel@tonic-gate } 40147c478bd9Sstevel@tonic-gate 40157c478bd9Sstevel@tonic-gate kmem_free(cookie, sizeof (struct ddi_umem_cookie)); 40167c478bd9Sstevel@tonic-gate } 40177c478bd9Sstevel@tonic-gate 40187c478bd9Sstevel@tonic-gate 40197c478bd9Sstevel@tonic-gate static int 40207c478bd9Sstevel@tonic-gate segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 40217c478bd9Sstevel@tonic-gate { 40227c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 40237c478bd9Sstevel@tonic-gate 40247c478bd9Sstevel@tonic-gate /* 40257c478bd9Sstevel@tonic-gate * It looks as if it is always mapped shared 40267c478bd9Sstevel@tonic-gate */ 40277c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID, 40287c478bd9Sstevel@tonic-gate "segdev_getmemid:start"); 40297c478bd9Sstevel@tonic-gate memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp); 40307c478bd9Sstevel@tonic-gate memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base); 40317c478bd9Sstevel@tonic-gate return (0); 40327c478bd9Sstevel@tonic-gate } 40337c478bd9Sstevel@tonic-gate 40347c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 40357c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t * 40367c478bd9Sstevel@tonic-gate segdev_getpolicy(struct seg *seg, caddr_t addr) 40377c478bd9Sstevel@tonic-gate { 40387c478bd9Sstevel@tonic-gate return (NULL); 40397c478bd9Sstevel@tonic-gate } 40407c478bd9Sstevel@tonic-gate 40411bd5c35fSelowe /*ARGSUSED*/ 40421bd5c35fSelowe static int 40431bd5c35fSelowe segdev_capable(struct seg *seg, segcapability_t capability) 40441bd5c35fSelowe { 40451bd5c35fSelowe return (0); 40461bd5c35fSelowe } 40471bd5c35fSelowe 40487c478bd9Sstevel@tonic-gate /* 40497c478bd9Sstevel@tonic-gate * ddi_umem_alloc() non-pageable quantum cache max size. 40507c478bd9Sstevel@tonic-gate * This is just a SWAG. 40517c478bd9Sstevel@tonic-gate */ 40527c478bd9Sstevel@tonic-gate #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE) 40537c478bd9Sstevel@tonic-gate 40547c478bd9Sstevel@tonic-gate /* 40557c478bd9Sstevel@tonic-gate * Initialize seg_dev from boot. This routine sets up the trash page 40567c478bd9Sstevel@tonic-gate * and creates the umem_np_arena used to back non-pageable memory 40577c478bd9Sstevel@tonic-gate * requests. 40587c478bd9Sstevel@tonic-gate */ 40597c478bd9Sstevel@tonic-gate void 40607c478bd9Sstevel@tonic-gate segdev_init(void) 40617c478bd9Sstevel@tonic-gate { 40627c478bd9Sstevel@tonic-gate struct seg kseg; 40637c478bd9Sstevel@tonic-gate 40647c478bd9Sstevel@tonic-gate umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE, 40657c478bd9Sstevel@tonic-gate devmap_alloc_pages, devmap_free_pages, heap_arena, 40667c478bd9Sstevel@tonic-gate DEVMAP_UMEM_QUANTUM, VM_SLEEP); 40677c478bd9Sstevel@tonic-gate 40687c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 40697c478bd9Sstevel@tonic-gate trashpp = page_create_va(&trashvp, 0, PAGESIZE, 40707c478bd9Sstevel@tonic-gate PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL); 40717c478bd9Sstevel@tonic-gate if (trashpp == NULL) 40727c478bd9Sstevel@tonic-gate panic("segdev_init: failed to create trash page"); 40737c478bd9Sstevel@tonic-gate pagezero(trashpp, 0, PAGESIZE); 40747c478bd9Sstevel@tonic-gate page_downgrade(trashpp); 40757c478bd9Sstevel@tonic-gate } 40767c478bd9Sstevel@tonic-gate 40777c478bd9Sstevel@tonic-gate /* 40787c478bd9Sstevel@tonic-gate * Invoke platform-dependent support routines so that /proc can have 40797c478bd9Sstevel@tonic-gate * the platform code deal with curious hardware. 40807c478bd9Sstevel@tonic-gate */ 40817c478bd9Sstevel@tonic-gate int 40827c478bd9Sstevel@tonic-gate segdev_copyfrom(struct seg *seg, 40837c478bd9Sstevel@tonic-gate caddr_t uaddr, const void *devaddr, void *kaddr, size_t len) 40847c478bd9Sstevel@tonic-gate { 40857c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 40867c478bd9Sstevel@tonic-gate struct snode *sp = VTOS(VTOCVP(sdp->vp)); 40877c478bd9Sstevel@tonic-gate 40887c478bd9Sstevel@tonic-gate return (e_ddi_copyfromdev(sp->s_dip, 40897c478bd9Sstevel@tonic-gate (off_t)(uaddr - seg->s_base), devaddr, kaddr, len)); 40907c478bd9Sstevel@tonic-gate } 40917c478bd9Sstevel@tonic-gate 40927c478bd9Sstevel@tonic-gate int 40937c478bd9Sstevel@tonic-gate segdev_copyto(struct seg *seg, 40947c478bd9Sstevel@tonic-gate caddr_t uaddr, const void *kaddr, void *devaddr, size_t len) 40957c478bd9Sstevel@tonic-gate { 40967c478bd9Sstevel@tonic-gate struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 40977c478bd9Sstevel@tonic-gate struct snode *sp = VTOS(VTOCVP(sdp->vp)); 40987c478bd9Sstevel@tonic-gate 40997c478bd9Sstevel@tonic-gate return (e_ddi_copytodev(sp->s_dip, 41007c478bd9Sstevel@tonic-gate (off_t)(uaddr - seg->s_base), kaddr, devaddr, len)); 41017c478bd9Sstevel@tonic-gate } 4102