xref: /illumos-gate/usr/src/uts/common/vm/seg_dev.c (revision c6f039c7)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
57b93957cSeota  * Common Development and Distribution License (the "License").
67b93957cSeota  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217b93957cSeota 
227c478bd9Sstevel@tonic-gate /*
2317965fd8SKrishnendu Sadhukhan - Sun Microsystems  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
2544bf619dSJohn Levon  * Copyright 2019 Joyent, Inc.
267c478bd9Sstevel@tonic-gate  */
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
297c478bd9Sstevel@tonic-gate /*	  All Rights Reserved  	*/
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate /*
327c478bd9Sstevel@tonic-gate  * University Copyright- Copyright (c) 1982, 1986, 1988
337c478bd9Sstevel@tonic-gate  * The Regents of the University of California
347c478bd9Sstevel@tonic-gate  * All Rights Reserved
357c478bd9Sstevel@tonic-gate  *
367c478bd9Sstevel@tonic-gate  * University Acknowledgment- Portions of this document are derived from
377c478bd9Sstevel@tonic-gate  * software developed by the University of California, Berkeley, and its
387c478bd9Sstevel@tonic-gate  * contributors.
397c478bd9Sstevel@tonic-gate  */
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate /*
427c478bd9Sstevel@tonic-gate  * VM - segment of a mapped device.
437c478bd9Sstevel@tonic-gate  *
447c478bd9Sstevel@tonic-gate  * This segment driver is used when mapping character special devices.
457c478bd9Sstevel@tonic-gate  */
467c478bd9Sstevel@tonic-gate 
477c478bd9Sstevel@tonic-gate #include <sys/types.h>
487c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
497c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
507c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
517c478bd9Sstevel@tonic-gate #include <sys/systm.h>
527c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
537c478bd9Sstevel@tonic-gate #include <sys/mman.h>
547c478bd9Sstevel@tonic-gate #include <sys/errno.h>
557c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
567c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
577c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
587c478bd9Sstevel@tonic-gate #include <sys/proc.h>
597c478bd9Sstevel@tonic-gate #include <sys/conf.h>
607c478bd9Sstevel@tonic-gate #include <sys/debug.h>
617c478bd9Sstevel@tonic-gate #include <sys/ddidevmap.h>
627b93957cSeota #include <sys/ddi_implfuncs.h>
637c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
647c478bd9Sstevel@tonic-gate 
657c478bd9Sstevel@tonic-gate #include <vm/page.h>
667c478bd9Sstevel@tonic-gate #include <vm/hat.h>
677c478bd9Sstevel@tonic-gate #include <vm/as.h>
687c478bd9Sstevel@tonic-gate #include <vm/seg.h>
697c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h>
707c478bd9Sstevel@tonic-gate #include <vm/seg_kp.h>
717c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
727c478bd9Sstevel@tonic-gate #include <vm/vpage.h>
737c478bd9Sstevel@tonic-gate 
747c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
757c478bd9Sstevel@tonic-gate #include <sys/esunddi.h>
767c478bd9Sstevel@tonic-gate #include <sys/fs/snode.h>
777c478bd9Sstevel@tonic-gate 
787b93957cSeota 
797c478bd9Sstevel@tonic-gate #if DEBUG
807c478bd9Sstevel@tonic-gate int segdev_debug;
817c478bd9Sstevel@tonic-gate #define	DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
827c478bd9Sstevel@tonic-gate #else
837c478bd9Sstevel@tonic-gate #define	DEBUGF(level, args)
847c478bd9Sstevel@tonic-gate #endif
857c478bd9Sstevel@tonic-gate 
867c478bd9Sstevel@tonic-gate /* Default timeout for devmap context management */
877c478bd9Sstevel@tonic-gate #define	CTX_TIMEOUT_VALUE 0
887c478bd9Sstevel@tonic-gate 
897c478bd9Sstevel@tonic-gate #define	HOLD_DHP_LOCK(dhp)  if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
907c478bd9Sstevel@tonic-gate 			{ mutex_enter(&dhp->dh_lock); }
917c478bd9Sstevel@tonic-gate 
927c478bd9Sstevel@tonic-gate #define	RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
937c478bd9Sstevel@tonic-gate 			{ mutex_exit(&dhp->dh_lock); }
947c478bd9Sstevel@tonic-gate 
957c478bd9Sstevel@tonic-gate #define	round_down_p2(a, s)	((a) & ~((s) - 1))
967c478bd9Sstevel@tonic-gate #define	round_up_p2(a, s)	(((a) + (s) - 1) & ~((s) - 1))
977c478bd9Sstevel@tonic-gate 
987c478bd9Sstevel@tonic-gate /*
997c478bd9Sstevel@tonic-gate  * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
1007c478bd9Sstevel@tonic-gate  * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
1017c478bd9Sstevel@tonic-gate  */
1027c478bd9Sstevel@tonic-gate #define	VA_PA_ALIGNED(uvaddr, paddr, pgsize)		\
1037c478bd9Sstevel@tonic-gate 	(((uvaddr | paddr) & (pgsize - 1)) == 0)
1047c478bd9Sstevel@tonic-gate #define	VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize)	\
1057c478bd9Sstevel@tonic-gate 	(((uvaddr ^ paddr) & (pgsize - 1)) == 0)
1067c478bd9Sstevel@tonic-gate 
1077c478bd9Sstevel@tonic-gate #define	vpgtob(n)	((n) * sizeof (struct vpage))	/* For brevity */
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate #define	VTOCVP(vp)	(VTOS(vp)->s_commonvp)	/* we "know" it's an snode */
1107c478bd9Sstevel@tonic-gate 
1117c478bd9Sstevel@tonic-gate static struct devmap_ctx *devmapctx_list = NULL;
1127c478bd9Sstevel@tonic-gate static struct devmap_softlock *devmap_slist = NULL;
1137c478bd9Sstevel@tonic-gate 
1147c478bd9Sstevel@tonic-gate /*
1157c478bd9Sstevel@tonic-gate  * mutex, vnode and page for the page of zeros we use for the trash mappings.
1167c478bd9Sstevel@tonic-gate  * One trash page is allocated on the first ddi_umem_setup call that uses it
1177c478bd9Sstevel@tonic-gate  * XXX Eventually, we may want to combine this with what segnf does when all
1187c478bd9Sstevel@tonic-gate  * hat layers implement HAT_NOFAULT.
1197c478bd9Sstevel@tonic-gate  *
1207c478bd9Sstevel@tonic-gate  * The trash page is used when the backing store for a userland mapping is
1217c478bd9Sstevel@tonic-gate  * removed but the application semantics do not take kindly to a SIGBUS.
1227c478bd9Sstevel@tonic-gate  * In that scenario, the applications pages are mapped to some dummy page
1237c478bd9Sstevel@tonic-gate  * which returns garbage on read and writes go into a common place.
1247c478bd9Sstevel@tonic-gate  * (Perfect for NO_FAULT semantics)
1257c478bd9Sstevel@tonic-gate  * The device driver is responsible to communicating to the app with some
1267c478bd9Sstevel@tonic-gate  * other mechanism that such remapping has happened and the app should take
1277c478bd9Sstevel@tonic-gate  * corrective action.
1287c478bd9Sstevel@tonic-gate  * We can also use an anonymous memory page as there is no requirement to
1297c478bd9Sstevel@tonic-gate  * keep the page locked, however this complicates the fault code. RFE.
1307c478bd9Sstevel@tonic-gate  */
1317c478bd9Sstevel@tonic-gate static struct vnode trashvp;
1327c478bd9Sstevel@tonic-gate static struct page *trashpp;
1337c478bd9Sstevel@tonic-gate 
1347c478bd9Sstevel@tonic-gate /* Non-pageable kernel memory is allocated from the umem_np_arena. */
1357c478bd9Sstevel@tonic-gate static vmem_t *umem_np_arena;
1367c478bd9Sstevel@tonic-gate 
1377c478bd9Sstevel@tonic-gate /* Set the cookie to a value we know will never be a valid umem_cookie */
1387c478bd9Sstevel@tonic-gate #define	DEVMAP_DEVMEM_COOKIE	((ddi_umem_cookie_t)0x1)
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate /*
1417c478bd9Sstevel@tonic-gate  * Macros to check if type of devmap handle
1427c478bd9Sstevel@tonic-gate  */
1437c478bd9Sstevel@tonic-gate #define	cookie_is_devmem(c)	\
1447c478bd9Sstevel@tonic-gate 	((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
1457c478bd9Sstevel@tonic-gate 
1467c478bd9Sstevel@tonic-gate #define	cookie_is_pmem(c)	\
1477c478bd9Sstevel@tonic-gate 	((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate #define	cookie_is_kpmem(c)	(!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
1507c478bd9Sstevel@tonic-gate 	((c)->type == KMEM_PAGEABLE))
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate #define	dhp_is_devmem(dhp)	\
1537c478bd9Sstevel@tonic-gate 	(cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
1547c478bd9Sstevel@tonic-gate 
1557c478bd9Sstevel@tonic-gate #define	dhp_is_pmem(dhp)	\
1567c478bd9Sstevel@tonic-gate 	(cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
1577c478bd9Sstevel@tonic-gate 
1587c478bd9Sstevel@tonic-gate #define	dhp_is_kpmem(dhp)	\
1597c478bd9Sstevel@tonic-gate 	(cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
1607c478bd9Sstevel@tonic-gate 
1617c478bd9Sstevel@tonic-gate /*
1627c478bd9Sstevel@tonic-gate  * Private seg op routines.
1637c478bd9Sstevel@tonic-gate  */
1647c478bd9Sstevel@tonic-gate static int	segdev_dup(struct seg *, struct seg *);
1657c478bd9Sstevel@tonic-gate static int	segdev_unmap(struct seg *, caddr_t, size_t);
1667c478bd9Sstevel@tonic-gate static void	segdev_free(struct seg *);
1677c478bd9Sstevel@tonic-gate static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
1687c478bd9Sstevel@tonic-gate 		    enum fault_type, enum seg_rw);
1697c478bd9Sstevel@tonic-gate static faultcode_t segdev_faulta(struct seg *, caddr_t);
1707c478bd9Sstevel@tonic-gate static int	segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
1717c478bd9Sstevel@tonic-gate static int	segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
1727c478bd9Sstevel@tonic-gate static void	segdev_badop(void);
1737c478bd9Sstevel@tonic-gate static int	segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
1747c478bd9Sstevel@tonic-gate static size_t	segdev_incore(struct seg *, caddr_t, size_t, char *);
1757c478bd9Sstevel@tonic-gate static int	segdev_lockop(struct seg *, caddr_t, size_t, int, int,
1767c478bd9Sstevel@tonic-gate 		    ulong_t *, size_t);
1777c478bd9Sstevel@tonic-gate static int	segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
1787c478bd9Sstevel@tonic-gate static u_offset_t	segdev_getoffset(struct seg *, caddr_t);
1797c478bd9Sstevel@tonic-gate static int	segdev_gettype(struct seg *, caddr_t);
1807c478bd9Sstevel@tonic-gate static int	segdev_getvp(struct seg *, caddr_t, struct vnode **);
1817c478bd9Sstevel@tonic-gate static int	segdev_advise(struct seg *, caddr_t, size_t, uint_t);
1827c478bd9Sstevel@tonic-gate static void	segdev_dump(struct seg *);
1837c478bd9Sstevel@tonic-gate static int	segdev_pagelock(struct seg *, caddr_t, size_t,
1847c478bd9Sstevel@tonic-gate 		    struct page ***, enum lock_type, enum seg_rw);
1857c478bd9Sstevel@tonic-gate static int	segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
1867c478bd9Sstevel@tonic-gate static int	segdev_getmemid(struct seg *, caddr_t, memid_t *);
1877c478bd9Sstevel@tonic-gate static lgrp_mem_policy_info_t	*segdev_getpolicy(struct seg *, caddr_t);
1881bd5c35fSelowe static int	segdev_capable(struct seg *, segcapability_t);
1897c478bd9Sstevel@tonic-gate 
1907c478bd9Sstevel@tonic-gate /*
1917c478bd9Sstevel@tonic-gate  * XXX	this struct is used by rootnex_map_fault to identify
1927c478bd9Sstevel@tonic-gate  *	the segment it has been passed. So if you make it
1937c478bd9Sstevel@tonic-gate  *	"static" you'll need to fix rootnex_map_fault.
1947c478bd9Sstevel@tonic-gate  */
1957c478bd9Sstevel@tonic-gate struct seg_ops segdev_ops = {
1967c478bd9Sstevel@tonic-gate 	segdev_dup,
1977c478bd9Sstevel@tonic-gate 	segdev_unmap,
1987c478bd9Sstevel@tonic-gate 	segdev_free,
1997c478bd9Sstevel@tonic-gate 	segdev_fault,
2007c478bd9Sstevel@tonic-gate 	segdev_faulta,
2017c478bd9Sstevel@tonic-gate 	segdev_setprot,
2027c478bd9Sstevel@tonic-gate 	segdev_checkprot,
2037c478bd9Sstevel@tonic-gate 	(int (*)())segdev_badop,	/* kluster */
2047c478bd9Sstevel@tonic-gate 	(size_t (*)(struct seg *))NULL,	/* swapout */
2057c478bd9Sstevel@tonic-gate 	segdev_sync,			/* sync */
2067c478bd9Sstevel@tonic-gate 	segdev_incore,
2077c478bd9Sstevel@tonic-gate 	segdev_lockop,			/* lockop */
2087c478bd9Sstevel@tonic-gate 	segdev_getprot,
2097c478bd9Sstevel@tonic-gate 	segdev_getoffset,
2107c478bd9Sstevel@tonic-gate 	segdev_gettype,
2117c478bd9Sstevel@tonic-gate 	segdev_getvp,
2127c478bd9Sstevel@tonic-gate 	segdev_advise,
2137c478bd9Sstevel@tonic-gate 	segdev_dump,
2147c478bd9Sstevel@tonic-gate 	segdev_pagelock,
2157c478bd9Sstevel@tonic-gate 	segdev_setpagesize,
2167c478bd9Sstevel@tonic-gate 	segdev_getmemid,
2177c478bd9Sstevel@tonic-gate 	segdev_getpolicy,
2181bd5c35fSelowe 	segdev_capable,
2199d12795fSRobert Mustacchi 	seg_inherit_notsup
2207c478bd9Sstevel@tonic-gate };
2217c478bd9Sstevel@tonic-gate 
2227c478bd9Sstevel@tonic-gate /*
2237c478bd9Sstevel@tonic-gate  * Private segdev support routines
2247c478bd9Sstevel@tonic-gate  */
2257c478bd9Sstevel@tonic-gate static struct segdev_data *sdp_alloc(void);
2267c478bd9Sstevel@tonic-gate 
2277c478bd9Sstevel@tonic-gate static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
2287c478bd9Sstevel@tonic-gate     size_t, enum seg_rw);
2297c478bd9Sstevel@tonic-gate 
2307c478bd9Sstevel@tonic-gate static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
2317c478bd9Sstevel@tonic-gate     struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
2327c478bd9Sstevel@tonic-gate 
2337c478bd9Sstevel@tonic-gate static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
2347c478bd9Sstevel@tonic-gate     size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
2357c478bd9Sstevel@tonic-gate 
2367c478bd9Sstevel@tonic-gate static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
2377c478bd9Sstevel@tonic-gate static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
2387c478bd9Sstevel@tonic-gate static void devmap_softlock_rele(devmap_handle_t *);
2397c478bd9Sstevel@tonic-gate static void devmap_ctx_rele(devmap_handle_t *);
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate static void devmap_ctxto(void *);
2427c478bd9Sstevel@tonic-gate 
2437c478bd9Sstevel@tonic-gate static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
2447c478bd9Sstevel@tonic-gate     caddr_t addr);
2457c478bd9Sstevel@tonic-gate 
2467c478bd9Sstevel@tonic-gate static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2477c478bd9Sstevel@tonic-gate     ulong_t *opfn, ulong_t *pagesize);
2487c478bd9Sstevel@tonic-gate 
2497c478bd9Sstevel@tonic-gate static void free_devmap_handle(devmap_handle_t *dhp);
2507c478bd9Sstevel@tonic-gate 
2517c478bd9Sstevel@tonic-gate static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
2527c478bd9Sstevel@tonic-gate     struct seg *newseg);
2537c478bd9Sstevel@tonic-gate 
2547c478bd9Sstevel@tonic-gate static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
2557c478bd9Sstevel@tonic-gate 
2567c478bd9Sstevel@tonic-gate static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
2577c478bd9Sstevel@tonic-gate 
2587c478bd9Sstevel@tonic-gate static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
2597c478bd9Sstevel@tonic-gate 
2607c478bd9Sstevel@tonic-gate static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2617c478bd9Sstevel@tonic-gate     offset_t off, size_t len, uint_t flags);
2627c478bd9Sstevel@tonic-gate 
2637c478bd9Sstevel@tonic-gate static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
2647c478bd9Sstevel@tonic-gate     caddr_t addr, size_t *llen, caddr_t *laddr);
2657c478bd9Sstevel@tonic-gate 
2667c478bd9Sstevel@tonic-gate static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
2677c478bd9Sstevel@tonic-gate 
2687c478bd9Sstevel@tonic-gate static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
2697c478bd9Sstevel@tonic-gate static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
2707c478bd9Sstevel@tonic-gate 
2717c478bd9Sstevel@tonic-gate static void *devmap_umem_alloc_np(size_t size, size_t flags);
2727c478bd9Sstevel@tonic-gate static void devmap_umem_free_np(void *addr, size_t size);
2737c478bd9Sstevel@tonic-gate 
2747c478bd9Sstevel@tonic-gate /*
2757c478bd9Sstevel@tonic-gate  * routines to lock and unlock underlying segkp segment for
2767c478bd9Sstevel@tonic-gate  * KMEM_PAGEABLE type cookies.
2777c478bd9Sstevel@tonic-gate  */
2787c478bd9Sstevel@tonic-gate static faultcode_t  acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
2797c478bd9Sstevel@tonic-gate static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
2807c478bd9Sstevel@tonic-gate 
2817c478bd9Sstevel@tonic-gate /*
2827c478bd9Sstevel@tonic-gate  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
2837c478bd9Sstevel@tonic-gate  * drivers with devmap_access callbacks
2847c478bd9Sstevel@tonic-gate  */
2857c478bd9Sstevel@tonic-gate static int devmap_softlock_enter(struct devmap_softlock *, size_t,
2867c478bd9Sstevel@tonic-gate 	enum fault_type);
2877c478bd9Sstevel@tonic-gate static void devmap_softlock_exit(struct devmap_softlock *, size_t,
2887c478bd9Sstevel@tonic-gate 	enum fault_type);
2897c478bd9Sstevel@tonic-gate 
2907c478bd9Sstevel@tonic-gate static kmutex_t devmapctx_lock;
2917c478bd9Sstevel@tonic-gate 
2927c478bd9Sstevel@tonic-gate static kmutex_t devmap_slock;
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate /*
2957c478bd9Sstevel@tonic-gate  * Initialize the thread callbacks and thread private data.
2967c478bd9Sstevel@tonic-gate  */
2977c478bd9Sstevel@tonic-gate static struct devmap_ctx *
devmap_ctxinit(dev_t dev,ulong_t id)2987c478bd9Sstevel@tonic-gate devmap_ctxinit(dev_t dev, ulong_t id)
2997c478bd9Sstevel@tonic-gate {
3007c478bd9Sstevel@tonic-gate 	struct devmap_ctx	*devctx;
3017c478bd9Sstevel@tonic-gate 	struct devmap_ctx	*tmp;
3027c478bd9Sstevel@tonic-gate 	dev_info_t		*dip;
3037c478bd9Sstevel@tonic-gate 
3047c478bd9Sstevel@tonic-gate 	tmp =  kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate 	mutex_enter(&devmapctx_lock);
3077c478bd9Sstevel@tonic-gate 
3087c478bd9Sstevel@tonic-gate 	dip = e_ddi_hold_devi_by_dev(dev, 0);
3097c478bd9Sstevel@tonic-gate 	ASSERT(dip != NULL);
3107c478bd9Sstevel@tonic-gate 	ddi_release_devi(dip);
3117c478bd9Sstevel@tonic-gate 
3127c478bd9Sstevel@tonic-gate 	for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
3137c478bd9Sstevel@tonic-gate 		if ((devctx->dip == dip) && (devctx->id == id))
3147c478bd9Sstevel@tonic-gate 			break;
3157c478bd9Sstevel@tonic-gate 
3167c478bd9Sstevel@tonic-gate 	if (devctx == NULL) {
3177c478bd9Sstevel@tonic-gate 		devctx = tmp;
3187c478bd9Sstevel@tonic-gate 		devctx->dip = dip;
3197c478bd9Sstevel@tonic-gate 		devctx->id = id;
3207c478bd9Sstevel@tonic-gate 		mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
3217c478bd9Sstevel@tonic-gate 		cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
3227c478bd9Sstevel@tonic-gate 		devctx->next = devmapctx_list;
3237c478bd9Sstevel@tonic-gate 		devmapctx_list = devctx;
3247c478bd9Sstevel@tonic-gate 	} else
3257c478bd9Sstevel@tonic-gate 		kmem_free(tmp, sizeof (struct devmap_ctx));
3267c478bd9Sstevel@tonic-gate 
3277c478bd9Sstevel@tonic-gate 	mutex_enter(&devctx->lock);
3287c478bd9Sstevel@tonic-gate 	devctx->refcnt++;
3297c478bd9Sstevel@tonic-gate 	mutex_exit(&devctx->lock);
3307c478bd9Sstevel@tonic-gate 	mutex_exit(&devmapctx_lock);
3317c478bd9Sstevel@tonic-gate 
3327c478bd9Sstevel@tonic-gate 	return (devctx);
3337c478bd9Sstevel@tonic-gate }
3347c478bd9Sstevel@tonic-gate 
3357c478bd9Sstevel@tonic-gate /*
3367c478bd9Sstevel@tonic-gate  * Timeout callback called if a CPU has not given up the device context
3377c478bd9Sstevel@tonic-gate  * within dhp->dh_timeout_length ticks
3387c478bd9Sstevel@tonic-gate  */
3397c478bd9Sstevel@tonic-gate static void
devmap_ctxto(void * data)3407c478bd9Sstevel@tonic-gate devmap_ctxto(void *data)
3417c478bd9Sstevel@tonic-gate {
3427c478bd9Sstevel@tonic-gate 	struct devmap_ctx *devctx = data;
3437c478bd9Sstevel@tonic-gate 
3447c478bd9Sstevel@tonic-gate 	TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
3457c478bd9Sstevel@tonic-gate 	    "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
3467c478bd9Sstevel@tonic-gate 	mutex_enter(&devctx->lock);
3477c478bd9Sstevel@tonic-gate 	/*
3487c478bd9Sstevel@tonic-gate 	 * Set oncpu = 0 so the next mapping trying to get the device context
3497c478bd9Sstevel@tonic-gate 	 * can.
3507c478bd9Sstevel@tonic-gate 	 */
3517c478bd9Sstevel@tonic-gate 	devctx->oncpu = 0;
3527c478bd9Sstevel@tonic-gate 	devctx->timeout = 0;
3537c478bd9Sstevel@tonic-gate 	cv_signal(&devctx->cv);
3547c478bd9Sstevel@tonic-gate 	mutex_exit(&devctx->lock);
3557c478bd9Sstevel@tonic-gate }
3567c478bd9Sstevel@tonic-gate 
3577c478bd9Sstevel@tonic-gate /*
3587c478bd9Sstevel@tonic-gate  * Create a device segment.
3597c478bd9Sstevel@tonic-gate  */
3607c478bd9Sstevel@tonic-gate int
segdev_create(struct seg ** segpp,void * argsp)361284ce987SPatrick Mooney segdev_create(struct seg **segpp, void *argsp)
3627c478bd9Sstevel@tonic-gate {
363284ce987SPatrick Mooney 	struct seg *seg = *segpp;
3647c478bd9Sstevel@tonic-gate 	struct segdev_data *sdp;
3657c478bd9Sstevel@tonic-gate 	struct segdev_crargs *a = (struct segdev_crargs *)argsp;
3667c478bd9Sstevel@tonic-gate 	devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
3677c478bd9Sstevel@tonic-gate 	int error;
3687c478bd9Sstevel@tonic-gate 
3697c478bd9Sstevel@tonic-gate 	/*
3707c478bd9Sstevel@tonic-gate 	 * Since the address space is "write" locked, we
3717c478bd9Sstevel@tonic-gate 	 * don't need the segment lock to protect "segdev" data.
3727c478bd9Sstevel@tonic-gate 	 */
373dc32d872SJosef 'Jeff' Sipek 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
3747c478bd9Sstevel@tonic-gate 
3757c478bd9Sstevel@tonic-gate 	hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
3767c478bd9Sstevel@tonic-gate 
3777c478bd9Sstevel@tonic-gate 	sdp = sdp_alloc();
3787c478bd9Sstevel@tonic-gate 
3797c478bd9Sstevel@tonic-gate 	sdp->mapfunc = a->mapfunc;
3807c478bd9Sstevel@tonic-gate 	sdp->offset = a->offset;
3817c478bd9Sstevel@tonic-gate 	sdp->prot = a->prot;
3827c478bd9Sstevel@tonic-gate 	sdp->maxprot = a->maxprot;
3837c478bd9Sstevel@tonic-gate 	sdp->type = a->type;
3847c478bd9Sstevel@tonic-gate 	sdp->pageprot = 0;
3857c478bd9Sstevel@tonic-gate 	sdp->softlockcnt = 0;
3867c478bd9Sstevel@tonic-gate 	sdp->vpage = NULL;
3877c478bd9Sstevel@tonic-gate 
3887c478bd9Sstevel@tonic-gate 	if (sdp->mapfunc == NULL)
3897c478bd9Sstevel@tonic-gate 		sdp->devmap_data = dhp;
3907c478bd9Sstevel@tonic-gate 	else
3917c478bd9Sstevel@tonic-gate 		sdp->devmap_data = dhp = NULL;
3927c478bd9Sstevel@tonic-gate 
3937c478bd9Sstevel@tonic-gate 	sdp->hat_flags = a->hat_flags;
3947c478bd9Sstevel@tonic-gate 	sdp->hat_attr = a->hat_attr;
3957c478bd9Sstevel@tonic-gate 
3967c478bd9Sstevel@tonic-gate 	/*
3977c478bd9Sstevel@tonic-gate 	 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
3987c478bd9Sstevel@tonic-gate 	 */
3997c478bd9Sstevel@tonic-gate 	ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
4007c478bd9Sstevel@tonic-gate 
4017c478bd9Sstevel@tonic-gate 	/*
4027c478bd9Sstevel@tonic-gate 	 * Hold shadow vnode -- segdev only deals with
4037c478bd9Sstevel@tonic-gate 	 * character (VCHR) devices. We use the common
4047c478bd9Sstevel@tonic-gate 	 * vp to hang pages on.
4057c478bd9Sstevel@tonic-gate 	 */
4067c478bd9Sstevel@tonic-gate 	sdp->vp = specfind(a->dev, VCHR);
4077c478bd9Sstevel@tonic-gate 	ASSERT(sdp->vp != NULL);
4087c478bd9Sstevel@tonic-gate 
4097c478bd9Sstevel@tonic-gate 	seg->s_ops = &segdev_ops;
4107c478bd9Sstevel@tonic-gate 	seg->s_data = sdp;
4117c478bd9Sstevel@tonic-gate 
4127c478bd9Sstevel@tonic-gate 	while (dhp != NULL) {
4137c478bd9Sstevel@tonic-gate 		dhp->dh_seg = seg;
4147c478bd9Sstevel@tonic-gate 		dhp = dhp->dh_next;
4157c478bd9Sstevel@tonic-gate 	}
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate 	/*
4187c478bd9Sstevel@tonic-gate 	 * Inform the vnode of the new mapping.
4197c478bd9Sstevel@tonic-gate 	 */
4207c478bd9Sstevel@tonic-gate 	/*
4217c478bd9Sstevel@tonic-gate 	 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
4227c478bd9Sstevel@tonic-gate 	 * dhp specific maxprot because spec_addmap does not use maxprot.
4237c478bd9Sstevel@tonic-gate 	 */
4247c478bd9Sstevel@tonic-gate 	error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
4257c478bd9Sstevel@tonic-gate 	    seg->s_as, seg->s_base, seg->s_size,
426da6c28aaSamw 	    sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
4277c478bd9Sstevel@tonic-gate 
4287c478bd9Sstevel@tonic-gate 	if (error != 0) {
4297c478bd9Sstevel@tonic-gate 		sdp->devmap_data = NULL;
4307c478bd9Sstevel@tonic-gate 		hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
4317c478bd9Sstevel@tonic-gate 		    HAT_UNLOAD_UNMAP);
43217965fd8SKrishnendu Sadhukhan - Sun Microsystems 	} else {
43317965fd8SKrishnendu Sadhukhan - Sun Microsystems 		/*
43417965fd8SKrishnendu Sadhukhan - Sun Microsystems 		 * Mappings of /dev/null don't count towards the VSZ of a
43517965fd8SKrishnendu Sadhukhan - Sun Microsystems 		 * process.  Mappings of /dev/null have no mapping type.
43617965fd8SKrishnendu Sadhukhan - Sun Microsystems 		 */
43717965fd8SKrishnendu Sadhukhan - Sun Microsystems 		if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
43817965fd8SKrishnendu Sadhukhan - Sun Microsystems 		    MAP_PRIVATE)) == 0) {
43917965fd8SKrishnendu Sadhukhan - Sun Microsystems 			seg->s_as->a_resvsize -= seg->s_size;
44017965fd8SKrishnendu Sadhukhan - Sun Microsystems 		}
4417c478bd9Sstevel@tonic-gate 	}
4427c478bd9Sstevel@tonic-gate 
4437c478bd9Sstevel@tonic-gate 	return (error);
4447c478bd9Sstevel@tonic-gate }
4457c478bd9Sstevel@tonic-gate 
4467c478bd9Sstevel@tonic-gate static struct segdev_data *
sdp_alloc(void)4477c478bd9Sstevel@tonic-gate sdp_alloc(void)
4487c478bd9Sstevel@tonic-gate {
4497c478bd9Sstevel@tonic-gate 	struct segdev_data *sdp;
4507c478bd9Sstevel@tonic-gate 
4517c478bd9Sstevel@tonic-gate 	sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
45244374aaeSsvemuri 	rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
4537c478bd9Sstevel@tonic-gate 
4547c478bd9Sstevel@tonic-gate 	return (sdp);
4557c478bd9Sstevel@tonic-gate }
4567c478bd9Sstevel@tonic-gate 
4577c478bd9Sstevel@tonic-gate /*
4587c478bd9Sstevel@tonic-gate  * Duplicate seg and return new segment in newseg.
4597c478bd9Sstevel@tonic-gate  */
4607c478bd9Sstevel@tonic-gate static int
segdev_dup(struct seg * seg,struct seg * newseg)4617c478bd9Sstevel@tonic-gate segdev_dup(struct seg *seg, struct seg *newseg)
4627c478bd9Sstevel@tonic-gate {
4637c478bd9Sstevel@tonic-gate 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4647c478bd9Sstevel@tonic-gate 	struct segdev_data *newsdp;
4657c478bd9Sstevel@tonic-gate 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
4667c478bd9Sstevel@tonic-gate 	size_t npages;
4677c478bd9Sstevel@tonic-gate 	int ret;
4687c478bd9Sstevel@tonic-gate 
4697c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
4707c478bd9Sstevel@tonic-gate 	    "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
4717c478bd9Sstevel@tonic-gate 
4727c478bd9Sstevel@tonic-gate 	DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
4737c478bd9Sstevel@tonic-gate 	    (void *)dhp, (void *)seg));
4747c478bd9Sstevel@tonic-gate 
4757c478bd9Sstevel@tonic-gate 	/*
4767c478bd9Sstevel@tonic-gate 	 * Since the address space is "write" locked, we
4777c478bd9Sstevel@tonic-gate 	 * don't need the segment lock to protect "segdev" data.
4787c478bd9Sstevel@tonic-gate 	 */
479dc32d872SJosef 'Jeff' Sipek 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
4807c478bd9Sstevel@tonic-gate 
4817c478bd9Sstevel@tonic-gate 	newsdp = sdp_alloc();
4827c478bd9Sstevel@tonic-gate 
4837c478bd9Sstevel@tonic-gate 	newseg->s_ops = seg->s_ops;
4847c478bd9Sstevel@tonic-gate 	newseg->s_data = (void *)newsdp;
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate 	VN_HOLD(sdp->vp);
4877c478bd9Sstevel@tonic-gate 	newsdp->vp 	= sdp->vp;
4887c478bd9Sstevel@tonic-gate 	newsdp->mapfunc = sdp->mapfunc;
4897c478bd9Sstevel@tonic-gate 	newsdp->offset	= sdp->offset;
4907c478bd9Sstevel@tonic-gate 	newsdp->pageprot = sdp->pageprot;
4917c478bd9Sstevel@tonic-gate 	newsdp->prot	= sdp->prot;
4927c478bd9Sstevel@tonic-gate 	newsdp->maxprot = sdp->maxprot;
4937c478bd9Sstevel@tonic-gate 	newsdp->type = sdp->type;
4947c478bd9Sstevel@tonic-gate 	newsdp->hat_attr = sdp->hat_attr;
4957c478bd9Sstevel@tonic-gate 	newsdp->hat_flags = sdp->hat_flags;
4967c478bd9Sstevel@tonic-gate 	newsdp->softlockcnt = 0;
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate 	/*
4997c478bd9Sstevel@tonic-gate 	 * Initialize per page data if the segment we are
5007c478bd9Sstevel@tonic-gate 	 * dup'ing has per page information.
5017c478bd9Sstevel@tonic-gate 	 */
5027c478bd9Sstevel@tonic-gate 	npages = seg_pages(newseg);
5037c478bd9Sstevel@tonic-gate 
5047c478bd9Sstevel@tonic-gate 	if (sdp->vpage != NULL) {
5057c478bd9Sstevel@tonic-gate 		size_t nbytes = vpgtob(npages);
5067c478bd9Sstevel@tonic-gate 
5077c478bd9Sstevel@tonic-gate 		newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
5087c478bd9Sstevel@tonic-gate 		bcopy(sdp->vpage, newsdp->vpage, nbytes);
5097c478bd9Sstevel@tonic-gate 	} else
5107c478bd9Sstevel@tonic-gate 		newsdp->vpage = NULL;
5117c478bd9Sstevel@tonic-gate 
5127c478bd9Sstevel@tonic-gate 	/*
5137c478bd9Sstevel@tonic-gate 	 * duplicate devmap handles
5147c478bd9Sstevel@tonic-gate 	 */
5157c478bd9Sstevel@tonic-gate 	if (dhp != NULL) {
5167c478bd9Sstevel@tonic-gate 		ret = devmap_handle_dup(dhp,
51744374aaeSsvemuri 		    (devmap_handle_t **)&newsdp->devmap_data, newseg);
5187c478bd9Sstevel@tonic-gate 		if (ret != 0) {
5197c478bd9Sstevel@tonic-gate 			TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
5207c478bd9Sstevel@tonic-gate 			    "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
5217c478bd9Sstevel@tonic-gate 			    ret, (void *)dhp, (void *)seg);
5227c478bd9Sstevel@tonic-gate 			DEBUGF(1, (CE_CONT,
5237c478bd9Sstevel@tonic-gate 			    "segdev_dup: ret %x dhp %p seg %p\n",
5247c478bd9Sstevel@tonic-gate 			    ret, (void *)dhp, (void *)seg));
5257c478bd9Sstevel@tonic-gate 			return (ret);
5267c478bd9Sstevel@tonic-gate 		}
5277c478bd9Sstevel@tonic-gate 	}
5287c478bd9Sstevel@tonic-gate 
5297c478bd9Sstevel@tonic-gate 	/*
5307c478bd9Sstevel@tonic-gate 	 * Inform the common vnode of the new mapping.
5317c478bd9Sstevel@tonic-gate 	 */
5327c478bd9Sstevel@tonic-gate 	return (VOP_ADDMAP(VTOCVP(newsdp->vp),
53344374aaeSsvemuri 	    newsdp->offset, newseg->s_as,
53444374aaeSsvemuri 	    newseg->s_base, newseg->s_size, newsdp->prot,
53544374aaeSsvemuri 	    newsdp->maxprot, sdp->type, CRED(), NULL));
5367c478bd9Sstevel@tonic-gate }
5377c478bd9Sstevel@tonic-gate 
5387c478bd9Sstevel@tonic-gate /*
5397c478bd9Sstevel@tonic-gate  * duplicate devmap handles
5407c478bd9Sstevel@tonic-gate  */
5417c478bd9Sstevel@tonic-gate static int
devmap_handle_dup(devmap_handle_t * dhp,devmap_handle_t ** new_dhp,struct seg * newseg)5427c478bd9Sstevel@tonic-gate devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
5437c478bd9Sstevel@tonic-gate     struct seg *newseg)
5447c478bd9Sstevel@tonic-gate {
5457c478bd9Sstevel@tonic-gate 	devmap_handle_t *newdhp_save = NULL;
5467c478bd9Sstevel@tonic-gate 	devmap_handle_t *newdhp = NULL;
5477c478bd9Sstevel@tonic-gate 	struct devmap_callback_ctl *callbackops;
5487c478bd9Sstevel@tonic-gate 
5497c478bd9Sstevel@tonic-gate 	while (dhp != NULL) {
5507c478bd9Sstevel@tonic-gate 		newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
5517c478bd9Sstevel@tonic-gate 
5527c478bd9Sstevel@tonic-gate 		/* Need to lock the original dhp while copying if REMAP */
5537c478bd9Sstevel@tonic-gate 		HOLD_DHP_LOCK(dhp);
5547c478bd9Sstevel@tonic-gate 		bcopy(dhp, newdhp, sizeof (devmap_handle_t));
5557c478bd9Sstevel@tonic-gate 		RELE_DHP_LOCK(dhp);
5567c478bd9Sstevel@tonic-gate 		newdhp->dh_seg = newseg;
5577c478bd9Sstevel@tonic-gate 		newdhp->dh_next = NULL;
5587c478bd9Sstevel@tonic-gate 		if (newdhp_save != NULL)
5597c478bd9Sstevel@tonic-gate 			newdhp_save->dh_next = newdhp;
5607c478bd9Sstevel@tonic-gate 		else
5617c478bd9Sstevel@tonic-gate 			*new_dhp = newdhp;
5627c478bd9Sstevel@tonic-gate 		newdhp_save = newdhp;
5637c478bd9Sstevel@tonic-gate 
5647c478bd9Sstevel@tonic-gate 		callbackops = &newdhp->dh_callbackops;
5657c478bd9Sstevel@tonic-gate 
5667c478bd9Sstevel@tonic-gate 		if (dhp->dh_softlock != NULL)
5677c478bd9Sstevel@tonic-gate 			newdhp->dh_softlock = devmap_softlock_init(
5687c478bd9Sstevel@tonic-gate 			    newdhp->dh_dev,
5697c478bd9Sstevel@tonic-gate 			    (ulong_t)callbackops->devmap_access);
5707c478bd9Sstevel@tonic-gate 		if (dhp->dh_ctx != NULL)
5717c478bd9Sstevel@tonic-gate 			newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
5727c478bd9Sstevel@tonic-gate 			    (ulong_t)callbackops->devmap_access);
5737c478bd9Sstevel@tonic-gate 
5747c478bd9Sstevel@tonic-gate 		/*
5757c478bd9Sstevel@tonic-gate 		 * Initialize dh_lock if we want to do remap.
5767c478bd9Sstevel@tonic-gate 		 */
5777c478bd9Sstevel@tonic-gate 		if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
5787c478bd9Sstevel@tonic-gate 			mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
5797c478bd9Sstevel@tonic-gate 			newdhp->dh_flags |= DEVMAP_LOCK_INITED;
5807c478bd9Sstevel@tonic-gate 		}
5817c478bd9Sstevel@tonic-gate 
5827c478bd9Sstevel@tonic-gate 		if (callbackops->devmap_dup != NULL) {
5837c478bd9Sstevel@tonic-gate 			int ret;
5847c478bd9Sstevel@tonic-gate 
5857c478bd9Sstevel@tonic-gate 			/*
5867c478bd9Sstevel@tonic-gate 			 * Call the dup callback so that the driver can
5877c478bd9Sstevel@tonic-gate 			 * duplicate its private data.
5887c478bd9Sstevel@tonic-gate 			 */
5897c478bd9Sstevel@tonic-gate 			ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
59044374aaeSsvemuri 			    (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
5917c478bd9Sstevel@tonic-gate 
5927c478bd9Sstevel@tonic-gate 			if (ret != 0) {
5937c478bd9Sstevel@tonic-gate 				/*
5947c478bd9Sstevel@tonic-gate 				 * We want to free up this segment as the driver
5957c478bd9Sstevel@tonic-gate 				 * has indicated that we can't dup it.  But we
5967c478bd9Sstevel@tonic-gate 				 * don't want to call the drivers, devmap_unmap,
5977c478bd9Sstevel@tonic-gate 				 * callback function as the driver does not
5987c478bd9Sstevel@tonic-gate 				 * think this segment exists. The caller of
5997c478bd9Sstevel@tonic-gate 				 * devmap_dup will call seg_free on newseg
6007c478bd9Sstevel@tonic-gate 				 * as it was the caller that allocated the
6017c478bd9Sstevel@tonic-gate 				 * segment.
6027c478bd9Sstevel@tonic-gate 				 */
6037c478bd9Sstevel@tonic-gate 				DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
6047c478bd9Sstevel@tonic-gate 				    "newdhp %p dhp %p\n", (void *)newdhp,
6057c478bd9Sstevel@tonic-gate 				    (void *)dhp));
6067c478bd9Sstevel@tonic-gate 				callbackops->devmap_unmap = NULL;
6077c478bd9Sstevel@tonic-gate 				return (ret);
6087c478bd9Sstevel@tonic-gate 			}
6097c478bd9Sstevel@tonic-gate 		}
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate 		dhp = dhp->dh_next;
6127c478bd9Sstevel@tonic-gate 	}
6137c478bd9Sstevel@tonic-gate 
6147c478bd9Sstevel@tonic-gate 	return (0);
6157c478bd9Sstevel@tonic-gate }
6167c478bd9Sstevel@tonic-gate 
6177c478bd9Sstevel@tonic-gate /*
6187c478bd9Sstevel@tonic-gate  * Split a segment at addr for length len.
6197c478bd9Sstevel@tonic-gate  */
6207c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6217c478bd9Sstevel@tonic-gate static int
segdev_unmap(struct seg * seg,caddr_t addr,size_t len)6227c478bd9Sstevel@tonic-gate segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
6237c478bd9Sstevel@tonic-gate {
6247c478bd9Sstevel@tonic-gate 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
6257c478bd9Sstevel@tonic-gate 	register struct segdev_data *nsdp;
6267c478bd9Sstevel@tonic-gate 	register struct seg *nseg;
6277c478bd9Sstevel@tonic-gate 	register size_t	opages;		/* old segment size in pages */
6287c478bd9Sstevel@tonic-gate 	register size_t	npages;		/* new segment size in pages */
6297c478bd9Sstevel@tonic-gate 	register size_t	dpages;		/* pages being deleted (unmapped) */
6307c478bd9Sstevel@tonic-gate 	register size_t	nbytes;
6317c478bd9Sstevel@tonic-gate 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
6327c478bd9Sstevel@tonic-gate 	devmap_handle_t *dhpp;
6337c478bd9Sstevel@tonic-gate 	devmap_handle_t *newdhp;
6347c478bd9Sstevel@tonic-gate 	struct devmap_callback_ctl *callbackops;
6357c478bd9Sstevel@tonic-gate 	caddr_t nbase;
6367c478bd9Sstevel@tonic-gate 	offset_t off;
6377c478bd9Sstevel@tonic-gate 	ulong_t nsize;
6387c478bd9Sstevel@tonic-gate 	size_t mlen, sz;
6397c478bd9Sstevel@tonic-gate 
6407c478bd9Sstevel@tonic-gate 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
6417c478bd9Sstevel@tonic-gate 	    "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
6427c478bd9Sstevel@tonic-gate 	    (void *)dhp, (void *)seg, (void *)addr, len);
6437c478bd9Sstevel@tonic-gate 
6447c478bd9Sstevel@tonic-gate 	DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
6457c478bd9Sstevel@tonic-gate 	    (void *)dhp, (void *)seg, (void *)addr, len));
6467c478bd9Sstevel@tonic-gate 
6477c478bd9Sstevel@tonic-gate 	/*
6487c478bd9Sstevel@tonic-gate 	 * Since the address space is "write" locked, we
6497c478bd9Sstevel@tonic-gate 	 * don't need the segment lock to protect "segdev" data.
6507c478bd9Sstevel@tonic-gate 	 */
651dc32d872SJosef 'Jeff' Sipek 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
6527c478bd9Sstevel@tonic-gate 
6537c478bd9Sstevel@tonic-gate 	if ((sz = sdp->softlockcnt) > 0) {
6547c478bd9Sstevel@tonic-gate 		/*
6557c478bd9Sstevel@tonic-gate 		 * Fail the unmap if pages are SOFTLOCKed through this mapping.
6567c478bd9Sstevel@tonic-gate 		 * softlockcnt is protected from change by the as write lock.
6577c478bd9Sstevel@tonic-gate 		 */
6587c478bd9Sstevel@tonic-gate 		TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
6597c478bd9Sstevel@tonic-gate 		    "segdev_unmap:error softlockcnt = %ld", sz);
6607c478bd9Sstevel@tonic-gate 		DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
6617c478bd9Sstevel@tonic-gate 		return (EAGAIN);
6627c478bd9Sstevel@tonic-gate 	}
6637c478bd9Sstevel@tonic-gate 
6647c478bd9Sstevel@tonic-gate 	/*
6657c478bd9Sstevel@tonic-gate 	 * Check for bad sizes
6667c478bd9Sstevel@tonic-gate 	 */
6677c478bd9Sstevel@tonic-gate 	if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
6687c478bd9Sstevel@tonic-gate 	    (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
6697c478bd9Sstevel@tonic-gate 		panic("segdev_unmap");
6707c478bd9Sstevel@tonic-gate 
6717c478bd9Sstevel@tonic-gate 	if (dhp != NULL) {
6727c478bd9Sstevel@tonic-gate 		devmap_handle_t *tdhp;
6737c478bd9Sstevel@tonic-gate 		/*
6747c478bd9Sstevel@tonic-gate 		 * If large page size was used in hat_devload(),
6757c478bd9Sstevel@tonic-gate 		 * the same page size must be used in hat_unload().
6767c478bd9Sstevel@tonic-gate 		 */
6777c478bd9Sstevel@tonic-gate 		dhpp = tdhp = devmap_find_handle(dhp, addr);
6787c478bd9Sstevel@tonic-gate 		while (tdhp != NULL) {
6797c478bd9Sstevel@tonic-gate 			if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
6807c478bd9Sstevel@tonic-gate 				break;
6817c478bd9Sstevel@tonic-gate 			}
6827c478bd9Sstevel@tonic-gate 			tdhp = tdhp->dh_next;
6837c478bd9Sstevel@tonic-gate 		}
6847c478bd9Sstevel@tonic-gate 		if (tdhp != NULL) {	/* found a dhp using large pages */
6857c478bd9Sstevel@tonic-gate 			size_t slen = len;
6867c478bd9Sstevel@tonic-gate 			size_t mlen;
6877c478bd9Sstevel@tonic-gate 			size_t soff;
6887c478bd9Sstevel@tonic-gate 
6897c478bd9Sstevel@tonic-gate 			soff = (ulong_t)(addr - dhpp->dh_uvaddr);
6907c478bd9Sstevel@tonic-gate 			while (slen != 0) {
6917c478bd9Sstevel@tonic-gate 				mlen = MIN(slen, (dhpp->dh_len - soff));
6927c478bd9Sstevel@tonic-gate 				hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
69344374aaeSsvemuri 				    dhpp->dh_len, HAT_UNLOAD_UNMAP);
6947c478bd9Sstevel@tonic-gate 				dhpp = dhpp->dh_next;
6957c478bd9Sstevel@tonic-gate 				ASSERT(slen >= mlen);
6967c478bd9Sstevel@tonic-gate 				slen -= mlen;
6977c478bd9Sstevel@tonic-gate 				soff = 0;
6987c478bd9Sstevel@tonic-gate 			}
6997c478bd9Sstevel@tonic-gate 		} else
7007c478bd9Sstevel@tonic-gate 			hat_unload(seg->s_as->a_hat, addr, len,
70144374aaeSsvemuri 			    HAT_UNLOAD_UNMAP);
7027c478bd9Sstevel@tonic-gate 	} else {
7037c478bd9Sstevel@tonic-gate 		/*
7047c478bd9Sstevel@tonic-gate 		 * Unload any hardware translations in the range
7057c478bd9Sstevel@tonic-gate 		 * to be taken out.
7067c478bd9Sstevel@tonic-gate 		 */
7077c478bd9Sstevel@tonic-gate 		hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
7087c478bd9Sstevel@tonic-gate 	}
7097c478bd9Sstevel@tonic-gate 
7107c478bd9Sstevel@tonic-gate 	/*
7117c478bd9Sstevel@tonic-gate 	 * get the user offset which will used in the driver callbacks
7127c478bd9Sstevel@tonic-gate 	 */
7137c478bd9Sstevel@tonic-gate 	off = sdp->offset + (offset_t)(addr - seg->s_base);
7147c478bd9Sstevel@tonic-gate 
7157c478bd9Sstevel@tonic-gate 	/*
7167c478bd9Sstevel@tonic-gate 	 * Inform the vnode of the unmapping.
7177c478bd9Sstevel@tonic-gate 	 */
7187c478bd9Sstevel@tonic-gate 	ASSERT(sdp->vp != NULL);
7197c478bd9Sstevel@tonic-gate 	(void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
72044374aaeSsvemuri 	    sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
7217c478bd9Sstevel@tonic-gate 
7227c478bd9Sstevel@tonic-gate 	/*
7237c478bd9Sstevel@tonic-gate 	 * Check for entire segment
7247c478bd9Sstevel@tonic-gate 	 */
7257c478bd9Sstevel@tonic-gate 	if (addr == seg->s_base && len == seg->s_size) {
7267c478bd9Sstevel@tonic-gate 		seg_free(seg);
7277c478bd9Sstevel@tonic-gate 		return (0);
7287c478bd9Sstevel@tonic-gate 	}
7297c478bd9Sstevel@tonic-gate 
7307c478bd9Sstevel@tonic-gate 	opages = seg_pages(seg);
7317c478bd9Sstevel@tonic-gate 	dpages = btop(len);
7327c478bd9Sstevel@tonic-gate 	npages = opages - dpages;
7337c478bd9Sstevel@tonic-gate 
7347c478bd9Sstevel@tonic-gate 	/*
7357c478bd9Sstevel@tonic-gate 	 * Check for beginning of segment
7367c478bd9Sstevel@tonic-gate 	 */
7377c478bd9Sstevel@tonic-gate 	if (addr == seg->s_base) {
7387c478bd9Sstevel@tonic-gate 		if (sdp->vpage != NULL) {
7397c478bd9Sstevel@tonic-gate 			register struct vpage *ovpage;
7407c478bd9Sstevel@tonic-gate 
7417c478bd9Sstevel@tonic-gate 			ovpage = sdp->vpage;	/* keep pointer to vpage */
7427c478bd9Sstevel@tonic-gate 
7437c478bd9Sstevel@tonic-gate 			nbytes = vpgtob(npages);
7447c478bd9Sstevel@tonic-gate 			sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
7457c478bd9Sstevel@tonic-gate 			bcopy(&ovpage[dpages], sdp->vpage, nbytes);
7467c478bd9Sstevel@tonic-gate 
7477c478bd9Sstevel@tonic-gate 			/* free up old vpage */
7487c478bd9Sstevel@tonic-gate 			kmem_free(ovpage, vpgtob(opages));
7497c478bd9Sstevel@tonic-gate 		}
7507c478bd9Sstevel@tonic-gate 
7517c478bd9Sstevel@tonic-gate 		/*
7527c478bd9Sstevel@tonic-gate 		 * free devmap handles from the beginning of the mapping.
7537c478bd9Sstevel@tonic-gate 		 */
7547c478bd9Sstevel@tonic-gate 		if (dhp != NULL)
7557c478bd9Sstevel@tonic-gate 			devmap_handle_unmap_head(dhp, len);
7567c478bd9Sstevel@tonic-gate 
7577c478bd9Sstevel@tonic-gate 		sdp->offset += (offset_t)len;
7587c478bd9Sstevel@tonic-gate 
7597c478bd9Sstevel@tonic-gate 		seg->s_base += len;
7607c478bd9Sstevel@tonic-gate 		seg->s_size -= len;
7617c478bd9Sstevel@tonic-gate 
7627c478bd9Sstevel@tonic-gate 		return (0);
7637c478bd9Sstevel@tonic-gate 	}
7647c478bd9Sstevel@tonic-gate 
7657c478bd9Sstevel@tonic-gate 	/*
7667c478bd9Sstevel@tonic-gate 	 * Check for end of segment
7677c478bd9Sstevel@tonic-gate 	 */
7687c478bd9Sstevel@tonic-gate 	if (addr + len == seg->s_base + seg->s_size) {
7697c478bd9Sstevel@tonic-gate 		if (sdp->vpage != NULL) {
7707c478bd9Sstevel@tonic-gate 			register struct vpage *ovpage;
7717c478bd9Sstevel@tonic-gate 
7727c478bd9Sstevel@tonic-gate 			ovpage = sdp->vpage;	/* keep pointer to vpage */
7737c478bd9Sstevel@tonic-gate 
7747c478bd9Sstevel@tonic-gate 			nbytes = vpgtob(npages);
7757c478bd9Sstevel@tonic-gate 			sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
7767c478bd9Sstevel@tonic-gate 			bcopy(ovpage, sdp->vpage, nbytes);
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate 			/* free up old vpage */
7797c478bd9Sstevel@tonic-gate 			kmem_free(ovpage, vpgtob(opages));
7807c478bd9Sstevel@tonic-gate 		}
7817c478bd9Sstevel@tonic-gate 		seg->s_size -= len;
7827c478bd9Sstevel@tonic-gate 
7837c478bd9Sstevel@tonic-gate 		/*
7847c478bd9Sstevel@tonic-gate 		 * free devmap handles from addr to the end of the mapping.
7857c478bd9Sstevel@tonic-gate 		 */
7867c478bd9Sstevel@tonic-gate 		if (dhp != NULL)
7877c478bd9Sstevel@tonic-gate 			devmap_handle_unmap_tail(dhp, addr);
7887c478bd9Sstevel@tonic-gate 
7897c478bd9Sstevel@tonic-gate 		return (0);
7907c478bd9Sstevel@tonic-gate 	}
7917c478bd9Sstevel@tonic-gate 
7927c478bd9Sstevel@tonic-gate 	/*
7937c478bd9Sstevel@tonic-gate 	 * The section to go is in the middle of the segment,
7947c478bd9Sstevel@tonic-gate 	 * have to make it into two segments.  nseg is made for
7957c478bd9Sstevel@tonic-gate 	 * the high end while seg is cut down at the low end.
7967c478bd9Sstevel@tonic-gate 	 */
7977c478bd9Sstevel@tonic-gate 	nbase = addr + len;				/* new seg base */
7987c478bd9Sstevel@tonic-gate 	nsize = (seg->s_base + seg->s_size) - nbase;	/* new seg size */
7997c478bd9Sstevel@tonic-gate 	seg->s_size = addr - seg->s_base;		/* shrink old seg */
8007c478bd9Sstevel@tonic-gate 	nseg = seg_alloc(seg->s_as, nbase, nsize);
8017c478bd9Sstevel@tonic-gate 	if (nseg == NULL)
8027c478bd9Sstevel@tonic-gate 		panic("segdev_unmap seg_alloc");
8037c478bd9Sstevel@tonic-gate 
8047c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
8057c478bd9Sstevel@tonic-gate 	    "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
8067c478bd9Sstevel@tonic-gate 	DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
8077c478bd9Sstevel@tonic-gate 	    (void *)seg, (void *)nseg));
8087c478bd9Sstevel@tonic-gate 	nsdp = sdp_alloc();
8097c478bd9Sstevel@tonic-gate 
8107c478bd9Sstevel@tonic-gate 	nseg->s_ops = seg->s_ops;
8117c478bd9Sstevel@tonic-gate 	nseg->s_data = (void *)nsdp;
8127c478bd9Sstevel@tonic-gate 
8137c478bd9Sstevel@tonic-gate 	VN_HOLD(sdp->vp);
8147c478bd9Sstevel@tonic-gate 	nsdp->mapfunc = sdp->mapfunc;
8157c478bd9Sstevel@tonic-gate 	nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
8167c478bd9Sstevel@tonic-gate 	nsdp->vp 	= sdp->vp;
8177c478bd9Sstevel@tonic-gate 	nsdp->pageprot = sdp->pageprot;
8187c478bd9Sstevel@tonic-gate 	nsdp->prot	= sdp->prot;
8197c478bd9Sstevel@tonic-gate 	nsdp->maxprot = sdp->maxprot;
8207c478bd9Sstevel@tonic-gate 	nsdp->type = sdp->type;
8217c478bd9Sstevel@tonic-gate 	nsdp->hat_attr = sdp->hat_attr;
8227c478bd9Sstevel@tonic-gate 	nsdp->hat_flags = sdp->hat_flags;
8237c478bd9Sstevel@tonic-gate 	nsdp->softlockcnt = 0;
8247c478bd9Sstevel@tonic-gate 
8257c478bd9Sstevel@tonic-gate 	/*
8267c478bd9Sstevel@tonic-gate 	 * Initialize per page data if the segment we are
8277c478bd9Sstevel@tonic-gate 	 * dup'ing has per page information.
8287c478bd9Sstevel@tonic-gate 	 */
8297c478bd9Sstevel@tonic-gate 	if (sdp->vpage != NULL) {
8307c478bd9Sstevel@tonic-gate 		/* need to split vpage into two arrays */
8317c478bd9Sstevel@tonic-gate 		register size_t nnbytes;
8327c478bd9Sstevel@tonic-gate 		register size_t nnpages;
8337c478bd9Sstevel@tonic-gate 		register struct vpage *ovpage;
8347c478bd9Sstevel@tonic-gate 
8357c478bd9Sstevel@tonic-gate 		ovpage = sdp->vpage;		/* keep pointer to vpage */
8367c478bd9Sstevel@tonic-gate 
8377c478bd9Sstevel@tonic-gate 		npages = seg_pages(seg);	/* seg has shrunk */
8387c478bd9Sstevel@tonic-gate 		nbytes = vpgtob(npages);
8397c478bd9Sstevel@tonic-gate 		nnpages = seg_pages(nseg);
8407c478bd9Sstevel@tonic-gate 		nnbytes = vpgtob(nnpages);
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate 		sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
8437c478bd9Sstevel@tonic-gate 		bcopy(ovpage, sdp->vpage, nbytes);
8447c478bd9Sstevel@tonic-gate 
8457c478bd9Sstevel@tonic-gate 		nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
8467c478bd9Sstevel@tonic-gate 		bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
8477c478bd9Sstevel@tonic-gate 
8487c478bd9Sstevel@tonic-gate 		/* free up old vpage */
8497c478bd9Sstevel@tonic-gate 		kmem_free(ovpage, vpgtob(opages));
8507c478bd9Sstevel@tonic-gate 	} else
8517c478bd9Sstevel@tonic-gate 		nsdp->vpage = NULL;
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 	/*
8547c478bd9Sstevel@tonic-gate 	 * unmap dhps.
8557c478bd9Sstevel@tonic-gate 	 */
8567c478bd9Sstevel@tonic-gate 	if (dhp == NULL) {
8577c478bd9Sstevel@tonic-gate 		nsdp->devmap_data = NULL;
8587c478bd9Sstevel@tonic-gate 		return (0);
8597c478bd9Sstevel@tonic-gate 	}
8607c478bd9Sstevel@tonic-gate 	while (dhp != NULL) {
8617c478bd9Sstevel@tonic-gate 		callbackops = &dhp->dh_callbackops;
8627c478bd9Sstevel@tonic-gate 		TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
8637c478bd9Sstevel@tonic-gate 		    "segdev_unmap: dhp=%p addr=%p", dhp, addr);
8647c478bd9Sstevel@tonic-gate 		DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
8657c478bd9Sstevel@tonic-gate 		    (void *)dhp, (void *)addr,
8667c478bd9Sstevel@tonic-gate 		    (void *)dhp->dh_uvaddr, dhp->dh_len));
8677c478bd9Sstevel@tonic-gate 
8687c478bd9Sstevel@tonic-gate 		if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
8697c478bd9Sstevel@tonic-gate 			dhpp = dhp->dh_next;
8707c478bd9Sstevel@tonic-gate 			dhp->dh_next = NULL;
8717c478bd9Sstevel@tonic-gate 			dhp = dhpp;
8727c478bd9Sstevel@tonic-gate 		} else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
8737c478bd9Sstevel@tonic-gate 			dhp = dhp->dh_next;
8747c478bd9Sstevel@tonic-gate 		} else if (addr > dhp->dh_uvaddr &&
87544374aaeSsvemuri 		    (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
8767c478bd9Sstevel@tonic-gate 			/*
8777c478bd9Sstevel@tonic-gate 			 * <addr, addr+len> is enclosed by dhp.
8787c478bd9Sstevel@tonic-gate 			 * create a newdhp that begins at addr+len and
8797c478bd9Sstevel@tonic-gate 			 * ends at dhp->dh_uvaddr+dhp->dh_len.
8807c478bd9Sstevel@tonic-gate 			 */
8817c478bd9Sstevel@tonic-gate 			newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
8827c478bd9Sstevel@tonic-gate 			HOLD_DHP_LOCK(dhp);
8837c478bd9Sstevel@tonic-gate 			bcopy(dhp, newdhp, sizeof (devmap_handle_t));
8847c478bd9Sstevel@tonic-gate 			RELE_DHP_LOCK(dhp);
8857c478bd9Sstevel@tonic-gate 			newdhp->dh_seg = nseg;
8867c478bd9Sstevel@tonic-gate 			newdhp->dh_next = dhp->dh_next;
8877c478bd9Sstevel@tonic-gate 			if (dhp->dh_softlock != NULL)
8887c478bd9Sstevel@tonic-gate 				newdhp->dh_softlock = devmap_softlock_init(
88944374aaeSsvemuri 				    newdhp->dh_dev,
89044374aaeSsvemuri 				    (ulong_t)callbackops->devmap_access);
8917c478bd9Sstevel@tonic-gate 			if (dhp->dh_ctx != NULL)
8927c478bd9Sstevel@tonic-gate 				newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
89344374aaeSsvemuri 				    (ulong_t)callbackops->devmap_access);
8947c478bd9Sstevel@tonic-gate 			if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
8957c478bd9Sstevel@tonic-gate 				mutex_init(&newdhp->dh_lock,
8967c478bd9Sstevel@tonic-gate 				    NULL, MUTEX_DEFAULT, NULL);
8977c478bd9Sstevel@tonic-gate 			}
8987c478bd9Sstevel@tonic-gate 			if (callbackops->devmap_unmap != NULL)
8997c478bd9Sstevel@tonic-gate 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
90044374aaeSsvemuri 				    off, len, dhp, &dhp->dh_pvtp,
90144374aaeSsvemuri 				    newdhp, &newdhp->dh_pvtp);
9027c478bd9Sstevel@tonic-gate 			mlen = len + (addr - dhp->dh_uvaddr);
9037c478bd9Sstevel@tonic-gate 			devmap_handle_reduce_len(newdhp, mlen);
9047c478bd9Sstevel@tonic-gate 			nsdp->devmap_data = newdhp;
9057c478bd9Sstevel@tonic-gate 			/* XX Changing len should recalculate LARGE flag */
9067c478bd9Sstevel@tonic-gate 			dhp->dh_len = addr - dhp->dh_uvaddr;
9077c478bd9Sstevel@tonic-gate 			dhpp = dhp->dh_next;
9087c478bd9Sstevel@tonic-gate 			dhp->dh_next = NULL;
9097c478bd9Sstevel@tonic-gate 			dhp = dhpp;
9107c478bd9Sstevel@tonic-gate 		} else if ((addr > dhp->dh_uvaddr) &&
91144374aaeSsvemuri 		    ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
9127c478bd9Sstevel@tonic-gate 			mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
9137c478bd9Sstevel@tonic-gate 			/*
9147c478bd9Sstevel@tonic-gate 			 * <addr, addr+len> spans over dhps.
9157c478bd9Sstevel@tonic-gate 			 */
9167c478bd9Sstevel@tonic-gate 			if (callbackops->devmap_unmap != NULL)
9177c478bd9Sstevel@tonic-gate 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
91844374aaeSsvemuri 				    off, mlen, (devmap_cookie_t *)dhp,
91944374aaeSsvemuri 				    &dhp->dh_pvtp, NULL, NULL);
9207c478bd9Sstevel@tonic-gate 			/* XX Changing len should recalculate LARGE flag */
9217c478bd9Sstevel@tonic-gate 			dhp->dh_len = addr - dhp->dh_uvaddr;
9227c478bd9Sstevel@tonic-gate 			dhpp = dhp->dh_next;
9237c478bd9Sstevel@tonic-gate 			dhp->dh_next = NULL;
9247c478bd9Sstevel@tonic-gate 			dhp = dhpp;
9257c478bd9Sstevel@tonic-gate 			nsdp->devmap_data = dhp;
9267c478bd9Sstevel@tonic-gate 		} else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
9277c478bd9Sstevel@tonic-gate 			/*
9287c478bd9Sstevel@tonic-gate 			 * dhp is enclosed by <addr, addr+len>.
9297c478bd9Sstevel@tonic-gate 			 */
9307c478bd9Sstevel@tonic-gate 			dhp->dh_seg = nseg;
9317c478bd9Sstevel@tonic-gate 			nsdp->devmap_data = dhp;
9327c478bd9Sstevel@tonic-gate 			dhp = devmap_handle_unmap(dhp);
9337c478bd9Sstevel@tonic-gate 			nsdp->devmap_data = dhp; /* XX redundant? */
9347c478bd9Sstevel@tonic-gate 		} else if (((addr + len) > dhp->dh_uvaddr) &&
93544374aaeSsvemuri 		    ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
9367c478bd9Sstevel@tonic-gate 			mlen = addr + len - dhp->dh_uvaddr;
9377c478bd9Sstevel@tonic-gate 			if (callbackops->devmap_unmap != NULL)
9387c478bd9Sstevel@tonic-gate 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
93944374aaeSsvemuri 				    dhp->dh_uoff, mlen, NULL,
94044374aaeSsvemuri 				    NULL, dhp, &dhp->dh_pvtp);
9417c478bd9Sstevel@tonic-gate 			devmap_handle_reduce_len(dhp, mlen);
9427c478bd9Sstevel@tonic-gate 			nsdp->devmap_data = dhp;
9437c478bd9Sstevel@tonic-gate 			dhp->dh_seg = nseg;
9447c478bd9Sstevel@tonic-gate 			dhp = dhp->dh_next;
9457c478bd9Sstevel@tonic-gate 		} else {
9467c478bd9Sstevel@tonic-gate 			dhp->dh_seg = nseg;
9477c478bd9Sstevel@tonic-gate 			dhp = dhp->dh_next;
9487c478bd9Sstevel@tonic-gate 		}
9497c478bd9Sstevel@tonic-gate 	}
9507c478bd9Sstevel@tonic-gate 	return (0);
9517c478bd9Sstevel@tonic-gate }
9527c478bd9Sstevel@tonic-gate 
9537c478bd9Sstevel@tonic-gate /*
9547c478bd9Sstevel@tonic-gate  * Utility function handles reducing the length of a devmap handle during unmap
9557c478bd9Sstevel@tonic-gate  * Note that is only used for unmapping the front portion of the handler,
9567c478bd9Sstevel@tonic-gate  * i.e., we are bumping up the offset/pfn etc up by len
9577c478bd9Sstevel@tonic-gate  * Do not use if reducing length at the tail.
9587c478bd9Sstevel@tonic-gate  */
9597c478bd9Sstevel@tonic-gate static void
devmap_handle_reduce_len(devmap_handle_t * dhp,size_t len)9607c478bd9Sstevel@tonic-gate devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
9617c478bd9Sstevel@tonic-gate {
9627c478bd9Sstevel@tonic-gate 	struct ddi_umem_cookie *cp;
9637c478bd9Sstevel@tonic-gate 	struct devmap_pmem_cookie *pcp;
9647c478bd9Sstevel@tonic-gate 	/*
9657c478bd9Sstevel@tonic-gate 	 * adjust devmap handle fields
9667c478bd9Sstevel@tonic-gate 	 */
9677c478bd9Sstevel@tonic-gate 	ASSERT(len < dhp->dh_len);
9687c478bd9Sstevel@tonic-gate 
9697c478bd9Sstevel@tonic-gate 	/* Make sure only page-aligned changes are done */
9707c478bd9Sstevel@tonic-gate 	ASSERT((len & PAGEOFFSET) == 0);
9717c478bd9Sstevel@tonic-gate 
9727c478bd9Sstevel@tonic-gate 	dhp->dh_len -= len;
9737c478bd9Sstevel@tonic-gate 	dhp->dh_uoff += (offset_t)len;
9747c478bd9Sstevel@tonic-gate 	dhp->dh_roff += (offset_t)len;
9757c478bd9Sstevel@tonic-gate 	dhp->dh_uvaddr += len;
9767c478bd9Sstevel@tonic-gate 	/* Need to grab dhp lock if REMAP */
9777c478bd9Sstevel@tonic-gate 	HOLD_DHP_LOCK(dhp);
9787c478bd9Sstevel@tonic-gate 	cp = dhp->dh_cookie;
9797c478bd9Sstevel@tonic-gate 	if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
9807c478bd9Sstevel@tonic-gate 		if (cookie_is_devmem(cp)) {
9817c478bd9Sstevel@tonic-gate 			dhp->dh_pfn += btop(len);
9827c478bd9Sstevel@tonic-gate 		} else if (cookie_is_pmem(cp)) {
9837c478bd9Sstevel@tonic-gate 			pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
9847c478bd9Sstevel@tonic-gate 			ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
98544374aaeSsvemuri 			    dhp->dh_roff < ptob(pcp->dp_npages));
9867c478bd9Sstevel@tonic-gate 		} else {
9877c478bd9Sstevel@tonic-gate 			ASSERT(dhp->dh_roff < cp->size);
9887c478bd9Sstevel@tonic-gate 			ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
98944374aaeSsvemuri 			    dhp->dh_cvaddr < (cp->cvaddr + cp->size));
9907c478bd9Sstevel@tonic-gate 			ASSERT((dhp->dh_cvaddr + len) <=
99144374aaeSsvemuri 			    (cp->cvaddr + cp->size));
9927c478bd9Sstevel@tonic-gate 
9937c478bd9Sstevel@tonic-gate 			dhp->dh_cvaddr += len;
9947c478bd9Sstevel@tonic-gate 		}
9957c478bd9Sstevel@tonic-gate 	}
9967c478bd9Sstevel@tonic-gate 	/* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
9977c478bd9Sstevel@tonic-gate 	RELE_DHP_LOCK(dhp);
9987c478bd9Sstevel@tonic-gate }
9997c478bd9Sstevel@tonic-gate 
10007c478bd9Sstevel@tonic-gate /*
10017c478bd9Sstevel@tonic-gate  * Free devmap handle, dhp.
10027c478bd9Sstevel@tonic-gate  * Return the next devmap handle on the linked list.
10037c478bd9Sstevel@tonic-gate  */
10047c478bd9Sstevel@tonic-gate static devmap_handle_t *
devmap_handle_unmap(devmap_handle_t * dhp)10057c478bd9Sstevel@tonic-gate devmap_handle_unmap(devmap_handle_t *dhp)
10067c478bd9Sstevel@tonic-gate {
10077c478bd9Sstevel@tonic-gate 	struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
10087c478bd9Sstevel@tonic-gate 	struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
10097c478bd9Sstevel@tonic-gate 	devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
10107c478bd9Sstevel@tonic-gate 
10117c478bd9Sstevel@tonic-gate 	ASSERT(dhp != NULL);
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate 	/*
10147c478bd9Sstevel@tonic-gate 	 * before we free up dhp, call the driver's devmap_unmap entry point
10157c478bd9Sstevel@tonic-gate 	 * to free resources allocated for this dhp.
10167c478bd9Sstevel@tonic-gate 	 */
10177c478bd9Sstevel@tonic-gate 	if (callbackops->devmap_unmap != NULL) {
10187c478bd9Sstevel@tonic-gate 		(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
101944374aaeSsvemuri 		    dhp->dh_len, NULL, NULL, NULL, NULL);
10207c478bd9Sstevel@tonic-gate 	}
10217c478bd9Sstevel@tonic-gate 
10227c478bd9Sstevel@tonic-gate 	if (dhpp == dhp) {	/* releasing first dhp, change sdp data */
10237c478bd9Sstevel@tonic-gate 		sdp->devmap_data = dhp->dh_next;
10247c478bd9Sstevel@tonic-gate 	} else {
10257c478bd9Sstevel@tonic-gate 		while (dhpp->dh_next != dhp) {
10267c478bd9Sstevel@tonic-gate 			dhpp = dhpp->dh_next;
10277c478bd9Sstevel@tonic-gate 		}
10287c478bd9Sstevel@tonic-gate 		dhpp->dh_next = dhp->dh_next;
10297c478bd9Sstevel@tonic-gate 	}
10307c478bd9Sstevel@tonic-gate 	dhpp = dhp->dh_next;	/* return value is next dhp in chain */
10317c478bd9Sstevel@tonic-gate 
10327c478bd9Sstevel@tonic-gate 	if (dhp->dh_softlock != NULL)
10337c478bd9Sstevel@tonic-gate 		devmap_softlock_rele(dhp);
10347c478bd9Sstevel@tonic-gate 
10357c478bd9Sstevel@tonic-gate 	if (dhp->dh_ctx != NULL)
10367c478bd9Sstevel@tonic-gate 		devmap_ctx_rele(dhp);
10377c478bd9Sstevel@tonic-gate 
10387c478bd9Sstevel@tonic-gate 	if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
10397c478bd9Sstevel@tonic-gate 		mutex_destroy(&dhp->dh_lock);
10407c478bd9Sstevel@tonic-gate 	}
10417c478bd9Sstevel@tonic-gate 	kmem_free(dhp, sizeof (devmap_handle_t));
10427c478bd9Sstevel@tonic-gate 
10437c478bd9Sstevel@tonic-gate 	return (dhpp);
10447c478bd9Sstevel@tonic-gate }
10457c478bd9Sstevel@tonic-gate 
10467c478bd9Sstevel@tonic-gate /*
10477c478bd9Sstevel@tonic-gate  * Free complete devmap handles from dhp for len bytes
10487c478bd9Sstevel@tonic-gate  * dhp can be either the first handle or a subsequent handle
10497c478bd9Sstevel@tonic-gate  */
10507c478bd9Sstevel@tonic-gate static void
devmap_handle_unmap_head(devmap_handle_t * dhp,size_t len)10517c478bd9Sstevel@tonic-gate devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
10527c478bd9Sstevel@tonic-gate {
10537c478bd9Sstevel@tonic-gate 	struct devmap_callback_ctl *callbackops;
10547c478bd9Sstevel@tonic-gate 
10557c478bd9Sstevel@tonic-gate 	/*
10567c478bd9Sstevel@tonic-gate 	 * free the devmap handles covered by len.
10577c478bd9Sstevel@tonic-gate 	 */
10587c478bd9Sstevel@tonic-gate 	while (len >= dhp->dh_len) {
10597c478bd9Sstevel@tonic-gate 		len -= dhp->dh_len;
10607c478bd9Sstevel@tonic-gate 		dhp = devmap_handle_unmap(dhp);
10617c478bd9Sstevel@tonic-gate 	}
10627c478bd9Sstevel@tonic-gate 	if (len != 0) {	/* partial unmap at head of first remaining dhp */
10637c478bd9Sstevel@tonic-gate 		callbackops = &dhp->dh_callbackops;
10647c478bd9Sstevel@tonic-gate 
10657c478bd9Sstevel@tonic-gate 		/*
10667c478bd9Sstevel@tonic-gate 		 * Call the unmap callback so the drivers can make
10677c478bd9Sstevel@tonic-gate 		 * adjustment on its private data.
10687c478bd9Sstevel@tonic-gate 		 */
10697c478bd9Sstevel@tonic-gate 		if (callbackops->devmap_unmap != NULL)
10707c478bd9Sstevel@tonic-gate 			(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
10717c478bd9Sstevel@tonic-gate 			    dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
10727c478bd9Sstevel@tonic-gate 		devmap_handle_reduce_len(dhp, len);
10737c478bd9Sstevel@tonic-gate 	}
10747c478bd9Sstevel@tonic-gate }
10757c478bd9Sstevel@tonic-gate 
10767c478bd9Sstevel@tonic-gate /*
10777c478bd9Sstevel@tonic-gate  * Free devmap handles to truncate  the mapping after addr
10787c478bd9Sstevel@tonic-gate  * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
10797c478bd9Sstevel@tonic-gate  *	Also could then use the routine in middle unmap case too
10807c478bd9Sstevel@tonic-gate  */
10817c478bd9Sstevel@tonic-gate static void
devmap_handle_unmap_tail(devmap_handle_t * dhp,caddr_t addr)10827c478bd9Sstevel@tonic-gate devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
10837c478bd9Sstevel@tonic-gate {
10847c478bd9Sstevel@tonic-gate 	register struct seg *seg = dhp->dh_seg;
10857c478bd9Sstevel@tonic-gate 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
10867c478bd9Sstevel@tonic-gate 	register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
10877c478bd9Sstevel@tonic-gate 	struct devmap_callback_ctl *callbackops;
10887c478bd9Sstevel@tonic-gate 	register devmap_handle_t *dhpp;
10897c478bd9Sstevel@tonic-gate 	size_t maplen;
10907c478bd9Sstevel@tonic-gate 	ulong_t off;
10917c478bd9Sstevel@tonic-gate 	size_t len;
10927c478bd9Sstevel@tonic-gate 
10937c478bd9Sstevel@tonic-gate 	maplen = (size_t)(addr - dhp->dh_uvaddr);
10947c478bd9Sstevel@tonic-gate 	d