xref: /illumos-gate/usr/src/uts/common/os/bp_map.c (revision 584b574a)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
57a1b360bScth  * Common Development and Distribution License (the "License").
67a1b360bScth  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2236945f79Smrj  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #include <sys/types.h>
277c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
287c478bd9Sstevel@tonic-gate #include <sys/systm.h>
297c478bd9Sstevel@tonic-gate #include <sys/mman.h>
307c478bd9Sstevel@tonic-gate #include <sys/buf.h>
317c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
337c478bd9Sstevel@tonic-gate #include <sys/debug.h>
347c478bd9Sstevel@tonic-gate #include <sys/machparam.h>
357c478bd9Sstevel@tonic-gate #include <vm/page.h>
367c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
372d3bf78eScth #include <vm/seg_kpm.h>
387c478bd9Sstevel@tonic-gate 
397c478bd9Sstevel@tonic-gate #ifdef __sparc
407c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h>
417c478bd9Sstevel@tonic-gate #define	BP_FLUSH(addr, size)	flush_instr_mem((void *)addr, size);
427c478bd9Sstevel@tonic-gate #else
437c478bd9Sstevel@tonic-gate #define	BP_FLUSH(addr, size)
447c478bd9Sstevel@tonic-gate #endif
457c478bd9Sstevel@tonic-gate 
4636945f79Smrj int bp_force_copy = 0;
4736945f79Smrj typedef enum {
4836945f79Smrj 	BP_COPYIN	= 0,
4936945f79Smrj 	BP_COPYOUT	= 1
5036945f79Smrj } bp_copydir_t;
5136945f79Smrj static int bp_copy_common(bp_copydir_t dir, struct buf *bp, void *driverbuf,
5236945f79Smrj     offset_t offset, size_t size);
5336945f79Smrj 
547c478bd9Sstevel@tonic-gate static vmem_t *bp_map_arena;
557c478bd9Sstevel@tonic-gate static size_t bp_align;
567c478bd9Sstevel@tonic-gate static uint_t bp_devload_flags = PROT_READ | PROT_WRITE | HAT_NOSYNC;
572d3bf78eScth int	bp_max_cache = 1 << 17;		/* 128K default; tunable */
582d3bf78eScth int	bp_mapin_kpm_enable = 1;	/* enable default; tunable */
597c478bd9Sstevel@tonic-gate 
607c478bd9Sstevel@tonic-gate static void *
bp_vmem_alloc(vmem_t * vmp,size_t size,int vmflag)617c478bd9Sstevel@tonic-gate bp_vmem_alloc(vmem_t *vmp, size_t size, int vmflag)
627c478bd9Sstevel@tonic-gate {
637c478bd9Sstevel@tonic-gate 	return (vmem_xalloc(vmp, size, bp_align, 0, 0, NULL, NULL, vmflag));
647c478bd9Sstevel@tonic-gate }
657c478bd9Sstevel@tonic-gate 
667c478bd9Sstevel@tonic-gate void
bp_init(size_t align,uint_t devload_flags)677c478bd9Sstevel@tonic-gate bp_init(size_t align, uint_t devload_flags)
687c478bd9Sstevel@tonic-gate {
697c478bd9Sstevel@tonic-gate 	bp_align = MAX(align, PAGESIZE);
707c478bd9Sstevel@tonic-gate 	bp_devload_flags |= devload_flags;
717c478bd9Sstevel@tonic-gate 
727c478bd9Sstevel@tonic-gate 	if (bp_align <= bp_max_cache)
737c478bd9Sstevel@tonic-gate 		bp_map_arena = vmem_create("bp_map", NULL, 0, bp_align,
747c478bd9Sstevel@tonic-gate 		    bp_vmem_alloc, vmem_free, heap_arena,
757c478bd9Sstevel@tonic-gate 		    MIN(8 * bp_align, bp_max_cache), VM_SLEEP);
767c478bd9Sstevel@tonic-gate }
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate /*
797c478bd9Sstevel@tonic-gate  * common routine so can be called with/without VM_SLEEP
807c478bd9Sstevel@tonic-gate  */
817c478bd9Sstevel@tonic-gate void *
bp_mapin_common(struct buf * bp,int flag)827c478bd9Sstevel@tonic-gate bp_mapin_common(struct buf *bp, int flag)
837c478bd9Sstevel@tonic-gate {
842d3bf78eScth 	struct as	*as;
852d3bf78eScth 	pfn_t		pfnum;
862d3bf78eScth 	page_t		*pp;
872d3bf78eScth 	page_t		**pplist;
882d3bf78eScth 	caddr_t		kaddr;
892d3bf78eScth 	caddr_t		addr;
902d3bf78eScth 	uintptr_t	off;
912d3bf78eScth 	size_t		size;
922d3bf78eScth 	pgcnt_t		npages;
932d3bf78eScth 	int		color;
947c478bd9Sstevel@tonic-gate 
95*584b574aSToomas Soome 	as = NULL;
967c478bd9Sstevel@tonic-gate 	/* return if already mapped in, no pageio/physio, or physio to kas */
977c478bd9Sstevel@tonic-gate 	if ((bp->b_flags & B_REMAPPED) ||
987c478bd9Sstevel@tonic-gate 	    !(bp->b_flags & (B_PAGEIO | B_PHYS)) ||
997c478bd9Sstevel@tonic-gate 	    (((bp->b_flags & (B_PAGEIO | B_PHYS)) == B_PHYS) &&
1007c478bd9Sstevel@tonic-gate 	    ((bp->b_proc == NULL) || (bp->b_proc->p_as == &kas))))
1017c478bd9Sstevel@tonic-gate 		return (bp->b_un.b_addr);
1027c478bd9Sstevel@tonic-gate 
1037c478bd9Sstevel@tonic-gate 	ASSERT((bp->b_flags & (B_PAGEIO | B_PHYS)) != (B_PAGEIO | B_PHYS));
1047c478bd9Sstevel@tonic-gate 
1052d3bf78eScth 	addr = (caddr_t)bp->b_un.b_addr;
1062d3bf78eScth 	off = (uintptr_t)addr & PAGEOFFSET;
1072d3bf78eScth 	size = P2ROUNDUP(bp->b_bcount + off, PAGESIZE);
1082d3bf78eScth 	npages = btop(size);
1092d3bf78eScth 
1102d3bf78eScth 	/* Fastpath single page IO to locked memory by using kpm. */
1112d3bf78eScth 	if ((bp->b_flags & (B_SHADOW | B_PAGEIO)) && (npages == 1) &&
1122d3bf78eScth 	    kpm_enable && bp_mapin_kpm_enable) {
1132d3bf78eScth 		if (bp->b_flags & B_SHADOW)
1142d3bf78eScth 			pp = *bp->b_shadow;
1152d3bf78eScth 		else
1162d3bf78eScth 			pp = bp->b_pages;
1172d3bf78eScth 		kaddr = hat_kpm_mapin(pp, NULL);
1182d3bf78eScth 		bp->b_un.b_addr = kaddr + off;
1192d3bf78eScth 		bp->b_flags |= B_REMAPPED;
1202d3bf78eScth 		return (bp->b_un.b_addr);
1212d3bf78eScth 	}
1222d3bf78eScth 
1237c478bd9Sstevel@tonic-gate 	/*
1247c478bd9Sstevel@tonic-gate 	 * Allocate kernel virtual space for remapping.
1257c478bd9Sstevel@tonic-gate 	 */
1267c478bd9Sstevel@tonic-gate 	color = bp_color(bp);
1277c478bd9Sstevel@tonic-gate 	ASSERT(color < bp_align);
1287c478bd9Sstevel@tonic-gate 
1297c478bd9Sstevel@tonic-gate 	if (bp_map_arena != NULL) {
1307c478bd9Sstevel@tonic-gate 		kaddr = (caddr_t)vmem_alloc(bp_map_arena,
1317c478bd9Sstevel@tonic-gate 		    P2ROUNDUP(color + size, bp_align), flag);
1327c478bd9Sstevel@tonic-gate 		if (kaddr == NULL)
1337c478bd9Sstevel@tonic-gate 			return (NULL);
1347c478bd9Sstevel@tonic-gate 		kaddr += color;
1357c478bd9Sstevel@tonic-gate 	} else {
1367c478bd9Sstevel@tonic-gate 		kaddr = vmem_xalloc(heap_arena, size, bp_align, color,
1377c478bd9Sstevel@tonic-gate 		    0, NULL, NULL, flag);
1387c478bd9Sstevel@tonic-gate 		if (kaddr == NULL)
1397c478bd9Sstevel@tonic-gate 			return (NULL);
1407c478bd9Sstevel@tonic-gate 	}
1417c478bd9Sstevel@tonic-gate 
1427c478bd9Sstevel@tonic-gate 	ASSERT(P2PHASE((uintptr_t)kaddr, bp_align) == color);
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate 	/*
1457c478bd9Sstevel@tonic-gate 	 * Map bp into the virtual space we just allocated.
1467c478bd9Sstevel@tonic-gate 	 */
1477c478bd9Sstevel@tonic-gate 	if (bp->b_flags & B_PAGEIO) {
1487c478bd9Sstevel@tonic-gate 		pp = bp->b_pages;
1492d3bf78eScth 		pplist = NULL;
1507c478bd9Sstevel@tonic-gate 	} else if (bp->b_flags & B_SHADOW) {
1512d3bf78eScth 		pp = NULL;
1527c478bd9Sstevel@tonic-gate 		pplist = bp->b_shadow;
1537c478bd9Sstevel@tonic-gate 	} else {
1542d3bf78eScth 		pp = NULL;
1552d3bf78eScth 		pplist = NULL;
1567c478bd9Sstevel@tonic-gate 		if (bp->b_proc == NULL || (as = bp->b_proc->p_as) == NULL)
1577c478bd9Sstevel@tonic-gate 			as = &kas;
1587c478bd9Sstevel@tonic-gate 	}
1597c478bd9Sstevel@tonic-gate 
1607c478bd9Sstevel@tonic-gate 	bp->b_flags |= B_REMAPPED;
1617c478bd9Sstevel@tonic-gate 	bp->b_un.b_addr = kaddr + off;
1627c478bd9Sstevel@tonic-gate 
1637c478bd9Sstevel@tonic-gate 	while (npages-- != 0) {
1647c478bd9Sstevel@tonic-gate 		if (pp) {
1657c478bd9Sstevel@tonic-gate 			pfnum = pp->p_pagenum;
1667c478bd9Sstevel@tonic-gate 			pp = pp->p_next;
1677c478bd9Sstevel@tonic-gate 		} else if (pplist == NULL) {
1682d3bf78eScth 			pfnum = hat_getpfnum(as->a_hat,
1692d3bf78eScth 			    (caddr_t)((uintptr_t)addr & MMU_PAGEMASK));
1702d3bf78eScth 			if (pfnum == PFN_INVALID)
1717c478bd9Sstevel@tonic-gate 				panic("bp_mapin_common: hat_getpfnum for"
1727c478bd9Sstevel@tonic-gate 				    " addr %p failed\n", (void *)addr);
1737c478bd9Sstevel@tonic-gate 			addr += PAGESIZE;
1747c478bd9Sstevel@tonic-gate 		} else {
1757c478bd9Sstevel@tonic-gate 			pfnum = (*pplist)->p_pagenum;
1767c478bd9Sstevel@tonic-gate 			pplist++;
1777c478bd9Sstevel@tonic-gate 		}
1787c478bd9Sstevel@tonic-gate 
1797c478bd9Sstevel@tonic-gate 		hat_devload(kas.a_hat, kaddr, PAGESIZE, pfnum,
1807c478bd9Sstevel@tonic-gate 		    bp_devload_flags, HAT_LOAD_LOCK);
1817c478bd9Sstevel@tonic-gate 
1827c478bd9Sstevel@tonic-gate 		kaddr += PAGESIZE;
1837c478bd9Sstevel@tonic-gate 	}
1847c478bd9Sstevel@tonic-gate 	return (bp->b_un.b_addr);
1857c478bd9Sstevel@tonic-gate }
1867c478bd9Sstevel@tonic-gate 
1877c478bd9Sstevel@tonic-gate /*
1887c478bd9Sstevel@tonic-gate  * Convert bp for pageio/physio to a kernel addressable location.
1897c478bd9Sstevel@tonic-gate  */
1907c478bd9Sstevel@tonic-gate void
bp_mapin(struct buf * bp)1917c478bd9Sstevel@tonic-gate bp_mapin(struct buf *bp)
1927c478bd9Sstevel@tonic-gate {
1937c478bd9Sstevel@tonic-gate 	(void) bp_mapin_common(bp, VM_SLEEP);
1947c478bd9Sstevel@tonic-gate }
1957c478bd9Sstevel@tonic-gate 
1967c478bd9Sstevel@tonic-gate /*
1977c478bd9Sstevel@tonic-gate  * Release all the resources associated with a previous bp_mapin() call.
1987c478bd9Sstevel@tonic-gate  */
1997c478bd9Sstevel@tonic-gate void
bp_mapout(struct buf * bp)2007c478bd9Sstevel@tonic-gate bp_mapout(struct buf *bp)
2017c478bd9Sstevel@tonic-gate {
2022d3bf78eScth 	caddr_t		addr;
2032d3bf78eScth 	uintptr_t	off;
2042d3bf78eScth 	uintptr_t	base;
2052d3bf78eScth 	uintptr_t	color;
2062d3bf78eScth 	size_t		size;
2072d3bf78eScth 	pgcnt_t		npages;
2082d3bf78eScth 	page_t		*pp;
2092d3bf78eScth 
2102d3bf78eScth 	if ((bp->b_flags & B_REMAPPED) == 0)
2112d3bf78eScth 		return;
2122d3bf78eScth 
2132d3bf78eScth 	addr = bp->b_un.b_addr;
2142d3bf78eScth 	off = (uintptr_t)addr & PAGEOFFSET;
2152d3bf78eScth 	size = P2ROUNDUP(bp->b_bcount + off, PAGESIZE);
2162d3bf78eScth 	npages = btop(size);
2172d3bf78eScth 
2182d3bf78eScth 	bp->b_un.b_addr = (caddr_t)off;		/* debugging aid */
2192d3bf78eScth 
2202d3bf78eScth 	if ((bp->b_flags & (B_SHADOW | B_PAGEIO)) && (npages == 1) &&
2212d3bf78eScth 	    kpm_enable && bp_mapin_kpm_enable) {
2222d3bf78eScth 		if (bp->b_flags & B_SHADOW)
2232d3bf78eScth 			pp = *bp->b_shadow;
2247c478bd9Sstevel@tonic-gate 		else
2252d3bf78eScth 			pp = bp->b_pages;
2267a1b360bScth 		addr = (caddr_t)((uintptr_t)addr & MMU_PAGEMASK);
2272d3bf78eScth 		hat_kpm_mapout(pp, NULL, addr);
2287c478bd9Sstevel@tonic-gate 		bp->b_flags &= ~B_REMAPPED;
2292d3bf78eScth 		return;
2307c478bd9Sstevel@tonic-gate 	}
2312d3bf78eScth 
2322d3bf78eScth 	base = (uintptr_t)addr & MMU_PAGEMASK;
2332d3bf78eScth 	BP_FLUSH(base, size);
2342d3bf78eScth 	hat_unload(kas.a_hat, (void *)base, size,
2352d3bf78eScth 	    HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK);
2362d3bf78eScth 	if (bp_map_arena != NULL) {
2372d3bf78eScth 		color = P2PHASE(base, bp_align);
2382d3bf78eScth 		vmem_free(bp_map_arena, (void *)(base - color),
2392d3bf78eScth 		    P2ROUNDUP(color + size, bp_align));
2402d3bf78eScth 	} else
2412d3bf78eScth 		vmem_free(heap_arena, (void *)base, size);
2422d3bf78eScth 	bp->b_flags &= ~B_REMAPPED;
2437c478bd9Sstevel@tonic-gate }
24436945f79Smrj 
24536945f79Smrj /*
24636945f79Smrj  * copy data from a KVA into a buf_t which may not be mapped in. offset
24736945f79Smrj  * is relative to the buf_t only.
24836945f79Smrj  */
24936945f79Smrj int
bp_copyout(void * driverbuf,struct buf * bp,offset_t offset,size_t size)25036945f79Smrj bp_copyout(void *driverbuf, struct buf *bp, offset_t offset, size_t size)
25136945f79Smrj {
25236945f79Smrj 	return (bp_copy_common(BP_COPYOUT, bp, driverbuf, offset, size));
25336945f79Smrj }
25436945f79Smrj 
25536945f79Smrj /*
25636945f79Smrj  * copy data from a buf_t which may not be mapped in, into a KVA.. offset
25736945f79Smrj  * is relative to the buf_t only.
25836945f79Smrj  */
25936945f79Smrj int
bp_copyin(struct buf * bp,void * driverbuf,offset_t offset,size_t size)26036945f79Smrj bp_copyin(struct buf *bp, void *driverbuf, offset_t offset, size_t size)
26136945f79Smrj {
26236945f79Smrj 	return (bp_copy_common(BP_COPYIN, bp, driverbuf, offset, size));
26336945f79Smrj }
26436945f79Smrj 
26536945f79Smrj 
26636945f79Smrj #define	BP_COPY(dir, driverbuf, baddr, sz)	\
26736945f79Smrj 	(dir == BP_COPYIN) ? \
26836945f79Smrj 	bcopy(baddr, driverbuf, sz) :  bcopy(driverbuf, baddr, sz)
26936945f79Smrj 
27036945f79Smrj static int
bp_copy_common(bp_copydir_t dir,struct buf * bp,void * driverbuf,offset_t offset,size_t size)27136945f79Smrj bp_copy_common(bp_copydir_t dir, struct buf *bp, void *driverbuf,
27236945f79Smrj     offset_t offset, size_t size)
27336945f79Smrj {
27436945f79Smrj 	page_t **pplist;
27536945f79Smrj 	uintptr_t poff;
27636945f79Smrj 	uintptr_t voff;
27736945f79Smrj 	struct as *as;
27836945f79Smrj 	caddr_t kaddr;
27936945f79Smrj 	caddr_t addr;
28036945f79Smrj 	page_t *page;
28136945f79Smrj 	size_t psize;
28236945f79Smrj 	page_t *pp;
28336945f79Smrj 	pfn_t pfn;
28436945f79Smrj 
28536945f79Smrj 	ASSERT((offset + size) <= bp->b_bcount);
286*584b574aSToomas Soome 	as = NULL;
28736945f79Smrj 
28836945f79Smrj 	/* if the buf_t already has a KVA, just do a bcopy */
28936945f79Smrj 	if (!(bp->b_flags & (B_PHYS | B_PAGEIO))) {
29036945f79Smrj 		BP_COPY(dir, driverbuf, bp->b_un.b_addr + offset, size);
29136945f79Smrj 		return (0);
29236945f79Smrj 	}
29336945f79Smrj 
29436945f79Smrj 	/* if we don't have kpm enabled, we need to do the slow path */
29536945f79Smrj 	if (!kpm_enable || bp_force_copy) {
29636945f79Smrj 		bp_mapin(bp);
29736945f79Smrj 		BP_COPY(dir, driverbuf, bp->b_un.b_addr + offset, size);
29836945f79Smrj 		bp_mapout(bp);
29936945f79Smrj 		return (0);
30036945f79Smrj 	}
30136945f79Smrj 
30236945f79Smrj 	/*
30336945f79Smrj 	 * kpm is enabled, and we need to map in the buf_t for the copy
30436945f79Smrj 	 */
30536945f79Smrj 
30636945f79Smrj 	/* setup pp, plist, and make sure 'as' is right */
30736945f79Smrj 	if (bp->b_flags & B_PAGEIO) {
30836945f79Smrj 		pp = bp->b_pages;
30936945f79Smrj 		pplist = NULL;
31036945f79Smrj 	} else if (bp->b_flags & B_SHADOW) {
31136945f79Smrj 		pp = NULL;
31236945f79Smrj 		pplist = bp->b_shadow;
31336945f79Smrj 	} else {
31436945f79Smrj 		pp = NULL;
31536945f79Smrj 		pplist = NULL;
31636945f79Smrj 		if (bp->b_proc == NULL || (as = bp->b_proc->p_as) == NULL) {
31736945f79Smrj 			as = &kas;
31836945f79Smrj 		}
31936945f79Smrj 	}
32036945f79Smrj 
32136945f79Smrj 	/*
32236945f79Smrj 	 * locals for the address, the offset into the first page, and the
32336945f79Smrj 	 * size of the first page we are going to copy.
32436945f79Smrj 	 */
32536945f79Smrj 	addr = (caddr_t)bp->b_un.b_addr;
32636945f79Smrj 	poff = (uintptr_t)addr & PAGEOFFSET;
32736945f79Smrj 	psize = MIN(PAGESIZE - poff, size);
32836945f79Smrj 
32936945f79Smrj 	/*
33036945f79Smrj 	 * we always start with a 0 offset into the driverbuf provided. The
33136945f79Smrj 	 * offset passed in only applies to the buf_t.
33236945f79Smrj 	 */
33336945f79Smrj 	voff = 0;
33436945f79Smrj 
33536945f79Smrj 	/* Loop until we've copied al the data */
33636945f79Smrj 	while (size > 0) {
33736945f79Smrj 
33836945f79Smrj 		/*
33936945f79Smrj 		 * for a pp or pplist, get the pfn, then go to the next page_t
34036945f79Smrj 		 * for the next time around the loop.
34136945f79Smrj 		 */
34236945f79Smrj 		if (pp) {
34336945f79Smrj 			page = pp;
34436945f79Smrj 			pp = pp->p_next;
34536945f79Smrj 		} else if (pplist != NULL) {
34636945f79Smrj 			page = (*pplist);
34736945f79Smrj 			pplist++;
34836945f79Smrj 
34936945f79Smrj 		/*
35036945f79Smrj 		 * We have a user VA. If we are going to copy this page, (e.g.
35136945f79Smrj 		 * the offset into the buf_t where we start to copy is
35236945f79Smrj 		 * within this page), get the pfn. Don't waste the cycles
35336945f79Smrj 		 * getting the pfn if we're not copying this page.
35436945f79Smrj 		 */
35536945f79Smrj 		} else if (offset < psize) {
35636945f79Smrj 			pfn = hat_getpfnum(as->a_hat,
35736945f79Smrj 			    (caddr_t)((uintptr_t)addr & PAGEMASK));
35836945f79Smrj 			if (pfn == PFN_INVALID) {
35936945f79Smrj 				return (-1);
36036945f79Smrj 			}
36136945f79Smrj 			page = page_numtopp_nolock(pfn);
36236945f79Smrj 			addr += psize - offset;
36336945f79Smrj 		} else {
36436945f79Smrj 			addr += psize;
36536945f79Smrj 		}
36636945f79Smrj 
36736945f79Smrj 		/*
36836945f79Smrj 		 * if we have an initial offset into the buf_t passed in,
36936945f79Smrj 		 * and it falls within the current page, account for it in
37036945f79Smrj 		 * the page size (how much we will copy) and the offset into the
37136945f79Smrj 		 * page (where we'll start copying from).
37236945f79Smrj 		 */
37336945f79Smrj 		if ((offset > 0) && (offset < psize)) {
37436945f79Smrj 			psize -= offset;
37536945f79Smrj 			poff += offset;
37636945f79Smrj 			offset = 0;
37736945f79Smrj 
37836945f79Smrj 		/*
37936945f79Smrj 		 * if we have an initial offset into the buf_t passed in,
38036945f79Smrj 		 * and it's not within the current page, skip this page.
38136945f79Smrj 		 * We don't have to worry about the first page offset and size
38236945f79Smrj 		 * anymore. psize will normally be PAGESIZE now unless we are
38336945f79Smrj 		 * on the last page.
38436945f79Smrj 		 */
38536945f79Smrj 		} else if (offset >= psize) {
38636945f79Smrj 			offset -= psize;
38736945f79Smrj 			psize = MIN(PAGESIZE, size);
38836945f79Smrj 			poff = 0;
38936945f79Smrj 			continue;
39036945f79Smrj 		}
39136945f79Smrj 
39236945f79Smrj 		/*
39336945f79Smrj 		 * get a kpm mapping to the page, them copy in/out of the
39436945f79Smrj 		 * page. update size left and offset into the driverbuf passed
39536945f79Smrj 		 * in for the next time around the loop.
39636945f79Smrj 		 */
39736945f79Smrj 		kaddr = hat_kpm_mapin(page, NULL) + poff;
39836945f79Smrj 		BP_COPY(dir, (void *)((uintptr_t)driverbuf + voff), kaddr,
39936945f79Smrj 		    psize);
40036945f79Smrj 		hat_kpm_mapout(page, NULL, kaddr - poff);
40136945f79Smrj 
40236945f79Smrj 		size -= psize;
40336945f79Smrj 		voff += psize;
40436945f79Smrj 
40536945f79Smrj 		poff = 0;
40636945f79Smrj 		psize = MIN(PAGESIZE, size);
40736945f79Smrj 	}
40836945f79Smrj 
40936945f79Smrj 	return (0);
41036945f79Smrj }
411