xref: /illumos-gate/usr/src/uts/common/vm/seg_dev.c (revision c6f039c7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  * Copyright 2019 Joyent, Inc.
26  */
27 
28 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
29 /*	  All Rights Reserved  	*/
30 
31 /*
32  * University Copyright- Copyright (c) 1982, 1986, 1988
33  * The Regents of the University of California
34  * All Rights Reserved
35  *
36  * University Acknowledgment- Portions of this document are derived from
37  * software developed by the University of California, Berkeley, and its
38  * contributors.
39  */
40 
41 /*
42  * VM - segment of a mapped device.
43  *
44  * This segment driver is used when mapping character special devices.
45  */
46 
47 #include <sys/types.h>
48 #include <sys/t_lock.h>
49 #include <sys/sysmacros.h>
50 #include <sys/vtrace.h>
51 #include <sys/systm.h>
52 #include <sys/vmsystm.h>
53 #include <sys/mman.h>
54 #include <sys/errno.h>
55 #include <sys/kmem.h>
56 #include <sys/cmn_err.h>
57 #include <sys/vnode.h>
58 #include <sys/proc.h>
59 #include <sys/conf.h>
60 #include <sys/debug.h>
61 #include <sys/ddidevmap.h>
62 #include <sys/ddi_implfuncs.h>
63 #include <sys/lgrp.h>
64 
65 #include <vm/page.h>
66 #include <vm/hat.h>
67 #include <vm/as.h>
68 #include <vm/seg.h>
69 #include <vm/seg_dev.h>
70 #include <vm/seg_kp.h>
71 #include <vm/seg_kmem.h>
72 #include <vm/vpage.h>
73 
74 #include <sys/sunddi.h>
75 #include <sys/esunddi.h>
76 #include <sys/fs/snode.h>
77 
78 
79 #if DEBUG
80 int segdev_debug;
81 #define	DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
82 #else
83 #define	DEBUGF(level, args)
84 #endif
85 
86 /* Default timeout for devmap context management */
87 #define	CTX_TIMEOUT_VALUE 0
88 
89 #define	HOLD_DHP_LOCK(dhp)  if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
90 			{ mutex_enter(&dhp->dh_lock); }
91 
92 #define	RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
93 			{ mutex_exit(&dhp->dh_lock); }
94 
95 #define	round_down_p2(a, s)	((a) & ~((s) - 1))
96 #define	round_up_p2(a, s)	(((a) + (s) - 1) & ~((s) - 1))
97 
98 /*
99  * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
100  * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
101  */
102 #define	VA_PA_ALIGNED(uvaddr, paddr, pgsize)		\
103 	(((uvaddr | paddr) & (pgsize - 1)) == 0)
104 #define	VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize)	\
105 	(((uvaddr ^ paddr) & (pgsize - 1)) == 0)
106 
107 #define	vpgtob(n)	((n) * sizeof (struct vpage))	/* For brevity */
108 
109 #define	VTOCVP(vp)	(VTOS(vp)->s_commonvp)	/* we "know" it's an snode */
110 
111 static struct devmap_ctx *devmapctx_list = NULL;
112 static struct devmap_softlock *devmap_slist = NULL;
113 
114 /*
115  * mutex, vnode and page for the page of zeros we use for the trash mappings.
116  * One trash page is allocated on the first ddi_umem_setup call that uses it
117  * XXX Eventually, we may want to combine this with what segnf does when all
118  * hat layers implement HAT_NOFAULT.
119  *
120  * The trash page is used when the backing store for a userland mapping is
121  * removed but the application semantics do not take kindly to a SIGBUS.
122  * In that scenario, the applications pages are mapped to some dummy page
123  * which returns garbage on read and writes go into a common place.
124  * (Perfect for NO_FAULT semantics)
125  * The device driver is responsible to communicating to the app with some
126  * other mechanism that such remapping has happened and the app should take
127  * corrective action.
128  * We can also use an anonymous memory page as there is no requirement to
129  * keep the page locked, however this complicates the fault code. RFE.
130  */
131 static struct vnode trashvp;
132 static struct page *trashpp;
133 
134 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
135 static vmem_t *umem_np_arena;
136 
137 /* Set the cookie to a value we know will never be a valid umem_cookie */
138 #define	DEVMAP_DEVMEM_COOKIE	((ddi_umem_cookie_t)0x1)
139 
140 /*
141  * Macros to check if type of devmap handle
142  */
143 #define	cookie_is_devmem(c)	\
144 	((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
145 
146 #define	cookie_is_pmem(c)	\
147 	((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
148 
149 #define	cookie_is_kpmem(c)	(!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
150 	((c)->type == KMEM_PAGEABLE))
151 
152 #define	dhp_is_devmem(dhp)	\
153 	(cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
154 
155 #define	dhp_is_pmem(dhp)	\
156 	(cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
157 
158 #define	dhp_is_kpmem(dhp)	\
159 	(cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
160 
161 /*
162  * Private seg op routines.
163  */
164 static int	segdev_dup(struct seg *, struct seg *);
165 static int	segdev_unmap(struct seg *, caddr_t, size_t);
166 static void	segdev_free(struct seg *);
167 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
168 		    enum fault_type, enum seg_rw);
169 static faultcode_t segdev_faulta(struct seg *, caddr_t);
170 static int	segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
171 static int	segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
172 static void	segdev_badop(void);
173 static int	segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
174 static size_t	segdev_incore(struct seg *, caddr_t, size_t, char *);
175 static int	segdev_lockop(struct seg *, caddr_t, size_t, int, int,
176 		    ulong_t *, size_t);
177 static int	segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
178 static u_offset_t	segdev_getoffset(struct seg *, caddr_t);
179 static int	segdev_gettype(struct seg *, caddr_t);
180 static int	segdev_getvp(struct seg *, caddr_t, struct vnode **);
181 static int	segdev_advise(struct seg *, caddr_t, size_t, uint_t);
182 static void	segdev_dump(struct seg *);
183 static int	segdev_pagelock(struct seg *, caddr_t, size_t,
184 		    struct page ***, enum lock_type, enum seg_rw);
185 static int	segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
186 static int	segdev_getmemid(struct seg *, caddr_t, memid_t *);
187 static lgrp_mem_policy_info_t	*segdev_getpolicy(struct seg *, caddr_t);
188 static int	segdev_capable(struct seg *, segcapability_t);
189 
190 /*
191  * XXX	this struct is used by rootnex_map_fault to identify
192  *	the segment it has been passed. So if you make it
193  *	"static" you'll need to fix rootnex_map_fault.
194  */
195 struct seg_ops segdev_ops = {
196 	segdev_dup,
197 	segdev_unmap,
198 	segdev_free,
199 	segdev_fault,
200 	segdev_faulta,
201 	segdev_setprot,
202 	segdev_checkprot,
203 	(int (*)())segdev_badop,	/* kluster */
204 	(size_t (*)(struct seg *))NULL,	/* swapout */
205 	segdev_sync,			/* sync */
206 	segdev_incore,
207 	segdev_lockop,			/* lockop */
208 	segdev_getprot,
209 	segdev_getoffset,
210 	segdev_gettype,
211 	segdev_getvp,
212 	segdev_advise,
213 	segdev_dump,
214 	segdev_pagelock,
215 	segdev_setpagesize,
216 	segdev_getmemid,
217 	segdev_getpolicy,
218 	segdev_capable,
219 	seg_inherit_notsup
220 };
221 
222 /*
223  * Private segdev support routines
224  */
225 static struct segdev_data *sdp_alloc(void);
226 
227 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
228     size_t, enum seg_rw);
229 
230 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
231     struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
232 
233 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
234     size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
235 
236 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
237 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
238 static void devmap_softlock_rele(devmap_handle_t *);
239 static void devmap_ctx_rele(devmap_handle_t *);
240 
241 static void devmap_ctxto(void *);
242 
243 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
244     caddr_t addr);
245 
246 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
247     ulong_t *opfn, ulong_t *pagesize);
248 
249 static void free_devmap_handle(devmap_handle_t *dhp);
250 
251 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
252     struct seg *newseg);
253 
254 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
255 
256 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
257 
258 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
259 
260 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
261     offset_t off, size_t len, uint_t flags);
262 
263 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
264     caddr_t addr, size_t *llen, caddr_t *laddr);
265 
266 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
267 
268 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
269 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
270 
271 static void *devmap_umem_alloc_np(size_t size, size_t flags);
272 static void devmap_umem_free_np(void *addr, size_t size);
273 
274 /*
275  * routines to lock and unlock underlying segkp segment for
276  * KMEM_PAGEABLE type cookies.
277  */
278 static faultcode_t  acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
279 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
280 
281 /*
282  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
283  * drivers with devmap_access callbacks
284  */
285 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
286 	enum fault_type);
287 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
288 	enum fault_type);
289 
290 static kmutex_t devmapctx_lock;
291 
292 static kmutex_t devmap_slock;
293 
294 /*
295  * Initialize the thread callbacks and thread private data.
296  */
297 static struct devmap_ctx *
devmap_ctxinit(dev_t dev,ulong_t id)298 devmap_ctxinit(dev_t dev, ulong_t id)
299 {
300 	struct devmap_ctx	*devctx;
301 	struct devmap_ctx	*tmp;
302 	dev_info_t		*dip;
303 
304 	tmp =  kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
305 
306 	mutex_enter(&devmapctx_lock);
307 
308 	dip = e_ddi_hold_devi_by_dev(dev, 0);
309 	ASSERT(dip != NULL);
310 	ddi_release_devi(dip);
311 
312 	for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
313 		if ((devctx->dip == dip) && (devctx->id == id))
314 			break;
315 
316 	if (devctx == NULL) {
317 		devctx = tmp;
318 		devctx->dip = dip;
319 		devctx->id = id;
320 		mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
321 		cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
322 		devctx->next = devmapctx_list;
323 		devmapctx_list = devctx;
324 	} else
325 		kmem_free(tmp, sizeof (struct devmap_ctx));
326 
327 	mutex_enter(&devctx->lock);
328 	devctx->refcnt++;
329 	mutex_exit(&devctx->lock);
330 	mutex_exit(&devmapctx_lock);
331 
332 	return (devctx);
333 }
334 
335 /*
336  * Timeout callback called if a CPU has not given up the device context
337  * within dhp->dh_timeout_length ticks
338  */
339 static void
devmap_ctxto(void * data)340 devmap_ctxto(void *data)
341 {
342 	struct devmap_ctx *devctx = data;
343 
344 	TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
345 	    "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
346 	mutex_enter(&devctx->lock);
347 	/*
348 	 * Set oncpu = 0 so the next mapping trying to get the device context
349 	 * can.
350 	 */
351 	devctx->oncpu = 0;
352 	devctx->timeout = 0;
353 	cv_signal(&devctx->cv);
354 	mutex_exit(&devctx->lock);
355 }
356 
357 /*
358  * Create a device segment.
359  */
360 int
segdev_create(struct seg ** segpp,void * argsp)361 segdev_create(struct seg **segpp, void *argsp)
362 {
363 	struct seg *seg = *segpp;
364 	struct segdev_data *sdp;
365 	struct segdev_crargs *a = (struct segdev_crargs *)argsp;
366 	devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
367 	int error;
368 
369 	/*
370 	 * Since the address space is "write" locked, we
371 	 * don't need the segment lock to protect "segdev" data.
372 	 */
373 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
374 
375 	hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
376 
377 	sdp = sdp_alloc();
378 
379 	sdp->mapfunc = a->mapfunc;
380 	sdp->offset = a->offset;
381 	sdp->prot = a->prot;
382 	sdp->maxprot = a->maxprot;
383 	sdp->type = a->type;
384 	sdp->pageprot = 0;
385 	sdp->softlockcnt = 0;
386 	sdp->vpage = NULL;
387 
388 	if (sdp->mapfunc == NULL)
389 		sdp->devmap_data = dhp;
390 	else
391 		sdp->devmap_data = dhp = NULL;
392 
393 	sdp->hat_flags = a->hat_flags;
394 	sdp->hat_attr = a->hat_attr;
395 
396 	/*
397 	 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
398 	 */
399 	ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
400 
401 	/*
402 	 * Hold shadow vnode -- segdev only deals with
403 	 * character (VCHR) devices. We use the common
404 	 * vp to hang pages on.
405 	 */
406 	sdp->vp = specfind(a->dev, VCHR);
407 	ASSERT(sdp->vp != NULL);
408 
409 	seg->s_ops = &segdev_ops;
410 	seg->s_data = sdp;
411 
412 	while (dhp != NULL) {
413 		dhp->dh_seg = seg;
414 		dhp = dhp->dh_next;
415 	}
416 
417 	/*
418 	 * Inform the vnode of the new mapping.
419 	 */
420 	/*
421 	 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
422 	 * dhp specific maxprot because spec_addmap does not use maxprot.
423 	 */
424 	error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
425 	    seg->s_as, seg->s_base, seg->s_size,
426 	    sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
427 
428 	if (error != 0) {
429 		sdp->devmap_data = NULL;
430 		hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
431 		    HAT_UNLOAD_UNMAP);
432 	} else {
433 		/*
434 		 * Mappings of /dev/null don't count towards the VSZ of a
435 		 * process.  Mappings of /dev/null have no mapping type.
436 		 */
437 		if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
438 		    MAP_PRIVATE)) == 0) {
439 			seg->s_as->a_resvsize -= seg->s_size;
440 		}
441 	}
442 
443 	return (error);
444 }
445 
446 static struct segdev_data *
sdp_alloc(void)447 sdp_alloc(void)
448 {
449 	struct segdev_data *sdp;
450 
451 	sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
452 	rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
453 
454 	return (sdp);
455 }
456 
457 /*
458  * Duplicate seg and return new segment in newseg.
459  */
460 static int
segdev_dup(struct seg * seg,struct seg * newseg)461 segdev_dup(struct seg *seg, struct seg *newseg)
462 {
463 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
464 	struct segdev_data *newsdp;
465 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
466 	size_t npages;
467 	int ret;
468 
469 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
470 	    "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
471 
472 	DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
473 	    (void *)dhp, (void *)seg));
474 
475 	/*
476 	 * Since the address space is "write" locked, we
477 	 * don't need the segment lock to protect "segdev" data.
478 	 */
479 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
480 
481 	newsdp = sdp_alloc();
482 
483 	newseg->s_ops = seg->s_ops;
484 	newseg->s_data = (void *)newsdp;
485 
486 	VN_HOLD(sdp->vp);
487 	newsdp->vp 	= sdp->vp;
488 	newsdp->mapfunc = sdp->mapfunc;
489 	newsdp->offset	= sdp->offset;
490 	newsdp->pageprot = sdp->pageprot;
491 	newsdp->prot	= sdp->prot;
492 	newsdp->maxprot = sdp->maxprot;
493 	newsdp->type = sdp->type;
494 	newsdp->hat_attr = sdp->hat_attr;
495 	newsdp->hat_flags = sdp->hat_flags;
496 	newsdp->softlockcnt = 0;
497 
498 	/*
499 	 * Initialize per page data if the segment we are
500 	 * dup'ing has per page information.
501 	 */
502 	npages = seg_pages(newseg);
503 
504 	if (sdp->vpage != NULL) {
505 		size_t nbytes = vpgtob(npages);
506 
507 		newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
508 		bcopy(sdp->vpage, newsdp->vpage, nbytes);
509 	} else
510 		newsdp->vpage = NULL;
511 
512 	/*
513 	 * duplicate devmap handles
514 	 */
515 	if (dhp != NULL) {
516 		ret = devmap_handle_dup(dhp,
517 		    (devmap_handle_t **)&newsdp->devmap_data, newseg);
518 		if (ret != 0) {
519 			TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
520 			    "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
521 			    ret, (void *)dhp, (void *)seg);
522 			DEBUGF(1, (CE_CONT,
523 			    "segdev_dup: ret %x dhp %p seg %p\n",
524 			    ret, (void *)dhp, (void *)seg));
525 			return (ret);
526 		}
527 	}
528 
529 	/*
530 	 * Inform the common vnode of the new mapping.
531 	 */
532 	return (VOP_ADDMAP(VTOCVP(newsdp->vp),
533 	    newsdp->offset, newseg->s_as,
534 	    newseg->s_base, newseg->s_size, newsdp->prot,
535 	    newsdp->maxprot, sdp->type, CRED(), NULL));
536 }
537 
538 /*
539  * duplicate devmap handles
540  */
541 static int
devmap_handle_dup(devmap_handle_t * dhp,devmap_handle_t ** new_dhp,struct seg * newseg)542 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
543     struct seg *newseg)
544 {
545 	devmap_handle_t *newdhp_save = NULL;
546 	devmap_handle_t *newdhp = NULL;
547 	struct devmap_callback_ctl *callbackops;
548 
549 	while (dhp != NULL) {
550 		newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
551 
552 		/* Need to lock the original dhp while copying if REMAP */
553 		HOLD_DHP_LOCK(dhp);
554 		bcopy(dhp, newdhp, sizeof (devmap_handle_t));
555 		RELE_DHP_LOCK(dhp);
556 		newdhp->dh_seg = newseg;
557 		newdhp->dh_next = NULL;
558 		if (newdhp_save != NULL)
559 			newdhp_save->dh_next = newdhp;
560 		else
561 			*new_dhp = newdhp;
562 		newdhp_save = newdhp;
563 
564 		callbackops = &newdhp->dh_callbackops;
565 
566 		if (dhp->dh_softlock != NULL)
567 			newdhp->dh_softlock = devmap_softlock_init(
568 			    newdhp->dh_dev,
569 			    (ulong_t)callbackops->devmap_access);
570 		if (dhp->dh_ctx != NULL)
571 			newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
572 			    (ulong_t)callbackops->devmap_access);
573 
574 		/*
575 		 * Initialize dh_lock if we want to do remap.
576 		 */
577 		if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
578 			mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
579 			newdhp->dh_flags |= DEVMAP_LOCK_INITED;
580 		}
581 
582 		if (callbackops->devmap_dup != NULL) {
583 			int ret;
584 
585 			/*
586 			 * Call the dup callback so that the driver can
587 			 * duplicate its private data.
588 			 */
589 			ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
590 			    (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
591 
592 			if (ret != 0) {
593 				/*
594 				 * We want to free up this segment as the driver
595 				 * has indicated that we can't dup it.  But we
596 				 * don't want to call the drivers, devmap_unmap,
597 				 * callback function as the driver does not
598 				 * think this segment exists. The caller of
599 				 * devmap_dup will call seg_free on newseg
600 				 * as it was the caller that allocated the
601 				 * segment.
602 				 */
603 				DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
604 				    "newdhp %p dhp %p\n", (void *)newdhp,
605 				    (void *)dhp));
606 				callbackops->devmap_unmap = NULL;
607 				return (ret);
608 			}
609 		}
610 
611 		dhp = dhp->dh_next;
612 	}
613 
614 	return (0);
615 }
616 
617 /*
618  * Split a segment at addr for length len.
619  */
620 /*ARGSUSED*/
621 static int
segdev_unmap(struct seg * seg,caddr_t addr,size_t len)622 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
623 {
624 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
625 	register struct segdev_data *nsdp;
626 	register struct seg *nseg;
627 	register size_t	opages;		/* old segment size in pages */
628 	register size_t	npages;		/* new segment size in pages */
629 	register size_t	dpages;		/* pages being deleted (unmapped) */
630 	register size_t	nbytes;
631 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
632 	devmap_handle_t *dhpp;
633 	devmap_handle_t *newdhp;
634 	struct devmap_callback_ctl *callbackops;
635 	caddr_t nbase;
636 	offset_t off;
637 	ulong_t nsize;
638 	size_t mlen, sz;
639 
640 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
641 	    "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
642 	    (void *)dhp, (void *)seg, (void *)addr, len);
643 
644 	DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
645 	    (void *)dhp, (void *)seg, (void *)addr, len));
646 
647 	/*
648 	 * Since the address space is "write" locked, we
649 	 * don't need the segment lock to protect "segdev" data.
650 	 */
651 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
652 
653 	if ((sz = sdp->softlockcnt) > 0) {
654 		/*
655 		 * Fail the unmap if pages are SOFTLOCKed through this mapping.
656 		 * softlockcnt is protected from change by the as write lock.
657 		 */
658 		TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
659 		    "segdev_unmap:error softlockcnt = %ld", sz);
660 		DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
661 		return (EAGAIN);
662 	}
663 
664 	/*
665 	 * Check for bad sizes
666 	 */
667 	if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
668 	    (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
669 		panic("segdev_unmap");
670 
671 	if (dhp != NULL) {
672 		devmap_handle_t *tdhp;
673 		/*
674 		 * If large page size was used in hat_devload(),
675 		 * the same page size must be used in hat_unload().
676 		 */
677 		dhpp = tdhp = devmap_find_handle(dhp, addr);
678 		while (tdhp != NULL) {
679 			if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
680 				break;
681 			}
682 			tdhp = tdhp->dh_next;
683 		}
684 		if (tdhp != NULL) {	/* found a dhp using large pages */
685 			size_t slen = len;
686 			size_t mlen;
687 			size_t soff;
688 
689 			soff = (ulong_t)(addr - dhpp->dh_uvaddr);
690 			while (slen != 0) {
691 				mlen = MIN(slen, (dhpp->dh_len - soff));
692 				hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
693 				    dhpp->dh_len, HAT_UNLOAD_UNMAP);
694 				dhpp = dhpp->dh_next;
695 				ASSERT(slen >= mlen);
696 				slen -= mlen;
697 				soff = 0;
698 			}
699 		} else
700 			hat_unload(seg->s_as->a_hat, addr, len,
701 			    HAT_UNLOAD_UNMAP);
702 	} else {
703 		/*
704 		 * Unload any hardware translations in the range
705 		 * to be taken out.
706 		 */
707 		hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
708 	}
709 
710 	/*
711 	 * get the user offset which will used in the driver callbacks
712 	 */
713 	off = sdp->offset + (offset_t)(addr - seg->s_base);
714 
715 	/*
716 	 * Inform the vnode of the unmapping.
717 	 */
718 	ASSERT(sdp->vp != NULL);
719 	(void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
720 	    sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
721 
722 	/*
723 	 * Check for entire segment
724 	 */
725 	if (addr == seg->s_base && len == seg->s_size) {
726 		seg_free(seg);
727 		return (0);
728 	}
729 
730 	opages = seg_pages(seg);
731 	dpages = btop(len);
732 	npages = opages - dpages;
733 
734 	/*
735 	 * Check for beginning of segment
736 	 */
737 	if (addr == seg->s_base) {
738 		if (sdp->vpage != NULL) {
739 			register struct vpage *ovpage;
740 
741 			ovpage = sdp->vpage;	/* keep pointer to vpage */
742 
743 			nbytes = vpgtob(npages);
744 			sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
745 			bcopy(&ovpage[dpages], sdp->vpage, nbytes);
746 
747 			/* free up old vpage */
748 			kmem_free(ovpage, vpgtob(opages));
749 		}
750 
751 		/*
752 		 * free devmap handles from the beginning of the mapping.
753 		 */
754 		if (dhp != NULL)
755 			devmap_handle_unmap_head(dhp, len);
756 
757 		sdp->offset += (offset_t)len;
758 
759 		seg->s_base += len;
760 		seg->s_size -= len;
761 
762 		return (0);
763 	}
764 
765 	/*
766 	 * Check for end of segment
767 	 */
768 	if (addr + len == seg->s_base + seg->s_size) {
769 		if (sdp->vpage != NULL) {
770 			register struct vpage *ovpage;
771 
772 			ovpage = sdp->vpage;	/* keep pointer to vpage */
773 
774 			nbytes = vpgtob(npages);
775 			sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
776 			bcopy(ovpage, sdp->vpage, nbytes);
777 
778 			/* free up old vpage */
779 			kmem_free(ovpage, vpgtob(opages));
780 		}
781 		seg->s_size -= len;
782 
783 		/*
784 		 * free devmap handles from addr to the end of the mapping.
785 		 */
786 		if (dhp != NULL)
787 			devmap_handle_unmap_tail(dhp, addr);
788 
789 		return (0);
790 	}
791 
792 	/*
793 	 * The section to go is in the middle of the segment,
794 	 * have to make it into two segments.  nseg is made for
795 	 * the high end while seg is cut down at the low end.
796 	 */
797 	nbase = addr + len;				/* new seg base */
798 	nsize = (seg->s_base + seg->s_size) - nbase;	/* new seg size */
799 	seg->s_size = addr - seg->s_base;		/* shrink old seg */
800 	nseg = seg_alloc(seg->s_as, nbase, nsize);
801 	if (nseg == NULL)
802 		panic("segdev_unmap seg_alloc");
803 
804 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
805 	    "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
806 	DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
807 	    (void *)seg, (void *)nseg));
808 	nsdp = sdp_alloc();
809 
810 	nseg->s_ops = seg->s_ops;
811 	nseg->s_data = (void *)nsdp;
812 
813 	VN_HOLD(sdp->vp);
814 	nsdp->mapfunc = sdp->mapfunc;
815 	nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
816 	nsdp->vp 	= sdp->vp;
817 	nsdp->pageprot = sdp->pageprot;
818 	nsdp->prot	= sdp->prot;
819 	nsdp->maxprot = sdp->maxprot;
820 	nsdp->type = sdp->type;
821 	nsdp->hat_attr = sdp->hat_attr;
822 	nsdp->hat_flags = sdp->hat_flags;
823 	nsdp->softlockcnt = 0;
824 
825 	/*
826 	 * Initialize per page data if the segment we are
827 	 * dup'ing has per page information.
828 	 */
829 	if (sdp->vpage != NULL) {
830 		/* need to split vpage into two arrays */
831 		register size_t nnbytes;
832 		register size_t nnpages;
833 		register struct vpage *ovpage;
834 
835 		ovpage = sdp->vpage;		/* keep pointer to vpage */
836 
837 		npages = seg_pages(seg);	/* seg has shrunk */
838 		nbytes = vpgtob(npages);
839 		nnpages = seg_pages(nseg);
840 		nnbytes = vpgtob(nnpages);
841 
842 		sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
843 		bcopy(ovpage, sdp->vpage, nbytes);
844 
845 		nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
846 		bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
847 
848 		/* free up old vpage */
849 		kmem_free(ovpage, vpgtob(opages));
850 	} else
851 		nsdp->vpage = NULL;
852 
853 	/*
854 	 * unmap dhps.
855 	 */
856 	if (dhp == NULL) {
857 		nsdp->devmap_data = NULL;
858 		return (0);
859 	}
860 	while (dhp != NULL) {
861 		callbackops = &dhp->dh_callbackops;
862 		TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
863 		    "segdev_unmap: dhp=%p addr=%p", dhp, addr);
864 		DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
865 		    (void *)dhp, (void *)addr,
866 		    (void *)dhp->dh_uvaddr, dhp->dh_len));
867 
868 		if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
869 			dhpp = dhp->dh_next;
870 			dhp->dh_next = NULL;
871 			dhp = dhpp;
872 		} else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
873 			dhp = dhp->dh_next;
874 		} else if (addr > dhp->dh_uvaddr &&
875 		    (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
876 			/*
877 			 * <addr, addr+len> is enclosed by dhp.
878 			 * create a newdhp that begins at addr+len and
879 			 * ends at dhp->dh_uvaddr+dhp->dh_len.
880 			 */
881 			newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
882 			HOLD_DHP_LOCK(dhp);
883 			bcopy(dhp, newdhp, sizeof (devmap_handle_t));
884 			RELE_DHP_LOCK(dhp);
885 			newdhp->dh_seg = nseg;
886 			newdhp->dh_next = dhp->dh_next;
887 			if (dhp->dh_softlock != NULL)
888 				newdhp->dh_softlock = devmap_softlock_init(
889 				    newdhp->dh_dev,
890 				    (ulong_t)callbackops->devmap_access);
891 			if (dhp->dh_ctx != NULL)
892 				newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
893 				    (ulong_t)callbackops->devmap_access);
894 			if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
895 				mutex_init(&newdhp->dh_lock,
896 				    NULL, MUTEX_DEFAULT, NULL);
897 			}
898 			if (callbackops->devmap_unmap != NULL)
899 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
900 				    off, len, dhp, &dhp->dh_pvtp,
901 				    newdhp, &newdhp->dh_pvtp);
902 			mlen = len + (addr - dhp->dh_uvaddr);
903 			devmap_handle_reduce_len(newdhp, mlen);
904 			nsdp->devmap_data = newdhp;
905 			/* XX Changing len should recalculate LARGE flag */
906 			dhp->dh_len = addr - dhp->dh_uvaddr;
907 			dhpp = dhp->dh_next;
908 			dhp->dh_next = NULL;
909 			dhp = dhpp;
910 		} else if ((addr > dhp->dh_uvaddr) &&
911 		    ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
912 			mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
913 			/*
914 			 * <addr, addr+len> spans over dhps.
915 			 */
916 			if (callbackops->devmap_unmap != NULL)
917 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
918 				    off, mlen, (devmap_cookie_t *)dhp,
919 				    &dhp->dh_pvtp, NULL, NULL);
920 			/* XX Changing len should recalculate LARGE flag */
921 			dhp->dh_len = addr - dhp->dh_uvaddr;
922 			dhpp = dhp->dh_next;
923 			dhp->dh_next = NULL;
924 			dhp = dhpp;
925 			nsdp->devmap_data = dhp;
926 		} else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
927 			/*
928 			 * dhp is enclosed by <addr, addr+len>.
929 			 */
930 			dhp->dh_seg = nseg;
931 			nsdp->devmap_data = dhp;
932 			dhp = devmap_handle_unmap(dhp);
933 			nsdp->devmap_data = dhp; /* XX redundant? */
934 		} else if (((addr + len) > dhp->dh_uvaddr) &&
935 		    ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
936 			mlen = addr + len - dhp->dh_uvaddr;
937 			if (callbackops->devmap_unmap != NULL)
938 				(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
939 				    dhp->dh_uoff, mlen, NULL,
940 				    NULL, dhp, &dhp->dh_pvtp);
941 			devmap_handle_reduce_len(dhp, mlen);
942 			nsdp->devmap_data = dhp;
943 			dhp->dh_seg = nseg;
944 			dhp = dhp->dh_next;
945 		} else {
946 			dhp->dh_seg = nseg;
947 			dhp = dhp->dh_next;
948 		}
949 	}
950 	return (0);
951 }
952 
953 /*
954  * Utility function handles reducing the length of a devmap handle during unmap
955  * Note that is only used for unmapping the front portion of the handler,
956  * i.e., we are bumping up the offset/pfn etc up by len
957  * Do not use if reducing length at the tail.
958  */
959 static void
devmap_handle_reduce_len(devmap_handle_t * dhp,size_t len)960 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
961 {
962 	struct ddi_umem_cookie *cp;
963 	struct devmap_pmem_cookie *pcp;
964 	/*
965 	 * adjust devmap handle fields
966 	 */
967 	ASSERT(len < dhp->dh_len);
968 
969 	/* Make sure only page-aligned changes are done */
970 	ASSERT((len & PAGEOFFSET) == 0);
971 
972 	dhp->dh_len -= len;
973 	dhp->dh_uoff += (offset_t)len;
974 	dhp->dh_roff += (offset_t)len;
975 	dhp->dh_uvaddr += len;
976 	/* Need to grab dhp lock if REMAP */
977 	HOLD_DHP_LOCK(dhp);
978 	cp = dhp->dh_cookie;
979 	if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
980 		if (cookie_is_devmem(cp)) {
981 			dhp->dh_pfn += btop(len);
982 		} else if (cookie_is_pmem(cp)) {
983 			pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
984 			ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
985 			    dhp->dh_roff < ptob(pcp->dp_npages));
986 		} else {
987 			ASSERT(dhp->dh_roff < cp->size);
988 			ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
989 			    dhp->dh_cvaddr < (cp->cvaddr + cp->size));
990 			ASSERT((dhp->dh_cvaddr + len) <=
991 			    (cp->cvaddr + cp->size));
992 
993 			dhp->dh_cvaddr += len;
994 		}
995 	}
996 	/* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
997 	RELE_DHP_LOCK(dhp);
998 }
999 
1000 /*
1001  * Free devmap handle, dhp.
1002  * Return the next devmap handle on the linked list.
1003  */
1004 static devmap_handle_t *
devmap_handle_unmap(devmap_handle_t * dhp)1005 devmap_handle_unmap(devmap_handle_t *dhp)
1006 {
1007 	struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1008 	struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1009 	devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1010 
1011 	ASSERT(dhp != NULL);
1012 
1013 	/*
1014 	 * before we free up dhp, call the driver's devmap_unmap entry point
1015 	 * to free resources allocated for this dhp.
1016 	 */
1017 	if (callbackops->devmap_unmap != NULL) {
1018 		(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1019 		    dhp->dh_len, NULL, NULL, NULL, NULL);
1020 	}
1021 
1022 	if (dhpp == dhp) {	/* releasing first dhp, change sdp data */
1023 		sdp->devmap_data = dhp->dh_next;
1024 	} else {
1025 		while (dhpp->dh_next != dhp) {
1026 			dhpp = dhpp->dh_next;
1027 		}
1028 		dhpp->dh_next = dhp->dh_next;
1029 	}
1030 	dhpp = dhp->dh_next;	/* return value is next dhp in chain */
1031 
1032 	if (dhp->dh_softlock != NULL)
1033 		devmap_softlock_rele(dhp);
1034 
1035 	if (dhp->dh_ctx != NULL)
1036 		devmap_ctx_rele(dhp);
1037 
1038 	if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1039 		mutex_destroy(&dhp->dh_lock);
1040 	}
1041 	kmem_free(dhp, sizeof (devmap_handle_t));
1042 
1043 	return (dhpp);
1044 }
1045 
1046 /*
1047  * Free complete devmap handles from dhp for len bytes
1048  * dhp can be either the first handle or a subsequent handle
1049  */
1050 static void
devmap_handle_unmap_head(devmap_handle_t * dhp,size_t len)1051 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1052 {
1053 	struct devmap_callback_ctl *callbackops;
1054 
1055 	/*
1056 	 * free the devmap handles covered by len.
1057 	 */
1058 	while (len >= dhp->dh_len) {
1059 		len -= dhp->dh_len;
1060 		dhp = devmap_handle_unmap(dhp);
1061 	}
1062 	if (len != 0) {	/* partial unmap at head of first remaining dhp */
1063 		callbackops = &dhp->dh_callbackops;
1064 
1065 		/*
1066 		 * Call the unmap callback so the drivers can make
1067 		 * adjustment on its private data.
1068 		 */
1069 		if (callbackops->devmap_unmap != NULL)
1070 			(*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1071 			    dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1072 		devmap_handle_reduce_len(dhp, len);
1073 	}
1074 }
1075 
1076 /*
1077  * Free devmap handles to truncate  the mapping after addr
1078  * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1079  *	Also could then use the routine in middle unmap case too
1080  */
1081 static void
devmap_handle_unmap_tail(devmap_handle_t * dhp,caddr_t addr)1082 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1083 {
1084 	register struct seg *seg = dhp->dh_seg;
1085 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1086 	register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1087 	struct devmap_callback_ctl *callbackops;
1088 	register devmap_handle_t *dhpp;
1089 	size_t maplen;
1090 	ulong_t off;
1091 	size_t len;
1092 
1093 	maplen = (size_t)(addr - dhp->dh_uvaddr);
1094 	dhph = devmap_find_handle(dhph, addr);
1095 
1096 	while (dhph != NULL) {
1097 		if (maplen == 0) {
1098 			dhph =  devmap_handle_unmap(dhph);
1099 		} else {
1100 			callbackops = &dhph->dh_callbackops;
1101 			len = dhph->dh_len - maplen;
1102 			off = (ulong_t)sdp->offset + (addr - seg->s_base);
1103 			/*
1104 			 * Call the unmap callback so the driver
1105 			 * can make adjustments on its private data.
1106 			 */
1107 			if (callbackops->devmap_unmap != NULL)
1108 				(*callbackops->devmap_unmap)(dhph,
1109 				    dhph->dh_pvtp, off, len,
1110 				    (devmap_cookie_t *)dhph,
1111 				    &dhph->dh_pvtp, NULL, NULL);
1112 			/* XXX Reducing len needs to recalculate LARGE flag */
1113 			dhph->dh_len = maplen;
1114 			maplen = 0;
1115 			dhpp = dhph->dh_next;
1116 			dhph->dh_next = NULL;
1117 			dhph = dhpp;
1118 		}
1119 	} /* end while */
1120 }
1121 
1122 /*
1123  * Free a segment.
1124  */
1125 static void
segdev_free(struct seg * seg)1126 segdev_free(struct seg *seg)
1127 {
1128 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1129 	devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1130 
1131 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1132 	    "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1133 	DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1134 	    (void *)dhp, (void *)seg));
1135 
1136 	/*
1137 	 * Since the address space is "write" locked, we
1138 	 * don't need the segment lock to protect "segdev" data.
1139 	 */
1140 	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
1141 
1142 	while (dhp != NULL)
1143 		dhp = devmap_handle_unmap(dhp);
1144 
1145 	VN_RELE(sdp->vp);
1146 	if (sdp->vpage != NULL)
1147 		kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1148 
1149 	rw_destroy(&sdp->lock);
1150 	kmem_free(sdp, sizeof (*sdp));
1151 }
1152 
1153 static void
free_devmap_handle(devmap_handle_t * dhp)1154 free_devmap_handle(devmap_handle_t *dhp)
1155 {
1156 	register devmap_handle_t *dhpp;
1157 
1158 	/*
1159 	 * free up devmap handle
1160 	 */
1161 	while (dhp != NULL) {
1162 		dhpp = dhp->dh_next;
1163 		if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1164 			mutex_destroy(&dhp->dh_lock);
1165 		}
1166 
1167 		if (dhp->dh_softlock != NULL)
1168 			devmap_softlock_rele(dhp);
1169 
1170 		if (dhp->dh_ctx != NULL)
1171 			devmap_ctx_rele(dhp);
1172 
1173 		kmem_free(dhp, sizeof (devmap_handle_t));
1174 		dhp = dhpp;
1175 	}
1176 }
1177 
1178 /*
1179  * routines to lock and unlock underlying segkp segment for
1180  * KMEM_PAGEABLE type cookies.
1181  * segkp only allows a single pending F_SOFTLOCK
1182  * we keep track of number of locks in the cookie so we can
1183  * have multiple pending faults and manage the calls to segkp.
1184  * RFE: if segkp supports either pagelock or can support multiple
1185  * calls to F_SOFTLOCK, then these routines can go away.
1186  *	If pagelock, segdev_faultpage can fault on a page by page basis
1187  *		and simplifies the code quite a bit.
1188  *	if multiple calls allowed but not partial ranges, then need for
1189  *	cookie->lock and locked count goes away, code can call as_fault directly
1190  */
1191 static faultcode_t
acquire_kpmem_lock(struct ddi_umem_cookie * cookie,size_t npages)1192 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1193 {
1194 	int err = 0;
1195 	ASSERT(cookie_is_kpmem(cookie));
1196 	/*
1197 	 * Fault in pages in segkp with F_SOFTLOCK.
1198 	 * We want to hold the lock until all pages have been loaded.
1199 	 * segkp only allows single caller to hold SOFTLOCK, so cookie
1200 	 * holds a count so we dont call into segkp multiple times
1201 	 */
1202 	mutex_enter(&cookie->lock);
1203 
1204 	/*
1205 	 * Check for overflow in locked field
1206 	 */
1207 	if ((UINT32_MAX - cookie->locked) < npages) {
1208 		err = FC_MAKE_ERR(ENOMEM);
1209 	} else if (cookie->locked == 0) {
1210 		/* First time locking */
1211 		err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1212 		    cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1213 	}
1214 	if (!err) {
1215 		cookie->locked += npages;
1216 	}
1217 	mutex_exit(&cookie->lock);
1218 	return (err);
1219 }
1220 
1221 static void
release_kpmem_lock(struct ddi_umem_cookie * cookie,size_t npages)1222 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1223 {
1224 	mutex_enter(&cookie->lock);
1225 	ASSERT(cookie_is_kpmem(cookie));
1226 	ASSERT(cookie->locked >= npages);
1227 	cookie->locked -= (uint_t)npages;
1228 	if (cookie->locked == 0) {
1229 		/* Last unlock */
1230 		if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1231 		    cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1232 			panic("segdev releasing kpmem lock %p", (void *)cookie);
1233 	}
1234 	mutex_exit(&cookie->lock);
1235 }
1236 
1237 /*
1238  * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1239  * drivers with devmap_access callbacks
1240  * slock->softlocked basically works like a rw lock
1241  *	-ve counts => F_SOFTLOCK in progress
1242  *	+ve counts => F_INVAL/F_PROT in progress
1243  * We allow only one F_SOFTLOCK at a time
1244  * but can have multiple pending F_INVAL/F_PROT calls
1245  *
1246  * This routine waits using cv_wait_sig so killing processes is more graceful
1247  * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1248  */
devmap_softlock_enter(struct devmap_softlock * slock,size_t npages,enum fault_type type)1249 static int devmap_softlock_enter(
1250 	struct devmap_softlock *slock,
1251 	size_t npages,
1252 	enum fault_type type)
1253 {
1254 	if (npages == 0)
1255 		return (0);
1256 	mutex_enter(&(slock->lock));
1257 	switch (type) {
1258 	case F_SOFTLOCK :
1259 		while (slock->softlocked) {
1260 			if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1261 				/* signalled */
1262 				mutex_exit(&(slock->lock));
1263 				return (EINTR);
1264 			}
1265 		}
1266 		slock->softlocked -= npages; /* -ve count => locked */
1267 		break;
1268 	case F_INVAL :
1269 	case F_PROT :
1270 		while (slock->softlocked < 0)
1271 			if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1272 				/* signalled */
1273 				mutex_exit(&(slock->lock));
1274 				return (EINTR);
1275 			}
1276 		slock->softlocked += npages; /* +ve count => f_invals */
1277 		break;
1278 	default:
1279 		ASSERT(0);
1280 	}
1281 	mutex_exit(&(slock->lock));
1282 	return (0);
1283 }
1284 
devmap_softlock_exit(struct devmap_softlock * slock,size_t npages,enum fault_type type)1285 static void devmap_softlock_exit(
1286 	struct devmap_softlock *slock,
1287 	size_t npages,
1288 	enum fault_type type)
1289 {
1290 	if (slock == NULL)
1291 		return;
1292 	mutex_enter(&(slock->lock));
1293 	switch (type) {
1294 	case F_SOFTLOCK :
1295 		ASSERT(-slock->softlocked >= npages);
1296 		slock->softlocked += npages;	/* -ve count is softlocked */
1297 		if (slock->softlocked == 0)
1298 			cv_signal(&slock->cv);
1299 		break;
1300 	case F_INVAL :
1301 	case F_PROT:
1302 		ASSERT(slock->softlocked >= npages);
1303 		slock->softlocked -= npages;
1304 		if (slock->softlocked == 0)
1305 			cv_signal(&slock->cv);
1306 		break;
1307 	default:
1308 		ASSERT(0);
1309 	}
1310 	mutex_exit(&(slock->lock));
1311 }
1312 
1313 /*
1314  * Do a F_SOFTUNLOCK call over the range requested.
1315  * The range must have already been F_SOFTLOCK'ed.
1316  * The segment lock should be held, (but not the segment private lock?)
1317  *  The softunlock code below does not adjust for large page sizes
1318  *	assumes the caller already did any addr/len adjustments for
1319  *	pagesize mappings before calling.
1320  */
1321 /*ARGSUSED*/
1322 static void
segdev_softunlock(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum seg_rw rw)1323 segdev_softunlock(
1324 	struct hat *hat,		/* the hat */
1325 	struct seg *seg,		/* seg_dev of interest */
1326 	caddr_t addr,			/* base address of range */
1327 	size_t len,			/* number of bytes */
1328 	enum seg_rw rw)			/* type of access at fault */
1329 {
1330 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1331 	devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1332 
1333 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1334 	    "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1335 	    dhp_head, sdp, addr, len);
1336 	DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1337 	    "addr %p len %lx\n",
1338 	    (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1339 
1340 	hat_unlock(hat, addr, len);
1341 
1342 	if (dhp_head != NULL) {
1343 		devmap_handle_t *dhp;
1344 		size_t mlen;
1345 		size_t tlen = len;
1346 		ulong_t off;
1347 
1348 		dhp = devmap_find_handle(dhp_head, addr);
1349 		ASSERT(dhp != NULL);
1350 
1351 		off = (ulong_t)(addr - dhp->dh_uvaddr);
1352 		while (tlen != 0) {
1353 			mlen = MIN(tlen, (dhp->dh_len - off));
1354 
1355 			/*
1356 			 * unlock segkp memory, locked during F_SOFTLOCK
1357 			 */
1358 			if (dhp_is_kpmem(dhp)) {
1359 				release_kpmem_lock(
1360 				    (struct ddi_umem_cookie *)dhp->dh_cookie,
1361 				    btopr(mlen));
1362 			}
1363 
1364 			/*
1365 			 * Do the softlock accounting for devmap_access
1366 			 */
1367 			if (dhp->dh_callbackops.devmap_access != NULL) {
1368 				devmap_softlock_exit(dhp->dh_softlock,
1369 				    btopr(mlen), F_SOFTLOCK);
1370 			}
1371 
1372 			tlen -= mlen;
1373 			dhp = dhp->dh_next;
1374 			off = 0;
1375 		}
1376 	}
1377 
1378 	mutex_enter(&freemem_lock);
1379 	ASSERT(sdp->softlockcnt >= btopr(len));
1380 	sdp->softlockcnt -= btopr(len);
1381 	mutex_exit(&freemem_lock);
1382 	if (sdp->softlockcnt == 0) {
1383 		/*
1384 		 * All SOFTLOCKS are gone. Wakeup any waiting
1385 		 * unmappers so they can try again to unmap.
1386 		 * Check for waiters first without the mutex
1387 		 * held so we don't always grab the mutex on
1388 		 * softunlocks.
1389 		 */
1390 		if (AS_ISUNMAPWAIT(seg->s_as)) {
1391 			mutex_enter(&seg->s_as->a_contents);
1392 			if (AS_ISUNMAPWAIT(seg->s_as)) {
1393 				AS_CLRUNMAPWAIT(seg->s_as);
1394 				cv_broadcast(&seg->s_as->a_cv);
1395 			}
1396 			mutex_exit(&seg->s_as->a_contents);
1397 		}
1398 	}
1399 
1400 }
1401 
1402 /*
1403  * Handle fault for a single page.
1404  * Done in a separate routine so we can handle errors more easily.
1405  * This routine is called only from segdev_faultpages()
1406  * when looping over the range of addresses requested. The segment lock is held.
1407  */
1408 static faultcode_t
segdev_faultpage(struct hat * hat,struct seg * seg,caddr_t addr,struct vpage * vpage,enum fault_type type,enum seg_rw rw,devmap_handle_t * dhp)1409 segdev_faultpage(
1410 	struct hat *hat,		/* the hat */
1411 	struct seg *seg,		/* seg_dev of interest */
1412 	caddr_t addr,			/* address in as */
1413 	struct vpage *vpage,		/* pointer to vpage for seg, addr */
1414 	enum fault_type type,		/* type of fault */
1415 	enum seg_rw rw,			/* type of access at fault */
1416 	devmap_handle_t *dhp)		/* devmap handle if any for this page */
1417 {
1418 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1419 	uint_t prot;
1420 	pfn_t pfnum = PFN_INVALID;
1421 	u_offset_t offset;
1422 	uint_t hat_flags;
1423 	dev_info_t *dip;
1424 
1425 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1426 	    "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1427 	DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1428 	    (void *)dhp, (void *)seg, (void *)addr));
1429 
1430 	/*
1431 	 * Initialize protection value for this page.
1432 	 * If we have per page protection values check it now.
1433 	 */
1434 	if (sdp->pageprot) {
1435 		uint_t protchk;
1436 
1437 		switch (rw) {
1438 		case S_READ:
1439 			protchk = PROT_READ;
1440 			break;
1441 		case S_WRITE:
1442 			protchk = PROT_WRITE;
1443 			break;
1444 		case S_EXEC:
1445 			protchk = PROT_EXEC;
1446 			break;
1447 		case S_OTHER:
1448 		default:
1449 			protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1450 			break;
1451 		}
1452 
1453 		prot = VPP_PROT(vpage);
1454 		if ((prot & protchk) == 0)
1455 			return (FC_PROT);	/* illegal access type */
1456 	} else {
1457 		prot = sdp->prot;
1458 		/* caller has already done segment level protection check */
1459 	}
1460 
1461 	if (type == F_SOFTLOCK) {
1462 		mutex_enter(&freemem_lock);
1463 		sdp->softlockcnt++;
1464 		mutex_exit(&freemem_lock);
1465 	}
1466 
1467 	hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1468 	offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1469 	/*
1470 	 * In the devmap framework, sdp->mapfunc is set to NULL.  we can get
1471 	 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1472 	 * seg->s_base.
1473 	 */
1474 	if (dhp == NULL) {
1475 		/* If segment has devmap_data, then dhp should be non-NULL */
1476 		ASSERT(sdp->devmap_data == NULL);
1477 		pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1478 		    (off_t)offset, prot);
1479 		prot |= sdp->hat_attr;
1480 	} else {
1481 		ulong_t off;
1482 		struct ddi_umem_cookie *cp;
1483 		struct devmap_pmem_cookie *pcp;
1484 
1485 		/* ensure the dhp passed in contains addr. */
1486 		ASSERT(dhp == devmap_find_handle(
1487 		    (devmap_handle_t *)sdp->devmap_data, addr));
1488 
1489 		off = addr - dhp->dh_uvaddr;
1490 
1491 		/*
1492 		 * This routine assumes that the caller makes sure that the
1493 		 * fields in dhp used below are unchanged due to remap during
1494 		 * this call. Caller does HOLD_DHP_LOCK if neeed
1495 		 */
1496 		cp = dhp->dh_cookie;
1497 		if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1498 			pfnum = PFN_INVALID;
1499 		} else if (cookie_is_devmem(cp)) {
1500 			pfnum = dhp->dh_pfn + btop(off);
1501 		} else if (cookie_is_pmem(cp)) {
1502 			pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1503 			ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1504 			    dhp->dh_roff < ptob(pcp->dp_npages));
1505 			pfnum = page_pptonum(
1506 			    pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1507 		} else {
1508 			ASSERT(dhp->dh_roff < cp->size);
1509 			ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1510 			    dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1511 			ASSERT((dhp->dh_cvaddr + off) <=
1512 			    (cp->cvaddr + cp->size));
1513 			ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1514 			    (cp->cvaddr + cp->size));
1515 
1516 			switch (cp->type) {
1517 			case UMEM_LOCKED :
1518 				if (cp->pparray != NULL) {
1519 					ASSERT((dhp->dh_roff &
1520 					    PAGEOFFSET) == 0);
1521 					pfnum = page_pptonum(
1522 					    cp->pparray[btop(off +
1523 					    dhp->dh_roff)]);
1524 				} else {
1525 					pfnum = hat_getpfnum(
1526 					    ((proc_t *)cp->procp)->p_as->a_hat,
1527 					    cp->cvaddr + off);
1528 				}
1529 			break;
1530 			case UMEM_TRASH :
1531 				pfnum = page_pptonum(trashpp);
1532 				/*
1533 				 * We should set hat_flags to HAT_NOFAULT also
1534 				 * However, not all hat layers implement this
1535 				 */
1536 				break;
1537 			case KMEM_PAGEABLE:
1538 			case KMEM_NON_PAGEABLE:
1539 				pfnum = hat_getpfnum(kas.a_hat,
1540 				    dhp->dh_cvaddr + off);
1541 				break;
1542 			default :
1543 				pfnum = PFN_INVALID;
1544 				break;
1545 			}
1546 		}
1547 		prot |= dhp->dh_hat_attr;
1548 	}
1549 	if (pfnum == PFN_INVALID) {
1550 		return (FC_MAKE_ERR(EFAULT));
1551 	}
1552 	/* prot should already be OR'ed in with hat_attributes if needed */
1553 
1554 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1555 	    "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1556 	    pfnum, pf_is_memory(pfnum), prot, hat_flags);
1557 	DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1558 	    "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1559 
1560 	if (pf_is_memory(pfnum) || (dhp != NULL)) {
1561 		/*
1562 		 * It's not _really_ required here to pass sdp->hat_flags
1563 		 * to hat_devload even though we do it.
1564 		 * This is because hat figures it out DEVMEM mappings
1565 		 * are non-consistent, anyway.
1566 		 */
1567 		hat_devload(hat, addr, PAGESIZE, pfnum,
1568 		    prot, hat_flags | sdp->hat_flags);
1569 		return (0);
1570 	}
1571 
1572 	/*
1573 	 * Fall through to the case where devmap is not used and need to call
1574 	 * up the device tree to set up the mapping
1575 	 */
1576 
1577 	dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1578 	ASSERT(dip);
1579 
1580 	/*
1581 	 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1582 	 * This is because this calls drivers which may not expect
1583 	 * prot to have any other values than PROT_ALL
1584 	 * The root nexus driver has a hack to peek into the segment
1585 	 * structure and then OR in sdp->hat_attr.
1586 	 * XX In case the bus_ops interfaces are ever revisited
1587 	 * we need to fix this. prot should include other hat attributes
1588 	 */
1589 	if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1590 	    (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1591 		return (FC_MAKE_ERR(EFAULT));
1592 	}
1593 	return (0);
1594 }
1595 
1596 static faultcode_t
segdev_fault(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw)1597 segdev_fault(
1598 	struct hat *hat,		/* the hat */
1599 	struct seg *seg,		/* the seg_dev of interest */
1600 	caddr_t addr,			/* the address of the fault */
1601 	size_t len,			/* the length of the range */
1602 	enum fault_type type,		/* type of fault */
1603 	enum seg_rw rw)			/* type of access at fault */
1604 {
1605 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1606 	devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1607 	devmap_handle_t *dhp;
1608 	struct devmap_softlock *slock = NULL;
1609 	ulong_t slpage = 0;
1610 	ulong_t off;
1611 	caddr_t maddr = addr;
1612 	int err;
1613 	int err_is_faultcode = 0;
1614 
1615 	TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1616 	    "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1617 	    (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1618 	DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1619 	    "addr %p len %lx type %x\n",
1620 	    (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1621 
1622 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
1623 
1624 	/* Handle non-devmap case */
1625 	if (dhp_head == NULL)
1626 		return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1627 
1628 	/* Find devmap handle */
1629 	if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1630 		return (FC_NOMAP);
1631 
1632 	/*
1633 	 * The seg_dev driver does not implement copy-on-write,
1634 	 * and always loads translations with maximal allowed permissions
1635 	 * but we got an fault trying to access the device.
1636 	 * Servicing the fault is not going to result in any better result
1637 	 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1638 	 *	faults, then the code below is written for that
1639 	 *	Pending resolution of the following:
1640 	 *	- determine if the F_INVAL/F_SOFTLOCK syncing
1641 	 *	is needed for F_PROT also or not. The code below assumes it does
1642 	 *	- If driver sees F_PROT and calls devmap_load with same type,
1643 	 *	then segdev_faultpages will fail with FC_PROT anyway, need to
1644 	 *	change that so calls from devmap_load to segdev_faultpages for
1645 	 *	F_PROT type are retagged to F_INVAL.
1646 	 * RFE: Today we dont have drivers that use devmap and want to handle
1647 	 *	F_PROT calls. The code in segdev_fault* is written to allow
1648 	 *	this case but is not tested. A driver that needs this capability
1649 	 *	should be able to remove the short-circuit case; resolve the
1650 	 *	above issues and "should" work.
1651 	 */
1652 	if (type == F_PROT) {
1653 		return (FC_PROT);
1654 	}
1655 
1656 	/*
1657 	 * Loop through dhp list calling devmap_access or segdev_faultpages for
1658 	 * each devmap handle.
1659 	 * drivers which implement devmap_access can interpose on faults and do
1660 	 * device-appropriate special actions before calling devmap_load.
1661 	 */
1662 
1663 	/*
1664 	 * Unfortunately, this simple loop has turned out to expose a variety
1665 	 * of complex problems which results in the following convoluted code.
1666 	 *
1667 	 * First, a desire to handle a serialization of F_SOFTLOCK calls
1668 	 * to the driver within the framework.
1669 	 *	This results in a dh_softlock structure that is on a per device
1670 	 *	(or device instance) basis and serializes devmap_access calls.
1671 	 *	Ideally we would need to do this for underlying
1672 	 *	memory/device regions that are being faulted on
1673 	 *	but that is hard to identify and with REMAP, harder
1674 	 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1675 	 * 	to F_SOFTLOCK calls to the driver.
1676 	 * These serializations are to simplify the driver programmer model.
1677 	 * To support these two features, the code first goes through the
1678 	 *	devmap handles and counts the pages (slpage) that are covered
1679 	 *	by devmap_access callbacks.
1680 	 * This part ends with a devmap_softlock_enter call
1681 	 *	which allows only one F_SOFTLOCK active on a device instance,
1682 	 *	but multiple F_INVAL/F_PROTs can be active except when a
1683 	 *	F_SOFTLOCK is active
1684 	 *
1685 	 * Next, we dont short-circuit the fault code upfront to call
1686 	 *	segdev_softunlock for F_SOFTUNLOCK, because we must use
1687 	 *	the same length when we softlock and softunlock.
1688 	 *
1689 	 *	-Hat layers may not support softunlocking lengths less than the
1690 	 *	original length when there is large page support.
1691 	 *	-kpmem locking is dependent on keeping the lengths same.
1692 	 *	-if drivers handled F_SOFTLOCK, they probably also expect to
1693 	 *		see an F_SOFTUNLOCK of the same length
1694 	 *	Hence, if extending lengths during softlock,
1695 	 *	softunlock has to make the same adjustments and goes through
1696 	 *	the same loop calling segdev_faultpages/segdev_softunlock
1697 	 *	But some of the synchronization and error handling is different
1698 	 */
1699 
1700 	if (type != F_SOFTUNLOCK) {
1701 		devmap_handle_t *dhpp = dhp;
1702 		size_t slen = len;
1703 
1704 		/*
1705 		 * Calculate count of pages that are :
1706 		 * a) within the (potentially extended) fault region
1707 		 * b) AND covered by devmap handle with devmap_access
1708 		 */
1709 		off = (ulong_t)(addr - dhpp->dh_uvaddr);
1710 		while (slen != 0) {
1711 			size_t mlen;
1712 
1713 			/*
1714 			 * Softlocking on a region that allows remap is
1715 			 * unsupported due to unresolved locking issues
1716 			 * XXX: unclear what these are?
1717 			 *	One potential is that if there is a pending
1718 			 *	softlock, then a remap should not be allowed
1719 			 *	until the unlock is done. This is easily
1720 			 *	fixed by returning error in devmap*remap on
1721 			 *	checking the dh->dh_softlock->softlocked value
1722 			 */
1723 			if ((type == F_SOFTLOCK) &&
1724 			    (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1725 				return (FC_NOSUPPORT);
1726 			}
1727 
1728 			mlen = MIN(slen, (dhpp->dh_len - off));
1729 			if (dhpp->dh_callbackops.devmap_access) {
1730 				size_t llen;
1731 				caddr_t laddr;
1732 				/*
1733 				 * use extended length for large page mappings
1734 				 */
1735 				HOLD_DHP_LOCK(dhpp);
1736 				if ((sdp->pageprot == 0) &&
1737 				    (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1738 					devmap_get_large_pgsize(dhpp,
1739 					    mlen, maddr, &llen, &laddr);
1740 				} else {
1741 					llen = mlen;
1742 				}
1743 				RELE_DHP_LOCK(dhpp);
1744 
1745 				slpage += btopr(llen);
1746 				slock = dhpp->dh_softlock;
1747 			}
1748 			maddr += mlen;
1749 			ASSERT(slen >= mlen);
1750 			slen -= mlen;
1751 			dhpp = dhpp->dh_next;
1752 			off = 0;
1753 		}
1754 		/*
1755 		 * synchonize with other faulting threads and wait till safe
1756 		 * devmap_softlock_enter might return due to signal in cv_wait
1757 		 *
1758 		 * devmap_softlock_enter has to be called outside of while loop
1759 		 * to prevent a deadlock if len spans over multiple dhps.
1760 		 * dh_softlock is based on device instance and if multiple dhps
1761 		 * use the same device instance, the second dhp's LOCK call
1762 		 * will hang waiting on the first to complete.
1763 		 * devmap_setup verifies that slocks in a dhp_chain are same.
1764 		 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1765 		 * 	F_INVAL/F_PROT, since we now allow multiple in parallel,
1766 		 *	we could have done the softlock_enter inside the loop
1767 		 *	and supported multi-dhp mappings with dissimilar devices
1768 		 */
1769 		if (err = devmap_softlock_enter(slock, slpage, type))
1770 			return (FC_MAKE_ERR(err));
1771 	}
1772 
1773 	/* reset 'maddr' to the start addr of the range of fault. */
1774 	maddr = addr;
1775 
1776 	/* calculate the offset corresponds to 'addr' in the first dhp. */
1777 	off = (ulong_t)(addr - dhp->dh_uvaddr);
1778 
1779 	/*
1780 	 * The fault length may span over multiple dhps.
1781 	 * Loop until the total length is satisfied.
1782 	 */
1783 	while (len != 0) {
1784 		size_t llen;
1785 		size_t mlen;
1786 		caddr_t laddr;
1787 
1788 		/*
1789 		 * mlen is the smaller of 'len' and the length
1790 		 * from addr to the end of mapping defined by dhp.
1791 		 */
1792 		mlen = MIN(len, (dhp->dh_len - off));
1793 
1794 		HOLD_DHP_LOCK(dhp);
1795 		/*
1796 		 * Pass the extended length and address to devmap_access
1797 		 * if large pagesize is used for loading address translations.
1798 		 */
1799 		if ((sdp->pageprot == 0) &&
1800 		    (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1801 			devmap_get_large_pgsize(dhp, mlen, maddr,
1802 			    &llen, &laddr);
1803 			ASSERT(maddr == addr || laddr == maddr);
1804 		} else {
1805 			llen = mlen;
1806 			laddr = maddr;
1807 		}
1808 
1809 		if (dhp->dh_callbackops.devmap_access != NULL) {
1810 			offset_t aoff;
1811 
1812 			aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1813 
1814 			/*
1815 			 * call driver's devmap_access entry point which will
1816 			 * call devmap_load/contextmgmt to load the translations
1817 			 *
1818 			 * We drop the dhp_lock before calling access so
1819 			 * drivers can call devmap_*_remap within access
1820 			 */
1821 			RELE_DHP_LOCK(dhp);
1822 
1823 			err = (*dhp->dh_callbackops.devmap_access)(
1824 			    dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1825 		} else {
1826 			/*
1827 			 * If no devmap_access entry point, then load mappings
1828 			 * hold dhp_lock across faultpages if REMAP
1829 			 */
1830 			err = segdev_faultpages(hat, seg, laddr, llen,
1831 			    type, rw, dhp);
1832 			err_is_faultcode = 1;
1833 			RELE_DHP_LOCK(dhp);
1834 		}
1835 
1836 		if (err) {
1837 			if ((type == F_SOFTLOCK) && (maddr > addr)) {
1838 				/*
1839 				 * If not first dhp, use
1840 				 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1841 				 * While this is recursion, it is incorrect to
1842 				 * call just segdev_softunlock
1843 				 * if we are using either large pages
1844 				 * or devmap_access. It will be more right
1845 				 * to go through the same loop as above
1846 				 * rather than call segdev_softunlock directly
1847 				 * It will use the right lenghths as well as
1848 				 * call into the driver devmap_access routines.
1849 				 */
1850 				size_t done = (size_t)(maddr - addr);
1851 				(void) segdev_fault(hat, seg, addr, done,
1852 				    F_SOFTUNLOCK, S_OTHER);
1853 				/*
1854 				 * reduce slpage by number of pages
1855 				 * released by segdev_softunlock
1856 				 */
1857 				ASSERT(slpage >= btopr(done));
1858 				devmap_softlock_exit(slock,
1859 				    slpage - btopr(done), type);
1860 			} else {
1861 				devmap_softlock_exit(slock, slpage, type);
1862 			}
1863 
1864 
1865 			/*
1866 			 * Segdev_faultpages() already returns a faultcode,
1867 			 * hence, result from segdev_faultpages() should be
1868 			 * returned directly.
1869 			 */
1870 			if (err_is_faultcode)
1871 				return (err);
1872 			return (FC_MAKE_ERR(err));
1873 		}
1874 
1875 		maddr += mlen;
1876 		ASSERT(len >= mlen);
1877 		len -= mlen;
1878 		dhp = dhp->dh_next;
1879 		off = 0;
1880 
1881 		ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1882 	}
1883 	/*
1884 	 * release the softlock count at end of fault
1885 	 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1886 	 */
1887 	if ((type == F_INVAL) || (type == F_PROT))
1888 		devmap_softlock_exit(slock, slpage, type);
1889 	return (0);
1890 }
1891 
1892 /*
1893  * segdev_faultpages
1894  *
1895  * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1896  * This routine assumes that the callers makes sure that the fields
1897  * in dhp used below are not changed due to remap during this call.
1898  * Caller does HOLD_DHP_LOCK if neeed
1899  * This routine returns a faultcode_t as a return value for segdev_fault.
1900  */
1901 static faultcode_t
segdev_faultpages(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw,devmap_handle_t * dhp)1902 segdev_faultpages(
1903 	struct hat *hat,		/* the hat */
1904 	struct seg *seg,		/* the seg_dev of interest */
1905 	caddr_t addr,			/* the address of the fault */
1906 	size_t len,			/* the length of the range */
1907 	enum fault_type type,		/* type of fault */
1908 	enum seg_rw rw,			/* type of access at fault */
1909 	devmap_handle_t *dhp)		/* devmap handle */
1910 {
1911 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1912 	register caddr_t a;
1913 	struct vpage *vpage;
1914 	struct ddi_umem_cookie *kpmem_cookie = NULL;
1915 	int err;
1916 
1917 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1918 	    "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1919 	    (void *)dhp, (void *)seg, (void *)addr, len);
1920 	DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1921 	    "dhp %p seg %p addr %p len %lx\n",
1922 	    (void *)dhp, (void *)seg, (void *)addr, len));
1923 
1924 	/*
1925 	 * The seg_dev driver does not implement copy-on-write,
1926 	 * and always loads translations with maximal allowed permissions
1927 	 * but we got an fault trying to access the device.
1928 	 * Servicing the fault is not going to result in any better result
1929 	 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1930 	 * This code should be removed and let the normal fault handling
1931 	 * take care of finding the error
1932 	 */
1933 	if (type == F_PROT) {
1934 		return (FC_PROT);
1935 	}
1936 
1937 	if (type == F_SOFTUNLOCK) {
1938 		segdev_softunlock(hat, seg, addr, len, rw);
1939 		return (0);
1940 	}
1941 
1942 	/*
1943 	 * For kernel pageable memory, fault/lock segkp pages
1944 	 * We hold this until the completion of this
1945 	 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1946 	 */
1947 	if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1948 		kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1949 		if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1950 			return (err);
1951 	}
1952 
1953 	/*
1954 	 * If we have the same protections for the entire segment,
1955 	 * insure that the access being attempted is legitimate.
1956 	 */
1957 	rw_enter(&sdp->lock, RW_READER);
1958 	if (sdp->pageprot == 0) {
1959 		uint_t protchk;
1960 
1961 		switch (rw) {
1962 		case S_READ:
1963 			protchk = PROT_READ;
1964 			break;
1965 		case S_WRITE:
1966 			protchk = PROT_WRITE;
1967 			break;
1968 		case S_EXEC:
1969 			protchk = PROT_EXEC;
1970 			break;
1971 		case S_OTHER:
1972 		default:
1973 			protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1974 			break;
1975 		}
1976 
1977 		if ((sdp->prot & protchk) == 0) {
1978 			rw_exit(&sdp->lock);
1979 			/* undo kpmem locking */
1980 			if (kpmem_cookie != NULL) {
1981 				release_kpmem_lock(kpmem_cookie, btopr(len));
1982 			}
1983 			return (FC_PROT);	/* illegal access type */
1984 		}
1985 	}
1986 
1987 	/*
1988 	 * we do a single hat_devload for the range if
1989 	 *   - devmap framework (dhp is not NULL),
1990 	 *   - pageprot == 0, i.e., no per-page protection set and
1991 	 *   - is device pages, irrespective of whether we are using large pages
1992 	 */
1993 	if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1994 		pfn_t pfnum;
1995 		uint_t hat_flags;
1996 
1997 		if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1998 			rw_exit(&sdp->lock);
1999 			return (FC_NOMAP);
2000 		}
2001 
2002 		if (type == F_SOFTLOCK) {
2003 			mutex_enter(&freemem_lock);
2004 			sdp->softlockcnt += btopr(len);
2005 			mutex_exit(&freemem_lock);
2006 		}
2007 
2008 		hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2009 		pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2010 		ASSERT(!pf_is_memory(pfnum));
2011 
2012 		hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2013 		    hat_flags | sdp->hat_flags);
2014 		rw_exit(&sdp->lock);
2015 		return (0);
2016 	}
2017 
2018 	/* Handle cases where we have to loop through fault handling per-page */
2019 
2020 	if (sdp->vpage == NULL)
2021 		vpage = NULL;
2022 	else
2023 		vpage = &sdp->vpage[seg_page(seg, addr)];
2024 
2025 	/* loop over the address range handling each fault */
2026 	for (a = addr; a < addr + len; a += PAGESIZE) {
2027 		if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2028 			break;
2029 		}
2030 		if (vpage != NULL)
2031 			vpage++;
2032 	}
2033 	rw_exit(&sdp->lock);
2034 	if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2035 		size_t done = (size_t)(a - addr); /* pages fault successfully */
2036 		if (done > 0) {
2037 			/* use softunlock for those pages */
2038 			segdev_softunlock(hat, seg, addr, done, S_OTHER);
2039 		}
2040 		if (kpmem_cookie != NULL) {
2041 			/* release kpmem lock for rest of pages */
2042 			ASSERT(len >= done);
2043 			release_kpmem_lock(kpmem_cookie, btopr(len - done));
2044 		}
2045 	} else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2046 		/* for non-SOFTLOCK cases, release kpmem */
2047 		release_kpmem_lock(kpmem_cookie, btopr(len));
2048 	}
2049 	return (err);
2050 }
2051 
2052 /*
2053  * Asynchronous page fault.  We simply do nothing since this
2054  * entry point is not supposed to load up the translation.
2055  */
2056 /*ARGSUSED*/
2057 static faultcode_t
segdev_faulta(struct seg * seg,caddr_t addr)2058 segdev_faulta(struct seg *seg, caddr_t addr)
2059 {
2060 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2061 	    "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2062 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2063 
2064 	return (0);
2065 }
2066 
2067 static int
segdev_setprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)2068 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2069 {
2070 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2071 	register devmap_handle_t *dhp;
2072 	register struct vpage *vp, *evp;
2073 	devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2074 	ulong_t off;
2075 	size_t mlen, sz;
2076 
2077 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2078 	    "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2079 	    (void *)seg, (void *)addr, len, prot);
2080 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2081 
2082 	if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2083 		/*
2084 		 * Fail the setprot if pages are SOFTLOCKed through this
2085 		 * mapping.
2086 		 * Softlockcnt is protected from change by the as read lock.
2087 		 */
2088 		TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2089 		    "segdev_setprot:error softlockcnt=%lx", sz);
2090 		DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2091 		return (EAGAIN);
2092 	}
2093 
2094 	if (dhp_head != NULL) {
2095 		if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2096 			return (EINVAL);
2097 
2098 		/*
2099 		 * check if violate maxprot.
2100 		 */
2101 		off = (ulong_t)(addr - dhp->dh_uvaddr);
2102 		mlen  = len;
2103 		while (dhp) {
2104 			if ((dhp->dh_maxprot & prot) != prot)
2105 				return (EACCES);	/* violated maxprot */
2106 
2107 			if (mlen > (dhp->dh_len - off)) {
2108 				mlen -= dhp->dh_len - off;
2109 				dhp = dhp->dh_next;
2110 				off = 0;
2111 			} else
2112 				break;
2113 		}
2114 	} else {
2115 		if ((sdp->maxprot & prot) != prot)
2116 			return (EACCES);
2117 	}
2118 
2119 	rw_enter(&sdp->lock, RW_WRITER);
2120 	if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2121 		if (sdp->prot == prot) {
2122 			rw_exit(&sdp->lock);
2123 			return (0);			/* all done */
2124 		}
2125 		sdp->prot = (uchar_t)prot;
2126 	} else {
2127 		sdp->pageprot = 1;
2128 		if (sdp->vpage == NULL) {
2129 			/*
2130 			 * First time through setting per page permissions,
2131 			 * initialize all the vpage structures to prot
2132 			 */
2133 			sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2134 			    KM_SLEEP);
2135 			evp = &sdp->vpage[seg_pages(seg)];
2136 			for (vp = sdp->vpage; vp < evp; vp++)
2137 				VPP_SETPROT(vp, sdp->prot);
2138 		}
2139 		/*
2140 		 * Now go change the needed vpages protections.
2141 		 */
2142 		evp = &sdp->vpage[seg_page(seg, addr + len)];
2143 		for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2144 			VPP_SETPROT(vp, prot);
2145 	}
2146 	rw_exit(&sdp->lock);
2147 
2148 	if (dhp_head != NULL) {
2149 		devmap_handle_t *tdhp;
2150 		/*
2151 		 * If large page size was used in hat_devload(),
2152 		 * the same page size must be used in hat_unload().
2153 		 */
2154 		dhp = tdhp = devmap_find_handle(dhp_head, addr);
2155 		while (tdhp != NULL) {
2156 			if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2157 				break;
2158 			}
2159 			tdhp = tdhp->dh_next;
2160 		}
2161 		if (tdhp) {
2162 			size_t slen = len;
2163 			size_t mlen;
2164 			size_t soff;
2165 
2166 			soff = (ulong_t)(addr - dhp->dh_uvaddr);
2167 			while (slen != 0) {
2168 				mlen = MIN(slen, (dhp->dh_len - soff));
2169 				hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2170 				    dhp->dh_len, HAT_UNLOAD);
2171 				dhp = dhp->dh_next;
2172 				ASSERT(slen >= mlen);
2173 				slen -= mlen;
2174 				soff = 0;
2175 			}
2176 			return (0);
2177 		}
2178 	}
2179 
2180 	if ((prot & ~PROT_USER) == PROT_NONE) {
2181 		hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2182 	} else {
2183 		/*
2184 		 * RFE: the segment should keep track of all attributes
2185 		 * allowing us to remove the deprecated hat_chgprot
2186 		 * and use hat_chgattr.
2187 		 */
2188 		hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2189 	}
2190 
2191 	return (0);
2192 }
2193 
2194 static int
segdev_checkprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)2195 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2196 {
2197 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2198 	struct vpage *vp, *evp;
2199 
2200 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2201 	    "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2202 	    (void *)seg, (void *)addr, len, prot);
2203 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2204 
2205 	/*
2206 	 * If segment protection can be used, simply check against them
2207 	 */
2208 	rw_enter(&sdp->lock, RW_READER);
2209 	if (sdp->pageprot == 0) {
2210 		register int err;
2211 
2212 		err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2213 		rw_exit(&sdp->lock);
2214 		return (err);
2215 	}
2216 
2217 	/*
2218 	 * Have to check down to the vpage level
2219 	 */
2220 	evp = &sdp->vpage[seg_page(seg, addr + len)];
2221 	for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2222 		if ((VPP_PROT(vp) & prot) != prot) {
2223 			rw_exit(&sdp->lock);
2224 			return (EACCES);
2225 		}
2226 	}
2227 	rw_exit(&sdp->lock);
2228 	return (0);
2229 }
2230 
2231 static int
segdev_getprot(struct seg * seg,caddr_t addr,size_t len,uint_t * protv)2232 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2233 {
2234 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2235 	size_t pgno;
2236 
2237 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2238 	    "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2239 	    (void *)seg, (void *)addr, len, (void *)protv);
2240 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2241 
2242 	pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2243 	if (pgno != 0) {
2244 		rw_enter(&sdp->lock, RW_READER);
2245 		if (sdp->pageprot == 0) {
2246 			do {
2247 				protv[--pgno] = sdp->prot;
2248 			} while (pgno != 0);
2249 		} else {
2250 			size_t pgoff = seg_page(seg, addr);
2251 
2252 			do {
2253 				pgno--;
2254 				protv[pgno] =
2255 				    VPP_PROT(&sdp->vpage[pgno + pgoff]);
2256 			} while (pgno != 0);
2257 		}
2258 		rw_exit(&sdp->lock);
2259 	}
2260 	return (0);
2261 }
2262 
2263 static u_offset_t
segdev_getoffset(register struct seg * seg,caddr_t addr)2264 segdev_getoffset(register struct seg *seg, caddr_t addr)
2265 {
2266 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2267 
2268 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2269 	    "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2270 
2271 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2272 
2273 	return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2274 }
2275 
2276 /*ARGSUSED*/
2277 static int
segdev_gettype(register struct seg * seg,caddr_t addr)2278 segdev_gettype(register struct seg *seg, caddr_t addr)
2279 {
2280 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2281 
2282 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2283 	    "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2284 
2285 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2286 
2287 	return (sdp->type);
2288 }
2289 
2290 
2291 /*ARGSUSED*/
2292 static int
segdev_getvp(register struct seg * seg,caddr_t addr,struct vnode ** vpp)2293 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2294 {
2295 	register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2296 
2297 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2298 	    "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2299 
2300 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2301 
2302 	/*
2303 	 * Note that this vp is the common_vp of the device, where the
2304 	 * pages are hung ..
2305 	 */
2306 	*vpp = VTOCVP(sdp->vp);
2307 
2308 	return (0);
2309 }
2310 
2311 static void
segdev_badop(void)2312 segdev_badop(void)
2313 {
2314 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2315 	    "segdev_badop:start");
2316 	panic("segdev_badop");
2317 	/*NOTREACHED*/
2318 }
2319 
2320 /*
2321  * segdev pages are not in the cache, and thus can't really be controlled.
2322  * Hence, syncs are simply always successful.
2323  */
2324 /*ARGSUSED*/
2325 static int
segdev_sync(struct seg * seg,caddr_t addr,size_t len,int attr,uint_t flags)2326 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2327 {
2328 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2329 
2330 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2331 
2332 	return (0);
2333 }
2334 
2335 /*
2336  * segdev pages are always "in core".
2337  */
2338 /*ARGSUSED*/
2339 static size_t
segdev_incore(struct seg * seg,caddr_t addr,size_t len,char * vec)2340 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2341 {
2342 	size_t v = 0;
2343 
2344 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2345 
2346 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2347 
2348 	for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2349 	    v += PAGESIZE)
2350 		*vec++ = 1;
2351 	return (v);
2352 }
2353 
2354 /*
2355  * segdev pages are not in the cache, and thus can't really be controlled.
2356  * Hence, locks are simply always successful.
2357  */
2358 /*ARGSUSED*/
2359 static int
segdev_lockop(struct seg * seg,caddr_t addr,size_t len,int attr,int op,ulong_t * lockmap,size_t pos)2360 segdev_lockop(struct seg *seg, caddr_t addr,
2361     size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2362 {
2363 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2364 
2365 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2366 
2367 	return (0);
2368 }
2369 
2370 /*
2371  * segdev pages are not in the cache, and thus can't really be controlled.
2372  * Hence, advise is simply always successful.
2373  */
2374 /*ARGSUSED*/
2375 static int
segdev_advise(struct seg * seg,caddr_t addr,size_t len,uint_t behav)2376 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2377 {
2378 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2379 
2380 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
2381 
2382 	return (0);
2383 }
2384 
2385 /*
2386  * segdev pages are not dumped, so we just return
2387  */
2388 static void
segdev_dump(struct seg * seg __unused)2389 segdev_dump(struct seg *seg __unused)
2390 {
2391 }
2392 
2393 /*
2394  * ddi_segmap_setup:	Used by drivers who wish specify mapping attributes
2395  *			for a segment.	Called from a drivers segmap(9E)
2396  *			routine.
2397  */
2398 /*ARGSUSED*/
2399 int
ddi_segmap_setup(dev_t dev,off_t offset,struct as * as,caddr_t * addrp,off_t len,uint_t prot,uint_t maxprot,uint_t flags,cred_t * cred,ddi_device_acc_attr_t * accattrp,uint_t rnumber)2400 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2401     off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2402     ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2403 {
2404 	struct segdev_crargs dev_a;
2405 	int (*mapfunc)(dev_t dev, off_t off, int prot);
2406 	uint_t hat_attr;
2407 	pfn_t pfn;
2408 	int error, i;
2409 
2410 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2411 	    "ddi_segmap_setup:start");
2412 
2413 	if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2414 		return (ENODEV);
2415 
2416 	/*
2417 	 * Character devices that support the d_mmap
2418 	 * interface can only be mmap'ed shared.
2419 	 */
2420 	if ((flags & MAP_TYPE) != MAP_SHARED)
2421 		return (EINVAL);
2422 
2423 	/*
2424 	 * Check that this region is indeed mappable on this platform.
2425 	 * Use the mapping function.
2426 	 */
2427 	if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2428 		return (ENXIO);
2429 
2430 	if (len == 0)
2431 		return (ENXIO);
2432 
2433 	/*
2434 	 * Check to ensure that the entire range is
2435 	 * legal and we are not trying to map in
2436 	 * more than the device will let us.
2437 	 */
2438 	/*
2439 	 * Save the pfn at offset here. This pfn will be
2440 	 * used later to get user address.
2441 	 */
2442 	pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset, maxprot);
2443 	if (pfn == PFN_INVALID)
2444 		return (ENXIO);
2445 
2446 	for (i = PAGESIZE; i < len; i += PAGESIZE) {
2447 		if (cdev_mmap(mapfunc, dev, offset + i, maxprot) == PFN_INVALID)
2448 			return (ENXIO);
2449 	}
2450 
2451 	as_rangelock(as);
2452 	/* Pick an address w/o worrying about any vac alignment constraints. */
2453 	error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2454 	if (error != 0) {
2455 		as_rangeunlock(as);
2456 		return (error);
2457 	}
2458 
2459 	dev_a.mapfunc = mapfunc;
2460 	dev_a.dev = dev;
2461 	dev_a.offset = (offset_t)offset;
2462 	dev_a.type = flags & MAP_TYPE;
2463 	dev_a.prot = (uchar_t)prot;
2464 	dev_a.maxprot = (uchar_t)maxprot;
2465 	dev_a.hat_attr = hat_attr;
2466 	dev_a.hat_flags = 0;
2467 	dev_a.devmap_data = NULL;
2468 
2469 	error = as_map(as, *addrp, len, segdev_create, &dev_a);
2470 	as_rangeunlock(as);
2471 	return (error);
2472 
2473 }
2474 
2475 /*ARGSUSED*/
2476 static int
segdev_pagelock(struct seg * seg,caddr_t addr,size_t len,struct page *** ppp,enum lock_type type,enum seg_rw rw)2477 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2478     struct page ***ppp, enum lock_type type, enum seg_rw rw)
2479 {
2480 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2481 	    "segdev_pagelock:start");
2482 	return (ENOTSUP);
2483 }
2484 
2485 /*ARGSUSED*/
2486 static int
segdev_setpagesize(struct seg * seg,caddr_t addr,size_t len,uint_t szc)2487 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2488     uint_t szc)
2489 {
2490 	return (ENOTSUP);
2491 }
2492 
2493 /*
2494  * devmap_device: Used by devmap framework to establish mapping
2495  *                called by devmap_seup(9F) during map setup time.
2496  */
2497 /*ARGSUSED*/
2498 static int
devmap_device(devmap_handle_t * dhp,struct as * as,caddr_t * addr,offset_t off,size_t len,uint_t flags)2499 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2500     offset_t off, size_t len, uint_t flags)
2501 {
2502 	devmap_handle_t *rdhp, *maxdhp;
2503 	struct segdev_crargs dev_a;
2504 	int	err;
2505 	uint_t maxprot = PROT_ALL;
2506 	offset_t offset = 0;
2507 	pfn_t pfn;
2508 	struct devmap_pmem_cookie *pcp;
2509 
2510 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2511 	    "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2512 	    (void *)dhp, (void *)addr, off, len);
2513 
2514 	DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2515 	    (void *)dhp, (void *)addr, off, len));
2516 
2517 	as_rangelock(as);
2518 	if ((flags & MAP_FIXED) == 0) {
2519 		offset_t aligned_off;
2520 
2521 		rdhp = maxdhp = dhp;
2522 		while (rdhp != NULL) {
2523 			maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2524 			    maxdhp : rdhp;
2525 			rdhp = rdhp->dh_next;
2526 			maxprot |= dhp->dh_maxprot;
2527 		}
2528 		offset = maxdhp->dh_uoff - dhp->dh_uoff;
2529 
2530 		/*
2531 		 * Use the dhp that has the
2532 		 * largest len to get user address.
2533 		 */
2534 		/*
2535 		 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2536 		 * use 0 which is as good as any other.
2537 		 */
2538 		if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2539 			aligned_off = (offset_t)0;
2540 		} else if (dhp_is_devmem(maxdhp)) {
2541 			aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2542 		} else if (dhp_is_pmem(maxdhp)) {
2543 			pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2544 			pfn = page_pptonum(
2545 			    pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2546 			aligned_off = (offset_t)ptob(pfn) - offset;
2547 		} else {
2548 			aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2549 			    offset;
2550 		}
2551 
2552 		/*
2553 		 * Pick an address aligned to dh_cookie.
2554 		 * for kernel memory/user memory, cookie is cvaddr.
2555 		 * for device memory, cookie is physical address.
2556 		 */
2557 		map_addr(addr, len, aligned_off, 1, flags);
2558 		if (*addr == NULL) {
2559 			as_rangeunlock(as);
2560 			return (ENOMEM);
2561 		}
2562 	} else {
2563 		/*
2564 		 * User-specified address; blow away any previous mappings.
2565 		 */
2566 		(void) as_unmap(as, *addr, len);
2567 	}
2568 
2569 	dev_a.mapfunc = NULL;
2570 	dev_a.dev = dhp->dh_dev;
2571 	dev_a.type = flags & MAP_TYPE;
2572 	dev_a.offset = off;
2573 	/*
2574 	 * sdp->maxprot has the least restrict protection of all dhps.
2575 	 */
2576 	dev_a.maxprot = maxprot;
2577 	dev_a.prot = dhp->dh_prot;
2578 	/*
2579 	 * devmap uses dhp->dh_hat_attr for hat.
2580 	 */
2581 	dev_a.hat_flags = 0;
2582 	dev_a.hat_attr = 0;
2583 	dev_a.devmap_data = (void *)dhp;
2584 
2585 	err = as_map(as, *addr, len, segdev_create, &dev_a);
2586 	as_rangeunlock(as);
2587 	return (err);
2588 }
2589 
2590 int
devmap_do_ctxmgt(devmap_cookie_t dhc,void * pvtp,offset_t off,size_t len,uint_t type,uint_t rw,int (* ctxmgt)(devmap_cookie_t,void *,offset_t,size_t,uint_t,uint_t))2591 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2592     uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2593     size_t, uint_t, uint_t))
2594 {
2595 	register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2596 	struct devmap_ctx *devctx;
2597 	int do_timeout = 0;
2598 	int ret;
2599 
2600 #ifdef lint
2601 	pvtp = pvtp;
2602 #endif
2603 
2604 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2605 	    "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2606 	    (void *)dhp, off, len);
2607 	DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2608 	    (void *)dhp, off, len));
2609 
2610 	if (ctxmgt == NULL)
2611 		return (FC_HWERR);
2612 
2613 	devctx = dhp->dh_ctx;
2614 
2615 	/*
2616 	 * If we are on an MP system with more than one cpu running
2617 	 * and if a thread on some CPU already has the context, wait
2618 	 * for it to finish if there is a hysteresis timeout.
2619 	 *
2620 	 * We call cv_wait() instead of cv_wait_sig() because
2621 	 * it does not matter much if it returned due to a signal
2622 	 * or due to a cv_signal() or cv_broadcast().  In either event
2623 	 * we need to complete the mapping otherwise the processes
2624 	 * will die with a SEGV.
2625 	 */
2626 	if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2627 		TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2628 		    "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2629 		    devctx, dhp);
2630 		do_timeout = 1;
2631 		mutex_enter(&devctx->lock);
2632 		while (devctx->oncpu)
2633 			cv_wait(&devctx->cv, &devctx->lock);
2634 		devctx->oncpu = 1;
2635 		mutex_exit(&devctx->lock);
2636 	}
2637 
2638 	/*
2639 	 * Call the contextmgt callback so that the driver can handle
2640 	 * the fault.
2641 	 */
2642 	ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2643 
2644 	/*
2645 	 * If devmap_access() returned -1, then there was a hardware
2646 	 * error so we need to convert the return value to something
2647 	 * that trap() will understand.  Otherwise, the return value
2648 	 * is already a fault code generated by devmap_unload()
2649 	 * or devmap_load().
2650 	 */
2651 	if (ret) {
2652 		TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2653 		    "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2654 		    ret, dhp, devctx);
2655 		DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2656 		    ret, (void *)dhp));
2657 		if (devctx->oncpu) {
2658 			mutex_enter(&devctx->lock);
2659 			devctx->oncpu = 0;
2660 			cv_signal(&devctx->cv);
2661 			mutex_exit(&devctx->lock);
2662 		}
2663 		return (FC_HWERR);
2664 	}
2665 
2666 	/*
2667 	 * Setup the timeout if we need to
2668 	 */
2669 	if (do_timeout) {
2670 		mutex_enter(&devctx->lock);
2671 		if (dhp->dh_timeout_length > 0) {
2672 			TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2673 			    "devmap_do_ctxmgt:timeout set");
2674 			devctx->timeout = timeout(devmap_ctxto,
2675 			    devctx, dhp->dh_timeout_length);
2676 		} else {
2677 			/*
2678 			 * We don't want to wait so set oncpu to
2679 			 * 0 and wake up anyone waiting.
2680 			 */
2681 			TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2682 			    "devmap_do_ctxmgt:timeout not set");
2683 			devctx->oncpu = 0;
2684 			cv_signal(&devctx->cv);
2685 		}
2686 		mutex_exit(&devctx->lock);
2687 	}
2688 
2689 	return (DDI_SUCCESS);
2690 }
2691 
2692 /*
2693  *                                       end of mapping
2694  *                    poff   fault_offset         |
2695  *            base     |        |                 |
2696  *              |      |        |                 |
2697  *              V      V        V                 V
2698  *  +-----------+---------------+-------+---------+-------+
2699  *              ^               ^       ^         ^
2700  *              |<--- offset--->|<-len->|         |
2701  *              |<--- dh_len(size of mapping) --->|
2702  *                     |<--  pg -->|
2703  *                              -->|rlen|<--
2704  */
2705 static ulong_t
devmap_roundup(devmap_handle_t * dhp,ulong_t offset,size_t len,ulong_t * opfn,ulong_t * pagesize)2706 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2707     ulong_t *opfn, ulong_t *pagesize)
2708 {
2709 	register int level;
2710 	ulong_t pg;
2711 	ulong_t poff;
2712 	ulong_t base;
2713 	caddr_t uvaddr;
2714 	long rlen;
2715 
2716 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2717 	    "devmap_roundup:start dhp=%p off=%lx len=%lx",
2718 	    (void *)dhp, offset, len);
2719 	DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2720 	    (void *)dhp, offset, len));
2721 
2722 	/*
2723 	 * get the max. pagesize that is aligned within the range
2724 	 * <dh_pfn, dh_pfn+offset>.
2725 	 *
2726 	 * The calculations below use physical address to ddetermine
2727 	 * the page size to use. The same calculations can use the
2728 	 * virtual address to determine the page size.
2729 	 */
2730 	pg = 0;
2731 	poff = 0;
2732 	base = (ulong_t)ptob(dhp->dh_pfn);
2733 	for (level = dhp->dh_mmulevel; level >= 0; level--) {
2734 		pg = page_get_pagesize(level);
2735 		poff = ((base + offset) & ~(pg - 1));
2736 		uvaddr = dhp->dh_uvaddr + (poff - base);
2737 		if ((poff >= base) &&
2738 		    ((poff + pg) <= (base + dhp->dh_len)) &&
2739 		    VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2740 			break;
2741 	}
2742 
2743 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2744 	    "devmap_roundup: base=%lx poff=%lx dhp=%p",
2745 	    base, poff, dhp);
2746 	DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2747 	    base, poff, dhp->dh_pfn));
2748 
2749 	ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2750 	ASSERT(level >= 0);
2751 
2752 	*pagesize = pg;
2753 	*opfn = dhp->dh_pfn + btop(poff - base);
2754 
2755 	rlen = len + offset - (poff - base + pg);
2756 
2757 	ASSERT(rlen < (long)len);
2758 
2759 	TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2760 	    "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2761 	    (void *)dhp, level, rlen, pagesize, opfn);
2762 	DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2763 	    "level %x rlen %lx psize %lx opfn %lx\n",
2764 	    (void *)dhp, level, rlen, *pagesize, *opfn));
2765 
2766 	return ((ulong_t)((rlen > 0) ? rlen : 0));
2767 }
2768 
2769 /*
2770  * find the dhp that contains addr.
2771  */
2772 static devmap_handle_t *
devmap_find_handle(devmap_handle_t * dhp_head,caddr_t addr)2773 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2774 {
2775 	devmap_handle_t *dhp;
2776 
2777 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2778 	    "devmap_find_handle:start");
2779 
2780 	dhp = dhp_head;
2781 	while (dhp) {
2782 		if (addr >= dhp->dh_uvaddr &&
2783 		    addr < (dhp->dh_uvaddr + dhp->dh_len))
2784 			return (dhp);
2785 		dhp = dhp->dh_next;
2786 	}
2787 
2788 	return ((devmap_handle_t *)NULL);
2789 }
2790 
2791 /*
2792  * devmap_unload:
2793  *			Marks a segdev segment or pages if offset->offset+len
2794  *			is not the entire segment as intercept and unloads the
2795  *			pages in the range offset -> offset+len.
2796  */
2797 int
devmap_unload(devmap_cookie_t dhc,offset_t offset,size_t len)2798 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2799 {
2800 	register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2801 	caddr_t	addr;
2802 	ulong_t	size;
2803 	ssize_t	soff;
2804 
2805 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2806 	    "devmap_unload:start dhp=%p offset=%llx len=%lx",
2807 	    (void *)dhp, offset, len);
2808 	DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2809 	    (void *)dhp, offset, len));
2810 
2811 	soff = (ssize_t)(offset - dhp->dh_uoff);
2812 	soff = round_down_p2(soff, PAGESIZE);
2813 	if (soff < 0 || soff >= dhp->dh_len)
2814 		return (FC_MAKE_ERR(EINVAL));
2815 
2816 	/*
2817 	 * Address and size must be page aligned.  Len is set to the
2818 	 * number of bytes in the number of pages that are required to
2819 	 * support len.  Offset is set to the byte offset of the first byte
2820 	 * of the page that contains offset.
2821 	 */
2822 	len = round_up_p2(len, PAGESIZE);
2823 
2824 	/*
2825 	 * If len is == 0, then calculate the size by getting
2826 	 * the number of bytes from offset to the end of the segment.
2827 	 */
2828 	if (len == 0)
2829 		size = dhp->dh_len - soff;
2830 	else {
2831 		size = len;
2832 		if ((soff + size) > dhp->dh_len)
2833 			return (FC_MAKE_ERR(EINVAL));
2834 	}
2835 
2836 	/*
2837 	 * The address is offset bytes from the base address of
2838 	 * the dhp.
2839 	 */
2840 	addr = (caddr_t)(soff + dhp->dh_uvaddr);
2841 
2842 	/*
2843 	 * If large page size was used in hat_devload(),
2844 	 * the same page size must be used in hat_unload().
2845 	 */
2846 	if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2847 		hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2848 		    dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2849 	} else {
2850 		hat_unload(dhp->dh_seg->s_as->a_hat,  addr, size,
2851 		    HAT_UNLOAD|HAT_UNLOAD_OTHER);
2852 	}
2853 
2854 	return (0);
2855 }
2856 
2857 /*
2858  * calculates the optimal page size that will be used for hat_devload().
2859  */
2860 static void
devmap_get_large_pgsize(devmap_handle_t * dhp,size_t len,caddr_t addr,size_t * llen,caddr_t * laddr)2861 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2862     size_t *llen, caddr_t *laddr)
2863 {
2864 	ulong_t off;
2865 	ulong_t pfn;
2866 	ulong_t pgsize;
2867 	uint_t first = 1;
2868 
2869 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2870 	    "devmap_get_large_pgsize:start");
2871 
2872 	/*
2873 	 * RFE - Code only supports large page mappings for devmem
2874 	 * This code could be changed in future if we want to support
2875 	 * large page mappings for kernel exported memory.
2876 	 */
2877 	ASSERT(dhp_is_devmem(dhp));
2878 	ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2879 
2880 	*llen = 0;
2881 	off = (ulong_t)(addr - dhp->dh_uvaddr);
2882 	while ((long)len > 0) {
2883 		/*
2884 		 * get the optimal pfn to minimize address translations.
2885 		 * devmap_roundup() returns residue bytes for next round
2886 		 * calculations.
2887 		 */
2888 		len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2889 
2890 		if (first) {
2891 			*laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2892 			first = 0;
2893 		}
2894 
2895 		*llen += pgsize;
2896 		off = ptob(pfn - dhp->dh_pfn) + pgsize;
2897 	}
2898 	/* Large page mapping len/addr cover more range than original fault */
2899 	ASSERT(*llen >= len && *laddr <= addr);
2900 	ASSERT((*laddr + *llen) >= (addr + len));
2901 }
2902 
2903 /*
2904  * Initialize the devmap_softlock structure.
2905  */
2906 static struct devmap_softlock *
devmap_softlock_init(dev_t dev,ulong_t id)2907 devmap_softlock_init(dev_t dev, ulong_t id)
2908 {
2909 	struct devmap_softlock *slock;
2910 	struct devmap_softlock *tmp;
2911 
2912 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2913 	    "devmap_softlock_init:start");
2914 
2915 	tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2916 	mutex_enter(&devmap_slock);
2917 
2918 	for (slock = devmap_slist; slock != NULL; slock = slock->next)
2919 		if ((slock->dev == dev) && (slock->id == id))
2920 			break;
2921 
2922 	if (slock == NULL) {
2923 		slock = tmp;
2924 		slock->dev = dev;
2925 		slock->id = id;
2926 		mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2927 		cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2928 		slock->next = devmap_slist;
2929 		devmap_slist = slock;
2930 	} else
2931 		kmem_free(tmp, sizeof (struct devmap_softlock));
2932 
2933 	mutex_enter(&slock->lock);
2934 	slock->refcnt++;
2935 	mutex_exit(&slock->lock);
2936 	mutex_exit(&devmap_slock);
2937 
2938 	return (slock);
2939 }
2940 
2941 /*
2942  * Wake up processes that sleep on softlocked.
2943  * Free dh_softlock if refcnt is 0.
2944  */
2945 static void
devmap_softlock_rele(devmap_handle_t * dhp)2946 devmap_softlock_rele(devmap_handle_t *dhp)
2947 {
2948 	struct devmap_softlock *slock = dhp->dh_softlock;
2949 	struct devmap_softlock *tmp;
2950 	struct devmap_softlock *parent;
2951 
2952 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2953 	    "devmap_softlock_rele:start");
2954 
2955 	mutex_enter(&devmap_slock);
2956 	mutex_enter(&slock->lock);
2957 
2958 	ASSERT(slock->refcnt > 0);
2959 
2960 	slock->refcnt--;
2961 
2962 	/*
2963 	 * If no one is using the device, free up the slock data.
2964 	 */
2965 	if (slock->refcnt == 0) {
2966 		slock->softlocked = 0;
2967 		cv_signal(&slock->cv);
2968 
2969 		if (devmap_slist == slock)
2970 			devmap_slist = slock->next;
2971 		else {
2972 			parent = devmap_slist;
2973 			for (tmp = devmap_slist->next; tmp != NULL;
2974 			    tmp = tmp->next) {
2975 				if (tmp == slock) {
2976 					parent->next = tmp->next;
2977 					break;
2978 				}
2979 				parent = tmp;
2980 			}
2981 		}
2982 		mutex_exit(&slock->lock);
2983 		mutex_destroy(&slock->lock);
2984 		cv_destroy(&slock->cv);
2985 		kmem_free(slock, sizeof (struct devmap_softlock));
2986 	} else
2987 		mutex_exit(&slock->lock);
2988 
2989 	mutex_exit(&devmap_slock);
2990 }
2991 
2992 /*
2993  * Wake up processes that sleep on dh_ctx->locked.
2994  * Free dh_ctx if refcnt is 0.
2995  */
2996 static void
devmap_ctx_rele(devmap_handle_t * dhp)2997 devmap_ctx_rele(devmap_handle_t *dhp)
2998 {
2999 	struct devmap_ctx *devctx = dhp->dh_ctx;
3000 	struct devmap_ctx *tmp;
3001 	struct devmap_ctx *parent;
3002 	timeout_id_t tid;
3003 
3004 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3005 	    "devmap_ctx_rele:start");
3006 
3007 	mutex_enter(&devmapctx_lock);
3008 	mutex_enter(&devctx->lock);
3009 
3010 	ASSERT(devctx->refcnt > 0);
3011 
3012 	devctx->refcnt--;
3013 
3014 	/*
3015 	 * If no one is using the device, free up the devctx data.
3016 	 */
3017 	if (devctx->refcnt == 0) {
3018 		/*
3019 		 * Untimeout any threads using this mapping as they are about
3020 		 * to go away.
3021 		 */
3022 		if (devctx->timeout != 0) {
3023 			TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3024 			    "devmap_ctx_rele:untimeout ctx->timeout");
3025 
3026 			tid = devctx->timeout;
3027 			mutex_exit(&devctx->lock);
3028 			(void) untimeout(tid);
3029 			mutex_enter(&devctx->lock);
3030 		}
3031 
3032 		devctx->oncpu = 0;
3033 		cv_signal(&devctx->cv);
3034 
3035 		if (devmapctx_list == devctx)
3036 			devmapctx_list = devctx->next;
3037 		else {
3038 			parent = devmapctx_list;
3039 			for (tmp = devmapctx_list->next; tmp != NULL;
3040 			    tmp = tmp->next) {
3041 				if (tmp == devctx) {
3042 					parent->next = tmp->next;
3043 					break;
3044 				}
3045 				parent = tmp;
3046 			}
3047 		}
3048 		mutex_exit(&devctx->lock);
3049 		mutex_destroy(&devctx->lock);
3050 		cv_destroy(&devctx->cv);
3051 		kmem_free(devctx, sizeof (struct devmap_ctx));
3052 	} else
3053 		mutex_exit(&devctx->lock);
3054 
3055 	mutex_exit(&devmapctx_lock);
3056 }
3057 
3058 /*
3059  * devmap_load:
3060  *			Marks a segdev segment or pages if offset->offset+len
3061  *			is not the entire segment as nointercept and faults in
3062  *			the pages in the range offset -> offset+len.
3063  */
3064 int
devmap_load(devmap_cookie_t dhc,offset_t offset,size_t len,uint_t type,uint_t rw)3065 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3066     uint_t rw)
3067 {
3068 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3069 	struct as *asp = dhp->dh_seg->s_as;
3070 	caddr_t	addr;
3071 	ulong_t	size;
3072 	ssize_t	soff;	/* offset from the beginning of the segment */
3073 	int rc;
3074 
3075 	TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3076 	    "devmap_load:start dhp=%p offset=%llx len=%lx",
3077 	    (void *)dhp, offset, len);
3078 
3079 	DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3080 	    (void *)dhp, offset, len));
3081 
3082 	/*
3083 	 *	Hat layer only supports devload to process' context for which
3084 	 *	the as lock is held. Verify here and return error if drivers
3085 	 *	inadvertently call devmap_load on a wrong devmap handle.
3086 	 */
3087 	if ((asp != &kas) && !AS_LOCK_HELD(asp))
3088 		return (FC_MAKE_ERR(EINVAL));
3089 
3090 	soff = (ssize_t)(offset - dhp->dh_uoff);
3091 	soff = round_down_p2(soff, PAGESIZE);
3092 	if (soff < 0 || soff >= dhp->dh_len)
3093 		return (FC_MAKE_ERR(EINVAL));
3094 
3095 	/*
3096 	 * Address and size must be page aligned.  Len is set to the
3097 	 * number of bytes in the number of pages that are required to
3098 	 * support len.  Offset is set to the byte offset of the first byte
3099 	 * of the page that contains offset.
3100 	 */
3101 	len = round_up_p2(len, PAGESIZE);
3102 
3103 	/*
3104 	 * If len == 0, then calculate the size by getting
3105 	 * the number of bytes from offset to the end of the segment.
3106 	 */
3107 	if (len == 0)
3108 		size = dhp->dh_len - soff;
3109 	else {
3110 		size = len;
3111 		if ((soff + size) > dhp->dh_len)
3112 			return (FC_MAKE_ERR(EINVAL));
3113 	}
3114 
3115 	/*
3116 	 * The address is offset bytes from the base address of
3117 	 * the segment.
3118 	 */
3119 	addr = (caddr_t)(soff + dhp->dh_uvaddr);
3120 
3121 	HOLD_DHP_LOCK(dhp);
3122 	rc = segdev_faultpages(asp->a_hat,
3123 	    dhp->dh_seg, addr, size, type, rw, dhp);
3124 	RELE_DHP_LOCK(dhp);
3125 	return (rc);
3126 }
3127 
3128 int
devmap_setup(dev_t dev,offset_t off,struct as * as,caddr_t * addrp,size_t len,uint_t prot,uint_t maxprot,uint_t flags,struct cred * cred)3129 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3130     size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3131 {
3132 	register devmap_handle_t *dhp;
3133 	int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3134 	    size_t *, uint_t);
3135 	int (*mmap)(dev_t, off_t, int);
3136 	struct devmap_callback_ctl *callbackops;
3137 	devmap_handle_t *dhp_head = NULL;
3138 	devmap_handle_t *dhp_prev = NULL;
3139 	devmap_handle_t *dhp_curr;
3140 	caddr_t addr;
3141 	int map_flag;
3142 	int ret;
3143 	ulong_t total_len;
3144 	size_t map_len;
3145 	size_t resid_len = len;
3146 	offset_t map_off = off;
3147 	struct devmap_softlock *slock = NULL;
3148 
3149 #ifdef lint
3150 	cred = cred;
3151 #endif
3152 
3153 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3154 	    "devmap_setup:start off=%llx len=%lx", off, len);
3155 	DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3156 	    off, len));
3157 
3158 	devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3159 	mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3160 
3161 	/*
3162 	 * driver must provide devmap(9E) entry point in cb_ops to use the
3163 	 * devmap framework.
3164 	 */
3165 	if (devmap == NULL || devmap == nulldev || devmap == nodev)
3166 		return (EINVAL);
3167 
3168 	/*
3169 	 * To protect from an inadvertent entry because the devmap entry point
3170 	 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3171 	 * mmap is NULL.
3172 	 */
3173 	map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3174 	if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3175 		return (EINVAL);
3176 
3177 	/*
3178 	 * devmap allows mmap(2) to map multiple registers.
3179 	 * one devmap_handle is created for each register mapped.
3180 	 */
3181 	for (total_len = 0; total_len < len; total_len += map_len) {
3182 		dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3183 
3184 		if (dhp_prev != NULL)
3185 			dhp_prev->dh_next = dhp;
3186 		else
3187 			dhp_head = dhp;
3188 		dhp_prev = dhp;
3189 
3190 		dhp->dh_prot = prot;
3191 		dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3192 		dhp->dh_dev = dev;
3193 		dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3194 		dhp->dh_uoff = map_off;
3195 
3196 		/*
3197 		 * Get mapping specific info from
3198 		 * the driver, such as rnumber, roff, len, callbackops,
3199 		 * accattrp and, if the mapping is for kernel memory,
3200 		 * ddi_umem_cookie.
3201 		 */
3202 		if ((ret = cdev_devmap(dev, dhp, map_off,
3203 		    resid_len, &map_len, get_udatamodel())) != 0) {
3204 			free_devmap_handle(dhp_head);
3205 			return (ENXIO);
3206 		}
3207 
3208 		if (map_len & PAGEOFFSET) {
3209 			free_devmap_handle(dhp_head);
3210 			return (EINVAL);
3211 		}
3212 
3213 		callbackops = &dhp->dh_callbackops;
3214 
3215 		if ((callbackops->devmap_access == NULL) ||
3216 		    (callbackops->devmap_access == nulldev) ||
3217 		    (callbackops->devmap_access == nodev)) {
3218 			/*
3219 			 * Normally devmap does not support MAP_PRIVATE unless
3220 			 * the drivers provide a valid devmap_access routine.
3221 			 */
3222 			if ((flags & MAP_PRIVATE) != 0) {
3223 				free_devmap_handle(dhp_head);
3224 				return (EINVAL);
3225 			}
3226 		} else {
3227 			/*
3228 			 * Initialize dhp_softlock and dh_ctx if the drivers
3229 			 * provide devmap_access.
3230 			 */
3231 			dhp->dh_softlock = devmap_softlock_init(dev,
3232 			    (ulong_t)callbackops->devmap_access);
3233 			dhp->dh_ctx = devmap_ctxinit(dev,
3234 			    (ulong_t)callbackops->devmap_access);
3235 
3236 			/*
3237 			 * segdev_fault can only work when all
3238 			 * dh_softlock in a multi-dhp mapping
3239 			 * are same. see comments in segdev_fault
3240 			 * This code keeps track of the first
3241 			 * dh_softlock allocated in slock and
3242 			 * compares all later allocations and if
3243 			 * not similar, returns an error.
3244 			 */
3245 			if (slock == NULL)
3246 				slock = dhp->dh_softlock;
3247 			if (slock != dhp->dh_softlock) {
3248 				free_devmap_handle(dhp_head);
3249 				return (ENOTSUP);
3250 			}
3251 		}
3252 
3253 		map_off += map_len;
3254 		resid_len -= map_len;
3255 	}
3256 
3257 	/*
3258 	 * get the user virtual address and establish the mapping between
3259 	 * uvaddr and device physical address.
3260 	 */
3261 	if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3262 	    != 0) {
3263 		/*
3264 		 * free devmap handles if error during the mapping.
3265 		 */
3266 		free_devmap_handle(dhp_head);
3267 
3268 		return (ret);
3269 	}
3270 
3271 	/*
3272 	 * call the driver's devmap_map callback to do more after the mapping,
3273 	 * such as to allocate driver private data for context management.
3274 	 */
3275 	dhp = dhp_head;
3276 	map_off = off;
3277 	addr = *addrp;
3278 	while (dhp != NULL) {
3279 		callbackops = &dhp->dh_callbackops;
3280 		dhp->dh_uvaddr = addr;
3281 		dhp_curr = dhp;
3282 		if (callbackops->devmap_map != NULL) {
3283 			ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3284 			    dev, flags, map_off,
3285 			    dhp->dh_len, &dhp->dh_pvtp);
3286 			if (ret != 0) {
3287 				struct segdev_data *sdp;
3288 
3289 				/*
3290 				 * call driver's devmap_unmap entry point
3291 				 * to free driver resources.
3292 				 */
3293 				dhp = dhp_head;
3294 				map_off = off;
3295 				while (dhp != dhp_curr) {
3296 					callbackops = &dhp->dh_callbackops;
3297 					if (callbackops->devmap_unmap != NULL) {
3298 						(*callbackops->devmap_unmap)(
3299 						    dhp, dhp->dh_pvtp,
3300 						    map_off, dhp->dh_len,
3301 						    NULL, NULL, NULL, NULL);
3302 					}
3303 					map_off += dhp->dh_len;
3304 					dhp = dhp->dh_next;
3305 				}
3306 				sdp = dhp_head->dh_seg->s_data;
3307 				sdp->devmap_data = NULL;
3308 				free_devmap_handle(dhp_head);
3309 				return (ENXIO);
3310 			}
3311 		}
3312 		map_off += dhp->dh_len;
3313 		addr += dhp->dh_len;
3314 		dhp = dhp->dh_next;
3315 	}
3316 
3317 	return (0);
3318 }
3319 
3320 int
ddi_devmap_segmap(dev_t dev,off_t off,ddi_as_handle_t as,caddr_t * addrp,off_t len,uint_t prot,uint_t maxprot,uint_t flags,struct cred * cred)3321 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3322     off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3323 {
3324 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3325 	    "devmap_segmap:start");
3326 	return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3327 	    (size_t)len, prot, maxprot, flags, cred));
3328 }
3329 
3330 /*
3331  * Called from devmap_devmem_setup/remap to see if can use large pages for
3332  * this device mapping.
3333  * Also calculate the max. page size for this mapping.
3334  * this page size will be used in fault routine for
3335  * optimal page size calculations.
3336  */
3337 static void
devmap_devmem_large_page_setup(devmap_handle_t * dhp)3338 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3339 {
3340 	ASSERT(dhp_is_devmem(dhp));
3341 	dhp->dh_mmulevel = 0;
3342 
3343 	/*
3344 	 * use large page size only if:
3345 	 *  1. device memory.
3346 	 *  2. mmu supports multiple page sizes,
3347 	 *  3. Driver did not disallow it
3348 	 *  4. dhp length is at least as big as the large pagesize
3349 	 *  5. the uvaddr and pfn are large pagesize aligned
3350 	 */
3351 	if (page_num_pagesizes() > 1 &&
3352 	    !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3353 		ulong_t base;
3354 		int level;
3355 
3356 		base = (ulong_t)ptob(dhp->dh_pfn);
3357 		for (level = 1; level < page_num_pagesizes(); level++) {
3358 			size_t pgsize = page_get_pagesize(level);
3359 			if ((dhp->dh_len < pgsize) ||
3360 			    (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3361 			    base, pgsize))) {
3362 				break;
3363 			}
3364 		}
3365 		dhp->dh_mmulevel = level - 1;
3366 	}
3367 	if (dhp->dh_mmulevel > 0) {
3368 		dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3369 	} else {
3370 		dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3371 	}
3372 }
3373 
3374 /*
3375  * Called by driver devmap routine to pass device specific info to
3376  * the framework.    used for device memory mapping only.
3377  */
3378 int
devmap_devmem_setup(devmap_cookie_t dhc,dev_info_t * dip,struct devmap_callback_ctl * callbackops,uint_t rnumber,offset_t roff,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3379 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3380     struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3381     size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3382 {
3383 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3384 	ddi_acc_handle_t handle;
3385 	ddi_map_req_t mr;
3386 	ddi_acc_hdl_t *hp;
3387 	int err;
3388 
3389 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3390 	    "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3391 	    (void *)dhp, roff, rnumber, (uint_t)len);
3392 	DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3393 	    "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3394 
3395 	/*
3396 	 * First to check if this function has been called for this dhp.
3397 	 */
3398 	if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3399 		return (DDI_FAILURE);
3400 
3401 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3402 		return (DDI_FAILURE);
3403 
3404 	if (flags & DEVMAP_MAPPING_INVALID) {
3405 		/*
3406 		 * Don't go up the tree to get pfn if the driver specifies
3407 		 * DEVMAP_MAPPING_INVALID in flags.
3408 		 *
3409 		 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3410 		 * remap permission.
3411 		 */
3412 		if (!(flags & DEVMAP_ALLOW_REMAP)) {
3413 			return (DDI_FAILURE);
3414 		}
3415 		dhp->dh_pfn = PFN_INVALID;
3416 	} else {
3417 		handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3418 		if (handle == NULL)
3419 			return (DDI_FAILURE);
3420 
3421 		hp = impl_acc_hdl_get(handle);
3422 		hp->ah_vers = VERS_ACCHDL;
3423 		hp->ah_dip = dip;
3424 		hp->ah_rnumber = rnumber;
3425 		hp->ah_offset = roff;
3426 		hp->ah_len = len;
3427 		if (accattrp != NULL)
3428 			hp->ah_acc = *accattrp;
3429 
3430 		mr.map_op = DDI_MO_MAP_LOCKED;
3431 		mr.map_type = DDI_MT_RNUMBER;
3432 		mr.map_obj.rnumber = rnumber;
3433 		mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3434 		mr.map_flags = DDI_MF_DEVICE_MAPPING;
3435 		mr.map_handlep = hp;
3436 		mr.map_vers = DDI_MAP_VERSION;
3437 
3438 		/*
3439 		 * up the device tree to get pfn.
3440 		 * The rootnex_map_regspec() routine in nexus drivers has been
3441 		 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3442 		 */
3443 		err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3444 		dhp->dh_hat_attr = hp->ah_hat_flags;
3445 		impl_acc_hdl_free(handle);
3446 
3447 		if (err)
3448 			return (DDI_FAILURE);
3449 	}
3450 	/* Should not be using devmem setup for memory pages */
3451 	ASSERT(!pf_is_memory(dhp->dh_pfn));
3452 
3453 	/* Only some of the flags bits are settable by the driver */
3454 	dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3455 	dhp->dh_len = ptob(btopr(len));
3456 
3457 	dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3458 	dhp->dh_roff = ptob(btop(roff));
3459 
3460 	/* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3461 	devmap_devmem_large_page_setup(dhp);
3462 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3463 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3464 
3465 
3466 	if (callbackops != NULL) {
3467 		bcopy(callbackops, &dhp->dh_callbackops,
3468 		    sizeof (struct devmap_callback_ctl));
3469 	}
3470 
3471 	/*
3472 	 * Initialize dh_lock if we want to do remap.
3473 	 */
3474 	if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3475 		mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3476 		dhp->dh_flags |= DEVMAP_LOCK_INITED;
3477 	}
3478 
3479 	dhp->dh_flags |= DEVMAP_SETUP_DONE;
3480 
3481 	return (DDI_SUCCESS);
3482 }
3483 
3484 int
devmap_devmem_remap(devmap_cookie_t dhc,dev_info_t * dip,uint_t rnumber,offset_t roff,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3485 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3486     uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3487     uint_t flags, ddi_device_acc_attr_t *accattrp)
3488 {
3489 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3490 	ddi_acc_handle_t handle;
3491 	ddi_map_req_t mr;
3492 	ddi_acc_hdl_t *hp;
3493 	pfn_t	pfn;
3494 	uint_t	hat_flags;
3495 	int	err;
3496 
3497 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3498 	    "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3499 	    (void *)dhp, roff, rnumber, (uint_t)len);
3500 	DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3501 	    "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3502 
3503 	/*
3504 	 * Return failure if setup has not been done or no remap permission
3505 	 * has been granted during the setup.
3506 	 */
3507 	if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3508 	    (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3509 		return (DDI_FAILURE);
3510 
3511 	/* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3512 	if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3513 		return (DDI_FAILURE);
3514 
3515 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3516 		return (DDI_FAILURE);
3517 
3518 	if (!(flags & DEVMAP_MAPPING_INVALID)) {
3519 		handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3520 		if (handle == NULL)
3521 			return (DDI_FAILURE);
3522 	}
3523 
3524 	HOLD_DHP_LOCK(dhp);
3525 
3526 	/*
3527 	 * Unload the old mapping, so next fault will setup the new mappings
3528 	 * Do this while holding the dhp lock so other faults dont reestablish
3529 	 * the mappings
3530 	 */
3531 	hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3532 	    dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3533 
3534 	if (flags & DEVMAP_MAPPING_INVALID) {
3535 		dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3536 		dhp->dh_pfn = PFN_INVALID;
3537 	} else {
3538 		/* clear any prior DEVMAP_MAPPING_INVALID flag */
3539 		dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3540 		hp = impl_acc_hdl_get(handle);
3541 		hp->ah_vers = VERS_ACCHDL;
3542 		hp->ah_dip = dip;
3543 		hp->ah_rnumber = rnumber;
3544 		hp->ah_offset = roff;
3545 		hp->ah_len = len;
3546 		if (accattrp != NULL)
3547 			hp->ah_acc = *accattrp;
3548 
3549 		mr.map_op = DDI_MO_MAP_LOCKED;
3550 		mr.map_type = DDI_MT_RNUMBER;
3551 		mr.map_obj.rnumber = rnumber;
3552 		mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3553 		mr.map_flags = DDI_MF_DEVICE_MAPPING;
3554 		mr.map_handlep = hp;
3555 		mr.map_vers = DDI_MAP_VERSION;
3556 
3557 		/*
3558 		 * up the device tree to get pfn.
3559 		 * The rootnex_map_regspec() routine in nexus drivers has been
3560 		 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3561 		 */
3562 		err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3563 		hat_flags = hp->ah_hat_flags;
3564 		impl_acc_hdl_free(handle);
3565 		if (err) {
3566 			RELE_DHP_LOCK(dhp);
3567 			return (DDI_FAILURE);
3568 		}
3569 		/*
3570 		 * Store result of ddi_map first in local variables, as we do
3571 		 * not want to overwrite the existing dhp with wrong data.
3572 		 */
3573 		dhp->dh_pfn = pfn;
3574 		dhp->dh_hat_attr = hat_flags;
3575 	}
3576 
3577 	/* clear the large page size flag */
3578 	dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3579 
3580 	dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3581 	dhp->dh_roff = ptob(btop(roff));
3582 
3583 	/* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3584 	devmap_devmem_large_page_setup(dhp);
3585 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3586 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3587 
3588 	RELE_DHP_LOCK(dhp);
3589 	return (DDI_SUCCESS);
3590 }
3591 
3592 /*
3593  * called by driver devmap routine to pass kernel virtual address  mapping
3594  * info to the framework.    used only for kernel memory
3595  * allocated from ddi_umem_alloc().
3596  */
3597 int
devmap_umem_setup(devmap_cookie_t dhc,dev_info_t * dip,struct devmap_callback_ctl * callbackops,ddi_umem_cookie_t cookie,offset_t off,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3598 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3599     struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3600     offset_t off, size_t len, uint_t maxprot, uint_t flags,
3601     ddi_device_acc_attr_t *accattrp)
3602 {
3603 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3604 	struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3605 
3606 #ifdef lint
3607 	dip = dip;
3608 #endif
3609 
3610 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3611 	    "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3612 	    (void *)dhp, off, cookie, len);
3613 	DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3614 	    "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3615 
3616 	if (cookie == NULL)
3617 		return (DDI_FAILURE);
3618 
3619 	/* For UMEM_TRASH, this restriction is not needed */
3620 	if ((off + len) > cp->size)
3621 		return (DDI_FAILURE);
3622 
3623 	/* check if the cache attributes are supported */
3624 	if (i_ddi_check_cache_attr(flags) == B_FALSE)
3625 		return (DDI_FAILURE);
3626 
3627 	/*
3628 	 * First to check if this function has been called for this dhp.
3629 	 */
3630 	if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3631 		return (DDI_FAILURE);
3632 
3633 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3634 		return (DDI_FAILURE);
3635 
3636 	if (flags & DEVMAP_MAPPING_INVALID) {
3637 		/*
3638 		 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3639 		 * remap permission.
3640 		 */
3641 		if (!(flags & DEVMAP_ALLOW_REMAP)) {
3642 			return (DDI_FAILURE);
3643 		}
3644 	} else {
3645 		dhp->dh_cookie = cookie;
3646 		dhp->dh_roff = ptob(btop(off));
3647 		dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3648 		/* set HAT cache attributes */
3649 		i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3650 		/* set HAT endianess attributes */
3651 		i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3652 	}
3653 
3654 	/*
3655 	 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3656 	 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3657 	 * create consistent mappings but our intention was to create
3658 	 * non-consistent mappings.
3659 	 *
3660 	 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3661 	 * mappings.
3662 	 *
3663 	 * kernel exported memory: hat figures it out it's memory and always
3664 	 * creates consistent mappings.
3665 	 *
3666 	 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3667 	 *
3668 	 * /dev/kmem: consistent mappings are created unless they are
3669 	 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3670 	 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3671 	 * mappings of /dev/kmem. See common/io/mem.c
3672 	 */
3673 
3674 	/* Only some of the flags bits are settable by the driver */
3675 	dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3676 
3677 	dhp->dh_len = ptob(btopr(len));
3678 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3679 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3680 
3681 	if (callbackops != NULL) {
3682 		bcopy(callbackops, &dhp->dh_callbackops,
3683 		    sizeof (struct devmap_callback_ctl));
3684 	}
3685 	/*
3686 	 * Initialize dh_lock if we want to do remap.
3687 	 */
3688 	if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3689 		mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3690 		dhp->dh_flags |= DEVMAP_LOCK_INITED;
3691 	}
3692 
3693 	dhp->dh_flags |= DEVMAP_SETUP_DONE;
3694 
3695 	return (DDI_SUCCESS);
3696 }
3697 
3698 int
devmap_umem_remap(devmap_cookie_t dhc,dev_info_t * dip,ddi_umem_cookie_t cookie,offset_t off,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3699 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3700     ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3701     uint_t flags, ddi_device_acc_attr_t *accattrp)
3702 {
3703 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3704 	struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3705 
3706 	TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3707 	    "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3708 	    (void *)dhp, off, cookie, len);
3709 	DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3710 	    "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3711 
3712 #ifdef lint
3713 	dip = dip;
3714 	accattrp = accattrp;
3715 #endif
3716 	/*
3717 	 * Reture failure if setup has not been done or no remap permission
3718 	 * has been granted during the setup.
3719 	 */
3720 	if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3721 	    (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3722 		return (DDI_FAILURE);
3723 
3724 	/* No flags supported for remap yet */
3725 	if (flags != 0)
3726 		return (DDI_FAILURE);
3727 
3728 	/* check if the cache attributes are supported */
3729 	if (i_ddi_check_cache_attr(flags) == B_FALSE)
3730 		return (DDI_FAILURE);
3731 
3732 	if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3733 		return (DDI_FAILURE);
3734 
3735 	/* For UMEM_TRASH, this restriction is not needed */
3736 	if ((off + len) > cp->size)
3737 		return (DDI_FAILURE);
3738 
3739 	HOLD_DHP_LOCK(dhp);
3740 	/*
3741 	 * Unload the old mapping, so next fault will setup the new mappings
3742 	 * Do this while holding the dhp lock so other faults dont reestablish
3743 	 * the mappings
3744 	 */
3745 	hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3746 	    dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3747 
3748 	dhp->dh_cookie = cookie;
3749 	dhp->dh_roff = ptob(btop(off));
3750 	dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3751 	/* set HAT cache attributes */
3752 	i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3753 	/* set HAT endianess attributes */
3754 	i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3755 
3756 	/* clear the large page size flag */
3757 	dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3758 
3759 	dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3760 	ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3761 	RELE_DHP_LOCK(dhp);
3762 	return (DDI_SUCCESS);
3763 }
3764 
3765 /*
3766  * to set timeout value for the driver's context management callback, e.g.
3767  * devmap_access().
3768  */
3769 void
devmap_set_ctx_timeout(devmap_cookie_t dhc,clock_t ticks)3770 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3771 {
3772 	devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3773 
3774 	TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3775 	    "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3776 	    (void *)dhp, ticks);
3777 	dhp->dh_timeout_length = ticks;
3778 }
3779 
3780 int
devmap_default_access(devmap_cookie_t dhp,void * pvtp,offset_t off,size_t len,uint_t type,uint_t rw)3781 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3782     size_t len, uint_t type, uint_t rw)
3783 {
3784 #ifdef lint
3785 	pvtp = pvtp;
3786 #endif
3787 
3788 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3789 	    "devmap_default_access:start");
3790 	return (devmap_load(dhp, off, len, type, rw));
3791 }
3792 
3793 /*
3794  * segkmem_alloc() wrapper to allocate memory which is both
3795  * non-relocatable (for DR) and sharelocked, since the rest
3796  * of this segment driver requires it.
3797  */
3798 static void *
devmap_alloc_pages(vmem_t * vmp,size_t size,int vmflag)3799 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3800 {
3801 	ASSERT(vmp != NULL);
3802 	ASSERT(kvseg.s_base != NULL);
3803 	vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3804 	return (segkmem_alloc(vmp, size, vmflag));
3805 }
3806 
3807 /*
3808  * This is where things are a bit incestuous with seg_kmem: unlike
3809  * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3810  * we need to do a bit of a dance around that to prevent duplication of
3811  * code until we decide to bite the bullet and implement a new kernel
3812  * segment for driver-allocated memory that is exported to user space.
3813  */
3814 static void
devmap_free_pages(vmem_t * vmp,void * inaddr,size_t size)3815 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3816 {
3817 	page_t *pp;
3818 	caddr_t addr = inaddr;
3819 	caddr_t eaddr;
3820 	pgcnt_t npages = btopr(size);
3821 
3822 	ASSERT(vmp != NULL);
3823 	ASSERT(kvseg.s_base != NULL);
3824 	ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3825 
3826 	hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3827 
3828 	for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3829 		/*
3830 		 * Use page_find() instead of page_lookup() to find the page
3831 		 * since we know that it is hashed and has a shared lock.
3832 		 */
3833 		pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3834 
3835 		if (pp == NULL)
3836 			panic("devmap_free_pages: page not found");
3837 		if (!page_tryupgrade(pp)) {
3838 			page_unlock(pp);
3839 			pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3840 			    SE_EXCL);
3841 			if (pp == NULL)
3842 				panic("devmap_free_pages: page already freed");
3843 		}
3844 		/* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3845 		pp->p_lckcnt = 0;
3846 		page_destroy(pp, 0);
3847 	}
3848 	page_unresv(npages);
3849 
3850 	if (vmp != NULL)
3851 		vmem_free(vmp, inaddr, size);
3852 }
3853 
3854 /*
3855  * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3856  * allocating non-pageable kmem in response to a ddi_umem_alloc()
3857  * default request. For now we allocate our own pages and we keep
3858  * them long-term sharelocked, since: A) the fault routines expect the
3859  * memory to already be locked; B) pageable umem is already long-term
3860  * locked; C) it's a lot of work to make it otherwise, particularly
3861  * since the nexus layer expects the pages to never fault. An RFE is to
3862  * not keep the pages long-term locked, but instead to be able to
3863  * take faults on them and simply look them up in kvp in case we
3864  * fault on them. Even then, we must take care not to let pageout
3865  * steal them from us since the data must remain resident; if we
3866  * do this we must come up with some way to pin the pages to prevent
3867  * faults while a driver is doing DMA to/from them.
3868  */
3869 static void *
devmap_umem_alloc_np(size_t size,size_t flags)3870 devmap_umem_alloc_np(size_t size, size_t flags)
3871 {
3872 	void *buf;
3873 	int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3874 
3875 	buf = vmem_alloc(umem_np_arena, size, vmflags);
3876 	if (buf != NULL)
3877 		bzero(buf, size);
3878 	return (buf);
3879 }
3880 
3881 static void
devmap_umem_free_np(void * addr,size_t size)3882 devmap_umem_free_np(void *addr, size_t size)
3883 {
3884 	vmem_free(umem_np_arena, addr, size);
3885 }
3886 
3887 /*
3888  * allocate page aligned kernel memory for exporting to user land.
3889  * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3890  * to find a user virtual address that is in same color as the address
3891  * allocated here.
3892  */
3893 void *
ddi_umem_alloc(size_t size,int flags,ddi_umem_cookie_t * cookie)3894 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3895 {
3896 	register size_t len = ptob(btopr(size));
3897 	void *buf = NULL;
3898 	struct ddi_umem_cookie *cp;
3899 	int iflags = 0;
3900 
3901 	*cookie = NULL;
3902 
3903 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3904 	    "devmap_umem_alloc:start");
3905 	if (len == 0)
3906 		return ((void *)NULL);
3907 
3908 	/*
3909 	 * allocate cookie
3910 	 */
3911 	if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3912 	    flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3913 		ASSERT(flags & DDI_UMEM_NOSLEEP);
3914 		return ((void *)NULL);
3915 	}
3916 
3917 	if (flags & DDI_UMEM_PAGEABLE) {
3918 		/* Only one of the flags is allowed */
3919 		ASSERT(!(flags & DDI_UMEM_TRASH));
3920 		/* initialize resource with 0 */
3921 		iflags = KPD_ZERO;
3922 
3923 		/*
3924 		 * to allocate unlocked pageable memory, use segkp_get() to
3925 		 * create a segkp segment.  Since segkp can only service kas,
3926 		 * other segment drivers such as segdev have to do
3927 		 * as_fault(segkp, SOFTLOCK) in its fault routine,
3928 		 */
3929 		if (flags & DDI_UMEM_NOSLEEP)
3930 			iflags |= KPD_NOWAIT;
3931 
3932 		if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3933 			kmem_free(cp, sizeof (struct ddi_umem_cookie));
3934 			return ((void *)NULL);
3935 		}
3936 		cp->type = KMEM_PAGEABLE;
3937 		mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3938 		cp->locked = 0;
3939 	} else if (flags & DDI_UMEM_TRASH) {
3940 		/* Only one of the flags is allowed */
3941 		ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3942 		cp->type = UMEM_TRASH;
3943 		buf = NULL;
3944 	} else {
3945 		if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3946 			kmem_free(cp, sizeof (struct ddi_umem_cookie));
3947 			return ((void *)NULL);
3948 		}
3949 
3950 		cp->type = KMEM_NON_PAGEABLE;
3951 	}
3952 
3953 	/*
3954 	 * need to save size here.  size will be used when
3955 	 * we do kmem_free.
3956 	 */
3957 	cp->size = len;
3958 	cp->cvaddr = (caddr_t)buf;
3959 
3960 	*cookie =  (void *)cp;
3961 	return (buf);
3962 }
3963 
3964 void
ddi_umem_free(ddi_umem_cookie_t cookie)3965 ddi_umem_free(ddi_umem_cookie_t cookie)
3966 {
3967 	struct ddi_umem_cookie *cp;
3968 
3969 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3970 	    "devmap_umem_free:start");
3971 
3972 	/*
3973 	 * if cookie is NULL, no effects on the system
3974 	 */
3975 	if (cookie == NULL)
3976 		return;
3977 
3978 	cp = (struct ddi_umem_cookie *)cookie;
3979 
3980 	switch (cp->type) {
3981 	case KMEM_PAGEABLE :
3982 		ASSERT(cp->cvaddr != NULL && cp->size != 0);
3983 		/*
3984 		 * Check if there are still any pending faults on the cookie
3985 		 * while the driver is deleting it,
3986 		 * XXX - could change to an ASSERT but wont catch errant drivers
3987 		 */
3988 		mutex_enter(&cp->lock);
3989 		if (cp->locked) {
3990 			mutex_exit(&cp->lock);
3991 			panic("ddi_umem_free for cookie with pending faults %p",
3992 			    (void *)cp);
3993 		}
3994 
3995 		segkp_release(segkp, cp->cvaddr);
3996 
3997 		/*
3998 		 * release mutex associated with this cookie.
3999 		 */
4000 		mutex_destroy(&cp->lock);
4001 		break;
4002 	case KMEM_NON_PAGEABLE :
4003 		ASSERT(cp->cvaddr != NULL && cp->size != 0);
4004 		devmap_umem_free_np(cp->cvaddr, cp->size);
4005 		break;
4006 	case UMEM_TRASH :
4007 		break;
4008 	case UMEM_LOCKED :
4009 		/* Callers should use ddi_umem_unlock for this type */
4010 		ddi_umem_unlock(cookie);
4011 		/* Frees the cookie too */
4012 		return;
4013 	default:
4014 		/* panic so we can diagnose the underlying cause */
4015 		panic("ddi_umem_free: illegal cookie type 0x%x\n",
4016 		    cp->type);
4017 	}
4018 
4019 	kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4020 }
4021 
4022 
4023 static int
segdev_getmemid(struct seg * seg,caddr_t addr,memid_t * memidp)4024 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4025 {
4026 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4027 
4028 	/*
4029 	 * It looks as if it is always mapped shared
4030 	 */
4031 	TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4032 	    "segdev_getmemid:start");
4033 	memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4034 	memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4035 	return (0);
4036 }
4037 
4038 /*ARGSUSED*/
4039 static lgrp_mem_policy_info_t *
segdev_getpolicy(struct seg * seg,caddr_t addr)4040 segdev_getpolicy(struct seg *seg, caddr_t addr)
4041 {
4042 	return (NULL);
4043 }
4044 
4045 /*ARGSUSED*/
4046 static int
segdev_capable(struct seg * seg,segcapability_t capability)4047 segdev_capable(struct seg *seg, segcapability_t capability)
4048 {
4049 	return (0);
4050 }
4051 
4052 /*
4053  * ddi_umem_alloc() non-pageable quantum cache max size.
4054  * This is just a SWAG.
4055  */
4056 #define	DEVMAP_UMEM_QUANTUM	(8*PAGESIZE)
4057 
4058 /*
4059  * Initialize seg_dev from boot. This routine sets up the trash page
4060  * and creates the umem_np_arena used to back non-pageable memory
4061  * requests.
4062  */
4063 void
segdev_init(void)4064 segdev_init(void)
4065 {
4066 	struct seg kseg;
4067 
4068 	umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4069 	    devmap_alloc_pages, devmap_free_pages, heap_arena,
4070 	    DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4071 
4072 	kseg.s_as = &kas;
4073 	trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4074 	    PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4075 	if (trashpp == NULL)
4076 		panic("segdev_init: failed to create trash page");
4077 	pagezero(trashpp, 0, PAGESIZE);
4078 	page_downgrade(trashpp);
4079 }
4080 
4081 /*
4082  * Invoke platform-dependent support routines so that /proc can have
4083  * the platform code deal with curious hardware.
4084  */
4085 int
segdev_copyfrom(struct seg * seg,caddr_t uaddr,const void * devaddr,void * kaddr,size_t len)4086 segdev_copyfrom(struct seg *seg,
4087     caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4088 {
4089 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4090 	struct snode *sp = VTOS(VTOCVP(sdp->vp));
4091 
4092 	return (e_ddi_copyfromdev(sp->s_dip,
4093 	    (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4094 }
4095 
4096 int
segdev_copyto(struct seg * seg,caddr_t uaddr,const void * kaddr,void * devaddr,size_t len)4097 segdev_copyto(struct seg *seg,
4098     caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4099 {
4100 	struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4101 	struct snode *sp = VTOS(VTOCVP(sdp->vp));
4102 
4103 	return (e_ddi_copytodev(sp->s_dip,
4104 	    (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4105 }
4106