xref: /illumos-gate/usr/src/uts/common/vm/vm_as.c (revision a98e9dbf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 /*
30  * University Copyright- Copyright (c) 1982, 1986, 1988
31  * The Regents of the University of California
32  * All Rights Reserved
33  *
34  * University Acknowledgment- Portions of this document are derived from
35  * software developed by the University of California, Berkeley, and its
36  * contributors.
37  */
38 
39 #pragma ident	"%Z%%M%	%I%	%E% SMI"
40 
41 /*
42  * VM - address spaces.
43  */
44 
45 #include <sys/types.h>
46 #include <sys/t_lock.h>
47 #include <sys/param.h>
48 #include <sys/errno.h>
49 #include <sys/systm.h>
50 #include <sys/mman.h>
51 #include <sys/sysmacros.h>
52 #include <sys/cpuvar.h>
53 #include <sys/sysinfo.h>
54 #include <sys/kmem.h>
55 #include <sys/vnode.h>
56 #include <sys/vmsystm.h>
57 #include <sys/cmn_err.h>
58 #include <sys/debug.h>
59 #include <sys/tnf_probe.h>
60 #include <sys/vtrace.h>
61 
62 #include <vm/hat.h>
63 #include <vm/xhat.h>
64 #include <vm/as.h>
65 #include <vm/seg.h>
66 #include <vm/seg_vn.h>
67 #include <vm/seg_dev.h>
68 #include <vm/seg_kmem.h>
69 #include <vm/seg_map.h>
70 #include <vm/seg_spt.h>
71 #include <vm/page.h>
72 
73 clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
74 
75 static struct kmem_cache *as_cache;
76 
77 static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
78 static void as_clearwatchprot(struct as *, caddr_t, size_t);
79 int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *);
80 
81 
82 /*
83  * Verifying the segment lists is very time-consuming; it may not be
84  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
85  */
86 #ifdef DEBUG
87 #define	VERIFY_SEGLIST
88 int do_as_verify = 0;
89 #endif
90 
91 /*
92  * Allocate a new callback data structure entry and fill in the events of
93  * interest, the address range of interest, and the callback argument.
94  * Link the entry on the as->a_callbacks list. A callback entry for the
95  * entire address space may be specified with vaddr = 0 and size = -1.
96  *
97  * CALLERS RESPONSIBILITY: If not calling from within the process context for
98  * the specified as, the caller must guarantee persistence of the specified as
99  * for the duration of this function (eg. pages being locked within the as
100  * will guarantee persistence).
101  */
102 int
103 as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
104 		caddr_t vaddr, size_t size, int sleepflag)
105 {
106 	struct as_callback 	*current_head, *cb;
107 	caddr_t 		saddr;
108 	size_t 			rsize;
109 
110 	/* callback function and an event are mandatory */
111 	if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
112 		return (EINVAL);
113 
114 	/* Adding a callback after as_free has been called is not allowed */
115 	if (as == &kas)
116 		return (ENOMEM);
117 
118 	/*
119 	 * vaddr = 0 and size = -1 is used to indicate that the callback range
120 	 * is the entire address space so no rounding is done in that case.
121 	 */
122 	if (size != -1) {
123 		saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
124 		rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
125 		    (size_t)saddr;
126 		/* check for wraparound */
127 		if (saddr + rsize < saddr)
128 			return (ENOMEM);
129 	} else {
130 		if (vaddr != 0)
131 			return (EINVAL);
132 		saddr = vaddr;
133 		rsize = size;
134 	}
135 
136 	/* Allocate and initialize a callback entry */
137 	cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
138 	if (cb == NULL)
139 		return (EAGAIN);
140 
141 	cb->ascb_func = cb_func;
142 	cb->ascb_arg = arg;
143 	cb->ascb_events = events;
144 	cb->ascb_saddr = saddr;
145 	cb->ascb_len = rsize;
146 
147 	/* Add the entry to the list */
148 	mutex_enter(&as->a_contents);
149 	current_head = as->a_callbacks;
150 	as->a_callbacks = cb;
151 	cb->ascb_next = current_head;
152 
153 	/*
154 	 * The call to this function may lose in a race with
155 	 * a pertinent event - eg. a thread does long term memory locking
156 	 * but before the callback is added another thread executes as_unmap.
157 	 * A broadcast here resolves that.
158 	 */
159 	if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
160 		AS_CLRUNMAPWAIT(as);
161 		cv_broadcast(&as->a_cv);
162 	}
163 
164 	mutex_exit(&as->a_contents);
165 	return (0);
166 }
167 
168 /*
169  * Search the callback list for an entry which pertains to arg.
170  *
171  * This is called from within the client upon completion of the callback.
172  * RETURN VALUES:
173  *	AS_CALLBACK_DELETED  (callback entry found and deleted)
174  *	AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
175  *	AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
176  *			entry will be made in as_do_callbacks)
177  *
178  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
179  * set, it indicates that as_do_callbacks is processing this entry.  The
180  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
181  * to unblock as_do_callbacks, in case it is blocked.
182  *
183  * CALLERS RESPONSIBILITY: If not calling from within the process context for
184  * the specified as, the caller must guarantee persistence of the specified as
185  * for the duration of this function (eg. pages being locked within the as
186  * will guarantee persistence).
187  */
188 uint_t
189 as_delete_callback(struct as *as, void *arg)
190 {
191 	struct as_callback **prevcb = &as->a_callbacks;
192 	struct as_callback *cb;
193 	uint_t rc = AS_CALLBACK_NOTFOUND;
194 
195 	mutex_enter(&as->a_contents);
196 	for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
197 		if (cb->ascb_arg != arg)
198 			continue;
199 
200 		/*
201 		 * If the events indicate AS_CALLBACK_CALLED, just clear
202 		 * AS_ALL_EVENT in the events field and wakeup the thread
203 		 * that may be waiting in as_do_callbacks.  as_do_callbacks
204 		 * will take care of removing this entry from the list.  In
205 		 * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
206 		 * (AS_CALLBACK_CALLED not set), just remove it from the
207 		 * list, return the memory and return AS_CALLBACK_DELETED.
208 		 */
209 		if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
210 			/* leave AS_CALLBACK_CALLED */
211 			cb->ascb_events &= ~AS_ALL_EVENT;
212 			rc = AS_CALLBACK_DELETE_DEFERRED;
213 			cv_broadcast(&as->a_cv);
214 		} else {
215 			*prevcb = cb->ascb_next;
216 			kmem_free(cb, sizeof (struct as_callback));
217 			rc = AS_CALLBACK_DELETED;
218 		}
219 		break;
220 	}
221 	mutex_exit(&as->a_contents);
222 	return (rc);
223 }
224 
225 /*
226  * Searches the as callback list for a matching entry.
227  * Returns a pointer to the first matching callback, or NULL if
228  * nothing is found.
229  * This function never sleeps so it is ok to call it with more
230  * locks held but the (required) a_contents mutex.
231  *
232  * See also comment on as_do_callbacks below.
233  */
234 static struct as_callback *
235 as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
236 			size_t event_len)
237 {
238 	struct as_callback	*cb;
239 
240 	ASSERT(MUTEX_HELD(&as->a_contents));
241 	for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
242 		/*
243 		 * If the callback has not already been called, then
244 		 * check if events or address range pertains.  An event_len
245 		 * of zero means do an unconditional callback.
246 		 */
247 		if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
248 		    ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
249 		    (event_addr + event_len < cb->ascb_saddr) ||
250 		    (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
251 			continue;
252 		}
253 		break;
254 	}
255 	return (cb);
256 }
257 
258 /*
259  * Executes a given callback and removes it from the callback list for
260  * this address space.
261  * This function may sleep so the caller must drop all locks except
262  * a_contents before calling this func.
263  *
264  * See also comments on as_do_callbacks below.
265  */
266 static void
267 as_execute_callback(struct as *as, struct as_callback *cb,
268 				uint_t events)
269 {
270 	struct as_callback **prevcb;
271 	void	*cb_arg;
272 
273 	ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
274 	cb->ascb_events |= AS_CALLBACK_CALLED;
275 	mutex_exit(&as->a_contents);
276 	(*cb->ascb_func)(as, cb->ascb_arg, events);
277 	mutex_enter(&as->a_contents);
278 	/*
279 	 * the callback function is required to delete the callback
280 	 * when the callback function determines it is OK for
281 	 * this thread to continue. as_delete_callback will clear
282 	 * the AS_ALL_EVENT in the events field when it is deleted.
283 	 * If the callback function called as_delete_callback,
284 	 * events will already be cleared and there will be no blocking.
285 	 */
286 	while ((cb->ascb_events & events) != 0) {
287 		cv_wait(&as->a_cv, &as->a_contents);
288 	}
289 	/*
290 	 * This entry needs to be taken off the list. Normally, the
291 	 * callback func itself does that, but unfortunately the list
292 	 * may have changed while the callback was running because the
293 	 * a_contents mutex was dropped and someone else other than the
294 	 * callback func itself could have called as_delete_callback,
295 	 * so we have to search to find this entry again.  The entry
296 	 * must have AS_CALLBACK_CALLED, and have the same 'arg'.
297 	 */
298 	cb_arg = cb->ascb_arg;
299 	prevcb = &as->a_callbacks;
300 	for (cb = as->a_callbacks; cb != NULL;
301 	    prevcb = &cb->ascb_next, cb = *prevcb) {
302 		if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
303 		    (cb_arg != cb->ascb_arg)) {
304 			continue;
305 		}
306 		*prevcb = cb->ascb_next;
307 		kmem_free(cb, sizeof (struct as_callback));
308 		break;
309 	}
310 }
311 
312 /*
313  * Check the callback list for a matching event and intersection of
314  * address range. If there is a match invoke the callback.  Skip an entry if:
315  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
316  *    - not event of interest
317  *    - not address range of interest
318  *
319  * An event_len of zero indicates a request for an unconditional callback
320  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
321  * a_contents lock must be dropped before a callback, so only one callback
322  * can be done before returning. Return -1 (true) if a callback was
323  * executed and removed from the list, else return 0 (false).
324  *
325  * The logically separate parts, i.e. finding a matching callback and
326  * executing a given callback have been separated into two functions
327  * so that they can be called with different sets of locks held beyond
328  * the always-required a_contents. as_find_callback does not sleep so
329  * it is ok to call it if more locks than a_contents (i.e. the a_lock
330  * rwlock) are held. as_execute_callback on the other hand may sleep
331  * so all locks beyond a_contents must be dropped by the caller if one
332  * does not want to end comatose.
333  */
334 static int
335 as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
336 			size_t event_len)
337 {
338 	struct as_callback *cb;
339 
340 	if ((cb = as_find_callback(as, events, event_addr, event_len))) {
341 		as_execute_callback(as, cb, events);
342 		return (-1);
343 	}
344 	return (0);
345 }
346 
347 /*
348  * Search for the segment containing addr. If a segment containing addr
349  * exists, that segment is returned.  If no such segment exists, and
350  * the list spans addresses greater than addr, then the first segment
351  * whose base is greater than addr is returned; otherwise, NULL is
352  * returned unless tail is true, in which case the last element of the
353  * list is returned.
354  *
355  * a_seglast is used to cache the last found segment for repeated
356  * searches to the same addr (which happens frequently).
357  */
358 struct seg *
359 as_findseg(struct as *as, caddr_t addr, int tail)
360 {
361 	struct seg *seg = as->a_seglast;
362 	avl_index_t where;
363 
364 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
365 
366 	if (seg != NULL &&
367 	    seg->s_base <= addr &&
368 	    addr < seg->s_base + seg->s_size)
369 		return (seg);
370 
371 	seg = avl_find(&as->a_segtree, &addr, &where);
372 	if (seg != NULL)
373 		return (as->a_seglast = seg);
374 
375 	seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
376 	if (seg == NULL && tail)
377 		seg = avl_last(&as->a_segtree);
378 	return (as->a_seglast = seg);
379 }
380 
381 #ifdef VERIFY_SEGLIST
382 /*
383  * verify that the linked list is coherent
384  */
385 static void
386 as_verify(struct as *as)
387 {
388 	struct seg *seg, *seglast, *p, *n;
389 	uint_t nsegs = 0;
390 
391 	if (do_as_verify == 0)
392 		return;
393 
394 	seglast = as->a_seglast;
395 
396 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
397 		ASSERT(seg->s_as == as);
398 		p = AS_SEGPREV(as, seg);
399 		n = AS_SEGNEXT(as, seg);
400 		ASSERT(p == NULL || p->s_as == as);
401 		ASSERT(p == NULL || p->s_base < seg->s_base);
402 		ASSERT(n == NULL || n->s_base > seg->s_base);
403 		ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
404 		if (seg == seglast)
405 			seglast = NULL;
406 		nsegs++;
407 	}
408 	ASSERT(seglast == NULL);
409 	ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
410 }
411 #endif /* VERIFY_SEGLIST */
412 
413 /*
414  * Add a new segment to the address space. The avl_find()
415  * may be expensive so we attempt to use last segment accessed
416  * in as_gap() as an insertion point.
417  */
418 int
419 as_addseg(struct as  *as, struct seg *newseg)
420 {
421 	struct seg *seg;
422 	caddr_t addr;
423 	caddr_t eaddr;
424 	avl_index_t where;
425 
426 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
427 
428 	as->a_updatedir = 1;	/* inform /proc */
429 	gethrestime(&as->a_updatetime);
430 
431 	if (as->a_lastgaphl != NULL) {
432 		struct seg *hseg = NULL;
433 		struct seg *lseg = NULL;
434 
435 		if (as->a_lastgaphl->s_base > newseg->s_base) {
436 			hseg = as->a_lastgaphl;
437 			lseg = AVL_PREV(&as->a_segtree, hseg);
438 		} else {
439 			lseg = as->a_lastgaphl;
440 			hseg = AVL_NEXT(&as->a_segtree, lseg);
441 		}
442 
443 		if (hseg && lseg && lseg->s_base < newseg->s_base &&
444 		    hseg->s_base > newseg->s_base) {
445 			avl_insert_here(&as->a_segtree, newseg, lseg,
446 			    AVL_AFTER);
447 			as->a_lastgaphl = NULL;
448 			as->a_seglast = newseg;
449 			return (0);
450 		}
451 		as->a_lastgaphl = NULL;
452 	}
453 
454 	addr = newseg->s_base;
455 	eaddr = addr + newseg->s_size;
456 again:
457 
458 	seg = avl_find(&as->a_segtree, &addr, &where);
459 
460 	if (seg == NULL)
461 		seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
462 
463 	if (seg == NULL)
464 		seg = avl_last(&as->a_segtree);
465 
466 	if (seg != NULL) {
467 		caddr_t base = seg->s_base;
468 
469 		/*
470 		 * If top of seg is below the requested address, then
471 		 * the insertion point is at the end of the linked list,
472 		 * and seg points to the tail of the list.  Otherwise,
473 		 * the insertion point is immediately before seg.
474 		 */
475 		if (base + seg->s_size > addr) {
476 			if (addr >= base || eaddr > base) {
477 #ifdef __sparc
478 				extern struct seg_ops segnf_ops;
479 
480 				/*
481 				 * no-fault segs must disappear if overlaid.
482 				 * XXX need new segment type so
483 				 * we don't have to check s_ops
484 				 */
485 				if (seg->s_ops == &segnf_ops) {
486 					seg_unmap(seg);
487 					goto again;
488 				}
489 #endif
490 				return (-1);	/* overlapping segment */
491 			}
492 		}
493 	}
494 	as->a_seglast = newseg;
495 	avl_insert(&as->a_segtree, newseg, where);
496 
497 #ifdef VERIFY_SEGLIST
498 	as_verify(as);
499 #endif
500 	return (0);
501 }
502 
503 struct seg *
504 as_removeseg(struct as *as, struct seg *seg)
505 {
506 	avl_tree_t *t;
507 
508 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
509 
510 	as->a_updatedir = 1;	/* inform /proc */
511 	gethrestime(&as->a_updatetime);
512 
513 	if (seg == NULL)
514 		return (NULL);
515 
516 	t = &as->a_segtree;
517 	if (as->a_seglast == seg)
518 		as->a_seglast = NULL;
519 	as->a_lastgaphl = NULL;
520 
521 	/*
522 	 * if this segment is at an address higher than
523 	 * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
524 	 */
525 	if (as->a_lastgap &&
526 	    (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
527 		as->a_lastgap = AVL_NEXT(t, seg);
528 
529 	/*
530 	 * remove the segment from the seg tree
531 	 */
532 	avl_remove(t, seg);
533 
534 #ifdef VERIFY_SEGLIST
535 	as_verify(as);
536 #endif
537 	return (seg);
538 }
539 
540 /*
541  * Find a segment containing addr.
542  */
543 struct seg *
544 as_segat(struct as *as, caddr_t addr)
545 {
546 	struct seg *seg = as->a_seglast;
547 
548 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
549 
550 	if (seg != NULL && seg->s_base <= addr &&
551 	    addr < seg->s_base + seg->s_size)
552 		return (seg);
553 
554 	seg = avl_find(&as->a_segtree, &addr, NULL);
555 	return (seg);
556 }
557 
558 /*
559  * Serialize all searches for holes in an address space to
560  * prevent two or more threads from allocating the same virtual
561  * address range.  The address space must not be "read/write"
562  * locked by the caller since we may block.
563  */
564 void
565 as_rangelock(struct as *as)
566 {
567 	mutex_enter(&as->a_contents);
568 	while (AS_ISCLAIMGAP(as))
569 		cv_wait(&as->a_cv, &as->a_contents);
570 	AS_SETCLAIMGAP(as);
571 	mutex_exit(&as->a_contents);
572 }
573 
574 /*
575  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
576  */
577 void
578 as_rangeunlock(struct as *as)
579 {
580 	mutex_enter(&as->a_contents);
581 	AS_CLRCLAIMGAP(as);
582 	cv_signal(&as->a_cv);
583 	mutex_exit(&as->a_contents);
584 }
585 
586 /*
587  * compar segments (or just an address) by segment address range
588  */
589 static int
590 as_segcompar(const void *x, const void *y)
591 {
592 	struct seg *a = (struct seg *)x;
593 	struct seg *b = (struct seg *)y;
594 
595 	if (a->s_base < b->s_base)
596 		return (-1);
597 	if (a->s_base >= b->s_base + b->s_size)
598 		return (1);
599 	return (0);
600 }
601 
602 
603 void
604 as_avlinit(struct as *as)
605 {
606 	avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
607 	    offsetof(struct seg, s_tree));
608 	avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
609 	    offsetof(struct watched_page, wp_link));
610 }
611 
612 /*ARGSUSED*/
613 static int
614 as_constructor(void *buf, void *cdrarg, int kmflags)
615 {
616 	struct as *as = buf;
617 
618 	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
619 	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
620 	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
621 	as_avlinit(as);
622 	return (0);
623 }
624 
625 /*ARGSUSED1*/
626 static void
627 as_destructor(void *buf, void *cdrarg)
628 {
629 	struct as *as = buf;
630 
631 	avl_destroy(&as->a_segtree);
632 	mutex_destroy(&as->a_contents);
633 	cv_destroy(&as->a_cv);
634 	rw_destroy(&as->a_lock);
635 }
636 
637 void
638 as_init(void)
639 {
640 	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
641 	    as_constructor, as_destructor, NULL, NULL, NULL, 0);
642 }
643 
644 /*
645  * Allocate and initialize an address space data structure.
646  * We call hat_alloc to allow any machine dependent
647  * information in the hat structure to be initialized.
648  */
649 struct as *
650 as_alloc(void)
651 {
652 	struct as *as;
653 
654 	as = kmem_cache_alloc(as_cache, KM_SLEEP);
655 
656 	as->a_flags		= 0;
657 	as->a_vbits		= 0;
658 	as->a_hrm		= NULL;
659 	as->a_seglast		= NULL;
660 	as->a_size		= 0;
661 	as->a_updatedir		= 0;
662 	gethrestime(&as->a_updatetime);
663 	as->a_objectdir		= NULL;
664 	as->a_sizedir		= 0;
665 	as->a_userlimit		= (caddr_t)USERLIMIT;
666 	as->a_lastgap		= NULL;
667 	as->a_lastgaphl		= NULL;
668 	as->a_callbacks		= NULL;
669 
670 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
671 	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
672 	AS_LOCK_EXIT(as, &as->a_lock);
673 
674 	as->a_xhat = NULL;
675 
676 	return (as);
677 }
678 
679 /*
680  * Free an address space data structure.
681  * Need to free the hat first and then
682  * all the segments on this as and finally
683  * the space for the as struct itself.
684  */
685 void
686 as_free(struct as *as)
687 {
688 	struct hat *hat = as->a_hat;
689 	struct seg *seg, *next;
690 	int called = 0;
691 
692 top:
693 	/*
694 	 * Invoke ALL callbacks. as_do_callbacks will do one callback
695 	 * per call, and not return (-1) until the callback has completed.
696 	 * When as_do_callbacks returns zero, all callbacks have completed.
697 	 */
698 	mutex_enter(&as->a_contents);
699 	while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0))
700 		;
701 
702 	/* This will prevent new XHATs from attaching to as */
703 	if (!called)
704 		AS_SETBUSY(as);
705 	mutex_exit(&as->a_contents);
706 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
707 
708 	if (!called) {
709 		called = 1;
710 		hat_free_start(hat);
711 		if (as->a_xhat != NULL)
712 			xhat_free_start_all(as);
713 	}
714 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
715 		int err;
716 
717 		next = AS_SEGNEXT(as, seg);
718 retry:
719 		err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
720 		if (err == EAGAIN) {
721 			mutex_enter(&as->a_contents);
722 			if (as->a_callbacks) {
723 				AS_LOCK_EXIT(as, &as->a_lock);
724 			} else if (!AS_ISNOUNMAPWAIT(as)) {
725 				/*
726 				 * Memory is currently locked. Wait for a
727 				 * cv_signal that it has been unlocked, then
728 				 * try the operation again.
729 				 */
730 				if (AS_ISUNMAPWAIT(as) == 0)
731 					cv_broadcast(&as->a_cv);
732 				AS_SETUNMAPWAIT(as);
733 				AS_LOCK_EXIT(as, &as->a_lock);
734 				while (AS_ISUNMAPWAIT(as))
735 					cv_wait(&as->a_cv, &as->a_contents);
736 			} else {
737 				/*
738 				 * We may have raced with
739 				 * segvn_reclaim()/segspt_reclaim(). In this
740 				 * case clean nounmapwait flag and retry since
741 				 * softlockcnt in this segment may be already
742 				 * 0.  We don't drop as writer lock so our
743 				 * number of retries without sleeping should
744 				 * be very small. See segvn_reclaim() for
745 				 * more comments.
746 				 */
747 				AS_CLRNOUNMAPWAIT(as);
748 				mutex_exit(&as->a_contents);
749 				goto retry;
750 			}
751 			mutex_exit(&as->a_contents);
752 			goto top;
753 		} else {
754 			/*
755 			 * We do not expect any other error return at this
756 			 * time. This is similar to an ASSERT in seg_unmap()
757 			 */
758 			ASSERT(err == 0);
759 		}
760 	}
761 	hat_free_end(hat);
762 	if (as->a_xhat != NULL)
763 		xhat_free_end_all(as);
764 	AS_LOCK_EXIT(as, &as->a_lock);
765 
766 	/* /proc stuff */
767 	ASSERT(avl_numnodes(&as->a_wpage) == 0);
768 	if (as->a_objectdir) {
769 		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
770 		as->a_objectdir = NULL;
771 		as->a_sizedir = 0;
772 	}
773 
774 	/*
775 	 * Free the struct as back to kmem.  Assert it has no segments.
776 	 */
777 	ASSERT(avl_numnodes(&as->a_segtree) == 0);
778 	kmem_cache_free(as_cache, as);
779 }
780 
781 int
782 as_dup(struct as *as, struct as **outas)
783 {
784 	struct as *newas;
785 	struct seg *seg, *newseg;
786 	int error;
787 
788 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
789 	as_clearwatch(as);
790 	newas = as_alloc();
791 	newas->a_userlimit = as->a_userlimit;
792 	AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER);
793 
794 	/* This will prevent new XHATs from attaching */
795 	mutex_enter(&as->a_contents);
796 	AS_SETBUSY(as);
797 	mutex_exit(&as->a_contents);
798 	mutex_enter(&newas->a_contents);
799 	AS_SETBUSY(newas);
800 	mutex_exit(&newas->a_contents);
801 
802 	(void) hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_SRD);
803 
804 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
805 
806 		if (seg->s_flags & S_PURGE)
807 			continue;
808 
809 		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
810 		if (newseg == NULL) {
811 			AS_LOCK_EXIT(newas, &newas->a_lock);
812 			as_setwatch(as);
813 			mutex_enter(&as->a_contents);
814 			AS_CLRBUSY(as);
815 			mutex_exit(&as->a_contents);
816 			AS_LOCK_EXIT(as, &as->a_lock);
817 			as_free(newas);
818 			return (-1);
819 		}
820 		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
821 			/*
822 			 * We call seg_free() on the new seg
823 			 * because the segment is not set up
824 			 * completely; i.e. it has no ops.
825 			 */
826 			as_setwatch(as);
827 			mutex_enter(&as->a_contents);
828 			AS_CLRBUSY(as);
829 			mutex_exit(&as->a_contents);
830 			AS_LOCK_EXIT(as, &as->a_lock);
831 			seg_free(newseg);
832 			AS_LOCK_EXIT(newas, &newas->a_lock);
833 			as_free(newas);
834 			return (error);
835 		}
836 		newas->a_size += seg->s_size;
837 	}
838 
839 	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
840 	if (as->a_xhat != NULL)
841 		error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL);
842 
843 	mutex_enter(&newas->a_contents);
844 	AS_CLRBUSY(newas);
845 	mutex_exit(&newas->a_contents);
846 	AS_LOCK_EXIT(newas, &newas->a_lock);
847 
848 	as_setwatch(as);
849 	mutex_enter(&as->a_contents);
850 	AS_CLRBUSY(as);
851 	mutex_exit(&as->a_contents);
852 	AS_LOCK_EXIT(as, &as->a_lock);
853 	if (error != 0) {
854 		as_free(newas);
855 		return (error);
856 	}
857 	*outas = newas;
858 	return (0);
859 }
860 
861 /*
862  * Handle a ``fault'' at addr for size bytes.
863  */
864 faultcode_t
865 as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
866 	enum fault_type type, enum seg_rw rw)
867 {
868 	struct seg *seg;
869 	caddr_t raddr;			/* rounded down addr */
870 	size_t rsize;			/* rounded up size */
871 	size_t ssize;
872 	faultcode_t res = 0;
873 	caddr_t addrsav;
874 	struct seg *segsav;
875 	int as_lock_held;
876 	klwp_t *lwp = ttolwp(curthread);
877 	int is_xhat = 0;
878 	int holding_wpage = 0;
879 	extern struct seg_ops   segdev_ops;
880 
881 
882 
883 	if (as->a_hat != hat) {
884 		/* This must be an XHAT then */
885 		is_xhat = 1;
886 
887 		if ((type != F_INVAL) || (as == &kas))
888 			return (FC_NOSUPPORT);
889 	}
890 
891 retry:
892 	if (!is_xhat) {
893 		/*
894 		 * Indicate that the lwp is not to be stopped while waiting
895 		 * for a pagefault.  This is to avoid deadlock while debugging
896 		 * a process via /proc over NFS (in particular).
897 		 */
898 		if (lwp != NULL)
899 			lwp->lwp_nostop++;
900 
901 		/*
902 		 * same length must be used when we softlock and softunlock.
903 		 * We don't support softunlocking lengths less than
904 		 * the original length when there is largepage support.
905 		 * See seg_dev.c for more comments.
906 		 */
907 		switch (type) {
908 
909 		case F_SOFTLOCK:
910 			CPU_STATS_ADD_K(vm, softlock, 1);
911 			break;
912 
913 		case F_SOFTUNLOCK:
914 			break;
915 
916 		case F_PROT:
917 			CPU_STATS_ADD_K(vm, prot_fault, 1);
918 			break;
919 
920 		case F_INVAL:
921 			CPU_STATS_ENTER_K();
922 			CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
923 			if (as == &kas)
924 				CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
925 			CPU_STATS_EXIT_K();
926 			break;
927 		}
928 	}
929 
930 	/* Kernel probe */
931 	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
932 	    tnf_opaque,	address,	addr,
933 	    tnf_fault_type,	fault_type,	type,
934 	    tnf_seg_access,	access,		rw);
935 
936 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
937 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
938 	    (size_t)raddr;
939 
940 	/*
941 	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
942 	 * correctness, but then we could be stuck holding this lock for
943 	 * a LONG time if the fault needs to be resolved on a slow
944 	 * filesystem, and then no-one will be able to exec new commands,
945 	 * as exec'ing requires the write lock on the as.
946 	 */
947 	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
948 	    raddr + size < segkmap->s_base + segkmap->s_size) {
949 		/*
950 		 * if (as==&kas), this can't be XHAT: we've already returned
951 		 * FC_NOSUPPORT.
952 		 */
953 		seg = segkmap;
954 		as_lock_held = 0;
955 	} else {
956 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
957 		if (is_xhat && avl_numnodes(&as->a_wpage) != 0) {
958 			/*
959 			 * Grab and hold the writers' lock on the as
960 			 * if the fault is to a watched page.
961 			 * This will keep CPUs from "peeking" at the
962 			 * address range while we're temporarily boosting
963 			 * the permissions for the XHAT device to
964 			 * resolve the fault in the segment layer.
965 			 *
966 			 * We could check whether faulted address
967 			 * is within a watched page and only then grab
968 			 * the writer lock, but this is simpler.
969 			 */
970 			AS_LOCK_EXIT(as, &as->a_lock);
971 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
972 		}
973 
974 		seg = as_segat(as, raddr);
975 		if (seg == NULL) {
976 			AS_LOCK_EXIT(as, &as->a_lock);
977 			if ((lwp != NULL) && (!is_xhat))
978 				lwp->lwp_nostop--;
979 			return (FC_NOMAP);
980 		}
981 
982 		as_lock_held = 1;
983 	}
984 
985 	addrsav = raddr;
986 	segsav = seg;
987 
988 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
989 		if (raddr >= seg->s_base + seg->s_size) {
990 			seg = AS_SEGNEXT(as, seg);
991 			if (seg == NULL || raddr != seg->s_base) {
992 				res = FC_NOMAP;
993 				break;
994 			}
995 		}
996 		if (raddr + rsize > seg->s_base + seg->s_size)
997 			ssize = seg->s_base + seg->s_size - raddr;
998 		else
999 			ssize = rsize;
1000 
1001 		if (!is_xhat || (seg->s_ops != &segdev_ops)) {
1002 
1003 			if (is_xhat && avl_numnodes(&as->a_wpage) != 0 &&
1004 			    pr_is_watchpage_as(raddr, rw, as)) {
1005 				/*
1006 				 * Handle watch pages.  If we're faulting on a
1007 				 * watched page from an X-hat, we have to
1008 				 * restore the original permissions while we
1009 				 * handle the fault.
1010 				 */
1011 				as_clearwatch(as);
1012 				holding_wpage = 1;
1013 			}
1014 
1015 			res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
1016 
1017 			/* Restore watchpoints */
1018 			if (holding_wpage) {
1019 				as_setwatch(as);
1020 				holding_wpage = 0;
1021 			}
1022 
1023 			if (res != 0)
1024 				break;
1025 		} else {
1026 			/* XHAT does not support seg_dev */
1027 			res = FC_NOSUPPORT;
1028 			break;
1029 		}
1030 	}
1031 
1032 	/*
1033 	 * If we were SOFTLOCKing and encountered a failure,
1034 	 * we must SOFTUNLOCK the range we already did. (Maybe we
1035 	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
1036 	 * right here...)
1037 	 */
1038 	if (res != 0 && type == F_SOFTLOCK) {
1039 		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
1040 			if (addrsav >= seg->s_base + seg->s_size)
1041 				seg = AS_SEGNEXT(as, seg);
1042 			ASSERT(seg != NULL);
1043 			/*
1044 			 * Now call the fault routine again to perform the
1045 			 * unlock using S_OTHER instead of the rw variable
1046 			 * since we never got a chance to touch the pages.
1047 			 */
1048 			if (raddr > seg->s_base + seg->s_size)
1049 				ssize = seg->s_base + seg->s_size - addrsav;
1050 			else
1051 				ssize = raddr - addrsav;
1052 			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
1053 			    F_SOFTUNLOCK, S_OTHER);
1054 		}
1055 	}
1056 	if (as_lock_held)
1057 		AS_LOCK_EXIT(as, &as->a_lock);
1058 	if ((lwp != NULL) && (!is_xhat))
1059 		lwp->lwp_nostop--;
1060 
1061 	/*
1062 	 * If the lower levels returned EDEADLK for a fault,
1063 	 * It means that we should retry the fault.  Let's wait
1064 	 * a bit also to let the deadlock causing condition clear.
1065 	 * This is part of a gross hack to work around a design flaw
1066 	 * in the ufs/sds logging code and should go away when the
1067 	 * logging code is re-designed to fix the problem. See bug
1068 	 * 4125102 for details of the problem.
1069 	 */
1070 	if (FC_ERRNO(res) == EDEADLK) {
1071 		delay(deadlk_wait);
1072 		res = 0;
1073 		goto retry;
1074 	}
1075 	return (res);
1076 }
1077 
1078 
1079 
1080 /*
1081  * Asynchronous ``fault'' at addr for size bytes.
1082  */
1083 faultcode_t
1084 as_faulta(struct as *as, caddr_t addr, size_t size)
1085 {
1086 	struct seg *seg;
1087 	caddr_t raddr;			/* rounded down addr */
1088 	size_t rsize;			/* rounded up size */
1089 	faultcode_t res = 0;
1090 	klwp_t *lwp = ttolwp(curthread);
1091 
1092 retry:
1093 	/*
1094 	 * Indicate that the lwp is not to be stopped while waiting
1095 	 * for a pagefault.  This is to avoid deadlock while debugging
1096 	 * a process via /proc over NFS (in particular).
1097 	 */
1098 	if (lwp != NULL)
1099 		lwp->lwp_nostop++;
1100 
1101 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1102 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1103 	    (size_t)raddr;
1104 
1105 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1106 	seg = as_segat(as, raddr);
1107 	if (seg == NULL) {
1108 		AS_LOCK_EXIT(as, &as->a_lock);
1109 		if (lwp != NULL)
1110 			lwp->lwp_nostop--;
1111 		return (FC_NOMAP);
1112 	}
1113 
1114 	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
1115 		if (raddr >= seg->s_base + seg->s_size) {
1116 			seg = AS_SEGNEXT(as, seg);
1117 			if (seg == NULL || raddr != seg->s_base) {
1118 				res = FC_NOMAP;
1119 				break;
1120 			}
1121 		}
1122 		res = SEGOP_FAULTA(seg, raddr);
1123 		if (res != 0)
1124 			break;
1125 	}
1126 	AS_LOCK_EXIT(as, &as->a_lock);
1127 	if (lwp != NULL)
1128 		lwp->lwp_nostop--;
1129 	/*
1130 	 * If the lower levels returned EDEADLK for a fault,
1131 	 * It means that we should retry the fault.  Let's wait
1132 	 * a bit also to let the deadlock causing condition clear.
1133 	 * This is part of a gross hack to work around a design flaw
1134 	 * in the ufs/sds logging code and should go away when the
1135 	 * logging code is re-designed to fix the problem. See bug
1136 	 * 4125102 for details of the problem.
1137 	 */
1138 	if (FC_ERRNO(res) == EDEADLK) {
1139 		delay(deadlk_wait);
1140 		res = 0;
1141 		goto retry;
1142 	}
1143 	return (res);
1144 }
1145 
1146 /*
1147  * Set the virtual mapping for the interval from [addr : addr + size)
1148  * in address space `as' to have the specified protection.
1149  * It is ok for the range to cross over several segments,
1150  * as long as they are contiguous.
1151  */
1152 int
1153 as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1154 {
1155 	struct seg *seg;
1156 	struct as_callback *cb;
1157 	size_t ssize;
1158 	caddr_t raddr;			/* rounded down addr */
1159 	size_t rsize;			/* rounded up size */
1160 	int error = 0, writer = 0;
1161 	caddr_t saveraddr;
1162 	size_t saversize;
1163 
1164 setprot_top:
1165 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1166 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1167 	    (size_t)raddr;
1168 
1169 	if (raddr + rsize < raddr)		/* check for wraparound */
1170 		return (ENOMEM);
1171 
1172 	saveraddr = raddr;
1173 	saversize = rsize;
1174 
1175 	/*
1176 	 * Normally we only lock the as as a reader. But
1177 	 * if due to setprot the segment driver needs to split
1178 	 * a segment it will return IE_RETRY. Therefore we re-acquire
1179 	 * the as lock as a writer so the segment driver can change
1180 	 * the seg list. Also the segment driver will return IE_RETRY
1181 	 * after it has changed the segment list so we therefore keep
1182 	 * locking as a writer. Since these opeartions should be rare
1183 	 * want to only lock as a writer when necessary.
1184 	 */
1185 	if (writer || avl_numnodes(&as->a_wpage) != 0) {
1186 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1187 	} else {
1188 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1189 	}
1190 
1191 	as_clearwatchprot(as, raddr, rsize);
1192 	seg = as_segat(as, raddr);
1193 	if (seg == NULL) {
1194 		as_setwatch(as);
1195 		AS_LOCK_EXIT(as, &as->a_lock);
1196 		return (ENOMEM);
1197 	}
1198 
1199 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1200 		if (raddr >= seg->s_base + seg->s_size) {
1201 			seg = AS_SEGNEXT(as, seg);
1202 			if (seg == NULL || raddr != seg->s_base) {
1203 				error = ENOMEM;
1204 				break;
1205 			}
1206 		}
1207 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1208 			ssize = seg->s_base + seg->s_size - raddr;
1209 		else
1210 			ssize = rsize;
1211 retry:
1212 		error = SEGOP_SETPROT(seg, raddr, ssize, prot);
1213 
1214 		if (error == IE_NOMEM) {
1215 			error = EAGAIN;
1216 			break;
1217 		}
1218 
1219 		if (error == IE_RETRY) {
1220 			AS_LOCK_EXIT(as, &as->a_lock);
1221 			writer = 1;
1222 			goto setprot_top;
1223 		}
1224 
1225 		if (error == EAGAIN) {
1226 			/*
1227 			 * Make sure we have a_lock as writer.
1228 			 */
1229 			if (writer == 0) {
1230 				AS_LOCK_EXIT(as, &as->a_lock);
1231 				writer = 1;
1232 				goto setprot_top;
1233 			}
1234 
1235 			/*
1236 			 * Memory is currently locked.  It must be unlocked
1237 			 * before this operation can succeed through a retry.
1238 			 * The possible reasons for locked memory and
1239 			 * corresponding strategies for unlocking are:
1240 			 * (1) Normal I/O
1241 			 *	wait for a signal that the I/O operation
1242 			 *	has completed and the memory is unlocked.
1243 			 * (2) Asynchronous I/O
1244 			 *	The aio subsystem does not unlock pages when
1245 			 *	the I/O is completed. Those pages are unlocked
1246 			 *	when the application calls aiowait/aioerror.
1247 			 *	So, to prevent blocking forever, cv_broadcast()
1248 			 *	is done to wake up aio_cleanup_thread.
1249 			 *	Subsequently, segvn_reclaim will be called, and
1250 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1251 			 * (3) Long term page locking:
1252 			 *	Drivers intending to have pages locked for a
1253 			 *	period considerably longer than for normal I/O
1254 			 *	(essentially forever) may have registered for a
1255 			 *	callback so they may unlock these pages on
1256 			 *	request. This is needed to allow this operation
1257 			 *	to succeed. Each entry on the callback list is
1258 			 *	examined. If the event or address range pertains
1259 			 *	the callback is invoked (unless it already is in
1260 			 *	progress). The a_contents lock must be dropped
1261 			 *	before the callback, so only one callback can
1262 			 *	be done at a time. Go to the top and do more
1263 			 *	until zero is returned. If zero is returned,
1264 			 *	either there were no callbacks for this event
1265 			 *	or they were already in progress.
1266 			 */
1267 			mutex_enter(&as->a_contents);
1268 			if (as->a_callbacks &&
1269 			    (cb = as_find_callback(as, AS_SETPROT_EVENT,
1270 			    seg->s_base, seg->s_size))) {
1271 				AS_LOCK_EXIT(as, &as->a_lock);
1272 				as_execute_callback(as, cb, AS_SETPROT_EVENT);
1273 			} else if (!AS_ISNOUNMAPWAIT(as)) {
1274 				if (AS_ISUNMAPWAIT(as) == 0)
1275 					cv_broadcast(&as->a_cv);
1276 				AS_SETUNMAPWAIT(as);
1277 				AS_LOCK_EXIT(as, &as->a_lock);
1278 				while (AS_ISUNMAPWAIT(as))
1279 					cv_wait(&as->a_cv, &as->a_contents);
1280 			} else {
1281 				/*
1282 				 * We may have raced with
1283 				 * segvn_reclaim()/segspt_reclaim(). In this
1284 				 * case clean nounmapwait flag and retry since
1285 				 * softlockcnt in this segment may be already
1286 				 * 0.  We don't drop as writer lock so our
1287 				 * number of retries without sleeping should
1288 				 * be very small. See segvn_reclaim() for
1289 				 * more comments.
1290 				 */
1291 				AS_CLRNOUNMAPWAIT(as);
1292 				mutex_exit(&as->a_contents);
1293 				goto retry;
1294 			}
1295 			mutex_exit(&as->a_contents);
1296 			goto setprot_top;
1297 		} else if (error != 0)
1298 			break;
1299 	}
1300 	if (error != 0) {
1301 		as_setwatch(as);
1302 	} else {
1303 		as_setwatchprot(as, saveraddr, saversize, prot);
1304 	}
1305 	AS_LOCK_EXIT(as, &as->a_lock);
1306 	return (error);
1307 }
1308 
1309 /*
1310  * Check to make sure that the interval [addr, addr + size)
1311  * in address space `as' has at least the specified protection.
1312  * It is ok for the range to cross over several segments, as long
1313  * as they are contiguous.
1314  */
1315 int
1316 as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
1317 {
1318 	struct seg *seg;
1319 	size_t ssize;
1320 	caddr_t raddr;			/* rounded down addr */
1321 	size_t rsize;			/* rounded up size */
1322 	int error = 0;
1323 
1324 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1325 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1326 	    (size_t)raddr;
1327 
1328 	if (raddr + rsize < raddr)		/* check for wraparound */
1329 		return (ENOMEM);
1330 
1331 	/*
1332 	 * This is ugly as sin...
1333 	 * Normally, we only acquire the address space readers lock.
1334 	 * However, if the address space has watchpoints present,
1335 	 * we must acquire the writer lock on the address space for
1336 	 * the benefit of as_clearwatchprot() and as_setwatchprot().
1337 	 */
1338 	if (avl_numnodes(&as->a_wpage) != 0)
1339 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1340 	else
1341 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1342 	as_clearwatchprot(as, raddr, rsize);
1343 	seg = as_segat(as, raddr);
1344 	if (seg == NULL) {
1345 		as_setwatch(as);
1346 		AS_LOCK_EXIT(as, &as->a_lock);
1347 		return (ENOMEM);
1348 	}
1349 
1350 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
1351 		if (raddr >= seg->s_base + seg->s_size) {
1352 			seg = AS_SEGNEXT(as, seg);
1353 			if (seg == NULL || raddr != seg->s_base) {
1354 				error = ENOMEM;
1355 				break;
1356 			}
1357 		}
1358 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
1359 			ssize = seg->s_base + seg->s_size - raddr;
1360 		else
1361 			ssize = rsize;
1362 
1363 		error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
1364 		if (error != 0)
1365 			break;
1366 	}
1367 	as_setwatch(as);
1368 	AS_LOCK_EXIT(as, &as->a_lock);
1369 	return (error);
1370 }
1371 
1372 int
1373 as_unmap(struct as *as, caddr_t addr, size_t size)
1374 {
1375 	struct seg *seg, *seg_next;
1376 	struct as_callback *cb;
1377 	caddr_t raddr, eaddr;
1378 	size_t ssize;
1379 	int err;
1380 
1381 top:
1382 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1383 	eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) &
1384 	    (uintptr_t)PAGEMASK);
1385 
1386 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1387 
1388 	as->a_updatedir = 1;	/* inform /proc */
1389 	gethrestime(&as->a_updatetime);
1390 
1391 	/*
1392 	 * Use as_findseg to find the first segment in the range, then
1393 	 * step through the segments in order, following s_next.
1394 	 */
1395 	as_clearwatchprot(as, raddr, eaddr - raddr);
1396 
1397 	for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
1398 		if (eaddr <= seg->s_base)
1399 			break;		/* eaddr was in a gap; all done */
1400 
1401 		/* this is implied by the test above */
1402 		ASSERT(raddr < eaddr);
1403 
1404 		if (raddr < seg->s_base)
1405 			raddr = seg->s_base; 	/* raddr was in a gap */
1406 
1407 		if (eaddr > (seg->s_base + seg->s_size))
1408 			ssize = seg->s_base + seg->s_size - raddr;
1409 		else
1410 			ssize = eaddr - raddr;
1411 
1412 		/*
1413 		 * Save next segment pointer since seg can be
1414 		 * destroyed during the segment unmap operation.
1415 		 */
1416 		seg_next = AS_SEGNEXT(as, seg);
1417 
1418 retry:
1419 		err = SEGOP_UNMAP(seg, raddr, ssize);
1420 		if (err == EAGAIN) {
1421 			/*
1422 			 * Memory is currently locked.  It must be unlocked
1423 			 * before this operation can succeed through a retry.
1424 			 * The possible reasons for locked memory and
1425 			 * corresponding strategies for unlocking are:
1426 			 * (1) Normal I/O
1427 			 *	wait for a signal that the I/O operation
1428 			 *	has completed and the memory is unlocked.
1429 			 * (2) Asynchronous I/O
1430 			 *	The aio subsystem does not unlock pages when
1431 			 *	the I/O is completed. Those pages are unlocked
1432 			 *	when the application calls aiowait/aioerror.
1433 			 *	So, to prevent blocking forever, cv_broadcast()
1434 			 *	is done to wake up aio_cleanup_thread.
1435 			 *	Subsequently, segvn_reclaim will be called, and
1436 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
1437 			 * (3) Long term page locking:
1438 			 *	Drivers intending to have pages locked for a
1439 			 *	period considerably longer than for normal I/O
1440 			 *	(essentially forever) may have registered for a
1441 			 *	callback so they may unlock these pages on
1442 			 *	request. This is needed to allow this operation
1443 			 *	to succeed. Each entry on the callback list is
1444 			 *	examined. If the event or address range pertains
1445 			 *	the callback is invoked (unless it already is in
1446 			 *	progress). The a_contents lock must be dropped
1447 			 *	before the callback, so only one callback can
1448 			 *	be done at a time. Go to the top and do more
1449 			 *	until zero is returned. If zero is returned,
1450 			 *	either there were no callbacks for this event
1451 			 *	or they were already in progress.
1452 			 */
1453 			mutex_enter(&as->a_contents);
1454 			if (as->a_callbacks &&
1455 			    (cb = as_find_callback(as, AS_UNMAP_EVENT,
1456 			    seg->s_base, seg->s_size))) {
1457 				AS_LOCK_EXIT(as, &as->a_lock);
1458 				as_execute_callback(as, cb, AS_UNMAP_EVENT);
1459 			} else if (!AS_ISNOUNMAPWAIT(as)) {
1460 				if (AS_ISUNMAPWAIT(as) == 0)
1461 					cv_broadcast(&as->a_cv);
1462 				AS_SETUNMAPWAIT(as);
1463 				AS_LOCK_EXIT(as, &as->a_lock);
1464 				while (AS_ISUNMAPWAIT(as))
1465 					cv_wait(&as->a_cv, &as->a_contents);
1466 			} else {
1467 				/*
1468 				 * We may have raced with
1469 				 * segvn_reclaim()/segspt_reclaim(). In this
1470 				 * case clean nounmapwait flag and retry since
1471 				 * softlockcnt in this segment may be already
1472 				 * 0.  We don't drop as writer lock so our
1473 				 * number of retries without sleeping should
1474 				 * be very small. See segvn_reclaim() for
1475 				 * more comments.
1476 				 */
1477 				AS_CLRNOUNMAPWAIT(as);
1478 				mutex_exit(&as->a_contents);
1479 				goto retry;
1480 			}
1481 			mutex_exit(&as->a_contents);
1482 			goto top;
1483 		} else if (err == IE_RETRY) {
1484 			AS_LOCK_EXIT(as, &as->a_lock);
1485 			goto top;
1486 		} else if (err) {
1487 			as_setwatch(as);
1488 			AS_LOCK_EXIT(as, &as->a_lock);
1489 			return (-1);
1490 		}
1491 
1492 		as->a_size -= ssize;
1493 		raddr += ssize;
1494 	}
1495 	AS_LOCK_EXIT(as, &as->a_lock);
1496 	return (0);
1497 }
1498 
1499 static int
1500 as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
1501     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1502 {
1503 	uint_t szc;
1504 	uint_t nszc;
1505 	int error;
1506 	caddr_t a;
1507 	caddr_t eaddr;
1508 	size_t segsize;
1509 	struct seg *seg;
1510 	size_t pgsz;
1511 	int do_off = (vn_a->vp != NULL || vn_a->amp != NULL);
1512 	uint_t save_szcvec;
1513 
1514 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1515 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1516 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1517 	ASSERT(vn_a->vp == NULL || vn_a->amp == NULL);
1518 	if (!do_off) {
1519 		vn_a->offset = 0;
1520 	}
1521 
1522 	if (szcvec <= 1) {
1523 		seg = seg_alloc(as, addr, size);
1524 		if (seg == NULL) {
1525 			return (ENOMEM);
1526 		}
1527 		vn_a->szc = 0;
1528 		error = (*crfp)(seg, vn_a);
1529 		if (error != 0) {
1530 			seg_free(seg);
1531 		} else {
1532 			as->a_size += size;
1533 		}
1534 		return (error);
1535 	}
1536 
1537 	eaddr = addr + size;
1538 	save_szcvec = szcvec;
1539 	szcvec >>= 1;
1540 	szc = 0;
1541 	nszc = 0;
1542 	while (szcvec) {
1543 		if ((szcvec & 0x1) == 0) {
1544 			nszc++;
1545 			szcvec >>= 1;
1546 			continue;
1547 		}
1548 		nszc++;
1549 		pgsz = page_get_pagesize(nszc);
1550 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
1551 		if (a != addr) {
1552 			ASSERT(a < eaddr);
1553 			segsize = a - addr;
1554 			seg = seg_alloc(as, addr, segsize);
1555 			if (seg == NULL) {
1556 				return (ENOMEM);
1557 			}
1558 			vn_a->szc = szc;
1559 			error = (*crfp)(seg, vn_a);
1560 			if (error != 0) {
1561 				seg_free(seg);
1562 				return (error);
1563 			}
1564 			as->a_size += segsize;
1565 			*segcreated = 1;
1566 			if (do_off) {
1567 				vn_a->offset += segsize;
1568 			}
1569 			addr = a;
1570 		}
1571 		szc = nszc;
1572 		szcvec >>= 1;
1573 	}
1574 
1575 	ASSERT(addr < eaddr);
1576 	szcvec = save_szcvec | 1; /* add 8K pages */
1577 	while (szcvec) {
1578 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
1579 		ASSERT(a >= addr);
1580 		if (a != addr) {
1581 			segsize = a - addr;
1582 			seg = seg_alloc(as, addr, segsize);
1583 			if (seg == NULL) {
1584 				return (ENOMEM);
1585 			}
1586 			vn_a->szc = szc;
1587 			error = (*crfp)(seg, vn_a);
1588 			if (error != 0) {
1589 				seg_free(seg);
1590 				return (error);
1591 			}
1592 			as->a_size += segsize;
1593 			*segcreated = 1;
1594 			if (do_off) {
1595 				vn_a->offset += segsize;
1596 			}
1597 			addr = a;
1598 		}
1599 		szcvec &= ~(1 << szc);
1600 		if (szcvec) {
1601 			szc = highbit(szcvec) - 1;
1602 			pgsz = page_get_pagesize(szc);
1603 		}
1604 	}
1605 	ASSERT(addr == eaddr);
1606 
1607 	return (0);
1608 }
1609 
1610 static int
1611 as_map_vnsegs(struct as *as, caddr_t addr, size_t size,
1612     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1613 {
1614 	uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA);
1615 	int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
1616 	uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1617 	    type, 0);
1618 	int error;
1619 	struct seg *seg;
1620 	struct vattr va;
1621 	u_offset_t eoff;
1622 	size_t save_size = 0;
1623 	extern size_t textrepl_size_thresh;
1624 
1625 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1626 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1627 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1628 	ASSERT(vn_a->vp != NULL);
1629 	ASSERT(vn_a->amp == NULL);
1630 
1631 again:
1632 	if (szcvec <= 1) {
1633 		seg = seg_alloc(as, addr, size);
1634 		if (seg == NULL) {
1635 			return (ENOMEM);
1636 		}
1637 		vn_a->szc = 0;
1638 		error = (*crfp)(seg, vn_a);
1639 		if (error != 0) {
1640 			seg_free(seg);
1641 		} else {
1642 			as->a_size += size;
1643 		}
1644 		return (error);
1645 	}
1646 
1647 	va.va_mask = AT_SIZE;
1648 	if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred, NULL) != 0) {
1649 		szcvec = 0;
1650 		goto again;
1651 	}
1652 	eoff = vn_a->offset & PAGEMASK;
1653 	if (eoff >= va.va_size) {
1654 		szcvec = 0;
1655 		goto again;
1656 	}
1657 	eoff += size;
1658 	if (btopr(va.va_size) < btopr(eoff)) {
1659 		save_size = size;
1660 		size = va.va_size - (vn_a->offset & PAGEMASK);
1661 		size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t);
1662 		szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1663 		    type, 0);
1664 		if (szcvec <= 1) {
1665 			size = save_size;
1666 			goto again;
1667 		}
1668 	}
1669 
1670 	if (size > textrepl_size_thresh) {
1671 		vn_a->flags |= _MAP_TEXTREPL;
1672 	}
1673 	error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a,
1674 	    segcreated);
1675 	if (error != 0) {
1676 		return (error);
1677 	}
1678 	if (save_size) {
1679 		addr += size;
1680 		size = save_size - size;
1681 		szcvec = 0;
1682 		goto again;
1683 	}
1684 	return (0);
1685 }
1686 
1687 /*
1688  * as_map_ansegs: shared or private anonymous memory.  Note that the flags
1689  * passed to map_pgszvec cannot be MAP_INITDATA, for anon.
1690  */
1691 static int
1692 as_map_ansegs(struct as *as, caddr_t addr, size_t size,
1693     int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated)
1694 {
1695 	uint_t szcvec;
1696 	uchar_t type;
1697 
1698 	ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE);
1699 	if (vn_a->type == MAP_SHARED) {
1700 		type = MAPPGSZC_SHM;
1701 	} else if (vn_a->type == MAP_PRIVATE) {
1702 		if (vn_a->szc == AS_MAP_HEAP) {
1703 			type = MAPPGSZC_HEAP;
1704 		} else if (vn_a->szc == AS_MAP_STACK) {
1705 			type = MAPPGSZC_STACK;
1706 		} else {
1707 			type = MAPPGSZC_PRIVM;
1708 		}
1709 	}
1710 	szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ?
1711 	    (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE),
1712 	    (vn_a->flags & MAP_TEXT), type, 0);
1713 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
1714 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
1715 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
1716 	ASSERT(vn_a->vp == NULL);
1717 
1718 	return (as_map_segvn_segs(as, addr, size, szcvec,
1719 	    crfp, vn_a, segcreated));
1720 }
1721 
1722 int
1723 as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp)
1724 {
1725 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1726 	return (as_map_locked(as, addr, size, crfp, argsp));
1727 }
1728 
1729 int
1730 as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(),
1731 		void *argsp)
1732 {
1733 	struct seg *seg = NULL;
1734 	caddr_t raddr;			/* rounded down addr */
1735 	size_t rsize;			/* rounded up size */
1736 	int error;
1737 	int unmap = 0;
1738 	struct proc *p = curproc;
1739 	struct segvn_crargs crargs;
1740 
1741 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
1742 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1743 	    (size_t)raddr;
1744 
1745 	/*
1746 	 * check for wrap around
1747 	 */
1748 	if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
1749 		AS_LOCK_EXIT(as, &as->a_lock);
1750 		return (ENOMEM);
1751 	}
1752 
1753 	as->a_updatedir = 1;	/* inform /proc */
1754 	gethrestime(&as->a_updatetime);
1755 
1756 	if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) {
1757 		AS_LOCK_EXIT(as, &as->a_lock);
1758 
1759 		(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p,
1760 		    RCA_UNSAFE_ALL);
1761 
1762 		return (ENOMEM);
1763 	}
1764 
1765 	if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) {
1766 		crargs = *(struct segvn_crargs *)argsp;
1767 		error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, &unmap);
1768 		if (error != 0) {
1769 			AS_LOCK_EXIT(as, &as->a_lock);
1770 			if (unmap) {
1771 				(void) as_unmap(as, addr, size);
1772 			}
1773 			return (error);
1774 		}
1775 	} else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) {
1776 		crargs = *(struct segvn_crargs *)argsp;
1777 		error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, &unmap);
1778 		if (error != 0) {
1779 			AS_LOCK_EXIT(as, &as->a_lock);
1780 			if (unmap) {
1781 				(void) as_unmap(as, addr, size);
1782 			}
1783 			return (error);
1784 		}
1785 	} else {
1786 		seg = seg_alloc(as, addr, size);
1787 		if (seg == NULL) {
1788 			AS_LOCK_EXIT(as, &as->a_lock);
1789 			return (ENOMEM);
1790 		}
1791 
1792 		error = (*crfp)(seg, argsp);
1793 		if (error != 0) {
1794 			seg_free(seg);
1795 			AS_LOCK_EXIT(as, &as->a_lock);
1796 			return (error);
1797 		}
1798 		/*
1799 		 * Add size now so as_unmap will work if as_ctl fails.
1800 		 */
1801 		as->a_size += rsize;
1802 	}
1803 
1804 	as_setwatch(as);
1805 
1806 	/*
1807 	 * If the address space is locked,
1808 	 * establish memory locks for the new segment.
1809 	 */
1810 	mutex_enter(&as->a_contents);
1811 	if (AS_ISPGLCK(as)) {
1812 		mutex_exit(&as->a_contents);
1813 		AS_LOCK_EXIT(as, &as->a_lock);
1814 		error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
1815 		if (error != 0)
1816 			(void) as_unmap(as, addr, size);
1817 	} else {
1818 		mutex_exit(&as->a_contents);
1819 		AS_LOCK_EXIT(as, &as->a_lock);
1820 	}
1821 	return (error);
1822 }
1823 
1824 
1825 /*
1826  * Delete all segments in the address space marked with S_PURGE.
1827  * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c).
1828  * These segments are deleted as a first step before calls to as_gap(), so
1829  * that they don't affect mmap() or shmat().
1830  */
1831 void
1832 as_purge(struct as *as)
1833 {
1834 	struct seg *seg;
1835 	struct seg *next_seg;
1836 
1837 	/*
1838 	 * the setting of NEEDSPURGE is protect by as_rangelock(), so
1839 	 * no need to grab a_contents mutex for this check
1840 	 */
1841 	if ((as->a_flags & AS_NEEDSPURGE) == 0)
1842 		return;
1843 
1844 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
1845 	next_seg = NULL;
1846 	seg = AS_SEGFIRST(as);
1847 	while (seg != NULL) {
1848 		next_seg = AS_SEGNEXT(as, seg);
1849 		if (seg->s_flags & S_PURGE)
1850 			SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
1851 		seg = next_seg;
1852 	}
1853 	AS_LOCK_EXIT(as, &as->a_lock);
1854 
1855 	mutex_enter(&as->a_contents);
1856 	as->a_flags &= ~AS_NEEDSPURGE;
1857 	mutex_exit(&as->a_contents);
1858 }
1859 
1860 /*
1861  * Find a hole within [*basep, *basep + *lenp), which contains a mappable
1862  * range of addresses at least "minlen" long, where the base of the range is
1863  * at "off" phase from an "align" boundary and there is space for a
1864  * "redzone"-sized redzone on eithe rside of the range.  Thus,
1865  * if align was 4M and off was 16k, the user wants a hole which will start
1866  * 16k into a 4M page.
1867  *
1868  * If flags specifies AH_HI, the hole will have the highest possible address
1869  * in the range.  We use the as->a_lastgap field to figure out where to
1870  * start looking for a gap.
1871  *
1872  * Otherwise, the gap will have the lowest possible address.
1873  *
1874  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
1875  *
1876  * If an adequate hole is found, *basep and *lenp are set to reflect the part of
1877  * the hole that is within range, and 0 is returned. On failure, -1 is returned.
1878  *
1879  * NOTE: This routine is not correct when base+len overflows caddr_t.
1880  */
1881 int
1882 as_gap_aligned(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp,
1883     uint_t flags, caddr_t addr, size_t align, size_t redzone, size_t off)
1884 {
1885 	caddr_t lobound = *basep;
1886 	caddr_t hibound = lobound + *lenp;
1887 	struct seg *lseg, *hseg;
1888 	caddr_t lo, hi;
1889 	int forward;
1890 	caddr_t save_base;
1891 	size_t save_len;
1892 
1893 	save_base = *basep;
1894 	save_len = *lenp;
1895 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
1896 	if (AS_SEGFIRST(as) == NULL) {
1897 		if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR,
1898 		    align, redzone, off)) {
1899 			AS_LOCK_EXIT(as, &as->a_lock);
1900 			return (0);
1901 		} else {
1902 			AS_LOCK_EXIT(as, &as->a_lock);
1903 			*basep = save_base;
1904 			*lenp = save_len;
1905 			return (-1);
1906 		}
1907 	}
1908 
1909 	/*
1910 	 * Set up to iterate over all the inter-segment holes in the given
1911 	 * direction.  lseg is NULL for the lowest-addressed hole and hseg is
1912 	 * NULL for the highest-addressed hole.  If moving backwards, we reset
1913 	 * sseg to denote the highest-addressed segment.
1914 	 */
1915 	forward = (flags & AH_DIR) == AH_LO;
1916 	if (forward) {
1917 		hseg = as_findseg(as, lobound, 1);
1918 		lseg = AS_SEGPREV(as, hseg);
1919 	} else {
1920 
1921 		/*
1922 		 * If allocating at least as much as the last allocation,
1923 		 * use a_lastgap's base as a better estimate of hibound.
1924 		 */
1925 		if (as->a_lastgap &&
1926 		    minlen >= as->a_lastgap->s_size &&
1927 		    hibound >= as->a_lastgap->s_base)
1928 			hibound = as->a_lastgap->s_base;
1929 
1930 		hseg = as_findseg(as, hibound, 1);
1931 		if (hseg->s_base + hseg->s_size < hibound) {
1932 			lseg = hseg;
1933 			hseg = NULL;
1934 		} else {
1935 			lseg = AS_SEGPREV(as, hseg);
1936 		}
1937 	}
1938 
1939 	for (;;) {
1940 		/*
1941 		 * Set lo and hi to the hole's boundaries.  (We should really
1942 		 * use MAXADDR in place of hibound in the expression below,
1943 		 * but can't express it easily; using hibound in its place is
1944 		 * harmless.)
1945 		 */
1946 		lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
1947 		hi = (hseg == NULL) ? hibound : hseg->s_base;
1948 		/*
1949 		 * If the iteration has moved past the interval from lobound
1950 		 * to hibound it's pointless to continue.
1951 		 */
1952 		if ((forward && lo > hibound) || (!forward && hi < lobound))
1953 			break;
1954 		else if (lo > hibound || hi < lobound)
1955 			goto cont;
1956 		/*
1957 		 * Candidate hole lies at least partially within the allowable
1958 		 * range.  Restrict it to fall completely within that range,
1959 		 * i.e., to [max(lo, lobound), min(hi, hibound)].
1960 		 */
1961 		if (lo < lobound)
1962 			lo = lobound;
1963 		if (hi > hibound)
1964 			hi = hibound;
1965 		/*
1966 		 * Verify that the candidate hole is big enough and meets
1967 		 * hardware constraints.
1968 		 */
1969 		*basep = lo;
1970 		*lenp = hi - lo;
1971 		if (valid_va_range_aligned(basep, lenp, minlen,
1972 		    forward ? AH_LO : AH_HI, align, redzone, off) &&
1973 		    ((flags & AH_CONTAIN) == 0 ||
1974 		    (*basep <= addr && *basep + *lenp > addr))) {
1975 			if (!forward)
1976 				as->a_lastgap = hseg;
1977 			if (hseg != NULL)
1978 				as->a_lastgaphl = hseg;
1979 			else
1980 				as->a_lastgaphl = lseg;
1981 			AS_LOCK_EXIT(as, &as->a_lock);
1982 			return (0);
1983 		}
1984 	cont:
1985 		/*
1986 		 * Move to the next hole.
1987 		 */
1988 		if (forward) {
1989 			lseg = hseg;
1990 			if (lseg == NULL)
1991 				break;
1992 			hseg = AS_SEGNEXT(as, hseg);
1993 		} else {
1994 			hseg = lseg;
1995 			if (hseg == NULL)
1996 				break;
1997 			lseg = AS_SEGPREV(as, lseg);
1998 		}
1999 	}
2000 	*basep = save_base;
2001 	*lenp = save_len;
2002 	AS_LOCK_EXIT(as, &as->a_lock);
2003 	return (-1);
2004 }
2005 
2006 /*
2007  * Find a hole of at least size minlen within [*basep, *basep + *lenp).
2008  *
2009  * If flags specifies AH_HI, the hole will have the highest possible address
2010  * in the range.  We use the as->a_lastgap field to figure out where to
2011  * start looking for a gap.
2012  *
2013  * Otherwise, the gap will have the lowest possible address.
2014  *
2015  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
2016  *
2017  * If an adequate hole is found, base and len are set to reflect the part of
2018  * the hole that is within range, and 0 is returned, otherwise,
2019  * -1 is returned.
2020  *
2021  * NOTE: This routine is not correct when base+len overflows caddr_t.
2022  */
2023 int
2024 as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags,
2025     caddr_t addr)
2026 {
2027 
2028 	return (as_gap_aligned(as, minlen, basep, lenp, flags, addr, 0, 0, 0));
2029 }
2030 
2031 /*
2032  * Return the next range within [base, base + len) that is backed
2033  * with "real memory".  Skip holes and non-seg_vn segments.
2034  * We're lazy and only return one segment at a time.
2035  */
2036 int
2037 as_memory(struct as *as, caddr_t *basep, size_t *lenp)
2038 {
2039 	extern struct seg_ops segspt_shmops;	/* needs a header file */
2040 	struct seg *seg;
2041 	caddr_t addr, eaddr;
2042 	caddr_t segend;
2043 
2044 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2045 
2046 	addr = *basep;
2047 	eaddr = addr + *lenp;
2048 
2049 	seg = as_findseg(as, addr, 0);
2050 	if (seg != NULL)
2051 		addr = MAX(seg->s_base, addr);
2052 
2053 	for (;;) {
2054 		if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
2055 			AS_LOCK_EXIT(as, &as->a_lock);
2056 			return (EINVAL);
2057 		}
2058 
2059 		if (seg->s_ops == &segvn_ops) {
2060 			segend = seg->s_base + seg->s_size;
2061 			break;
2062 		}
2063 
2064 		/*
2065 		 * We do ISM by looking into the private data
2066 		 * to determine the real size of the segment.
2067 		 */
2068 		if (seg->s_ops == &segspt_shmops) {
2069 			segend = seg->s_base + spt_realsize(seg);
2070 			if (addr < segend)
2071 				break;
2072 		}
2073 
2074 		seg = AS_SEGNEXT(as, seg);
2075 
2076 		if (seg != NULL)
2077 			addr = seg->s_base;
2078 	}
2079 
2080 	*basep = addr;
2081 
2082 	if (segend > eaddr)
2083 		*lenp = eaddr - addr;
2084 	else
2085 		*lenp = segend - addr;
2086 
2087 	AS_LOCK_EXIT(as, &as->a_lock);
2088 	return (0);
2089 }
2090 
2091 /*
2092  * Swap the pages associated with the address space as out to
2093  * secondary storage, returning the number of bytes actually
2094  * swapped.
2095  *
2096  * The value returned is intended to correlate well with the process's
2097  * memory requirements.  Its usefulness for this purpose depends on
2098  * how well the segment-level routines do at returning accurate
2099  * information.
2100  */
2101 size_t
2102 as_swapout(struct as *as)
2103 {
2104 	struct seg *seg;
2105 	size_t swpcnt = 0;
2106 
2107 	/*
2108 	 * Kernel-only processes have given up their address
2109 	 * spaces.  Of course, we shouldn't be attempting to
2110 	 * swap out such processes in the first place...
2111 	 */
2112 	if (as == NULL)
2113 		return (0);
2114 
2115 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2116 
2117 	/* Prevent XHATs from attaching */
2118 	mutex_enter(&as->a_contents);
2119 	AS_SETBUSY(as);
2120 	mutex_exit(&as->a_contents);
2121 
2122 
2123 	/*
2124 	 * Free all mapping resources associated with the address
2125 	 * space.  The segment-level swapout routines capitalize
2126 	 * on this unmapping by scavanging pages that have become
2127 	 * unmapped here.
2128 	 */
2129 	hat_swapout(as->a_hat);
2130 	if (as->a_xhat != NULL)
2131 		xhat_swapout_all(as);
2132 
2133 	mutex_enter(&as->a_contents);
2134 	AS_CLRBUSY(as);
2135 	mutex_exit(&as->a_contents);
2136 
2137 	/*
2138 	 * Call the swapout routines of all segments in the address
2139 	 * space to do the actual work, accumulating the amount of
2140 	 * space reclaimed.
2141 	 */
2142 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
2143 		struct seg_ops *ov = seg->s_ops;
2144 
2145 		/*
2146 		 * We have to check to see if the seg has
2147 		 * an ops vector because the seg may have
2148 		 * been in the middle of being set up when
2149 		 * the process was picked for swapout.
2150 		 */
2151 		if ((ov != NULL) && (ov->swapout != NULL))
2152 			swpcnt += SEGOP_SWAPOUT(seg);
2153 	}
2154 	AS_LOCK_EXIT(as, &as->a_lock);
2155 	return (swpcnt);
2156 }
2157 
2158 /*
2159  * Determine whether data from the mappings in interval [addr, addr + size)
2160  * are in the primary memory (core) cache.
2161  */
2162 int
2163 as_incore(struct as *as, caddr_t addr,
2164     size_t size, char *vec, size_t *sizep)
2165 {
2166 	struct seg *seg;
2167 	size_t ssize;
2168 	caddr_t raddr;		/* rounded down addr */
2169 	size_t rsize;		/* rounded up size */
2170 	size_t isize;			/* iteration size */
2171 	int error = 0;		/* result, assume success */
2172 
2173 	*sizep = 0;
2174 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2175 	rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
2176 	    (size_t)raddr;
2177 
2178 	if (raddr + rsize < raddr)		/* check for wraparound */
2179 		return (ENOMEM);
2180 
2181 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2182 	seg = as_segat(as, raddr);
2183 	if (seg == NULL) {
2184 		AS_LOCK_EXIT(as, &as->a_lock);
2185 		return (-1);
2186 	}
2187 
2188 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2189 		if (raddr >= seg->s_base + seg->s_size) {
2190 			seg = AS_SEGNEXT(as, seg);
2191 			if (seg == NULL || raddr != seg->s_base) {
2192 				error = -1;
2193 				break;
2194 			}
2195 		}
2196 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2197 			ssize = seg->s_base + seg->s_size - raddr;
2198 		else
2199 			ssize = rsize;
2200 		*sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
2201 		if (isize != ssize) {
2202 			error = -1;
2203 			break;
2204 		}
2205 		vec += btopr(ssize);
2206 	}
2207 	AS_LOCK_EXIT(as, &as->a_lock);
2208 	return (error);
2209 }
2210 
2211 static void
2212 as_segunlock(struct seg *seg, caddr_t addr, int attr,
2213 	ulong_t *bitmap, size_t position, size_t npages)
2214 {
2215 	caddr_t	range_start;
2216 	size_t	pos1 = position;
2217 	size_t	pos2;
2218 	size_t	size;
2219 	size_t  end_pos = npages + position;
2220 
2221 	while (bt_range(bitmap, &pos1, &pos2, end_pos)) {
2222 		size = ptob((pos2 - pos1));
2223 		range_start = (caddr_t)((uintptr_t)addr +
2224 		    ptob(pos1 - position));
2225 
2226 		(void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK,
2227 		    (ulong_t *)NULL, (size_t)NULL);
2228 		pos1 = pos2;
2229 	}
2230 }
2231 
2232 static void
2233 as_unlockerr(struct as *as, int attr, ulong_t *mlock_map,
2234 	caddr_t raddr, size_t rsize)
2235 {
2236 	struct seg *seg = as_segat(as, raddr);
2237 	size_t ssize;
2238 
2239 	while (rsize != 0) {
2240 		if (raddr >= seg->s_base + seg->s_size)
2241 			seg = AS_SEGNEXT(as, seg);
2242 
2243 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2244 			ssize = seg->s_base + seg->s_size - raddr;
2245 		else
2246 			ssize = rsize;
2247 
2248 		as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize));
2249 
2250 		rsize -= ssize;
2251 		raddr += ssize;
2252 	}
2253 }
2254 
2255 /*
2256  * Cache control operations over the interval [addr, addr + size) in
2257  * address space "as".
2258  */
2259 /*ARGSUSED*/
2260 int
2261 as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
2262     uintptr_t arg, ulong_t *lock_map, size_t pos)
2263 {
2264 	struct seg *seg;	/* working segment */
2265 	caddr_t raddr;		/* rounded down addr */
2266 	caddr_t initraddr;	/* saved initial rounded down addr */
2267 	size_t rsize;		/* rounded up size */
2268 	size_t initrsize;	/* saved initial rounded up size */
2269 	size_t ssize;		/* size of seg */
2270 	int error = 0;			/* result */
2271 	size_t mlock_size;	/* size of bitmap */
2272 	ulong_t *mlock_map;	/* pointer to bitmap used */
2273 				/* to represent the locked */
2274 				/* pages. */
2275 retry:
2276 	if (error == IE_RETRY)
2277 		AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2278 	else
2279 		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2280 
2281 	/*
2282 	 * If these are address space lock/unlock operations, loop over
2283 	 * all segments in the address space, as appropriate.
2284 	 */
2285 	if (func == MC_LOCKAS) {
2286 		size_t npages, idx;
2287 		size_t rlen = 0;	/* rounded as length */
2288 
2289 		idx = pos;
2290 
2291 		if (arg & MCL_FUTURE) {
2292 			mutex_enter(&as->a_contents);
2293 			AS_SETPGLCK(as);
2294 			mutex_exit(&as->a_contents);
2295 		}
2296 		if ((arg & MCL_CURRENT) == 0) {
2297 			AS_LOCK_EXIT(as, &as->a_lock);
2298 			return (0);
2299 		}
2300 
2301 		seg = AS_SEGFIRST(as);
2302 		if (seg == NULL) {
2303 			AS_LOCK_EXIT(as, &as->a_lock);
2304 			return (0);
2305 		}
2306 
2307 		do {
2308 			raddr = (caddr_t)((uintptr_t)seg->s_base &
2309 			    (uintptr_t)PAGEMASK);
2310 			rlen += (((uintptr_t)(seg->s_base + seg->s_size) +
2311 			    PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr;
2312 		} while ((seg = AS_SEGNEXT(as, seg)) != NULL);
2313 
2314 		mlock_size = BT_BITOUL(btopr(rlen));
2315 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2316 		    sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2317 				AS_LOCK_EXIT(as, &as->a_lock);
2318 				return (EAGAIN);
2319 		}
2320 
2321 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2322 			error = SEGOP_LOCKOP(seg, seg->s_base,
2323 			    seg->s_size, attr, MC_LOCK, mlock_map, pos);
2324 			if (error != 0)
2325 				break;
2326 			pos += seg_pages(seg);
2327 		}
2328 
2329 		if (error) {
2330 			for (seg = AS_SEGFIRST(as); seg != NULL;
2331 			    seg = AS_SEGNEXT(as, seg)) {
2332 
2333 				raddr = (caddr_t)((uintptr_t)seg->s_base &
2334 				    (uintptr_t)PAGEMASK);
2335 				npages = seg_pages(seg);
2336 				as_segunlock(seg, raddr, attr, mlock_map,
2337 				    idx, npages);
2338 				idx += npages;
2339 			}
2340 		}
2341 
2342 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2343 		AS_LOCK_EXIT(as, &as->a_lock);
2344 		goto lockerr;
2345 	} else if (func == MC_UNLOCKAS) {
2346 		mutex_enter(&as->a_contents);
2347 		AS_CLRPGLCK(as);
2348 		mutex_exit(&as->a_contents);
2349 
2350 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2351 			error = SEGOP_LOCKOP(seg, seg->s_base,
2352 			    seg->s_size, attr, MC_UNLOCK, NULL, 0);
2353 			if (error != 0)
2354 				break;
2355 		}
2356 
2357 		AS_LOCK_EXIT(as, &as->a_lock);
2358 		goto lockerr;
2359 	}
2360 
2361 	/*
2362 	 * Normalize addresses and sizes.
2363 	 */
2364 	initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2365 	initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2366 	    (size_t)raddr;
2367 
2368 	if (raddr + rsize < raddr) {		/* check for wraparound */
2369 		AS_LOCK_EXIT(as, &as->a_lock);
2370 		return (ENOMEM);
2371 	}
2372 
2373 	/*
2374 	 * Get initial segment.
2375 	 */
2376 	if ((seg = as_segat(as, raddr)) == NULL) {
2377 		AS_LOCK_EXIT(as, &as->a_lock);
2378 		return (ENOMEM);
2379 	}
2380 
2381 	if (func == MC_LOCK) {
2382 		mlock_size = BT_BITOUL(btopr(rsize));
2383 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2384 		    sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2385 				AS_LOCK_EXIT(as, &as->a_lock);
2386 				return (EAGAIN);
2387 		}
2388 	}
2389 
2390 	/*
2391 	 * Loop over all segments.  If a hole in the address range is
2392 	 * discovered, then fail.  For each segment, perform the appropriate
2393 	 * control operation.
2394 	 */
2395 	while (rsize != 0) {
2396 
2397 		/*
2398 		 * Make sure there's no hole, calculate the portion
2399 		 * of the next segment to be operated over.
2400 		 */
2401 		if (raddr >= seg->s_base + seg->s_size) {
2402 			seg = AS_SEGNEXT(as, seg);
2403 			if (seg == NULL || raddr != seg->s_base) {
2404 				if (func == MC_LOCK) {
2405 					as_unlockerr(as, attr, mlock_map,
2406 					    initraddr, initrsize - rsize);
2407 					kmem_free(mlock_map,
2408 					    mlock_size * sizeof (ulong_t));
2409 				}
2410 				AS_LOCK_EXIT(as, &as->a_lock);
2411 				return (ENOMEM);
2412 			}
2413 		}
2414 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
2415 			ssize = seg->s_base + seg->s_size - raddr;
2416 		else
2417 			ssize = rsize;
2418 
2419 		/*
2420 		 * Dispatch on specific function.
2421 		 */
2422 		switch (func) {
2423 
2424 		/*
2425 		 * Synchronize cached data from mappings with backing
2426 		 * objects.
2427 		 */
2428 		case MC_SYNC:
2429 			if (error = SEGOP_SYNC(seg, raddr, ssize,
2430 			    attr, (uint_t)arg)) {
2431 				AS_LOCK_EXIT(as, &as->a_lock);
2432 				return (error);
2433 			}
2434 			break;
2435 
2436 		/*
2437 		 * Lock pages in memory.
2438 		 */
2439 		case MC_LOCK:
2440 			if (error = SEGOP_LOCKOP(seg, raddr, ssize,
2441 			    attr, func, mlock_map, pos)) {
2442 				as_unlockerr(as, attr, mlock_map, initraddr,
2443 				    initrsize - rsize + ssize);
2444 				kmem_free(mlock_map, mlock_size *
2445 				    sizeof (ulong_t));
2446 				AS_LOCK_EXIT(as, &as->a_lock);
2447 				goto lockerr;
2448 			}
2449 			break;
2450 
2451 		/*
2452 		 * Unlock mapped pages.
2453 		 */
2454 		case MC_UNLOCK:
2455 			(void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
2456 			    (ulong_t *)NULL, (size_t)NULL);
2457 			break;
2458 
2459 		/*
2460 		 * Store VM advise for mapped pages in segment layer.
2461 		 */
2462 		case MC_ADVISE:
2463 			error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg);
2464 
2465 			/*
2466 			 * Check for regular errors and special retry error
2467 			 */
2468 			if (error) {
2469 				if (error == IE_RETRY) {
2470 					/*
2471 					 * Need to acquire writers lock, so
2472 					 * have to drop readers lock and start
2473 					 * all over again
2474 					 */
2475 					AS_LOCK_EXIT(as, &as->a_lock);
2476 					goto retry;
2477 				} else if (error == IE_REATTACH) {
2478 					/*
2479 					 * Find segment for current address
2480 					 * because current segment just got
2481 					 * split or concatenated
2482 					 */
2483 					seg = as_segat(as, raddr);
2484 					if (seg == NULL) {
2485 						AS_LOCK_EXIT(as, &as->a_lock);
2486 						return (ENOMEM);
2487 					}
2488 				} else {
2489 					/*
2490 					 * Regular error
2491 					 */
2492 					AS_LOCK_EXIT(as, &as->a_lock);
2493 					return (error);
2494 				}
2495 			}
2496 			break;
2497 
2498 		/*
2499 		 * Can't happen.
2500 		 */
2501 		default:
2502 			panic("as_ctl: bad operation %d", func);
2503 			/*NOTREACHED*/
2504 		}
2505 
2506 		rsize -= ssize;
2507 		raddr += ssize;
2508 	}
2509 
2510 	if (func == MC_LOCK)
2511 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2512 	AS_LOCK_EXIT(as, &as->a_lock);
2513 	return (0);
2514 lockerr:
2515 
2516 	/*
2517 	 * If the lower levels returned EDEADLK for a segment lockop,
2518 	 * it means that we should retry the operation.  Let's wait
2519 	 * a bit also to let the deadlock causing condition clear.
2520 	 * This is part of a gross hack to work around a design flaw
2521 	 * in the ufs/sds logging code and should go away when the
2522 	 * logging code is re-designed to fix the problem. See bug
2523 	 * 4125102 for details of the problem.
2524 	 */
2525 	if (error == EDEADLK) {
2526 		delay(deadlk_wait);
2527 		error = 0;
2528 		goto retry;
2529 	}
2530 	return (error);
2531 }
2532 
2533 /*
2534  * Special code for exec to move the stack segment from its interim
2535  * place in the old address to the right place in the new address space.
2536  */
2537 /*ARGSUSED*/
2538 int
2539 as_exec(struct as *oas, caddr_t ostka, size_t stksz,
2540     struct as *nas, caddr_t nstka, uint_t hatflag)
2541 {
2542 	struct seg *stkseg;
2543 
2544 	AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER);
2545 	stkseg = as_segat(oas, ostka);
2546 	stkseg = as_removeseg(oas, stkseg);
2547 	ASSERT(stkseg != NULL);
2548 	ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz);
2549 	stkseg->s_as = nas;
2550 	stkseg->s_base = nstka;
2551 
2552 	/*
2553 	 * It's ok to lock the address space we are about to exec to.
2554 	 */
2555 	AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER);
2556 	ASSERT(avl_numnodes(&nas->a_wpage) == 0);
2557 	nas->a_size += stkseg->s_size;
2558 	oas->a_size -= stkseg->s_size;
2559 	(void) as_addseg(nas, stkseg);
2560 	AS_LOCK_EXIT(nas, &nas->a_lock);
2561 	AS_LOCK_EXIT(oas, &oas->a_lock);
2562 	return (0);
2563 }
2564 
2565 int
2566 fc_decode(faultcode_t fault_err)
2567 {
2568 	int error = 0;
2569 
2570 	switch (FC_CODE(fault_err)) {
2571 	case FC_OBJERR:
2572 		error = FC_ERRNO(fault_err);
2573 		break;
2574 	case FC_PROT:
2575 		error = EACCES;
2576 		break;
2577 	default:
2578 		error = EFAULT;
2579 		break;
2580 	}
2581 	return (error);
2582 }
2583 
2584 /*
2585  * Pagelock pages from a range that spans more than 1 segment.  Obtain shadow
2586  * lists from each segment and copy them to one contiguous shadow list (plist)
2587  * as expected by the caller.  Save pointers to per segment shadow lists at
2588  * the tail of plist so that they can be used during as_pageunlock().
2589  */
2590 static int
2591 as_pagelock_segs(struct as *as, struct seg *seg, struct page ***ppp,
2592     caddr_t addr, size_t size, enum seg_rw rw)
2593 {
2594 	caddr_t sv_addr = addr;
2595 	size_t sv_size = size;
2596 	struct seg *sv_seg = seg;
2597 	ulong_t segcnt = 1;
2598 	ulong_t cnt;
2599 	size_t ssize;
2600 	pgcnt_t npages = btop(size);
2601 	page_t **plist;
2602 	page_t **pl;
2603 	int error;
2604 	caddr_t eaddr;
2605 	faultcode_t fault_err = 0;
2606 	pgcnt_t pl_off;
2607 	extern struct seg_ops segspt_shmops;
2608 
2609 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
2610 	ASSERT(seg != NULL);
2611 	ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size);
2612 	ASSERT(addr + size > seg->s_base + seg->s_size);
2613 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2614 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2615 
2616 	/*
2617 	 * Count the number of segments covered by the range we are about to
2618 	 * lock. The segment count is used to size the shadow list we return
2619 	 * back to the caller.
2620 	 */
2621 	for (; size != 0; size -= ssize, addr += ssize) {
2622 		if (addr >= seg->s_base + seg->s_size) {
2623 
2624 			seg = AS_SEGNEXT(as, seg);
2625 			if (seg == NULL || addr != seg->s_base) {
2626 				AS_LOCK_EXIT(as, &as->a_lock);
2627 				return (EFAULT);
2628 			}
2629 			/*
2630 			 * Do a quick check if subsequent segments
2631 			 * will most likely support pagelock.
2632 			 */
2633 			if (seg->s_ops == &segvn_ops) {
2634 				vnode_t *vp;
2635 
2636 				if (SEGOP_GETVP(seg, addr, &vp) != 0 ||
2637 				    vp != NULL) {
2638 					AS_LOCK_EXIT(as, &as->a_lock);
2639 					goto slow;
2640 				}
2641 			} else if (seg->s_ops != &segspt_shmops) {
2642 				AS_LOCK_EXIT(as, &as->a_lock);
2643 				goto slow;
2644 			}
2645 			segcnt++;
2646 		}
2647 		if (addr + size > seg->s_base + seg->s_size) {
2648 			ssize = seg->s_base + seg->s_size - addr;
2649 		} else {
2650 			ssize = size;
2651 		}
2652 	}
2653 	ASSERT(segcnt > 1);
2654 
2655 	plist = kmem_zalloc((npages + segcnt) * sizeof (page_t *), KM_SLEEP);
2656 
2657 	addr = sv_addr;
2658 	size = sv_size;
2659 	seg = sv_seg;
2660 
2661 	for (cnt = 0, pl_off = 0; size != 0; size -= ssize, addr += ssize) {
2662 		if (addr >= seg->s_base + seg->s_size) {
2663 			seg = AS_SEGNEXT(as, seg);
2664 			ASSERT(seg != NULL && addr == seg->s_base);
2665 			cnt++;
2666 			ASSERT(cnt < segcnt);
2667 		}
2668 		if (addr + size > seg->s_base + seg->s_size) {
2669 			ssize = seg->s_base + seg->s_size - addr;
2670 		} else {
2671 			ssize = size;
2672 		}
2673 		pl = &plist[npages + cnt];
2674 		error = SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2675 		    L_PAGELOCK, rw);
2676 		if (error) {
2677 			break;
2678 		}
2679 		ASSERT(plist[npages + cnt] != NULL);
2680 		ASSERT(pl_off + btop(ssize) <= npages);
2681 		bcopy(plist[npages + cnt], &plist[pl_off],
2682 		    btop(ssize) * sizeof (page_t *));
2683 		pl_off += btop(ssize);
2684 	}
2685 
2686 	if (size == 0) {
2687 		AS_LOCK_EXIT(as, &as->a_lock);
2688 		ASSERT(cnt == segcnt - 1);
2689 		*ppp = plist;
2690 		return (0);
2691 	}
2692 
2693 	/*
2694 	 * one of pagelock calls failed. The error type is in error variable.
2695 	 * Unlock what we've locked so far and retry with F_SOFTLOCK if error
2696 	 * type is either EFAULT or ENOTSUP. Otherwise just return the error
2697 	 * back to the caller.
2698 	 */
2699 
2700 	eaddr = addr;
2701 	seg = sv_seg;
2702 
2703 	for (cnt = 0, addr = sv_addr; addr < eaddr; addr += ssize) {
2704 		if (addr >= seg->s_base + seg->s_size) {
2705 			seg = AS_SEGNEXT(as, seg);
2706 			ASSERT(seg != NULL && addr == seg->s_base);
2707 			cnt++;
2708 			ASSERT(cnt < segcnt);
2709 		}
2710 		if (eaddr > seg->s_base + seg->s_size) {
2711 			ssize = seg->s_base + seg->s_size - addr;
2712 		} else {
2713 			ssize = eaddr - addr;
2714 		}
2715 		pl = &plist[npages + cnt];
2716 		ASSERT(*pl != NULL);
2717 		(void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2718 		    L_PAGEUNLOCK, rw);
2719 	}
2720 
2721 	AS_LOCK_EXIT(as, &as->a_lock);
2722 
2723 	kmem_free(plist, (npages + segcnt) * sizeof (page_t *));
2724 
2725 	if (error != ENOTSUP && error != EFAULT) {
2726 		return (error);
2727 	}
2728 
2729 slow:
2730 	/*
2731 	 * If we are here because pagelock failed due to the need to cow fault
2732 	 * in the pages we want to lock F_SOFTLOCK will do this job and in
2733 	 * next as_pagelock() call for this address range pagelock will
2734 	 * hopefully succeed.
2735 	 */
2736 	fault_err = as_fault(as->a_hat, as, sv_addr, sv_size, F_SOFTLOCK, rw);
2737 	if (fault_err != 0) {
2738 		return (fc_decode(fault_err));
2739 	}
2740 	*ppp = NULL;
2741 
2742 	return (0);
2743 }
2744 
2745 /*
2746  * lock pages in a given address space. Return shadow list. If
2747  * the list is NULL, the MMU mapping is also locked.
2748  */
2749 int
2750 as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
2751     size_t size, enum seg_rw rw)
2752 {
2753 	size_t rsize;
2754 	caddr_t raddr;
2755 	faultcode_t fault_err;
2756 	struct seg *seg;
2757 	int err;
2758 
2759 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
2760 	    "as_pagelock_start: addr %p size %ld", addr, size);
2761 
2762 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2763 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2764 	    (size_t)raddr;
2765 
2766 	/*
2767 	 * if the request crosses two segments let
2768 	 * as_fault handle it.
2769 	 */
2770 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2771 
2772 	seg = as_segat(as, raddr);
2773 	if (seg == NULL) {
2774 		AS_LOCK_EXIT(as, &as->a_lock);
2775 		return (EFAULT);
2776 	}
2777 	ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size);
2778 	if (raddr + rsize > seg->s_base + seg->s_size) {
2779 		return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw));
2780 	}
2781 	if (raddr + rsize <= raddr) {
2782 		AS_LOCK_EXIT(as, &as->a_lock);
2783 		return (EFAULT);
2784 	}
2785 
2786 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
2787 	    "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
2788 
2789 	/*
2790 	 * try to lock pages and pass back shadow list
2791 	 */
2792 	err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
2793 
2794 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
2795 
2796 	AS_LOCK_EXIT(as, &as->a_lock);
2797 
2798 	if (err == 0 || (err != ENOTSUP && err != EFAULT)) {
2799 		return (err);
2800 	}
2801 
2802 	/*
2803 	 * Use F_SOFTLOCK to lock the pages because pagelock failed either due
2804 	 * to no pagelock support for this segment or pages need to be cow
2805 	 * faulted in. If fault is needed F_SOFTLOCK will do this job for
2806 	 * this as_pagelock() call and in the next as_pagelock() call for the
2807 	 * same address range pagelock call will hopefull succeed.
2808 	 */
2809 	fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
2810 	if (fault_err != 0) {
2811 		return (fc_decode(fault_err));
2812 	}
2813 	*ppp = NULL;
2814 
2815 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
2816 	return (0);
2817 }
2818 
2819 /*
2820  * unlock pages locked by as_pagelock_segs().  Retrieve per segment shadow
2821  * lists from the end of plist and call pageunlock interface for each segment.
2822  * Drop as lock and free plist.
2823  */
2824 static void
2825 as_pageunlock_segs(struct as *as, struct seg *seg, caddr_t addr, size_t size,
2826     struct page **plist, enum seg_rw rw)
2827 {
2828 	ulong_t cnt;
2829 	caddr_t eaddr = addr + size;
2830 	pgcnt_t npages = btop(size);
2831 	size_t ssize;
2832 	page_t **pl;
2833 
2834 	ASSERT(AS_LOCK_HELD(as, &as->a_lock));
2835 	ASSERT(seg != NULL);
2836 	ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size);
2837 	ASSERT(addr + size > seg->s_base + seg->s_size);
2838 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2839 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2840 	ASSERT(plist != NULL);
2841 
2842 	for (cnt = 0; addr < eaddr; addr += ssize) {
2843 		if (addr >= seg->s_base + seg->s_size) {
2844 			seg = AS_SEGNEXT(as, seg);
2845 			ASSERT(seg != NULL && addr == seg->s_base);
2846 			cnt++;
2847 		}
2848 		if (eaddr > seg->s_base + seg->s_size) {
2849 			ssize = seg->s_base + seg->s_size - addr;
2850 		} else {
2851 			ssize = eaddr - addr;
2852 		}
2853 		pl = &plist[npages + cnt];
2854 		ASSERT(*pl != NULL);
2855 		(void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2856 		    L_PAGEUNLOCK, rw);
2857 	}
2858 	ASSERT(cnt > 0);
2859 	AS_LOCK_EXIT(as, &as->a_lock);
2860 
2861 	cnt++;
2862 	kmem_free(plist, (npages + cnt) * sizeof (page_t *));
2863 }
2864 
2865 /*
2866  * unlock pages in a given address range
2867  */
2868 void
2869 as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
2870     enum seg_rw rw)
2871 {
2872 	struct seg *seg;
2873 	size_t rsize;
2874 	caddr_t raddr;
2875 
2876 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
2877 	    "as_pageunlock_start: addr %p size %ld", addr, size);
2878 
2879 	/*
2880 	 * if the shadow list is NULL, as_pagelock was
2881 	 * falling back to as_fault
2882 	 */
2883 	if (pp == NULL) {
2884 		(void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
2885 		return;
2886 	}
2887 
2888 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2889 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2890 	    (size_t)raddr;
2891 
2892 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
2893 	seg = as_segat(as, raddr);
2894 	ASSERT(seg != NULL);
2895 
2896 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
2897 	    "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
2898 
2899 	ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size);
2900 	if (raddr + rsize <= seg->s_base + seg->s_size) {
2901 		SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
2902 	} else {
2903 		as_pageunlock_segs(as, seg, raddr, rsize, pp, rw);
2904 		return;
2905 	}
2906 	AS_LOCK_EXIT(as, &as->a_lock);
2907 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
2908 }
2909 
2910 int
2911 as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
2912     boolean_t wait)
2913 {
2914 	struct seg *seg;
2915 	size_t ssize;
2916 	caddr_t raddr;			/* rounded down addr */
2917 	size_t rsize;			/* rounded up size */
2918 	int error = 0;
2919 	size_t pgsz = page_get_pagesize(szc);
2920 
2921 setpgsz_top:
2922 	if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) {
2923 		return (EINVAL);
2924 	}
2925 
2926 	raddr = addr;
2927 	rsize = size;
2928 
2929 	if (raddr + rsize < raddr)		/* check for wraparound */
2930 		return (ENOMEM);
2931 
2932 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
2933 	as_clearwatchprot(as, raddr, rsize);
2934 	seg = as_segat(as, raddr);
2935 	if (seg == NULL) {
2936 		as_setwatch(as);
2937 		AS_LOCK_EXIT(as, &as->a_lock);
2938 		return (ENOMEM);
2939 	}
2940 
2941 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
2942 		if (raddr >= seg->s_base + seg->s_size) {
2943 			seg = AS_SEGNEXT(as, seg);
2944 			if (seg == NULL || raddr != seg->s_base) {
2945 				error = ENOMEM;
2946 				break;
2947 			}
2948 		}
2949 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
2950 			ssize = seg->s_base + seg->s_size - raddr;
2951 		} else {
2952 			ssize = rsize;
2953 		}
2954 
2955 retry:
2956 		error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
2957 
2958 		if (error == IE_NOMEM) {
2959 			error = EAGAIN;
2960 			break;
2961 		}
2962 
2963 		if (error == IE_RETRY) {
2964 			AS_LOCK_EXIT(as, &as->a_lock);
2965 			goto setpgsz_top;
2966 		}
2967 
2968 		if (error == ENOTSUP) {
2969 			error = EINVAL;
2970 			break;
2971 		}
2972 
2973 		if (wait && (error == EAGAIN)) {
2974 			/*
2975 			 * Memory is currently locked.  It must be unlocked
2976 			 * before this operation can succeed through a retry.
2977 			 * The possible reasons for locked memory and
2978 			 * corresponding strategies for unlocking are:
2979 			 * (1) Normal I/O
2980 			 *	wait for a signal that the I/O operation
2981 			 *	has completed and the memory is unlocked.
2982 			 * (2) Asynchronous I/O
2983 			 *	The aio subsystem does not unlock pages when
2984 			 *	the I/O is completed. Those pages are unlocked
2985 			 *	when the application calls aiowait/aioerror.
2986 			 *	So, to prevent blocking forever, cv_broadcast()
2987 			 *	is done to wake up aio_cleanup_thread.
2988 			 *	Subsequently, segvn_reclaim will be called, and
2989 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
2990 			 * (3) Long term page locking:
2991 			 *	This is not relevant for as_setpagesize()
2992 			 *	because we cannot change the page size for
2993 			 *	driver memory. The attempt to do so will
2994 			 *	fail with a different error than EAGAIN so
2995 			 *	there's no need to trigger as callbacks like
2996 			 *	as_unmap, as_setprot or as_free would do.
2997 			 */
2998 			mutex_enter(&as->a_contents);
2999 			if (!AS_ISNOUNMAPWAIT(as)) {
3000 				if (AS_ISUNMAPWAIT(as) == 0) {
3001 					cv_broadcast(&as->a_cv);
3002 				}
3003 				AS_SETUNMAPWAIT(as);
3004 				AS_LOCK_EXIT(as, &as->a_lock);
3005 				while (AS_ISUNMAPWAIT(as)) {
3006 					cv_wait(&as->a_cv, &as->a_contents);
3007 				}
3008 			} else {
3009 				/*
3010 				 * We may have raced with
3011 				 * segvn_reclaim()/segspt_reclaim(). In this
3012 				 * case clean nounmapwait flag and retry since
3013 				 * softlockcnt in this segment may be already
3014 				 * 0.  We don't drop as writer lock so our
3015 				 * number of retries without sleeping should
3016 				 * be very small. See segvn_reclaim() for
3017 				 * more comments.
3018 				 */
3019 				AS_CLRNOUNMAPWAIT(as);
3020 				mutex_exit(&as->a_contents);
3021 				goto retry;
3022 			}
3023 			mutex_exit(&as->a_contents);
3024 			goto setpgsz_top;
3025 		} else if (error != 0) {
3026 			break;
3027 		}
3028 	}
3029 	as_setwatch(as);
3030 	AS_LOCK_EXIT(as, &as->a_lock);
3031 	return (error);
3032 }
3033 
3034 /*
3035  * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments
3036  * in its chunk where s_szc is less than the szc we want to set.
3037  */
3038 static int
3039 as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
3040     int *retry)
3041 {
3042 	struct seg *seg;
3043 	size_t ssize;
3044 	int error;
3045 
3046 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3047 
3048 	seg = as_segat(as, raddr);
3049 	if (seg == NULL) {
3050 		panic("as_iset3_default_lpsize: no seg");
3051 	}
3052 
3053 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
3054 		if (raddr >= seg->s_base + seg->s_size) {
3055 			seg = AS_SEGNEXT(as, seg);
3056 			if (seg == NULL || raddr != seg->s_base) {
3057 				panic("as_iset3_default_lpsize: as changed");
3058 			}
3059 		}
3060 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3061 			ssize = seg->s_base + seg->s_size - raddr;
3062 		} else {
3063 			ssize = rsize;
3064 		}
3065 
3066 		if (szc > seg->s_szc) {
3067 			error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
3068 			/* Only retry on EINVAL segments that have no vnode. */
3069 			if (error == EINVAL) {
3070 				vnode_t *vp = NULL;
3071 				if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) &&
3072 				    (SEGOP_GETVP(seg, raddr, &vp) != 0 ||
3073 				    vp == NULL)) {
3074 					*retry = 1;
3075 				} else {
3076 					*retry = 0;
3077 				}
3078 			}
3079 			if (error) {
3080 				return (error);
3081 			}
3082 		}
3083 	}
3084 	return (0);
3085 }
3086 
3087 /*
3088  * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the
3089  * pagesize on each segment in its range, but if any fails with EINVAL,
3090  * then it reduces the pagesizes to the next size in the bitmap and
3091  * retries as_iset3_default_lpsize(). The reason why the code retries
3092  * smaller allowed sizes on EINVAL is because (a) the anon offset may not
3093  * match the bigger sizes, and (b) it's hard to get this offset (to begin
3094  * with) to pass to map_pgszcvec().
3095  */
3096 static int
3097 as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc,
3098     uint_t szcvec)
3099 {
3100 	int error;
3101 	int retry;
3102 
3103 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3104 
3105 	for (;;) {
3106 		error = as_iset3_default_lpsize(as, addr, size, szc, &retry);
3107 		if (error == EINVAL && retry) {
3108 			szcvec &= ~(1 << szc);
3109 			if (szcvec <= 1) {
3110 				return (EINVAL);
3111 			}
3112 			szc = highbit(szcvec) - 1;
3113 		} else {
3114 			return (error);
3115 		}
3116 	}
3117 }
3118 
3119 /*
3120  * as_iset1_default_lpsize() breaks its chunk into areas where existing
3121  * segments have a smaller szc than we want to set. For each such area,
3122  * it calls as_iset2_default_lpsize()
3123  */
3124 static int
3125 as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
3126     uint_t szcvec)
3127 {
3128 	struct seg *seg;
3129 	size_t ssize;
3130 	caddr_t setaddr = raddr;
3131 	size_t setsize = 0;
3132 	int set;
3133 	int error;
3134 
3135 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3136 
3137 	seg = as_segat(as, raddr);
3138 	if (seg == NULL) {
3139 		panic("as_iset1_default_lpsize: no seg");
3140 	}
3141 	if (seg->s_szc < szc) {
3142 		set = 1;
3143 	} else {
3144 		set = 0;
3145 	}
3146 
3147 	for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3148 		if (raddr >= seg->s_base + seg->s_size) {
3149 			seg = AS_SEGNEXT(as, seg);
3150 			if (seg == NULL || raddr != seg->s_base) {
3151 				panic("as_iset1_default_lpsize: as changed");
3152 			}
3153 			if (seg->s_szc >= szc && set) {
3154 				ASSERT(setsize != 0);
3155 				error = as_iset2_default_lpsize(as,
3156 				    setaddr, setsize, szc, szcvec);
3157 				if (error) {
3158 					return (error);
3159 				}
3160 				set = 0;
3161 			} else if (seg->s_szc < szc && !set) {
3162 				setaddr = raddr;
3163 				setsize = 0;
3164 				set = 1;
3165 			}
3166 		}
3167 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3168 			ssize = seg->s_base + seg->s_size - raddr;
3169 		} else {
3170 			ssize = rsize;
3171 		}
3172 	}
3173 	error = 0;
3174 	if (set) {
3175 		ASSERT(setsize != 0);
3176 		error = as_iset2_default_lpsize(as, setaddr, setsize,
3177 		    szc, szcvec);
3178 	}
3179 	return (error);
3180 }
3181 
3182 /*
3183  * as_iset_default_lpsize() breaks its chunk according to the size code bitmap
3184  * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each
3185  * chunk to as_iset1_default_lpsize().
3186  */
3187 static int
3188 as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags,
3189     int type)
3190 {
3191 	int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
3192 	uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr,
3193 	    flags, rtype, 1);
3194 	uint_t szc;
3195 	uint_t nszc;
3196 	int error;
3197 	caddr_t a;
3198 	caddr_t eaddr;
3199 	size_t segsize;
3200 	size_t pgsz;
3201 	uint_t save_szcvec;
3202 
3203 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3204 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
3205 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
3206 
3207 	szcvec &= ~1;
3208 	if (szcvec <= 1) {	/* skip if base page size */
3209 		return (0);
3210 	}
3211 
3212 	/* Get the pagesize of the first larger page size. */
3213 	szc = lowbit(szcvec) - 1;
3214 	pgsz = page_get_pagesize(szc);
3215 	eaddr = addr + size;
3216 	addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
3217 	eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
3218 
3219 	save_szcvec = szcvec;
3220 	szcvec >>= (szc + 1);
3221 	nszc = szc;
3222 	while (szcvec) {
3223 		if ((szcvec & 0x1) == 0) {
3224 			nszc++;
3225 			szcvec >>= 1;
3226 			continue;
3227 		}
3228 		nszc++;
3229 		pgsz = page_get_pagesize(nszc);
3230 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
3231 		if (a != addr) {
3232 			ASSERT(szc > 0);
3233 			ASSERT(a < eaddr);
3234 			segsize = a - addr;
3235 			error = as_iset1_default_lpsize(as, addr, segsize, szc,
3236 			    save_szcvec);
3237 			if (error) {
3238 				return (error);
3239 			}
3240 			addr = a;
3241 		}
3242 		szc = nszc;
3243 		szcvec >>= 1;
3244 	}
3245 
3246 	ASSERT(addr < eaddr);
3247 	szcvec = save_szcvec;
3248 	while (szcvec) {
3249 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
3250 		ASSERT(a >= addr);
3251 		if (a != addr) {
3252 			ASSERT(szc > 0);
3253 			segsize = a - addr;
3254 			error = as_iset1_default_lpsize(as, addr, segsize, szc,
3255 			    save_szcvec);
3256 			if (error) {
3257 				return (error);
3258 			}
3259 			addr = a;
3260 		}
3261 		szcvec &= ~(1 << szc);
3262 		if (szcvec) {
3263 			szc = highbit(szcvec) - 1;
3264 			pgsz = page_get_pagesize(szc);
3265 		}
3266 	}
3267 	ASSERT(addr == eaddr);
3268 
3269 	return (0);
3270 }
3271 
3272 /*
3273  * Set the default large page size for the range. Called via memcntl with
3274  * page size set to 0. as_set_default_lpsize breaks the range down into
3275  * chunks with the same type/flags, ignores-non segvn segments, and passes
3276  * each chunk to as_iset_default_lpsize().
3277  */
3278 int
3279 as_set_default_lpsize(struct as *as, caddr_t addr, size_t size)
3280 {
3281 	struct seg *seg;
3282 	caddr_t raddr;
3283 	size_t rsize;
3284 	size_t ssize;
3285 	int rtype, rflags;
3286 	int stype, sflags;
3287 	int error;
3288 	caddr_t	setaddr;
3289 	size_t setsize;
3290 	int segvn;
3291 
3292 	if (size == 0)
3293 		return (0);
3294 
3295 	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
3296 again:
3297 	error = 0;
3298 
3299 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3300 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
3301 	    (size_t)raddr;
3302 
3303 	if (raddr + rsize < raddr) {		/* check for wraparound */
3304 		AS_LOCK_EXIT(as, &as->a_lock);
3305 		return (ENOMEM);
3306 	}
3307 	as_clearwatchprot(as, raddr, rsize);
3308 	seg = as_segat(as, raddr);
3309 	if (seg == NULL) {
3310 		as_setwatch(as);
3311 		AS_LOCK_EXIT(as, &as->a_lock);
3312 		return (ENOMEM);
3313 	}
3314 	if (seg->s_ops == &segvn_ops) {
3315 		rtype = SEGOP_GETTYPE(seg, addr);
3316 		rflags = rtype & (MAP_TEXT | MAP_INITDATA);
3317 		rtype = rtype & (MAP_SHARED | MAP_PRIVATE);
3318 		segvn = 1;
3319 	} else {
3320 		segvn = 0;
3321 	}
3322 	setaddr = raddr;
3323 	setsize = 0;
3324 
3325 	for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3326 		if (raddr >= (seg->s_base + seg->s_size)) {
3327 			seg = AS_SEGNEXT(as, seg);
3328 			if (seg == NULL || raddr != seg->s_base) {
3329 				error = ENOMEM;
3330 				break;
3331 			}
3332 			if (seg->s_ops == &segvn_ops) {
3333 				stype = SEGOP_GETTYPE(seg, raddr);
3334 				sflags = stype & (MAP_TEXT | MAP_INITDATA);
3335 				stype &= (MAP_SHARED | MAP_PRIVATE);
3336 				if (segvn && (rflags != sflags ||
3337 				    rtype != stype)) {
3338 					/*
3339 					 * The next segment is also segvn but
3340 					 * has different flags and/or type.
3341 					 */
3342 					ASSERT(setsize != 0);
3343 					error = as_iset_default_lpsize(as,
3344 					    setaddr, setsize, rflags, rtype);
3345 					if (error) {
3346 						break;
3347 					}
3348 					rflags = sflags;
3349 					rtype = stype;
3350 					setaddr = raddr;
3351 					setsize = 0;
3352 				} else if (!segvn) {
3353 					rflags = sflags;
3354 					rtype = stype;
3355 					setaddr = raddr;
3356 					setsize = 0;
3357 					segvn = 1;
3358 				}
3359 			} else if (segvn) {
3360 				/* The next segment is not segvn. */
3361 				ASSERT(setsize != 0);
3362 				error = as_iset_default_lpsize(as,
3363 				    setaddr, setsize, rflags, rtype);
3364 				if (error) {
3365 					break;
3366 				}
3367 				segvn = 0;
3368 			}
3369 		}
3370 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3371 			ssize = seg->s_base + seg->s_size - raddr;
3372 		} else {
3373 			ssize = rsize;
3374 		}
3375 	}
3376 	if (error == 0 && segvn) {
3377 		/* The last chunk when rsize == 0. */
3378 		ASSERT(setsize != 0);
3379 		error = as_iset_default_lpsize(as, setaddr, setsize,
3380 		    rflags, rtype);
3381 	}
3382 
3383 	if (error == IE_RETRY) {
3384 		goto again;
3385 	} else if (error == IE_NOMEM) {
3386 		error = EAGAIN;
3387 	} else if (error == ENOTSUP) {
3388 		error = EINVAL;
3389 	} else if (error == EAGAIN) {
3390 		mutex_enter(&as->a_contents);
3391 		if (!AS_ISNOUNMAPWAIT(as)) {
3392 			if (AS_ISUNMAPWAIT(as) == 0) {
3393 				cv_broadcast(&as->a_cv);
3394 			}
3395 			AS_SETUNMAPWAIT(as);
3396 			AS_LOCK_EXIT(as, &as->a_lock);
3397 			while (AS_ISUNMAPWAIT(as)) {
3398 				cv_wait(&as->a_cv, &as->a_contents);
3399 			}
3400 			mutex_exit(&as->a_contents);
3401 			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
3402 		} else {
3403 			/*
3404 			 * We may have raced with
3405 			 * segvn_reclaim()/segspt_reclaim(). In this case
3406 			 * clean nounmapwait flag and retry since softlockcnt
3407 			 * in this segment may be already 0.  We don't drop as
3408 			 * writer lock so our number of retries without
3409 			 * sleeping should be very small. See segvn_reclaim()
3410 			 * for more comments.
3411 			 */
3412 			AS_CLRNOUNMAPWAIT(as);
3413 			mutex_exit(&as->a_contents);
3414 		}
3415 		goto again;
3416 	}
3417 
3418 	as_setwatch(as);
3419 	AS_LOCK_EXIT(as, &as->a_lock);
3420 	return (error);
3421 }
3422 
3423 /*
3424  * Setup all of the uninitialized watched pages that we can.
3425  */
3426 void
3427 as_setwatch(struct as *as)
3428 {
3429 	struct watched_page *pwp;
3430 	struct seg *seg;
3431 	caddr_t vaddr;
3432 	uint_t prot;
3433 	int  err, retrycnt;
3434 
3435 	if (avl_numnodes(&as->a_wpage) == 0)
3436 		return;
3437 
3438 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3439 
3440 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
3441 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
3442 		retrycnt = 0;
3443 	retry:
3444 		vaddr = pwp->wp_vaddr;
3445 		if (pwp->wp_oprot != 0 ||	/* already set up */
3446 		    (seg = as_segat(as, vaddr)) == NULL ||
3447 		    SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
3448 			continue;
3449 
3450 		pwp->wp_oprot = prot;
3451 		if (pwp->wp_read)
3452 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3453 		if (pwp->wp_write)
3454 			prot &= ~PROT_WRITE;
3455 		if (pwp->wp_exec)
3456 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3457 		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) {
3458 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
3459 			if (err == IE_RETRY) {
3460 				pwp->wp_oprot = 0;
3461 				ASSERT(retrycnt == 0);
3462 				retrycnt++;
3463 				goto retry;
3464 			}
3465 		}
3466 		pwp->wp_prot = prot;
3467 	}
3468 }
3469 
3470 /*
3471  * Clear all of the watched pages in the address space.
3472  */
3473 void
3474 as_clearwatch(struct as *as)
3475 {
3476 	struct watched_page *pwp;
3477 	struct seg *seg;
3478 	caddr_t vaddr;
3479 	uint_t prot;
3480 	int err, retrycnt;
3481 
3482 	if (avl_numnodes(&as->a_wpage) == 0)
3483 		return;
3484 
3485 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3486 
3487 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
3488 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
3489 		retrycnt = 0;
3490 	retry:
3491 		vaddr = pwp->wp_vaddr;
3492 		if (pwp->wp_oprot == 0 ||	/* not set up */
3493 		    (seg = as_segat(as, vaddr)) == NULL)
3494 			continue;
3495 
3496 		if ((prot = pwp->wp_oprot) != pwp->wp_prot) {
3497 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
3498 			if (err == IE_RETRY) {
3499 				ASSERT(retrycnt == 0);
3500 				retrycnt++;
3501 				goto retry;
3502 			}
3503 		}
3504 		pwp->wp_oprot = 0;
3505 		pwp->wp_prot = 0;
3506 	}
3507 }
3508 
3509 /*
3510  * Force a new setup for all the watched pages in the range.
3511  */
3512 static void
3513 as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
3514 {
3515 	struct watched_page *pwp;
3516 	struct watched_page tpw;
3517 	caddr_t eaddr = addr + size;
3518 	caddr_t vaddr;
3519 	struct seg *seg;
3520 	int err, retrycnt;
3521 	uint_t	wprot;
3522 	avl_index_t where;
3523 
3524 	if (avl_numnodes(&as->a_wpage) == 0)
3525 		return;
3526 
3527 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3528 
3529 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3530 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
3531 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
3532 
3533 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
3534 		retrycnt = 0;
3535 		vaddr = pwp->wp_vaddr;
3536 
3537 		wprot = prot;
3538 		if (pwp->wp_read)
3539 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3540 		if (pwp->wp_write)
3541 			wprot &= ~PROT_WRITE;
3542 		if (pwp->wp_exec)
3543 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
3544 		if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) {
3545 		retry:
3546 			seg = as_segat(as, vaddr);
3547 			if (seg == NULL) {
3548 				panic("as_setwatchprot: no seg");
3549 				/*NOTREACHED*/
3550 			}
3551 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot);
3552 			if (err == IE_RETRY) {
3553 				ASSERT(retrycnt == 0);
3554 				retrycnt++;
3555 				goto retry;
3556 			}
3557 		}
3558 		pwp->wp_oprot = prot;
3559 		pwp->wp_prot = wprot;
3560 
3561 		pwp = AVL_NEXT(&as->a_wpage, pwp);
3562 	}
3563 }
3564 
3565 /*
3566  * Clear all of the watched pages in the range.
3567  */
3568 static void
3569 as_clearwatchprot(struct as *as, caddr_t addr, size_t size)
3570 {
3571 	caddr_t eaddr = addr + size;
3572 	struct watched_page *pwp;
3573 	struct watched_page tpw;
3574 	uint_t prot;
3575 	struct seg *seg;
3576 	int err, retrycnt;
3577 	avl_index_t where;
3578 
3579 	if (avl_numnodes(&as->a_wpage) == 0)
3580 		return;
3581 
3582 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3583 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
3584 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
3585 
3586 	ASSERT(AS_WRITE_HELD(as, &as->a_lock));
3587 
3588 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
3589 
3590 		if ((prot = pwp->wp_oprot) != 0) {
3591 			retrycnt = 0;
3592 
3593 			if (prot != pwp->wp_prot) {
3594 			retry:
3595 				seg = as_segat(as, pwp->wp_vaddr);
3596 				if (seg == NULL)
3597 					continue;
3598 				err = SEGOP_SETPROT(seg, pwp->wp_vaddr,
3599 				    PAGESIZE, prot);
3600 				if (err == IE_RETRY) {
3601 					ASSERT(retrycnt == 0);
3602 					retrycnt++;
3603 					goto retry;
3604 
3605 				}
3606 			}
3607 			pwp->wp_oprot = 0;
3608 			pwp->wp_prot = 0;
3609 		}
3610 
3611 		pwp = AVL_NEXT(&as->a_wpage, pwp);
3612 	}
3613 }
3614 
3615 void
3616 as_signal_proc(struct as *as, k_siginfo_t *siginfo)
3617 {
3618 	struct proc *p;
3619 
3620 	mutex_enter(&pidlock);
3621 	for (p = practive; p; p = p->p_next) {
3622 		if (p->p_as == as) {
3623 			mutex_enter(&p->p_lock);
3624 			if (p->p_as == as)
3625 				sigaddq(p, NULL, siginfo, KM_NOSLEEP);
3626 			mutex_exit(&p->p_lock);
3627 		}
3628 	}
3629 	mutex_exit(&pidlock);
3630 }
3631 
3632 /*
3633  * return memory object ID
3634  */
3635 int
3636 as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
3637 {
3638 	struct seg	*seg;
3639 	int		sts;
3640 
3641 	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
3642 	seg = as_segat(as, addr);
3643 	if (seg == NULL) {
3644 		AS_LOCK_EXIT(as, &as->a_lock);
3645 		return (EFAULT);
3646 	}
3647 	/*
3648 	 * catch old drivers which may not support getmemid
3649 	 */
3650 	if (seg->s_ops->getmemid == NULL) {
3651 		AS_LOCK_EXIT(as, &as->a_lock);
3652 		return (ENODEV);
3653 	}
3654 
3655 	sts = SEGOP_GETMEMID(seg, addr, memidp);
3656 
3657 	AS_LOCK_EXIT(as, &as->a_lock);
3658 	return (sts);
3659 }
3660