xref: /illumos-gate/usr/src/uts/common/vm/vm_as.c (revision 284ce987)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
502ff05a9Svsakar  * Common Development and Distribution License (the "License").
602ff05a9Svsakar  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2217965fd8SKrishnendu Sadhukhan - Sun Microsystems  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
24*284ce987SPatrick Mooney  * Copyright 2018 Joyent, Inc.
256430daecSBrandon Baker  * Copyright (c) 2016 by Delphix. All rights reserved.
267c478bd9Sstevel@tonic-gate  */
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
297c478bd9Sstevel@tonic-gate /*	  All Rights Reserved  	*/
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate /*
327c478bd9Sstevel@tonic-gate  * University Copyright- Copyright (c) 1982, 1986, 1988
337c478bd9Sstevel@tonic-gate  * The Regents of the University of California
347c478bd9Sstevel@tonic-gate  * All Rights Reserved
357c478bd9Sstevel@tonic-gate  *
367c478bd9Sstevel@tonic-gate  * University Acknowledgment- Portions of this document are derived from
377c478bd9Sstevel@tonic-gate  * software developed by the University of California, Berkeley, and its
387c478bd9Sstevel@tonic-gate  * contributors.
397c478bd9Sstevel@tonic-gate  */
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate /*
427c478bd9Sstevel@tonic-gate  * VM - address spaces.
437c478bd9Sstevel@tonic-gate  */
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate #include <sys/types.h>
467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
477c478bd9Sstevel@tonic-gate #include <sys/param.h>
487c478bd9Sstevel@tonic-gate #include <sys/errno.h>
497c478bd9Sstevel@tonic-gate #include <sys/systm.h>
507c478bd9Sstevel@tonic-gate #include <sys/mman.h>
517c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
527c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
537c478bd9Sstevel@tonic-gate #include <sys/sysinfo.h>
547c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
557c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
567c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
577c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
587c478bd9Sstevel@tonic-gate #include <sys/debug.h>
597c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h>
607c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
617c478bd9Sstevel@tonic-gate 
627c478bd9Sstevel@tonic-gate #include <vm/hat.h>
637c478bd9Sstevel@tonic-gate #include <vm/as.h>
647c478bd9Sstevel@tonic-gate #include <vm/seg.h>
657c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h>
667c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h>
677c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
687c478bd9Sstevel@tonic-gate #include <vm/seg_map.h>
697c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h>
70*284ce987SPatrick Mooney #include <vm/seg_hole.h>
717c478bd9Sstevel@tonic-gate #include <vm/page.h>
727c478bd9Sstevel@tonic-gate 
737c478bd9Sstevel@tonic-gate clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate static struct kmem_cache *as_cache;
767c478bd9Sstevel@tonic-gate 
777c478bd9Sstevel@tonic-gate static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
787c478bd9Sstevel@tonic-gate static void as_clearwatchprot(struct as *, caddr_t, size_t);
797c478bd9Sstevel@tonic-gate 
807c478bd9Sstevel@tonic-gate 
817c478bd9Sstevel@tonic-gate /*
827c478bd9Sstevel@tonic-gate  * Verifying the segment lists is very time-consuming; it may not be
837c478bd9Sstevel@tonic-gate  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
847c478bd9Sstevel@tonic-gate  */
857c478bd9Sstevel@tonic-gate #ifdef DEBUG
867c478bd9Sstevel@tonic-gate #define	VERIFY_SEGLIST
877c478bd9Sstevel@tonic-gate int do_as_verify = 0;
887c478bd9Sstevel@tonic-gate #endif
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate /*
917c478bd9Sstevel@tonic-gate  * Allocate a new callback data structure entry and fill in the events of
927c478bd9Sstevel@tonic-gate  * interest, the address range of interest, and the callback argument.
937c478bd9Sstevel@tonic-gate  * Link the entry on the as->a_callbacks list. A callback entry for the
947c478bd9Sstevel@tonic-gate  * entire address space may be specified with vaddr = 0 and size = -1.
957c478bd9Sstevel@tonic-gate  *
967c478bd9Sstevel@tonic-gate  * CALLERS RESPONSIBILITY: If not calling from within the process context for
977c478bd9Sstevel@tonic-gate  * the specified as, the caller must guarantee persistence of the specified as
987c478bd9Sstevel@tonic-gate  * for the duration of this function (eg. pages being locked within the as
997c478bd9Sstevel@tonic-gate  * will guarantee persistence).
1007c478bd9Sstevel@tonic-gate  */
1017c478bd9Sstevel@tonic-gate int
1027c478bd9Sstevel@tonic-gate as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
1036430daecSBrandon Baker     caddr_t vaddr, size_t size, int sleepflag)
1047c478bd9Sstevel@tonic-gate {
1057c478bd9Sstevel@tonic-gate 	struct as_callback 	*current_head, *cb;
1067c478bd9Sstevel@tonic-gate 	caddr_t 		saddr;
1077c478bd9Sstevel@tonic-gate 	size_t 			rsize;
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 	/* callback function and an event are mandatory */
1107c478bd9Sstevel@tonic-gate 	if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
1117c478bd9Sstevel@tonic-gate 		return (EINVAL);
1127c478bd9Sstevel@tonic-gate 
1137c478bd9Sstevel@tonic-gate 	/* Adding a callback after as_free has been called is not allowed */
1147c478bd9Sstevel@tonic-gate 	if (as == &kas)
1157c478bd9Sstevel@tonic-gate 		return (ENOMEM);
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate 	/*
1187c478bd9Sstevel@tonic-gate 	 * vaddr = 0 and size = -1 is used to indicate that the callback range
1197c478bd9Sstevel@tonic-gate 	 * is the entire address space so no rounding is done in that case.
1207c478bd9Sstevel@tonic-gate 	 */
1217c478bd9Sstevel@tonic-gate 	if (size != -1) {
1227c478bd9Sstevel@tonic-gate 		saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
1237c478bd9Sstevel@tonic-gate 		rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
124843e1988Sjohnlev 		    (size_t)saddr;
1257c478bd9Sstevel@tonic-gate 		/* check for wraparound */
1267c478bd9Sstevel@tonic-gate 		if (saddr + rsize < saddr)
1277c478bd9Sstevel@tonic-gate 			return (ENOMEM);
1287c478bd9Sstevel@tonic-gate 	} else {
1297c478bd9Sstevel@tonic-gate 		if (vaddr != 0)
1307c478bd9Sstevel@tonic-gate 			return (EINVAL);
1317c478bd9Sstevel@tonic-gate 		saddr = vaddr;
1327c478bd9Sstevel@tonic-gate 		rsize = size;
1337c478bd9Sstevel@tonic-gate 	}
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate 	/* Allocate and initialize a callback entry */
1367c478bd9Sstevel@tonic-gate 	cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
1377c478bd9Sstevel@tonic-gate 	if (cb == NULL)
1387c478bd9Sstevel@tonic-gate 		return (EAGAIN);
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate 	cb->ascb_func = cb_func;
1417c478bd9Sstevel@tonic-gate 	cb->ascb_arg = arg;
1427c478bd9Sstevel@tonic-gate 	cb->ascb_events = events;
1437c478bd9Sstevel@tonic-gate 	cb->ascb_saddr = saddr;
1447c478bd9Sstevel@tonic-gate 	cb->ascb_len = rsize;
1457c478bd9Sstevel@tonic-gate 
1467c478bd9Sstevel@tonic-gate 	/* Add the entry to the list */
1477c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
1487c478bd9Sstevel@tonic-gate 	current_head = as->a_callbacks;
1497c478bd9Sstevel@tonic-gate 	as->a_callbacks = cb;
1507c478bd9Sstevel@tonic-gate 	cb->ascb_next = current_head;
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate 	/*
1537c478bd9Sstevel@tonic-gate 	 * The call to this function may lose in a race with
1547c478bd9Sstevel@tonic-gate 	 * a pertinent event - eg. a thread does long term memory locking
1557c478bd9Sstevel@tonic-gate 	 * but before the callback is added another thread executes as_unmap.
1567c478bd9Sstevel@tonic-gate 	 * A broadcast here resolves that.
1577c478bd9Sstevel@tonic-gate 	 */
1587c478bd9Sstevel@tonic-gate 	if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
1597c478bd9Sstevel@tonic-gate 		AS_CLRUNMAPWAIT(as);
1607c478bd9Sstevel@tonic-gate 		cv_broadcast(&as->a_cv);
1617c478bd9Sstevel@tonic-gate 	}
1627c478bd9Sstevel@tonic-gate 
1637c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
1647c478bd9Sstevel@tonic-gate 	return (0);
1657c478bd9Sstevel@tonic-gate }
1667c478bd9Sstevel@tonic-gate 
1677c478bd9Sstevel@tonic-gate /*
1687c478bd9Sstevel@tonic-gate  * Search the callback list for an entry which pertains to arg.
1697c478bd9Sstevel@tonic-gate  *
1707c478bd9Sstevel@tonic-gate  * This is called from within the client upon completion of the callback.
1717c478bd9Sstevel@tonic-gate  * RETURN VALUES:
1727c478bd9Sstevel@tonic-gate  *	AS_CALLBACK_DELETED  (callback entry found and deleted)
1737c478bd9Sstevel@tonic-gate  *	AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
1747c478bd9Sstevel@tonic-gate  *	AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
1757c478bd9Sstevel@tonic-gate  *			entry will be made in as_do_callbacks)
1767c478bd9Sstevel@tonic-gate  *
1777c478bd9Sstevel@tonic-gate  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
1787c478bd9Sstevel@tonic-gate  * set, it indicates that as_do_callbacks is processing this entry.  The
1797c478bd9Sstevel@tonic-gate  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
1807c478bd9Sstevel@tonic-gate  * to unblock as_do_callbacks, in case it is blocked.
1817c478bd9Sstevel@tonic-gate  *
1827c478bd9Sstevel@tonic-gate  * CALLERS RESPONSIBILITY: If not calling from within the process context for
1837c478bd9Sstevel@tonic-gate  * the specified as, the caller must guarantee persistence of the specified as
1847c478bd9Sstevel@tonic-gate  * for the duration of this function (eg. pages being locked within the as
1857c478bd9Sstevel@tonic-gate  * will guarantee persistence).
1867c478bd9Sstevel@tonic-gate  */
1877c478bd9Sstevel@tonic-gate uint_t
1887c478bd9Sstevel@tonic-gate as_delete_callback(struct as *as, void *arg)
1897c478bd9Sstevel@tonic-gate {
1907c478bd9Sstevel@tonic-gate 	struct as_callback **prevcb = &as->a_callbacks;
1917c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
1927c478bd9Sstevel@tonic-gate 	uint_t rc = AS_CALLBACK_NOTFOUND;
1937c478bd9Sstevel@tonic-gate 
1947c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
1957c478bd9Sstevel@tonic-gate 	for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
1967c478bd9Sstevel@tonic-gate 		if (cb->ascb_arg != arg)
1977c478bd9Sstevel@tonic-gate 			continue;
1987c478bd9Sstevel@tonic-gate 
1997c478bd9Sstevel@tonic-gate 		/*
2007c478bd9Sstevel@tonic-gate 		 * If the events indicate AS_CALLBACK_CALLED, just clear
2017c478bd9Sstevel@tonic-gate 		 * AS_ALL_EVENT in the events field and wakeup the thread
2027c478bd9Sstevel@tonic-gate 		 * that may be waiting in as_do_callbacks.  as_do_callbacks
2037c478bd9Sstevel@tonic-gate 		 * will take care of removing this entry from the list.  In
2047c478bd9Sstevel@tonic-gate 		 * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
2057c478bd9Sstevel@tonic-gate 		 * (AS_CALLBACK_CALLED not set), just remove it from the
2067c478bd9Sstevel@tonic-gate 		 * list, return the memory and return AS_CALLBACK_DELETED.
2077c478bd9Sstevel@tonic-gate 		 */
2087c478bd9Sstevel@tonic-gate 		if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
2097c478bd9Sstevel@tonic-gate 			/* leave AS_CALLBACK_CALLED */
2107c478bd9Sstevel@tonic-gate 			cb->ascb_events &= ~AS_ALL_EVENT;
2117c478bd9Sstevel@tonic-gate 			rc = AS_CALLBACK_DELETE_DEFERRED;
2127c478bd9Sstevel@tonic-gate 			cv_broadcast(&as->a_cv);
2137c478bd9Sstevel@tonic-gate 		} else {
2147c478bd9Sstevel@tonic-gate 			*prevcb = cb->ascb_next;
2157c478bd9Sstevel@tonic-gate 			kmem_free(cb, sizeof (struct as_callback));
2167c478bd9Sstevel@tonic-gate 			rc = AS_CALLBACK_DELETED;
2177c478bd9Sstevel@tonic-gate 		}
2187c478bd9Sstevel@tonic-gate 		break;
2197c478bd9Sstevel@tonic-gate 	}
2207c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
2217c478bd9Sstevel@tonic-gate 	return (rc);
2227c478bd9Sstevel@tonic-gate }
2237c478bd9Sstevel@tonic-gate 
2247c478bd9Sstevel@tonic-gate /*
2257c478bd9Sstevel@tonic-gate  * Searches the as callback list for a matching entry.
2267c478bd9Sstevel@tonic-gate  * Returns a pointer to the first matching callback, or NULL if
2277c478bd9Sstevel@tonic-gate  * nothing is found.
2287c478bd9Sstevel@tonic-gate  * This function never sleeps so it is ok to call it with more
2297c478bd9Sstevel@tonic-gate  * locks held but the (required) a_contents mutex.
2307c478bd9Sstevel@tonic-gate  *
2317c478bd9Sstevel@tonic-gate  * See also comment on as_do_callbacks below.
2327c478bd9Sstevel@tonic-gate  */
2337c478bd9Sstevel@tonic-gate static struct as_callback *
2347c478bd9Sstevel@tonic-gate as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
2356430daecSBrandon Baker     size_t event_len)
2367c478bd9Sstevel@tonic-gate {
2377c478bd9Sstevel@tonic-gate 	struct as_callback	*cb;
2387c478bd9Sstevel@tonic-gate 
2397c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&as->a_contents));
2407c478bd9Sstevel@tonic-gate 	for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
2417c478bd9Sstevel@tonic-gate 		/*
2427c478bd9Sstevel@tonic-gate 		 * If the callback has not already been called, then
2437c478bd9Sstevel@tonic-gate 		 * check if events or address range pertains.  An event_len
2447c478bd9Sstevel@tonic-gate 		 * of zero means do an unconditional callback.
2457c478bd9Sstevel@tonic-gate 		 */
2467c478bd9Sstevel@tonic-gate 		if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
2477c478bd9Sstevel@tonic-gate 		    ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
2487c478bd9Sstevel@tonic-gate 		    (event_addr + event_len < cb->ascb_saddr) ||
2497c478bd9Sstevel@tonic-gate 		    (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
2507c478bd9Sstevel@tonic-gate 			continue;
2517c478bd9Sstevel@tonic-gate 		}
2527c478bd9Sstevel@tonic-gate 		break;
2537c478bd9Sstevel@tonic-gate 	}
2547c478bd9Sstevel@tonic-gate 	return (cb);
2557c478bd9Sstevel@tonic-gate }
2567c478bd9Sstevel@tonic-gate 
2577c478bd9Sstevel@tonic-gate /*
2587c478bd9Sstevel@tonic-gate  * Executes a given callback and removes it from the callback list for
2597c478bd9Sstevel@tonic-gate  * this address space.
2607c478bd9Sstevel@tonic-gate  * This function may sleep so the caller must drop all locks except
2617c478bd9Sstevel@tonic-gate  * a_contents before calling this func.
2627c478bd9Sstevel@tonic-gate  *
2637c478bd9Sstevel@tonic-gate  * See also comments on as_do_callbacks below.
2647c478bd9Sstevel@tonic-gate  */
2657c478bd9Sstevel@tonic-gate static void
2667c478bd9Sstevel@tonic-gate as_execute_callback(struct as *as, struct as_callback *cb,
2676430daecSBrandon Baker     uint_t events)
2687c478bd9Sstevel@tonic-gate {
2697c478bd9Sstevel@tonic-gate 	struct as_callback **prevcb;
2707c478bd9Sstevel@tonic-gate 	void	*cb_arg;
2717c478bd9Sstevel@tonic-gate 
2727c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
2737c478bd9Sstevel@tonic-gate 	cb->ascb_events |= AS_CALLBACK_CALLED;
2747c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
2757c478bd9Sstevel@tonic-gate 	(*cb->ascb_func)(as, cb->ascb_arg, events);
2767c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
2777c478bd9Sstevel@tonic-gate 	/*
2787c478bd9Sstevel@tonic-gate 	 * the callback function is required to delete the callback
2797c478bd9Sstevel@tonic-gate 	 * when the callback function determines it is OK for
2807c478bd9Sstevel@tonic-gate 	 * this thread to continue. as_delete_callback will clear
2817c478bd9Sstevel@tonic-gate 	 * the AS_ALL_EVENT in the events field when it is deleted.
2827c478bd9Sstevel@tonic-gate 	 * If the callback function called as_delete_callback,
2837c478bd9Sstevel@tonic-gate 	 * events will already be cleared and there will be no blocking.
2847c478bd9Sstevel@tonic-gate 	 */
2857c478bd9Sstevel@tonic-gate 	while ((cb->ascb_events & events) != 0) {
2867c478bd9Sstevel@tonic-gate 		cv_wait(&as->a_cv, &as->a_contents);
2877c478bd9Sstevel@tonic-gate 	}
2887c478bd9Sstevel@tonic-gate 	/*
2897c478bd9Sstevel@tonic-gate 	 * This entry needs to be taken off the list. Normally, the
2907c478bd9Sstevel@tonic-gate 	 * callback func itself does that, but unfortunately the list
2917c478bd9Sstevel@tonic-gate 	 * may have changed while the callback was running because the
2927c478bd9Sstevel@tonic-gate 	 * a_contents mutex was dropped and someone else other than the
2937c478bd9Sstevel@tonic-gate 	 * callback func itself could have called as_delete_callback,
2947c478bd9Sstevel@tonic-gate 	 * so we have to search to find this entry again.  The entry
2957c478bd9Sstevel@tonic-gate 	 * must have AS_CALLBACK_CALLED, and have the same 'arg'.
2967c478bd9Sstevel@tonic-gate 	 */
2977c478bd9Sstevel@tonic-gate 	cb_arg = cb->ascb_arg;
2987c478bd9Sstevel@tonic-gate 	prevcb = &as->a_callbacks;
2997c478bd9Sstevel@tonic-gate 	for (cb = as->a_callbacks; cb != NULL;
3007c478bd9Sstevel@tonic-gate 	    prevcb = &cb->ascb_next, cb = *prevcb) {
3017c478bd9Sstevel@tonic-gate 		if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
3027c478bd9Sstevel@tonic-gate 		    (cb_arg != cb->ascb_arg)) {
3037c478bd9Sstevel@tonic-gate 			continue;
3047c478bd9Sstevel@tonic-gate 		}
3057c478bd9Sstevel@tonic-gate 		*prevcb = cb->ascb_next;
3067c478bd9Sstevel@tonic-gate 		kmem_free(cb, sizeof (struct as_callback));
3077c478bd9Sstevel@tonic-gate 		break;
3087c478bd9Sstevel@tonic-gate 	}
3097c478bd9Sstevel@tonic-gate }
3107c478bd9Sstevel@tonic-gate 
3117c478bd9Sstevel@tonic-gate /*
3127c478bd9Sstevel@tonic-gate  * Check the callback list for a matching event and intersection of
3137c478bd9Sstevel@tonic-gate  * address range. If there is a match invoke the callback.  Skip an entry if:
3147c478bd9Sstevel@tonic-gate  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
3157c478bd9Sstevel@tonic-gate  *    - not event of interest
3167c478bd9Sstevel@tonic-gate  *    - not address range of interest
3177c478bd9Sstevel@tonic-gate  *
3187c478bd9Sstevel@tonic-gate  * An event_len of zero indicates a request for an unconditional callback
3197c478bd9Sstevel@tonic-gate  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
3207c478bd9Sstevel@tonic-gate  * a_contents lock must be dropped before a callback, so only one callback
3217c478bd9Sstevel@tonic-gate  * can be done before returning. Return -1 (true) if a callback was
3227c478bd9Sstevel@tonic-gate  * executed and removed from the list, else return 0 (false).
3237c478bd9Sstevel@tonic-gate  *
3247c478bd9Sstevel@tonic-gate  * The logically separate parts, i.e. finding a matching callback and
3257c478bd9Sstevel@tonic-gate  * executing a given callback have been separated into two functions
3267c478bd9Sstevel@tonic-gate  * so that they can be called with different sets of locks held beyond
3277c478bd9Sstevel@tonic-gate  * the always-required a_contents. as_find_callback does not sleep so
3287c478bd9Sstevel@tonic-gate  * it is ok to call it if more locks than a_contents (i.e. the a_lock
3297c478bd9Sstevel@tonic-gate  * rwlock) are held. as_execute_callback on the other hand may sleep
3307c478bd9Sstevel@tonic-gate  * so all locks beyond a_contents must be dropped by the caller if one
3317c478bd9Sstevel@tonic-gate  * does not want to end comatose.
3327c478bd9Sstevel@tonic-gate  */
3337c478bd9Sstevel@tonic-gate static int
3347c478bd9Sstevel@tonic-gate as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
3356430daecSBrandon Baker     size_t event_len)
3367c478bd9Sstevel@tonic-gate {
3377c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
3387c478bd9Sstevel@tonic-gate 
3397c478bd9Sstevel@tonic-gate 	if ((cb = as_find_callback(as, events, event_addr, event_len))) {
3407c478bd9Sstevel@tonic-gate 		as_execute_callback(as, cb, events);
3417c478bd9Sstevel@tonic-gate 		return (-1);
3427c478bd9Sstevel@tonic-gate 	}
3437c478bd9Sstevel@tonic-gate 	return (0);
3447c478bd9Sstevel@tonic-gate }
3457c478bd9Sstevel@tonic-gate 
3467c478bd9Sstevel@tonic-gate /*
3477c478bd9Sstevel@tonic-gate  * Search for the segment containing addr. If a segment containing addr
3487c478bd9Sstevel@tonic-gate  * exists, that segment is returned.  If no such segment exists, and
3497c478bd9Sstevel@tonic-gate  * the list spans addresses greater than addr, then the first segment
3507c478bd9Sstevel@tonic-gate  * whose base is greater than addr is returned; otherwise, NULL is
3517c478bd9Sstevel@tonic-gate  * returned unless tail is true, in which case the last element of the
3527c478bd9Sstevel@tonic-gate  * list is returned.
3537c478bd9Sstevel@tonic-gate  *
3547c478bd9Sstevel@tonic-gate  * a_seglast is used to cache the last found segment for repeated
3557c478bd9Sstevel@tonic-gate  * searches to the same addr (which happens frequently).
3567c478bd9Sstevel@tonic-gate  */
3577c478bd9Sstevel@tonic-gate struct seg *
3587c478bd9Sstevel@tonic-gate as_findseg(struct as *as, caddr_t addr, int tail)
3597c478bd9Sstevel@tonic-gate {
3607c478bd9Sstevel@tonic-gate 	struct seg *seg = as->a_seglast;
3617c478bd9Sstevel@tonic-gate 	avl_index_t where;
3627c478bd9Sstevel@tonic-gate 
363dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_LOCK_HELD(as));
3647c478bd9Sstevel@tonic-gate 
3657c478bd9Sstevel@tonic-gate 	if (seg != NULL &&
3667c478bd9Sstevel@tonic-gate 	    seg->s_base <= addr &&
3677c478bd9Sstevel@tonic-gate 	    addr < seg->s_base + seg->s_size)
3687c478bd9Sstevel@tonic-gate 		return (seg);
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate 	seg = avl_find(&as->a_segtree, &addr, &where);
3717c478bd9Sstevel@tonic-gate 	if (seg != NULL)
3727c478bd9Sstevel@tonic-gate 		return (as->a_seglast = seg);
3737c478bd9Sstevel@tonic-gate 
3747c478bd9Sstevel@tonic-gate 	seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
3757c478bd9Sstevel@tonic-gate 	if (seg == NULL && tail)
3767c478bd9Sstevel@tonic-gate 		seg = avl_last(&as->a_segtree);
3777c478bd9Sstevel@tonic-gate 	return (as->a_seglast = seg);
3787c478bd9Sstevel@tonic-gate }
3797c478bd9Sstevel@tonic-gate 
3807c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST
3817c478bd9Sstevel@tonic-gate /*
3827c478bd9Sstevel@tonic-gate  * verify that the linked list is coherent
3837c478bd9Sstevel@tonic-gate  */
3847c478bd9Sstevel@tonic-gate static void
3857c478bd9Sstevel@tonic-gate as_verify(struct as *as)
3867c478bd9Sstevel@tonic-gate {
3877c478bd9Sstevel@tonic-gate 	struct seg *seg, *seglast, *p, *n;
3887c478bd9Sstevel@tonic-gate 	uint_t nsegs = 0;
3897c478bd9Sstevel@tonic-gate 
3907c478bd9Sstevel@tonic-gate 	if (do_as_verify == 0)
3917c478bd9Sstevel@tonic-gate 		return;
3927c478bd9Sstevel@tonic-gate 
3937c478bd9Sstevel@tonic-gate 	seglast = as->a_seglast;
3947c478bd9Sstevel@tonic-gate 
3957c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
3967c478bd9Sstevel@tonic-gate 		ASSERT(seg->s_as == as);
3977c478bd9Sstevel@tonic-gate 		p = AS_SEGPREV(as, seg);
3987c478bd9Sstevel@tonic-gate 		n = AS_SEGNEXT(as, seg);
3997c478bd9Sstevel@tonic-gate 		ASSERT(p == NULL || p->s_as == as);
4007c478bd9Sstevel@tonic-gate 		ASSERT(p == NULL || p->s_base < seg->s_base);
4017c478bd9Sstevel@tonic-gate 		ASSERT(n == NULL || n->s_base > seg->s_base);
4027c478bd9Sstevel@tonic-gate 		ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
4037c478bd9Sstevel@tonic-gate 		if (seg == seglast)
4047c478bd9Sstevel@tonic-gate 			seglast = NULL;
4057c478bd9Sstevel@tonic-gate 		nsegs++;
4067c478bd9Sstevel@tonic-gate 	}
4077c478bd9Sstevel@tonic-gate 	ASSERT(seglast == NULL);
4087c478bd9Sstevel@tonic-gate 	ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
4097c478bd9Sstevel@tonic-gate }
4107c478bd9Sstevel@tonic-gate #endif /* VERIFY_SEGLIST */
4117c478bd9Sstevel@tonic-gate 
4127c478bd9Sstevel@tonic-gate /*
4137c478bd9Sstevel@tonic-gate  * Add a new segment to the address space. The avl_find()
4147c478bd9Sstevel@tonic-gate  * may be expensive so we attempt to use last segment accessed
4157c478bd9Sstevel@tonic-gate  * in as_gap() as an insertion point.
4167c478bd9Sstevel@tonic-gate  */
4177c478bd9Sstevel@tonic-gate int
4187c478bd9Sstevel@tonic-gate as_addseg(struct as  *as, struct seg *newseg)
4197c478bd9Sstevel@tonic-gate {
4207c478bd9Sstevel@tonic-gate 	struct seg *seg;
4217c478bd9Sstevel@tonic-gate 	caddr_t addr;
4227c478bd9Sstevel@tonic-gate 	caddr_t eaddr;
4237c478bd9Sstevel@tonic-gate 	avl_index_t where;
4247c478bd9Sstevel@tonic-gate 
425dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate 	as->a_updatedir = 1;	/* inform /proc */
4287c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate 	if (as->a_lastgaphl != NULL) {
4317c478bd9Sstevel@tonic-gate 		struct seg *hseg = NULL;
4327c478bd9Sstevel@tonic-gate 		struct seg *lseg = NULL;
4337c478bd9Sstevel@tonic-gate 
4347c478bd9Sstevel@tonic-gate 		if (as->a_lastgaphl->s_base > newseg->s_base) {
4357c478bd9Sstevel@tonic-gate 			hseg = as->a_lastgaphl;
4367c478bd9Sstevel@tonic-gate 			lseg = AVL_PREV(&as->a_segtree, hseg);
4377c478bd9Sstevel@tonic-gate 		} else {
4387c478bd9Sstevel@tonic-gate 			lseg = as->a_lastgaphl;
4397c478bd9Sstevel@tonic-gate 			hseg = AVL_NEXT(&as->a_segtree, lseg);
4407c478bd9Sstevel@tonic-gate 		}
4417c478bd9Sstevel@tonic-gate 
4427c478bd9Sstevel@tonic-gate 		if (hseg && lseg && lseg->s_base < newseg->s_base &&
4437c478bd9Sstevel@tonic-gate 		    hseg->s_base > newseg->s_base) {
4447c478bd9Sstevel@tonic-gate 			avl_insert_here(&as->a_segtree, newseg, lseg,
4457c478bd9Sstevel@tonic-gate 			    AVL_AFTER);
4467c478bd9Sstevel@tonic-gate 			as->a_lastgaphl = NULL;
4477c478bd9Sstevel@tonic-gate 			as->a_seglast = newseg;
4487c478bd9Sstevel@tonic-gate 			return (0);
4497c478bd9Sstevel@tonic-gate 		}
4507c478bd9Sstevel@tonic-gate 		as->a_lastgaphl = NULL;
4517c478bd9Sstevel@tonic-gate 	}
4527c478bd9Sstevel@tonic-gate 
4537c478bd9Sstevel@tonic-gate 	addr = newseg->s_base;
4547c478bd9Sstevel@tonic-gate 	eaddr = addr + newseg->s_size;
4557c478bd9Sstevel@tonic-gate again:
4567c478bd9Sstevel@tonic-gate 
4577c478bd9Sstevel@tonic-gate 	seg = avl_find(&as->a_segtree, &addr, &where);
4587c478bd9Sstevel@tonic-gate 
4597c478bd9Sstevel@tonic-gate 	if (seg == NULL)
4607c478bd9Sstevel@tonic-gate 		seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
4617c478bd9Sstevel@tonic-gate 
4627c478bd9Sstevel@tonic-gate 	if (seg == NULL)
4637c478bd9Sstevel@tonic-gate 		seg = avl_last(&as->a_segtree);
4647c478bd9Sstevel@tonic-gate 
4657c478bd9Sstevel@tonic-gate 	if (seg != NULL) {
4667c478bd9Sstevel@tonic-gate 		caddr_t base = seg->s_base;
4677c478bd9Sstevel@tonic-gate 
4687c478bd9Sstevel@tonic-gate 		/*
4697c478bd9Sstevel@tonic-gate 		 * If top of seg is below the requested address, then
4707c478bd9Sstevel@tonic-gate 		 * the insertion point is at the end of the linked list,
4717c478bd9Sstevel@tonic-gate 		 * and seg points to the tail of the list.  Otherwise,
4727c478bd9Sstevel@tonic-gate 		 * the insertion point is immediately before seg.
4737c478bd9Sstevel@tonic-gate 		 */
4747c478bd9Sstevel@tonic-gate 		if (base + seg->s_size > addr) {
4757c478bd9Sstevel@tonic-gate 			if (addr >= base || eaddr > base) {
4767c478bd9Sstevel@tonic-gate #ifdef __sparc
4777c478bd9Sstevel@tonic-gate 				extern struct seg_ops segnf_ops;
4787c478bd9Sstevel@tonic-gate 
4797c478bd9Sstevel@tonic-gate 				/*
4807c478bd9Sstevel@tonic-gate 				 * no-fault segs must disappear if overlaid.
4817c478bd9Sstevel@tonic-gate 				 * XXX need new segment type so
4827c478bd9Sstevel@tonic-gate 				 * we don't have to check s_ops
4837c478bd9Sstevel@tonic-gate 				 */
4847c478bd9Sstevel@tonic-gate 				if (seg->s_ops == &segnf_ops) {
4857c478bd9Sstevel@tonic-gate 					seg_unmap(seg);
4867c478bd9Sstevel@tonic-gate 					goto again;
4877c478bd9Sstevel@tonic-gate 				}
4887c478bd9Sstevel@tonic-gate #endif
4897c478bd9Sstevel@tonic-gate 				return (-1);	/* overlapping segment */
4907c478bd9Sstevel@tonic-gate 			}
4917c478bd9Sstevel@tonic-gate 		}
4927c478bd9Sstevel@tonic-gate 	}
4937c478bd9Sstevel@tonic-gate 	as->a_seglast = newseg;
4947c478bd9Sstevel@tonic-gate 	avl_insert(&as->a_segtree, newseg, where);
4957c478bd9Sstevel@tonic-gate 
4967c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST
4977c478bd9Sstevel@tonic-gate 	as_verify(as);
4987c478bd9Sstevel@tonic-gate #endif
4997c478bd9Sstevel@tonic-gate 	return (0);
5007c478bd9Sstevel@tonic-gate }
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate struct seg *
5037c478bd9Sstevel@tonic-gate as_removeseg(struct as *as, struct seg *seg)
5047c478bd9Sstevel@tonic-gate {
5057c478bd9Sstevel@tonic-gate 	avl_tree_t *t;
5067c478bd9Sstevel@tonic-gate 
507dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate 	as->a_updatedir = 1;	/* inform /proc */
5107c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
5117c478bd9Sstevel@tonic-gate 
5127c478bd9Sstevel@tonic-gate 	if (seg == NULL)
5137c478bd9Sstevel@tonic-gate 		return (NULL);
5147c478bd9Sstevel@tonic-gate 
5157c478bd9Sstevel@tonic-gate 	t = &as->a_segtree;
5167c478bd9Sstevel@tonic-gate 	if (as->a_seglast == seg)
5177c478bd9Sstevel@tonic-gate 		as->a_seglast = NULL;
5187c478bd9Sstevel@tonic-gate 	as->a_lastgaphl = NULL;
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate 	/*
5217c478bd9Sstevel@tonic-gate 	 * if this segment is at an address higher than
5227c478bd9Sstevel@tonic-gate 	 * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
5237c478bd9Sstevel@tonic-gate 	 */
5247c478bd9Sstevel@tonic-gate 	if (as->a_lastgap &&
5257c478bd9Sstevel@tonic-gate 	    (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
5267c478bd9Sstevel@tonic-gate 		as->a_lastgap = AVL_NEXT(t, seg);
5277c478bd9Sstevel@tonic-gate 
5287c478bd9Sstevel@tonic-gate 	/*
5297c478bd9Sstevel@tonic-gate 	 * remove the segment from the seg tree
5307c478bd9Sstevel@tonic-gate 	 */
5317c478bd9Sstevel@tonic-gate 	avl_remove(t, seg);
5327c478bd9Sstevel@tonic-gate 
5337c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST
5347c478bd9Sstevel@tonic-gate 	as_verify(as);
5357c478bd9Sstevel@tonic-gate #endif
5367c478bd9Sstevel@tonic-gate 	return (seg);
5377c478bd9Sstevel@tonic-gate }
5387c478bd9Sstevel@tonic-gate 
5397c478bd9Sstevel@tonic-gate /*
5407c478bd9Sstevel@tonic-gate  * Find a segment containing addr.
5417c478bd9Sstevel@tonic-gate  */
5427c478bd9Sstevel@tonic-gate struct seg *
5437c478bd9Sstevel@tonic-gate as_segat(struct as *as, caddr_t addr)
5447c478bd9Sstevel@tonic-gate {
5457c478bd9Sstevel@tonic-gate 	struct seg *seg = as->a_seglast;
5467c478bd9Sstevel@tonic-gate 
547dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_LOCK_HELD(as));
5487c478bd9Sstevel@tonic-gate 
5497c478bd9Sstevel@tonic-gate 	if (seg != NULL && seg->s_base <= addr &&
5507c478bd9Sstevel@tonic-gate 	    addr < seg->s_base + seg->s_size)
5517c478bd9Sstevel@tonic-gate 		return (seg);
5527c478bd9Sstevel@tonic-gate 
5537c478bd9Sstevel@tonic-gate 	seg = avl_find(&as->a_segtree, &addr, NULL);
5547c478bd9Sstevel@tonic-gate 	return (seg);
5557c478bd9Sstevel@tonic-gate }
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate /*
5587c478bd9Sstevel@tonic-gate  * Serialize all searches for holes in an address space to
5597c478bd9Sstevel@tonic-gate  * prevent two or more threads from allocating the same virtual
5607c478bd9Sstevel@tonic-gate  * address range.  The address space must not be "read/write"
5617c478bd9Sstevel@tonic-gate  * locked by the caller since we may block.
5627c478bd9Sstevel@tonic-gate  */
5637c478bd9Sstevel@tonic-gate void
5647c478bd9Sstevel@tonic-gate as_rangelock(struct as *as)
5657c478bd9Sstevel@tonic-gate {
5667c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
5677c478bd9Sstevel@tonic-gate 	while (AS_ISCLAIMGAP(as))
5687c478bd9Sstevel@tonic-gate 		cv_wait(&as->a_cv, &as->a_contents);
5697c478bd9Sstevel@tonic-gate 	AS_SETCLAIMGAP(as);
5707c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
5717c478bd9Sstevel@tonic-gate }
5727c478bd9Sstevel@tonic-gate 
5737c478bd9Sstevel@tonic-gate /*
5747c478bd9Sstevel@tonic-gate  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
5757c478bd9Sstevel@tonic-gate  */
5767c478bd9Sstevel@tonic-gate void
5777c478bd9Sstevel@tonic-gate as_rangeunlock(struct as *as)
5787c478bd9Sstevel@tonic-gate {
5797c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
5807c478bd9Sstevel@tonic-gate 	AS_CLRCLAIMGAP(as);
5817c478bd9Sstevel@tonic-gate 	cv_signal(&as->a_cv);
5827c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
5837c478bd9Sstevel@tonic-gate }
5847c478bd9Sstevel@tonic-gate 
5857c478bd9Sstevel@tonic-gate /*
5867c478bd9Sstevel@tonic-gate  * compar segments (or just an address) by segment address range
5877c478bd9Sstevel@tonic-gate  */
5887c478bd9Sstevel@tonic-gate static int
5897c478bd9Sstevel@tonic-gate as_segcompar(const void *x, const void *y)
5907c478bd9Sstevel@tonic-gate {
5917c478bd9Sstevel@tonic-gate 	struct seg *a = (struct seg *)x;
5927c478bd9Sstevel@tonic-gate 	struct seg *b = (struct seg *)y;
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 	if (a->s_base < b->s_base)
5957c478bd9Sstevel@tonic-gate 		return (-1);
5967c478bd9Sstevel@tonic-gate 	if (a->s_base >= b->s_base + b->s_size)
5977c478bd9Sstevel@tonic-gate 		return (1);
5987c478bd9Sstevel@tonic-gate 	return (0);
5997c478bd9Sstevel@tonic-gate }
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate 
6027c478bd9Sstevel@tonic-gate void
6037c478bd9Sstevel@tonic-gate as_avlinit(struct as *as)
6047c478bd9Sstevel@tonic-gate {
6057c478bd9Sstevel@tonic-gate 	avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
6067c478bd9Sstevel@tonic-gate 	    offsetof(struct seg, s_tree));
6077c478bd9Sstevel@tonic-gate 	avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
6087c478bd9Sstevel@tonic-gate 	    offsetof(struct watched_page, wp_link));
6097c478bd9Sstevel@tonic-gate }
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6127c478bd9Sstevel@tonic-gate static int
6137c478bd9Sstevel@tonic-gate as_constructor(void *buf, void *cdrarg, int kmflags)
6147c478bd9Sstevel@tonic-gate {
6157c478bd9Sstevel@tonic-gate 	struct as *as = buf;
6167c478bd9Sstevel@tonic-gate 
6177c478bd9Sstevel@tonic-gate 	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
6187c478bd9Sstevel@tonic-gate 	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
6197c478bd9Sstevel@tonic-gate 	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
6207c478bd9Sstevel@tonic-gate 	as_avlinit(as);
6217c478bd9Sstevel@tonic-gate 	return (0);
6227c478bd9Sstevel@tonic-gate }
6237c478bd9Sstevel@tonic-gate 
6247c478bd9Sstevel@tonic-gate /*ARGSUSED1*/
6257c478bd9Sstevel@tonic-gate static void
6267c478bd9Sstevel@tonic-gate as_destructor(void *buf, void *cdrarg)
6277c478bd9Sstevel@tonic-gate {
6287c478bd9Sstevel@tonic-gate 	struct as *as = buf;
6297c478bd9Sstevel@tonic-gate 
6307c478bd9Sstevel@tonic-gate 	avl_destroy(&as->a_segtree);
6317c478bd9Sstevel@tonic-gate 	mutex_destroy(&as->a_contents);
6327c478bd9Sstevel@tonic-gate 	cv_destroy(&as->a_cv);
6337c478bd9Sstevel@tonic-gate 	rw_destroy(&as->a_lock);
6347c478bd9Sstevel@tonic-gate }
6357c478bd9Sstevel@tonic-gate 
6367c478bd9Sstevel@tonic-gate void
6377c478bd9Sstevel@tonic-gate as_init(void)
6387c478bd9Sstevel@tonic-gate {
6397c478bd9Sstevel@tonic-gate 	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
640843e1988Sjohnlev 	    as_constructor, as_destructor, NULL, NULL, NULL, 0);
6417c478bd9Sstevel@tonic-gate }
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate /*
6447c478bd9Sstevel@tonic-gate  * Allocate and initialize an address space data structure.
6457c478bd9Sstevel@tonic-gate  * We call hat_alloc to allow any machine dependent
6467c478bd9Sstevel@tonic-gate  * information in the hat structure to be initialized.
6477c478bd9Sstevel@tonic-gate  */
6487c478bd9Sstevel@tonic-gate struct as *
6497c478bd9Sstevel@tonic-gate as_alloc(void)
6507c478bd9Sstevel@tonic-gate {
6517c478bd9Sstevel@tonic-gate 	struct as *as;
6527c478bd9Sstevel@tonic-gate 
6537c478bd9Sstevel@tonic-gate 	as = kmem_cache_alloc(as_cache, KM_SLEEP);
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 	as->a_flags		= 0;
6567c478bd9Sstevel@tonic-gate 	as->a_vbits		= 0;
6577c478bd9Sstevel@tonic-gate 	as->a_hrm		= NULL;
6587c478bd9Sstevel@tonic-gate 	as->a_seglast		= NULL;
6597c478bd9Sstevel@tonic-gate 	as->a_size		= 0;
66040688216SSudheer A 	as->a_resvsize		= 0;
6617c478bd9Sstevel@tonic-gate 	as->a_updatedir		= 0;
6627c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
6637c478bd9Sstevel@tonic-gate 	as->a_objectdir		= NULL;
6647c478bd9Sstevel@tonic-gate 	as->a_sizedir		= 0;
6657c478bd9Sstevel@tonic-gate 	as->a_userlimit		= (caddr_t)USERLIMIT;
6667c478bd9Sstevel@tonic-gate 	as->a_lastgap		= NULL;
6677c478bd9Sstevel@tonic-gate 	as->a_lastgaphl		= NULL;
6687c478bd9Sstevel@tonic-gate 	as->a_callbacks		= NULL;
6696430daecSBrandon Baker 	as->a_proc		= NULL;
6707c478bd9Sstevel@tonic-gate 
671dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
6727c478bd9Sstevel@tonic-gate 	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
673dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
6747c478bd9Sstevel@tonic-gate 
6757c478bd9Sstevel@tonic-gate 	return (as);
6767c478bd9Sstevel@tonic-gate }
6777c478bd9Sstevel@tonic-gate 
6787c478bd9Sstevel@tonic-gate /*
6797c478bd9Sstevel@tonic-gate  * Free an address space data structure.
6807c478bd9Sstevel@tonic-gate  * Need to free the hat first and then
6817c478bd9Sstevel@tonic-gate  * all the segments on this as and finally
6827c478bd9Sstevel@tonic-gate  * the space for the as struct itself.
6837c478bd9Sstevel@tonic-gate  */
6847c478bd9Sstevel@tonic-gate void
6857c478bd9Sstevel@tonic-gate as_free(struct as *as)
6867c478bd9Sstevel@tonic-gate {
6877c478bd9Sstevel@tonic-gate 	struct hat *hat = as->a_hat;
6887c478bd9Sstevel@tonic-gate 	struct seg *seg, *next;
6890d5ae8c1SJosef 'Jeff' Sipek 	boolean_t free_started = B_FALSE;
6907c478bd9Sstevel@tonic-gate 
6917c478bd9Sstevel@tonic-gate top:
6927c478bd9Sstevel@tonic-gate 	/*
6937c478bd9Sstevel@tonic-gate 	 * Invoke ALL callbacks. as_do_callbacks will do one callback
6947c478bd9Sstevel@tonic-gate 	 * per call, and not return (-1) until the callback has completed.
6957c478bd9Sstevel@tonic-gate 	 * When as_do_callbacks returns zero, all callbacks have completed.
6967c478bd9Sstevel@tonic-gate 	 */
6977c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
698843e1988Sjohnlev 	while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0))
699843e1988Sjohnlev 		;
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
702dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
7037c478bd9Sstevel@tonic-gate 
7040d5ae8c1SJosef 'Jeff' Sipek 	if (!free_started) {
7050d5ae8c1SJosef 'Jeff' Sipek 		free_started = B_TRUE;
7067c478bd9Sstevel@tonic-gate 		hat_free_start(hat);
7077c478bd9Sstevel@tonic-gate 	}
7087c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
7097c478bd9Sstevel@tonic-gate 		int err;
7107c478bd9Sstevel@tonic-gate 
7117c478bd9Sstevel@tonic-gate 		next = AS_SEGNEXT(as, seg);
712a98e9dbfSaguzovsk retry:
7137c478bd9Sstevel@tonic-gate 		err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
7147c478bd9Sstevel@tonic-gate 		if (err == EAGAIN) {
7157c478bd9Sstevel@tonic-gate 			mutex_enter(&as->a_contents);
7167c478bd9Sstevel@tonic-gate 			if (as->a_callbacks) {
717dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
718a98e9dbfSaguzovsk 			} else if (!AS_ISNOUNMAPWAIT(as)) {
7197c478bd9Sstevel@tonic-gate 				/*
7207c478bd9Sstevel@tonic-gate 				 * Memory is currently locked. Wait for a
7217c478bd9Sstevel@tonic-gate 				 * cv_signal that it has been unlocked, then
7227c478bd9Sstevel@tonic-gate 				 * try the operation again.
7237c478bd9Sstevel@tonic-gate 				 */
7247c478bd9Sstevel@tonic-gate 				if (AS_ISUNMAPWAIT(as) == 0)
7257c478bd9Sstevel@tonic-gate 					cv_broadcast(&as->a_cv);
7267c478bd9Sstevel@tonic-gate 				AS_SETUNMAPWAIT(as);
727dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
7287c478bd9Sstevel@tonic-gate 				while (AS_ISUNMAPWAIT(as))
7297c478bd9Sstevel@tonic-gate 					cv_wait(&as->a_cv, &as->a_contents);
730a98e9dbfSaguzovsk 			} else {
731a98e9dbfSaguzovsk 				/*
732a98e9dbfSaguzovsk 				 * We may have raced with
733a98e9dbfSaguzovsk 				 * segvn_reclaim()/segspt_reclaim(). In this
734a98e9dbfSaguzovsk 				 * case clean nounmapwait flag and retry since
735a98e9dbfSaguzovsk 				 * softlockcnt in this segment may be already
736a98e9dbfSaguzovsk 				 * 0.  We don't drop as writer lock so our
737a98e9dbfSaguzovsk 				 * number of retries without sleeping should
738a98e9dbfSaguzovsk 				 * be very small. See segvn_reclaim() for
739a98e9dbfSaguzovsk 				 * more comments.
740a98e9dbfSaguzovsk 				 */
741a98e9dbfSaguzovsk 				AS_CLRNOUNMAPWAIT(as);
742a98e9dbfSaguzovsk 				mutex_exit(&as->a_contents);
743a98e9dbfSaguzovsk 				goto retry;
7447c478bd9Sstevel@tonic-gate 			}
7457c478bd9Sstevel@tonic-gate 			mutex_exit(&as->a_contents);
7467c478bd9Sstevel@tonic-gate 			goto top;
7477c478bd9Sstevel@tonic-gate 		} else {
7487c478bd9Sstevel@tonic-gate 			/*
7497c478bd9Sstevel@tonic-gate 			 * We do not expect any other error return at this
7507c478bd9Sstevel@tonic-gate 			 * time. This is similar to an ASSERT in seg_unmap()
7517c478bd9Sstevel@tonic-gate 			 */
7527c478bd9Sstevel@tonic-gate 			ASSERT(err == 0);
7537c478bd9Sstevel@tonic-gate 		}
7547c478bd9Sstevel@tonic-gate 	}
7557c478bd9Sstevel@tonic-gate 	hat_free_end(hat);
756dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
7577c478bd9Sstevel@tonic-gate 
7587c478bd9Sstevel@tonic-gate 	/* /proc stuff */
7597c478bd9Sstevel@tonic-gate 	ASSERT(avl_numnodes(&as->a_wpage) == 0);
7607c478bd9Sstevel@tonic-gate 	if (as->a_objectdir) {
7617c478bd9Sstevel@tonic-gate 		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
7627c478bd9Sstevel@tonic-gate 		as->a_objectdir = NULL;
7637c478bd9Sstevel@tonic-gate 		as->a_sizedir = 0;
7647c478bd9Sstevel@tonic-gate 	}
7657c478bd9Sstevel@tonic-gate 
7667c478bd9Sstevel@tonic-gate 	/*
7677c478bd9Sstevel@tonic-gate 	 * Free the struct as back to kmem.  Assert it has no segments.
7687c478bd9Sstevel@tonic-gate 	 */
7697c478bd9Sstevel@tonic-gate 	ASSERT(avl_numnodes(&as->a_segtree) == 0);
7707c478bd9Sstevel@tonic-gate 	kmem_cache_free(as_cache, as);
7717c478bd9Sstevel@tonic-gate }
7727c478bd9Sstevel@tonic-gate 
7737c478bd9Sstevel@tonic-gate int
774bb5ca623SVamsi Nagineni as_dup(struct as *as, struct proc *forkedproc)
7757c478bd9Sstevel@tonic-gate {
7767c478bd9Sstevel@tonic-gate 	struct as *newas;
7777c478bd9Sstevel@tonic-gate 	struct seg *seg, *newseg;
77840688216SSudheer A 	size_t	purgesize = 0;
7797c478bd9Sstevel@tonic-gate 	int error;
7807c478bd9Sstevel@tonic-gate 
781dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
7827c478bd9Sstevel@tonic-gate 	as_clearwatch(as);
7837c478bd9Sstevel@tonic-gate 	newas = as_alloc();
7847c478bd9Sstevel@tonic-gate 	newas->a_userlimit = as->a_userlimit;
785bb5ca623SVamsi Nagineni 	newas->a_proc = forkedproc;
786250f6acdSvk 
787dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(newas, RW_WRITER);
7887c478bd9Sstevel@tonic-gate 
78905d3dc4bSpaulsan 	(void) hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_SRD);
7907c478bd9Sstevel@tonic-gate 
7917c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
7927c478bd9Sstevel@tonic-gate 
79340688216SSudheer A 		if (seg->s_flags & S_PURGE) {
79440688216SSudheer A 			purgesize += seg->s_size;
7957c478bd9Sstevel@tonic-gate 			continue;
79640688216SSudheer A 		}
7977c478bd9Sstevel@tonic-gate 
7987c478bd9Sstevel@tonic-gate 		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
7997c478bd9Sstevel@tonic-gate 		if (newseg == NULL) {
800dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(newas);
8017c478bd9Sstevel@tonic-gate 			as_setwatch(as);
802dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
8037c478bd9Sstevel@tonic-gate 			as_free(newas);
8047c478bd9Sstevel@tonic-gate 			return (-1);
8057c478bd9Sstevel@tonic-gate 		}
8067c478bd9Sstevel@tonic-gate 		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
8077c478bd9Sstevel@tonic-gate 			/*
8087c478bd9Sstevel@tonic-gate 			 * We call seg_free() on the new seg
8097c478bd9Sstevel@tonic-gate 			 * because the segment is not set up
8107c478bd9Sstevel@tonic-gate 			 * completely; i.e. it has no ops.
8117c478bd9Sstevel@tonic-gate 			 */
8127c478bd9Sstevel@tonic-gate 			as_setwatch(as);
813dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
8147c478bd9Sstevel@tonic-gate 			seg_free(newseg);
815dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(newas);
8167c478bd9Sstevel@tonic-gate 			as_free(newas);
8177c478bd9Sstevel@tonic-gate 			return (error);
8187c478bd9Sstevel@tonic-gate 		}
819*284ce987SPatrick Mooney 		if ((newseg->s_flags & S_HOLE) == 0) {
820*284ce987SPatrick Mooney 			newas->a_size += seg->s_size;
821*284ce987SPatrick Mooney 		}
8227c478bd9Sstevel@tonic-gate 	}
82340688216SSudheer A 	newas->a_resvsize = as->a_resvsize - purgesize;
8247c478bd9Sstevel@tonic-gate 
8257c478bd9Sstevel@tonic-gate 	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
8267c478bd9Sstevel@tonic-gate 
827dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(newas);
8287c478bd9Sstevel@tonic-gate 
8297c478bd9Sstevel@tonic-gate 	as_setwatch(as);
830dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
8317c478bd9Sstevel@tonic-gate 	if (error != 0) {
8327c478bd9Sstevel@tonic-gate 		as_free(newas);
8337c478bd9Sstevel@tonic-gate 		return (error);
8347c478bd9Sstevel@tonic-gate 	}
835bb5ca623SVamsi Nagineni 	forkedproc->p_as = newas;
8367c478bd9Sstevel@tonic-gate 	return (0);
8377c478bd9Sstevel@tonic-gate }
8387c478bd9Sstevel@tonic-gate 
8397c478bd9Sstevel@tonic-gate /*
8407c478bd9Sstevel@tonic-gate  * Handle a ``fault'' at addr for size bytes.
8417c478bd9Sstevel@tonic-gate  */
8427c478bd9Sstevel@tonic-gate faultcode_t
8437c478bd9Sstevel@tonic-gate as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
8446430daecSBrandon Baker     enum fault_type type, enum seg_rw rw)
8457c478bd9Sstevel@tonic-gate {
8467c478bd9Sstevel@tonic-gate 	struct seg *seg;
8477c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
8487c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
8497c478bd9Sstevel@tonic-gate 	size_t ssize;
8507c478bd9Sstevel@tonic-gate 	faultcode_t res = 0;
8517c478bd9Sstevel@tonic-gate 	caddr_t addrsav;
8527c478bd9Sstevel@tonic-gate 	struct seg *segsav;
8537c478bd9Sstevel@tonic-gate 	int as_lock_held;
8547c478bd9Sstevel@tonic-gate 	klwp_t *lwp = ttolwp(curthread);
8557c478bd9Sstevel@tonic-gate 
8567c478bd9Sstevel@tonic-gate 
8577c478bd9Sstevel@tonic-gate 
8587c478bd9Sstevel@tonic-gate retry:
8590d5ae8c1SJosef 'Jeff' Sipek 	/*
8600d5ae8c1SJosef 'Jeff' Sipek 	 * Indicate that the lwp is not to be stopped while waiting for a
8610d5ae8c1SJosef 'Jeff' Sipek 	 * pagefault.  This is to avoid deadlock while debugging a process
8620d5ae8c1SJosef 'Jeff' Sipek 	 * via /proc over NFS (in particular).
8630d5ae8c1SJosef 'Jeff' Sipek 	 */
8640d5ae8c1SJosef 'Jeff' Sipek 	if (lwp != NULL)
8650d5ae8c1SJosef 'Jeff' Sipek 		lwp->lwp_nostop++;
8667c478bd9Sstevel@tonic-gate 
8670d5ae8c1SJosef 'Jeff' Sipek 	/*
8680d5ae8c1SJosef 'Jeff' Sipek 	 * same length must be used when we softlock and softunlock.  We
8690d5ae8c1SJosef 'Jeff' Sipek 	 * don't support softunlocking lengths less than the original length
8700d5ae8c1SJosef 'Jeff' Sipek 	 * when there is largepage support.  See seg_dev.c for more
8710d5ae8c1SJosef 'Jeff' Sipek 	 * comments.
8720d5ae8c1SJosef 'Jeff' Sipek 	 */
8730d5ae8c1SJosef 'Jeff' Sipek 	switch (type) {
8747c478bd9Sstevel@tonic-gate 
8750d5ae8c1SJosef 'Jeff' Sipek 	case F_SOFTLOCK:
8760d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ADD_K(vm, softlock, 1);
8770d5ae8c1SJosef 'Jeff' Sipek 		break;
8787c478bd9Sstevel@tonic-gate 
8790d5ae8c1SJosef 'Jeff' Sipek 	case F_SOFTUNLOCK:
8800d5ae8c1SJosef 'Jeff' Sipek 		break;
8817c478bd9Sstevel@tonic-gate 
8820d5ae8c1SJosef 'Jeff' Sipek 	case F_PROT:
8830d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ADD_K(vm, prot_fault, 1);
8840d5ae8c1SJosef 'Jeff' Sipek 		break;
8857c478bd9Sstevel@tonic-gate 
8860d5ae8c1SJosef 'Jeff' Sipek 	case F_INVAL:
8870d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ENTER_K();
8880d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
8890d5ae8c1SJosef 'Jeff' Sipek 		if (as == &kas)
8900d5ae8c1SJosef 'Jeff' Sipek 			CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
8910d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_EXIT_K();
8920d5ae8c1SJosef 'Jeff' Sipek 		break;
8937c478bd9Sstevel@tonic-gate 	}
8947c478bd9Sstevel@tonic-gate 
8957c478bd9Sstevel@tonic-gate 	/* Kernel probe */
8967c478bd9Sstevel@tonic-gate 	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
897843e1988Sjohnlev 	    tnf_opaque,	address,	addr,
898843e1988Sjohnlev 	    tnf_fault_type,	fault_type,	type,
899843e1988Sjohnlev 	    tnf_seg_access,	access,		rw);
9007c478bd9Sstevel@tonic-gate 
9017c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
9027c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
903843e1988Sjohnlev 	    (size_t)raddr;
9047c478bd9Sstevel@tonic-gate 
9057c478bd9Sstevel@tonic-gate 	/*
9067c478bd9Sstevel@tonic-gate 	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
9077c478bd9Sstevel@tonic-gate 	 * correctness, but then we could be stuck holding this lock for
9087c478bd9Sstevel@tonic-gate 	 * a LONG time if the fault needs to be resolved on a slow
9097c478bd9Sstevel@tonic-gate 	 * filesystem, and then no-one will be able to exec new commands,
9107c478bd9Sstevel@tonic-gate 	 * as exec'ing requires the write lock on the as.
9117c478bd9Sstevel@tonic-gate 	 */
9127c478bd9Sstevel@tonic-gate 	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
9137c478bd9Sstevel@tonic-gate 	    raddr + size < segkmap->s_base + segkmap->s_size) {
9147c478bd9Sstevel@tonic-gate 		seg = segkmap;
9157c478bd9Sstevel@tonic-gate 		as_lock_held = 0;
9167c478bd9Sstevel@tonic-gate 	} else {
917dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_READER);
9187c478bd9Sstevel@tonic-gate 
9197c478bd9Sstevel@tonic-gate 		seg = as_segat(as, raddr);
9207c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
921dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
9220d5ae8c1SJosef 'Jeff' Sipek 			if (lwp != NULL)
9237c478bd9Sstevel@tonic-gate 				lwp->lwp_nostop--;
9247c478bd9Sstevel@tonic-gate 			return (FC_NOMAP);
9257c478bd9Sstevel@tonic-gate 		}
9267c478bd9Sstevel@tonic-gate 
9277c478bd9Sstevel@tonic-gate 		as_lock_held = 1;
9287c478bd9Sstevel@tonic-gate 	}
9297c478bd9Sstevel@tonic-gate 
9307c478bd9Sstevel@tonic-gate 	addrsav = raddr;
9317c478bd9Sstevel@tonic-gate 	segsav = seg;
9327c478bd9Sstevel@tonic-gate 
9337c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
9347c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
9357c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
9367c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
9377c478bd9Sstevel@tonic-gate 				res = FC_NOMAP;
9387c478bd9Sstevel@tonic-gate 				break;
9397c478bd9Sstevel@tonic-gate 			}
9407c478bd9Sstevel@tonic-gate 		}
9417c478bd9Sstevel@tonic-gate 		if (raddr + rsize > seg->s_base + seg->s_size)
9427c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
9437c478bd9Sstevel@tonic-gate 		else
9447c478bd9Sstevel@tonic-gate 			ssize = rsize;
9457c478bd9Sstevel@tonic-gate 
9460d5ae8c1SJosef 'Jeff' Sipek 		res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
9470d5ae8c1SJosef 'Jeff' Sipek 		if (res != 0)
9487c478bd9Sstevel@tonic-gate 			break;
9497c478bd9Sstevel@tonic-gate 	}
9507c478bd9Sstevel@tonic-gate 
9517c478bd9Sstevel@tonic-gate 	/*
9527c478bd9Sstevel@tonic-gate 	 * If we were SOFTLOCKing and encountered a failure,
9537c478bd9Sstevel@tonic-gate 	 * we must SOFTUNLOCK the range we already did. (Maybe we
9547c478bd9Sstevel@tonic-gate 	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
9557c478bd9Sstevel@tonic-gate 	 * right here...)
9567c478bd9Sstevel@tonic-gate 	 */
9577c478bd9Sstevel@tonic-gate 	if (res != 0 && type == F_SOFTLOCK) {
9587c478bd9Sstevel@tonic-gate 		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
9597c478bd9Sstevel@tonic-gate 			if (addrsav >= seg->s_base + seg->s_size)
9607c478bd9Sstevel@tonic-gate 				seg = AS_SEGNEXT(as, seg);
9617c478bd9Sstevel@tonic-gate 			ASSERT(seg != NULL);
9627c478bd9Sstevel@tonic-gate 			/*
9637c478bd9Sstevel@tonic-gate 			 * Now call the fault routine again to perform the
9647c478bd9Sstevel@tonic-gate 			 * unlock using S_OTHER instead of the rw variable
9657c478bd9Sstevel@tonic-gate 			 * since we never got a chance to touch the pages.
9667c478bd9Sstevel@tonic-gate 			 */
9677c478bd9Sstevel@tonic-gate 			if (raddr > seg->s_base + seg->s_size)
9687c478bd9Sstevel@tonic-gate 				ssize = seg->s_base + seg->s_size - addrsav;
9697c478bd9Sstevel@tonic-gate 			else
9707c478bd9Sstevel@tonic-gate 				ssize = raddr - addrsav;
9717c478bd9Sstevel@tonic-gate 			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
9727c478bd9Sstevel@tonic-gate 			    F_SOFTUNLOCK, S_OTHER);
9737c478bd9Sstevel@tonic-gate 		}
9747c478bd9Sstevel@tonic-gate 	}
9757c478bd9Sstevel@tonic-gate 	if (as_lock_held)
976dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
9770d5ae8c1SJosef 'Jeff' Sipek 	if (lwp != NULL)
9787c478bd9Sstevel@tonic-gate 		lwp->lwp_nostop--;
9798548bf79Snr 
9807c478bd9Sstevel@tonic-gate 	/*
9817c478bd9Sstevel@tonic-gate 	 * If the lower levels returned EDEADLK for a fault,
9827c478bd9Sstevel@tonic-gate 	 * It means that we should retry the fault.  Let's wait
9837c478bd9Sstevel@tonic-gate 	 * a bit also to let the deadlock causing condition clear.
9847c478bd9Sstevel@tonic-gate 	 * This is part of a gross hack to work around a design flaw
9857c478bd9Sstevel@tonic-gate 	 * in the ufs/sds logging code and should go away when the
9867c478bd9Sstevel@tonic-gate 	 * logging code is re-designed to fix the problem. See bug
9877c478bd9Sstevel@tonic-gate 	 * 4125102 for details of the problem.
9887c478bd9Sstevel@tonic-gate 	 */
9897c478bd9Sstevel@tonic-gate 	if (FC_ERRNO(res) == EDEADLK) {
9907c478bd9Sstevel@tonic-gate 		delay(deadlk_wait);
9917c478bd9Sstevel@tonic-gate 		res = 0;
9927c478bd9Sstevel@tonic-gate 		goto retry;
9937c478bd9Sstevel@tonic-gate 	}
9947c478bd9Sstevel@tonic-gate 	return (res);
9957c478bd9Sstevel@tonic-gate }
9967c478bd9Sstevel@tonic-gate 
9977c478bd9Sstevel@tonic-gate 
9987c478bd9Sstevel@tonic-gate 
9997c478bd9Sstevel@tonic-gate /*
10007c478bd9Sstevel@tonic-gate  * Asynchronous ``fault'' at addr for size bytes.
10017c478bd9Sstevel@tonic-gate  */
10027c478bd9Sstevel@tonic-gate faultcode_t
10037c478bd9Sstevel@tonic-gate as_faulta(struct as *as, caddr_t addr, size_t size)
10047c478bd9Sstevel@tonic-gate {
10057c478bd9Sstevel@tonic-gate 	struct seg *seg;
10067c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
10077c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
10087c478bd9Sstevel@tonic-gate 	faultcode_t res = 0;
10097c478bd9Sstevel@tonic-gate 	klwp_t *lwp = ttolwp(curthread);
10107c478bd9Sstevel@tonic-gate 
10117c478bd9Sstevel@tonic-gate retry:
10127c478bd9Sstevel@tonic-gate 	/*
10137c478bd9Sstevel@tonic-gate 	 * Indicate that the lwp is not to be stopped while waiting
10147c478bd9Sstevel@tonic-gate 	 * for a pagefault.  This is to avoid deadlock while debugging
10157c478bd9Sstevel@tonic-gate 	 * a process via /proc over NFS (in particular).
10167c478bd9Sstevel@tonic-gate 	 */
10178548bf79Snr 	if (lwp != NULL)
10187c478bd9Sstevel@tonic-gate 		lwp->lwp_nostop++;
10197c478bd9Sstevel@tonic-gate 
10207c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
10217c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1022843e1988Sjohnlev 	    (size_t)raddr;
10237c478bd9Sstevel@tonic-gate 
1024dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
10257c478bd9Sstevel@tonic-gate 	seg = as_segat(as, raddr);
10267c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
1027dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
10288548bf79Snr 		if (lwp != NULL)
10297c478bd9Sstevel@tonic-gate 			lwp->lwp_nostop--;
10307c478bd9Sstevel@tonic-gate 		return (FC_NOMAP);
10317c478bd9Sstevel@tonic-gate 	}
10327c478bd9Sstevel@tonic-gate 
10337c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
10347c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
10357c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
10367c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
10377c478bd9Sstevel@tonic-gate 				res = FC_NOMAP;
10387c478bd9Sstevel@tonic-gate 				break;
10397c478bd9Sstevel@tonic-gate 			}
10407c478bd9Sstevel@tonic-gate 		}
10417c478bd9Sstevel@tonic-gate 		res = SEGOP_FAULTA(seg, raddr);
10427c478bd9Sstevel@tonic-gate 		if (res != 0)
10437c478bd9Sstevel@tonic-gate 			break;
10447c478bd9Sstevel@tonic-gate 	}
1045dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
10468548bf79Snr 	if (lwp != NULL)
10477c478bd9Sstevel@tonic-gate 		lwp->lwp_nostop--;
10487c478bd9Sstevel@tonic-gate 	/*
10497c478bd9Sstevel@tonic-gate 	 * If the lower levels returned EDEADLK for a fault,
10507c478bd9Sstevel@tonic-gate 	 * It means that we should retry the fault.  Let's wait
10517c478bd9Sstevel@tonic-gate 	 * a bit also to let the deadlock causing condition clear.
10527c478bd9Sstevel@tonic-gate 	 * This is part of a gross hack to work around a design flaw
10537c478bd9Sstevel@tonic-gate 	 * in the ufs/sds logging code and should go away when the
10547c478bd9Sstevel@tonic-gate 	 * logging code is re-designed to fix the problem. See bug
10557c478bd9Sstevel@tonic-gate 	 * 4125102 for details of the problem.
10567c478bd9Sstevel@tonic-gate 	 */
10577c478bd9Sstevel@tonic-gate 	if (FC_ERRNO(res) == EDEADLK) {
10587c478bd9Sstevel@tonic-gate 		delay(deadlk_wait);
10597c478bd9Sstevel@tonic-gate 		res = 0;
10607c478bd9Sstevel@tonic-gate 		goto retry;
10617c478bd9Sstevel@tonic-gate 	}
10627c478bd9Sstevel@tonic-gate 	return (res);
10637c478bd9Sstevel@tonic-gate }
10647c478bd9Sstevel@tonic-gate 
10657c478bd9Sstevel@tonic-gate /*
10667c478bd9Sstevel@tonic-gate  * Set the virtual mapping for the interval from [addr : addr + size)
10677c478bd9Sstevel@tonic-gate  * in address space `as' to have the specified protection.
10687c478bd9Sstevel@tonic-gate  * It is ok for the range to cross over several segments,
10697c478bd9Sstevel@tonic-gate  * as long as they are contiguous.
10707c478bd9Sstevel@tonic-gate  */
10717c478bd9Sstevel@tonic-gate int
10727c478bd9Sstevel@tonic-gate as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
10737c478bd9Sstevel@tonic-gate {
10747c478bd9Sstevel@tonic-gate 	struct seg *seg;
10757c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
10767c478bd9Sstevel@tonic-gate 	size_t ssize;
10777c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
10787c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
10797c478bd9Sstevel@tonic-gate 	int error = 0, writer = 0;
10807c478bd9Sstevel@tonic-gate 	caddr_t saveraddr;
10817c478bd9Sstevel@tonic-gate 	size_t saversize;
10827c478bd9Sstevel@tonic-gate 
10837c478bd9Sstevel@tonic-gate setprot_top:
10847c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
10857c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1086843e1988Sjohnlev 	    (size_t)raddr;
10877c478bd9Sstevel@tonic-gate 
10887c478bd9Sstevel@tonic-gate 	if (raddr + rsize < raddr)		/* check for wraparound */
10897c478bd9Sstevel@tonic-gate 		return (ENOMEM);
10907c478bd9Sstevel@tonic-gate 
10917c478bd9Sstevel@tonic-gate 	saveraddr = raddr;
10927c478bd9Sstevel@tonic-gate 	saversize = rsize;
10937c478bd9Sstevel@tonic-gate 
10947c478bd9Sstevel@tonic-gate 	/*
10957c478bd9Sstevel@tonic-gate 	 * Normally we only lock the as as a reader. But
10967c478bd9Sstevel@tonic-gate 	 * if due to setprot the segment driver needs to split
1097da6c28aaSamw 	 * a segment it will return IE_RETRY. Therefore we re-acquire
10987c478bd9Sstevel@tonic-gate 	 * the as lock as a writer so the segment driver can change
10997c478bd9Sstevel@tonic-gate 	 * the seg list. Also the segment driver will return IE_RETRY
11007c478bd9Sstevel@tonic-gate 	 * after it has changed the segment list so we therefore keep
11017c478bd9Sstevel@tonic-gate 	 * locking as a writer. Since these opeartions should be rare
11027c478bd9Sstevel@tonic-gate 	 * want to only lock as a writer when necessary.
11037c478bd9Sstevel@tonic-gate 	 */
11047c478bd9Sstevel@tonic-gate 	if (writer || avl_numnodes(&as->a_wpage) != 0) {
1105dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_WRITER);
11067c478bd9Sstevel@tonic-gate 	} else {
1107dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_READER);
11087c478bd9Sstevel@tonic-gate 	}
11097c478bd9Sstevel@tonic-gate 
11107c478bd9Sstevel@tonic-gate 	as_clearwatchprot(as, raddr, rsize);
11117c478bd9Sstevel@tonic-gate 	seg = as_segat(as, raddr);
11127c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
11137c478bd9Sstevel@tonic-gate 		as_setwatch(as);
1114dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
11157c478bd9Sstevel@tonic-gate 		return (ENOMEM);
11167c478bd9Sstevel@tonic-gate 	}
11177c478bd9Sstevel@tonic-gate 
11187c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
11197c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
11207c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
11217c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
11227c478bd9Sstevel@tonic-gate 				error = ENOMEM;
11237c478bd9Sstevel@tonic-gate 				break;
11247c478bd9Sstevel@tonic-gate 			}
11257c478bd9Sstevel@tonic-gate 		}
11267c478bd9Sstevel@tonic-gate 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
11277c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
11287c478bd9Sstevel@tonic-gate 		else
11297c478bd9Sstevel@tonic-gate 			ssize = rsize;
1130a98e9dbfSaguzovsk retry:
11317c478bd9Sstevel@tonic-gate 		error = SEGOP_SETPROT(seg, raddr, ssize, prot);
11327c478bd9Sstevel@tonic-gate 
11337c478bd9Sstevel@tonic-gate 		if (error == IE_NOMEM) {
11347c478bd9Sstevel@tonic-gate 			error = EAGAIN;
11357c478bd9Sstevel@tonic-gate 			break;
11367c478bd9Sstevel@tonic-gate 		}
11377c478bd9Sstevel@tonic-gate 
11387c478bd9Sstevel@tonic-gate 		if (error == IE_RETRY) {
1139dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
11407c478bd9Sstevel@tonic-gate 			writer = 1;
11417c478bd9Sstevel@tonic-gate 			goto setprot_top;
11427c478bd9Sstevel@tonic-gate 		}
11437c478bd9Sstevel@tonic-gate 
11447c478bd9Sstevel@tonic-gate 		if (error == EAGAIN) {
11457c478bd9Sstevel@tonic-gate 			/*
11467c478bd9Sstevel@tonic-gate 			 * Make sure we have a_lock as writer.
11477c478bd9Sstevel@tonic-gate 			 */
11487c478bd9Sstevel@tonic-gate 			if (writer == 0) {
1149dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
11507c478bd9Sstevel@tonic-gate 				writer = 1;
11517c478bd9Sstevel@tonic-gate 				goto setprot_top;
11527c478bd9Sstevel@tonic-gate 			}
11537c478bd9Sstevel@tonic-gate 
11547c478bd9Sstevel@tonic-gate 			/*
11557c478bd9Sstevel@tonic-gate 			 * Memory is currently locked.  It must be unlocked
11567c478bd9Sstevel@tonic-gate 			 * before this operation can succeed through a retry.
11577c478bd9Sstevel@tonic-gate 			 * The possible reasons for locked memory and
11587c478bd9Sstevel@tonic-gate 			 * corresponding strategies for unlocking are:
11597c478bd9Sstevel@tonic-gate 			 * (1) Normal I/O
11607c478bd9Sstevel@tonic-gate 			 *	wait for a signal that the I/O operation
11617c478bd9Sstevel@tonic-gate 			 *	has completed and the memory is unlocked.
11627c478bd9Sstevel@tonic-gate 			 * (2) Asynchronous I/O
11637c478bd9Sstevel@tonic-gate 			 *	The aio subsystem does not unlock pages when
11647c478bd9Sstevel@tonic-gate 			 *	the I/O is completed. Those pages are unlocked
11657c478bd9Sstevel@tonic-gate 			 *	when the application calls aiowait/aioerror.
11667c478bd9Sstevel@tonic-gate 			 *	So, to prevent blocking forever, cv_broadcast()
11677c478bd9Sstevel@tonic-gate 			 *	is done to wake up aio_cleanup_thread.
11687c478bd9Sstevel@tonic-gate 			 *	Subsequently, segvn_reclaim will be called, and
11697c478bd9Sstevel@tonic-gate 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
11707c478bd9Sstevel@tonic-gate 			 * (3) Long term page locking:
11717c478bd9Sstevel@tonic-gate 			 *	Drivers intending to have pages locked for a
11727c478bd9Sstevel@tonic-gate 			 *	period considerably longer than for normal I/O
11737c478bd9Sstevel@tonic-gate 			 *	(essentially forever) may have registered for a
11747c478bd9Sstevel@tonic-gate 			 *	callback so they may unlock these pages on
11757c478bd9Sstevel@tonic-gate 			 *	request. This is needed to allow this operation
11767c478bd9Sstevel@tonic-gate 			 *	to succeed. Each entry on the callback list is
11777c478bd9Sstevel@tonic-gate 			 *	examined. If the event or address range pertains
11787c478bd9Sstevel@tonic-gate 			 *	the callback is invoked (unless it already is in
11797c478bd9Sstevel@tonic-gate 			 *	progress). The a_contents lock must be dropped
11807c478bd9Sstevel@tonic-gate 			 *	before the callback, so only one callback can
11817c478bd9Sstevel@tonic-gate 			 *	be done at a time. Go to the top and do more
11827c478bd9Sstevel@tonic-gate 			 *	until zero is returned. If zero is returned,
11837c478bd9Sstevel@tonic-gate 			 *	either there were no callbacks for this event
11847c478bd9Sstevel@tonic-gate 			 *	or they were already in progress.
11857c478bd9Sstevel@tonic-gate 			 */
11867c478bd9Sstevel@tonic-gate 			mutex_enter(&as->a_contents);
11877c478bd9Sstevel@tonic-gate 			if (as->a_callbacks &&
1188843e1988Sjohnlev 			    (cb = as_find_callback(as, AS_SETPROT_EVENT,
1189843e1988Sjohnlev 			    seg->s_base, seg->s_size))) {
1190dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
11917c478bd9Sstevel@tonic-gate 				as_execute_callback(as, cb, AS_SETPROT_EVENT);
1192a98e9dbfSaguzovsk 			} else if (!AS_ISNOUNMAPWAIT(as)) {
11937c478bd9Sstevel@tonic-gate 				if (AS_ISUNMAPWAIT(as) == 0)
11947c478bd9Sstevel@tonic-gate 					cv_broadcast(&as->a_cv);
11957c478bd9Sstevel@tonic-gate 				AS_SETUNMAPWAIT(as);
1196dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
11977c478bd9Sstevel@tonic-gate 				while (AS_ISUNMAPWAIT(as))
11987c478bd9Sstevel@tonic-gate 					cv_wait(&as->a_cv, &as->a_contents);
1199a98e9dbfSaguzovsk 			} else {
1200a98e9dbfSaguzovsk 				/*
1201a98e9dbfSaguzovsk 				 * We may have raced with
1202a98e9dbfSaguzovsk 				 * segvn_reclaim()/segspt_reclaim(). In this
1203a98e9dbfSaguzovsk 				 * case clean nounmapwait flag and retry since
1204a98e9dbfSaguzovsk 				 * softlockcnt in this segment may be already
1205a98e9dbfSaguzovsk 				 * 0.  We don't drop as writer lock so our
1206a98e9dbfSaguzovsk 				 * number of retries without sleeping should
1207a98e9dbfSaguzovsk 				 * be very small. See segvn_reclaim() for
1208a98e9dbfSaguzovsk 				 * more comments.
1209a98e9dbfSaguzovsk 				 */
1210a98e9dbfSaguzovsk 				AS_CLRNOUNMAPWAIT(as);
1211a98e9dbfSaguzovsk 				mutex_exit(&as->a_contents);
1212a98e9dbfSaguzovsk 				goto retry;
12137c478bd9Sstevel@tonic-gate 			}
12147c478bd9Sstevel@tonic-gate 			mutex_exit(&as->a_contents);
12157c478bd9Sstevel@tonic-gate 			goto setprot_top;
12167c478bd9Sstevel@tonic-gate 		} else if (error != 0)
12177c478bd9Sstevel@tonic-gate 			break;
12187c478bd9Sstevel@tonic-gate 	}
12197c478bd9Sstevel@tonic-gate 	if (error != 0) {
12207c478bd9Sstevel@tonic-gate 		as_setwatch(as);
12217c478bd9Sstevel@tonic-gate 	} else {
12227c478bd9Sstevel@tonic-gate 		as_setwatchprot(as, saveraddr, saversize, prot);
12237c478bd9Sstevel@tonic-gate 	}
1224dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
12257c478bd9Sstevel@tonic-gate 	return (error);
12267c478bd9Sstevel@tonic-gate }
12277c478bd9Sstevel@tonic-gate 
12287c478bd9Sstevel@tonic-gate /*
12297c478bd9Sstevel@tonic-gate  * Check to make sure that the interval [addr, addr + size)
12307c478bd9Sstevel@tonic-gate  * in address space `as' has at least the specified protection.
12317c478bd9Sstevel@tonic-gate  * It is ok for the range to cross over several segments, as long
12327c478bd9Sstevel@tonic-gate  * as they are contiguous.
12337c478bd9Sstevel@tonic-gate  */
12347c478bd9Sstevel@tonic-gate int
12357c478bd9Sstevel@tonic-gate as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
12367c478bd9Sstevel@tonic-gate {
12377c478bd9Sstevel@tonic-gate 	struct seg *seg;
12387c478bd9Sstevel@tonic-gate 	size_t ssize;
12397c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
12407c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
12417c478bd9Sstevel@tonic-gate 	int error = 0;
12427c478bd9Sstevel@tonic-gate 
12437c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
12447c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1245843e1988Sjohnlev 	    (size_t)raddr;
12467c478bd9Sstevel@tonic-gate 
12477c478bd9Sstevel@tonic-gate 	if (raddr + rsize < raddr)		/* check for wraparound */
12487c478bd9Sstevel@tonic-gate 		return (ENOMEM);
12497c478bd9Sstevel@tonic-gate 
12507c478bd9Sstevel@tonic-gate 	/*
12517c478bd9Sstevel@tonic-gate 	 * This is ugly as sin...
12527c478bd9Sstevel@tonic-gate 	 * Normally, we only acquire the address space readers lock.
12537c478bd9Sstevel@tonic-gate 	 * However, if the address space has watchpoints present,
12547c478bd9Sstevel@tonic-gate 	 * we must acquire the writer lock on the address space for
12557c478bd9Sstevel@tonic-gate 	 * the benefit of as_clearwatchprot() and as_setwatchprot().
12567c478bd9Sstevel@tonic-gate 	 */
12577c478bd9Sstevel@tonic-gate 	if (avl_numnodes(&as->a_wpage) != 0)
1258dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_WRITER);
12597c478bd9Sstevel@tonic-gate 	else
1260dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_READER);
12617c478bd9Sstevel@tonic-gate 	as_clearwatchprot(as, raddr, rsize);
12627c478bd9Sstevel@tonic-gate 	seg = as_segat(as, raddr);
12637c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
12647c478bd9Sstevel@tonic-gate 		as_setwatch(as);
1265dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
12667c478bd9Sstevel@tonic-gate 		return (ENOMEM);
12677c478bd9Sstevel@tonic-gate 	}
12687c478bd9Sstevel@tonic-gate 
12697c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
12707c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
12717c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
12727c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
12737c478bd9Sstevel@tonic-gate 				error = ENOMEM;
12747c478bd9Sstevel@tonic-gate 				break;
12757c478bd9Sstevel@tonic-gate 			}
12767c478bd9Sstevel@tonic-gate 		}
12777c478bd9Sstevel@tonic-gate 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
12787c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
12797c478bd9Sstevel@tonic-gate 		else
12807c478bd9Sstevel@tonic-gate 			ssize = rsize;
12817c478bd9Sstevel@tonic-gate 
12827c478bd9Sstevel@tonic-gate 		error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
12837c478bd9Sstevel@tonic-gate 		if (error != 0)
12847c478bd9Sstevel@tonic-gate 			break;
12857c478bd9Sstevel@tonic-gate 	}
12867c478bd9Sstevel@tonic-gate 	as_setwatch(as);
1287dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
12887c478bd9Sstevel@tonic-gate 	return (error);
12897c478bd9Sstevel@tonic-gate }
12907c478bd9Sstevel@tonic-gate 
12917c478bd9Sstevel@tonic-gate int
12927c478bd9Sstevel@tonic-gate as_unmap(struct as *as, caddr_t addr, size_t size)
12937c478bd9Sstevel@tonic-gate {
12947c478bd9Sstevel@tonic-gate 	struct seg *seg, *seg_next;
12957c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
12967c478bd9Sstevel@tonic-gate 	caddr_t raddr, eaddr;
129740688216SSudheer A 	size_t ssize, rsize = 0;
12987c478bd9Sstevel@tonic-gate 	int err;
12997c478bd9Sstevel@tonic-gate 
13007c478bd9Sstevel@tonic-gate top:
13017c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
13027c478bd9Sstevel@tonic-gate 	eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) &
13037c478bd9Sstevel@tonic-gate 	    (uintptr_t)PAGEMASK);
13047c478bd9Sstevel@tonic-gate 
1305dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
13067c478bd9Sstevel@tonic-gate 
13077c478bd9Sstevel@tonic-gate 	as->a_updatedir = 1;	/* inform /proc */
13087c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
13097c478bd9Sstevel@tonic-gate 
13107c478bd9Sstevel@tonic-gate 	/*
13117c478bd9Sstevel@tonic-gate 	 * Use as_findseg to find the first segment in the range, then
13127c478bd9Sstevel@tonic-gate 	 * step through the segments in order, following s_next.
13137c478bd9Sstevel@tonic-gate 	 */
13147c478bd9Sstevel@tonic-gate 	as_clearwatchprot(as, raddr, eaddr - raddr);
13157c478bd9Sstevel@tonic-gate 
13167c478bd9Sstevel@tonic-gate 	for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
1317*284ce987SPatrick Mooney 		const boolean_t is_hole = ((seg->s_flags & S_HOLE) != 0);
1318*284ce987SPatrick Mooney 
13197c478bd9Sstevel@tonic-gate 		if (eaddr <= seg->s_base)
13207c478bd9Sstevel@tonic-gate 			break;		/* eaddr was in a gap; all done */
13217c478bd9Sstevel@tonic-gate 
13227c478bd9Sstevel@tonic-gate 		/* this is implied by the test above */
13237c478bd9Sstevel@tonic-gate 		ASSERT(raddr < eaddr);
13247c478bd9Sstevel@tonic-gate 
13257c478bd9Sstevel@tonic-gate 		if (raddr < seg->s_base)
13267c478bd9Sstevel@tonic-gate 			raddr = seg->s_base; 	/* raddr was in a gap */
13277c478bd9Sstevel@tonic-gate 
13287c478bd9Sstevel@tonic-gate 		if (eaddr > (seg->s_base + seg->s_size))
13297c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
13307c478bd9Sstevel@tonic-gate 		else
13317c478bd9Sstevel@tonic-gate 			ssize = eaddr - raddr;
13327c478bd9Sstevel@tonic-gate 
13337c478bd9Sstevel@tonic-gate 		/*
13347c478bd9Sstevel@tonic-gate 		 * Save next segment pointer since seg can be
13357c478bd9Sstevel@tonic-gate 		 * destroyed during the segment unmap operation.
13367c478bd9Sstevel@tonic-gate 		 */
13377c478bd9Sstevel@tonic-gate 		seg_next = AS_SEGNEXT(as, seg);
13387c478bd9Sstevel@tonic-gate 
133940688216SSudheer A 		/*
134040688216SSudheer A 		 * We didn't count /dev/null mappings, so ignore them here.
134140688216SSudheer A 		 * We'll handle MAP_NORESERVE cases in segvn_unmap(). (Again,
134240688216SSudheer A 		 * we have to do this check here while we have seg.)
134340688216SSudheer A 		 */
134417965fd8SKrishnendu Sadhukhan - Sun Microsystems 		rsize = 0;
134540688216SSudheer A 		if (!SEG_IS_DEVNULL_MAPPING(seg) &&
134640688216SSudheer A 		    !SEG_IS_PARTIAL_RESV(seg))
134740688216SSudheer A 			rsize = ssize;
134840688216SSudheer A 
1349a98e9dbfSaguzovsk retry:
13507c478bd9Sstevel@tonic-gate 		err = SEGOP_UNMAP(seg, raddr, ssize);
13517c478bd9Sstevel@tonic-gate 		if (err == EAGAIN) {
13527c478bd9Sstevel@tonic-gate 			/*
13537c478bd9Sstevel@tonic-gate 			 * Memory is currently locked.  It must be unlocked
13547c478bd9Sstevel@tonic-gate 			 * before this operation can succeed through a retry.
13557c478bd9Sstevel@tonic-gate 			 * The possible reasons for locked memory and
13567c478bd9Sstevel@tonic-gate 			 * corresponding strategies for unlocking are:
13577c478bd9Sstevel@tonic-gate 			 * (1) Normal I/O
13587c478bd9Sstevel@tonic-gate 			 *	wait for a signal that the I/O operation
13597c478bd9Sstevel@tonic-gate 			 *	has completed and the memory is unlocked.
13607c478bd9Sstevel@tonic-gate 			 * (2) Asynchronous I/O
13617c478bd9Sstevel@tonic-gate 			 *	The aio subsystem does not unlock pages when
13627c478bd9Sstevel@tonic-gate 			 *	the I/O is completed. Those pages are unlocked
13637c478bd9Sstevel@tonic-gate 			 *	when the application calls aiowait/aioerror.
13647c478bd9Sstevel@tonic-gate 			 *	So, to prevent blocking forever, cv_broadcast()
13657c478bd9Sstevel@tonic-gate 			 *	is done to wake up aio_cleanup_thread.
13667c478bd9Sstevel@tonic-gate 			 *	Subsequently, segvn_reclaim will be called, and
13677c478bd9Sstevel@tonic-gate 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
13687c478bd9Sstevel@tonic-gate 			 * (3) Long term page locking:
13697c478bd9Sstevel@tonic-gate 			 *	Drivers intending to have pages locked for a
13707c478bd9Sstevel@tonic-gate 			 *	period considerably longer than for normal I/O
13717c478bd9Sstevel@tonic-gate 			 *	(essentially forever) may have registered for a
13727c478bd9Sstevel@tonic-gate 			 *	callback so they may unlock these pages on
13737c478bd9Sstevel@tonic-gate 			 *	request. This is needed to allow this operation
13747c478bd9Sstevel@tonic-gate 			 *	to succeed. Each entry on the callback list is
13757c478bd9Sstevel@tonic-gate 			 *	examined. If the event or address range pertains
13767c478bd9Sstevel@tonic-gate 			 *	the callback is invoked (unless it already is in
13777c478bd9Sstevel@tonic-gate 			 *	progress). The a_contents lock must be dropped
13787c478bd9Sstevel@tonic-gate 			 *	before the callback, so only one callback can
13797c478bd9Sstevel@tonic-gate 			 *	be done at a time. Go to the top and do more
13807c478bd9Sstevel@tonic-gate 			 *	until zero is returned. If zero is returned,
13817c478bd9Sstevel@tonic-gate 			 *	either there were no callbacks for this event
13827c478bd9Sstevel@tonic-gate 			 *	or they were already in progress.
13837c478bd9Sstevel@tonic-gate 			 */
13847c478bd9Sstevel@tonic-gate 			mutex_enter(&as->a_contents);
13857c478bd9Sstevel@tonic-gate 			if (as->a_callbacks &&
1386843e1988Sjohnlev 			    (cb = as_find_callback(as, AS_UNMAP_EVENT,
1387843e1988Sjohnlev 			    seg->s_base, seg->s_size))) {
1388dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
13897c478bd9Sstevel@tonic-gate 				as_execute_callback(as, cb, AS_UNMAP_EVENT);
1390a98e9dbfSaguzovsk 			} else if (!AS_ISNOUNMAPWAIT(as)) {
13917c478bd9Sstevel@tonic-gate 				if (AS_ISUNMAPWAIT(as) == 0)
13927c478bd9Sstevel@tonic-gate 					cv_broadcast(&as->a_cv);
13937c478bd9Sstevel@tonic-gate 				AS_SETUNMAPWAIT(as);
1394dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
13957c478bd9Sstevel@tonic-gate 				while (AS_ISUNMAPWAIT(as))
13967c478bd9Sstevel@tonic-gate 					cv_wait(&as->a_cv, &as->a_contents);
1397a98e9dbfSaguzovsk 			} else {
1398a98e9dbfSaguzovsk 				/*
1399a98e9dbfSaguzovsk 				 * We may have raced with
1400a98e9dbfSaguzovsk 				 * segvn_reclaim()/segspt_reclaim(). In this
1401a98e9dbfSaguzovsk 				 * case clean nounmapwait flag and retry since
1402a98e9dbfSaguzovsk 				 * softlockcnt in this segment may be already
1403a98e9dbfSaguzovsk 				 * 0.  We don't drop as writer lock so our
1404a98e9dbfSaguzovsk 				 * number of retries without sleeping should
1405a98e9dbfSaguzovsk 				 * be very small. See segvn_reclaim() for
1406a98e9dbfSaguzovsk 				 * more comments.
1407a98e9dbfSaguzovsk 				 */
1408a98e9dbfSaguzovsk 				AS_CLRNOUNMAPWAIT(as);
1409a98e9dbfSaguzovsk 				mutex_exit(&as->a_contents);
1410a98e9dbfSaguzovsk 				goto retry;
14117c478bd9Sstevel@tonic-gate 			}
14127c478bd9Sstevel@tonic-gate 			mutex_exit(&as->a_contents);
14137c478bd9Sstevel@tonic-gate 			goto top;
14147c478bd9Sstevel@tonic-gate 		} else if (err == IE_RETRY) {
1415dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
14167c478bd9Sstevel@tonic-gate 			goto top;
14177c478bd9Sstevel@tonic-gate 		} else if (err) {
14187c478bd9Sstevel@tonic-gate 			as_setwatch(as);
1419dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
14207c478bd9Sstevel@tonic-gate 			return (-1);
14217c478bd9Sstevel@tonic-gate 		}
14227c478bd9Sstevel@tonic-gate 
1423*284ce987SPatrick Mooney 		if (!is_hole) {
1424*284ce987SPatrick Mooney 			as->a_size -= ssize;
1425*284ce987SPatrick Mooney 			if (rsize)
1426*284ce987SPatrick Mooney 				as->a_resvsize -= rsize;
1427*284ce987SPatrick Mooney 		}
14287c478bd9Sstevel@tonic-gate 		raddr += ssize;
14297c478bd9Sstevel@tonic-gate 	}
1430dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
14317c478bd9Sstevel@tonic-gate 	return (0);
14327c478bd9Sstevel@tonic-gate }
14337c478bd9Sstevel@tonic-gate 
14347c478bd9Sstevel@tonic-gate static int
143507b65a64Saguzovsk as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec,
1436*284ce987SPatrick Mooney     segcreate_func_t crfp, struct segvn_crargs *vn_a, boolean_t *segcreated)
14377c478bd9Sstevel@tonic-gate {
1438*284ce987SPatrick Mooney 	uint_t szc, nszc, save_szcvec;
14397c478bd9Sstevel@tonic-gate 	int error;
1440*284ce987SPatrick Mooney 	caddr_t a, eaddr;
14417c478bd9Sstevel@tonic-gate 	size_t pgsz;
1442*284ce987SPatrick Mooney 	const boolean_t do_off = (vn_a->vp != NULL || vn_a->amp != NULL);
14437c478bd9Sstevel@tonic-gate 
1444dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
14457c478bd9Sstevel@tonic-gate 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
14467c478bd9Sstevel@tonic-gate 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
144707b65a64Saguzovsk 	ASSERT(vn_a->vp == NULL || vn_a->amp == NULL);
1448*284ce987SPatrick Mooney 
144907b65a64Saguzovsk 	if (!do_off) {
145007b65a64Saguzovsk 		vn_a->offset = 0;
145107b65a64Saguzovsk 	}
14527c478bd9Sstevel@tonic-gate 
14537c478bd9Sstevel@tonic-gate 	if (szcvec <= 1) {
1454*284ce987SPatrick Mooney 		struct seg *seg, *segref;
1455*284ce987SPatrick Mooney 
1456*284ce987SPatrick Mooney 		seg = segref = seg_alloc(as, addr, size);
14577c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
14587c478bd9Sstevel@tonic-gate 			return (ENOMEM);
14597c478bd9Sstevel@tonic-gate 		}
14607c478bd9Sstevel@tonic-gate 		vn_a->szc = 0;
1461*284ce987SPatrick Mooney 		error = (*crfp)(&seg, vn_a);
14627c478bd9Sstevel@tonic-gate 		if (error != 0) {
1463*284ce987SPatrick Mooney 			VERIFY3P(seg, ==, segref);
14647c478bd9Sstevel@tonic-gate 			seg_free(seg);
14653a30c6acSsusans 		} else {
14663a30c6acSsusans 			as->a_size += size;
146717965fd8SKrishnendu Sadhukhan - Sun Microsystems 			as->a_resvsize += size;
14687c478bd9Sstevel@tonic-gate 		}
14697c478bd9Sstevel@tonic-gate 		return (error);
14707c478bd9Sstevel@tonic-gate 	}
14717c478bd9Sstevel@tonic-gate 
14727c478bd9Sstevel@tonic-gate 	eaddr = addr + size;
14737c478bd9Sstevel@tonic-gate 	save_szcvec = szcvec;
14747c478bd9Sstevel@tonic-gate 	szcvec >>= 1;
14757c478bd9Sstevel@tonic-gate 	szc = 0;
14767c478bd9Sstevel@tonic-gate 	nszc = 0;
14777c478bd9Sstevel@tonic-gate 	while (szcvec) {
14787c478bd9Sstevel@tonic-gate 		if ((szcvec & 0x1) == 0) {
14797c478bd9Sstevel@tonic-gate 			nszc++;
14807c478bd9Sstevel@tonic-gate 			szcvec >>= 1;
14817c478bd9Sstevel@tonic-gate 			continue;
14827c478bd9Sstevel@tonic-gate 		}
14837c478bd9Sstevel@tonic-gate 		nszc++;
14847c478bd9Sstevel@tonic-gate 		pgsz = page_get_pagesize(nszc);
14857c478bd9Sstevel@tonic-gate 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
14867c478bd9Sstevel@tonic-gate 		if (a != addr) {
1487*284ce987SPatrick Mooney 			struct seg *seg, *segref;
1488*284ce987SPatrick Mooney 			size_t segsize;
1489*284ce987SPatrick Mooney 
14907c478bd9Sstevel@tonic-gate 			ASSERT(a < eaddr);
1491*284ce987SPatrick Mooney 
14927c478bd9Sstevel@tonic-gate 			segsize = a - addr;
1493*284ce987SPatrick Mooney 			seg = segref = seg_alloc(as, addr, segsize);
14947c478bd9Sstevel@tonic-gate 			if (seg == NULL) {
14957c478bd9Sstevel@tonic-gate 				return (ENOMEM);
14967c478bd9Sstevel@tonic-gate 			}
14977c478bd9Sstevel@tonic-gate 			vn_a->szc = szc;
1498*284ce987SPatrick Mooney 			error = (*crfp)(&seg, vn_a);
14997c478bd9Sstevel@tonic-gate 			if (error != 0) {
1500*284ce987SPatrick Mooney 				VERIFY3P(seg, ==, segref);
15017c478bd9Sstevel@tonic-gate 				seg_free(seg);
15027c478bd9Sstevel@tonic-gate 				return (error);
15037c478bd9Sstevel@tonic-gate 			}
15043a30c6acSsusans 			as->a_size += segsize;
150517965fd8SKrishnendu Sadhukhan - Sun Microsystems 			as->a_resvsize += segsize;
1506*284ce987SPatrick Mooney 			*segcreated = B_TRUE;
150707b65a64Saguzovsk 			if (do_off) {
150807b65a64Saguzovsk 				vn_a->offset += segsize;
150907b65a64Saguzovsk 			}
15107c478bd9Sstevel@tonic-gate 			addr = a;
15117c478bd9Sstevel@tonic-gate 		}
15127c478bd9Sstevel@tonic-gate 		szc = nszc;
15137c478bd9Sstevel@tonic-gate 		szcvec >>= 1;
15147c478bd9Sstevel@tonic-gate 	}
15157c478bd9Sstevel@tonic-gate 
15167c478bd9Sstevel@tonic-gate 	ASSERT(addr < eaddr);
15177c478bd9Sstevel@tonic-gate 	szcvec = save_szcvec | 1; /* add 8K pages */
15187c478bd9Sstevel@tonic-gate 	while (szcvec) {
15197c478bd9Sstevel@tonic-gate 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
15207c478bd9Sstevel@tonic-gate 		ASSERT(a >= addr);
15217c478bd9Sstevel@tonic-gate 		if (a != addr) {
1522*284ce987SPatrick Mooney 			struct seg *seg, *segref;
1523*284ce987SPatrick Mooney 			size_t segsize;
1524*284ce987SPatrick Mooney 
15257c478bd9Sstevel@tonic-gate 			segsize = a - addr;
1526*284ce987SPatrick Mooney 			seg = segref = seg_alloc(as, addr, segsize);
15277c478bd9Sstevel@tonic-gate 			if (seg == NULL) {
15287c478bd9Sstevel@tonic-gate 				return (ENOMEM);
15297c478bd9Sstevel@tonic-gate 			}
15307c478bd9Sstevel@tonic-gate 			vn_a->szc = szc;
1531*284ce987SPatrick Mooney 			error = (*crfp)(&seg, vn_a);
15327c478bd9Sstevel@tonic-gate 			if (error != 0) {
1533*284ce987SPatrick Mooney 				VERIFY3P(seg, ==, segref);
15347c478bd9Sstevel@tonic-gate 				seg_free(seg);
15357c478bd9Sstevel@tonic-gate 				return (error);
15367c478bd9Sstevel@tonic-gate 			}
15373a30c6acSsusans 			as->a_size += segsize;
153817965fd8SKrishnendu Sadhukhan - Sun Microsystems 			as->a_resvsize += segsize;
1539*284ce987SPatrick Mooney 			*segcreated = B_TRUE;
154007b65a64Saguzovsk 			if (do_off) {
154107b65a64Saguzovsk 				vn_a->offset += segsize;
154207b65a64Saguzovsk 			}
15437c478bd9Sstevel@tonic-gate 			addr = a;
15447c478bd9Sstevel@tonic-gate 		}
15457c478bd9Sstevel@tonic-gate 		szcvec &= ~(1 << szc);
15467c478bd9Sstevel@tonic-gate 		if (szcvec) {
15477c478bd9Sstevel@tonic-gate 			szc = highbit(szcvec) - 1;
15487c478bd9Sstevel@tonic-gate 			pgsz = page_get_pagesize(szc);
15497c478bd9Sstevel@tonic-gate 		}
15507c478bd9Sstevel@tonic-gate 	}
15517c478bd9Sstevel@tonic-gate 	ASSERT(addr == eaddr);
15527c478bd9Sstevel@tonic-gate 
155307b65a64Saguzovsk 	return (0);
155407b65a64Saguzovsk }
155507b65a64Saguzovsk 
155607b65a64Saguzovsk static int
155707b65a64Saguzovsk as_map_vnsegs(struct as *as, caddr_t addr, size_t size,
1558*284ce987SPatrick Mooney     segcreate_func_t crfp, struct segvn_crargs *vn_a, boolean_t *segcreated)
155907b65a64Saguzovsk {
1560ec25b48fSsusans 	uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA);
1561ec25b48fSsusans 	int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
1562ec25b48fSsusans 	uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1563ec25b48fSsusans 	    type, 0);
156407b65a64Saguzovsk 	int error;
156507b65a64Saguzovsk 	struct vattr va;
156607b65a64Saguzovsk 	u_offset_t eoff;
156707b65a64Saguzovsk 	size_t save_size = 0;
15682cb27123Saguzovsk 	extern size_t textrepl_size_thresh;
156907b65a64Saguzovsk 
1570dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
157107b65a64Saguzovsk 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
157207b65a64Saguzovsk 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
157307b65a64Saguzovsk 	ASSERT(vn_a->vp != NULL);
157407b65a64Saguzovsk 	ASSERT(vn_a->amp == NULL);
157507b65a64Saguzovsk 
157607b65a64Saguzovsk again:
157707b65a64Saguzovsk 	if (szcvec <= 1) {
1578*284ce987SPatrick Mooney 		struct seg *seg, *segref;
1579*284ce987SPatrick Mooney 
1580*284ce987SPatrick Mooney 		seg = segref = seg_alloc(as, addr, size);
158107b65a64Saguzovsk 		if (seg == NULL) {
158207b65a64Saguzovsk 			return (ENOMEM);
158307b65a64Saguzovsk 		}
158407b65a64Saguzovsk 		vn_a->szc = 0;
1585*284ce987SPatrick Mooney 		error = (*crfp)(&seg, vn_a);
158607b65a64Saguzovsk 		if (error != 0) {
1587*284ce987SPatrick Mooney 			VERIFY3P(seg, ==, segref);
158807b65a64Saguzovsk 			seg_free(seg);
15893a30c6acSsusans 		} else {
15903a30c6acSsusans 			as->a_size += size;
159117965fd8SKrishnendu Sadhukhan - Sun Microsystems 			as->a_resvsize += size;
159207b65a64Saguzovsk 		}
159307b65a64Saguzovsk 		return (error);
159407b65a64Saguzovsk 	}
159507b65a64Saguzovsk 
159607b65a64Saguzovsk 	va.va_mask = AT_SIZE;
1597da6c28aaSamw 	if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred, NULL) != 0) {
159807b65a64Saguzovsk 		szcvec = 0;
159907b65a64Saguzovsk 		goto again;
160007b65a64Saguzovsk 	}
160107b65a64Saguzovsk 	eoff = vn_a->offset & PAGEMASK;
160207b65a64Saguzovsk 	if (eoff >= va.va_size) {
160307b65a64Saguzovsk 		szcvec = 0;
160407b65a64Saguzovsk 		goto again;
160507b65a64Saguzovsk 	}
160607b65a64Saguzovsk 	eoff += size;
160707b65a64Saguzovsk 	if (btopr(va.va_size) < btopr(eoff)) {
160807b65a64Saguzovsk 		save_size = size;
160907b65a64Saguzovsk 		size = va.va_size - (vn_a->offset & PAGEMASK);
161007b65a64Saguzovsk 		size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t);
1611ec25b48fSsusans 		szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags,
1612ec25b48fSsusans 		    type, 0);
161307b65a64Saguzovsk 		if (szcvec <= 1) {
161407b65a64Saguzovsk 			size = save_size;
161507b65a64Saguzovsk 			goto again;
161607b65a64Saguzovsk 		}
161707b65a64Saguzovsk 	}
161807b65a64Saguzovsk 
16192cb27123Saguzovsk 	if (size > textrepl_size_thresh) {
16202cb27123Saguzovsk 		vn_a->flags |= _MAP_TEXTREPL;
16212cb27123Saguzovsk 	}
162207b65a64Saguzovsk 	error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a,
162307b65a64Saguzovsk 	    segcreated);
162407b65a64Saguzovsk 	if (error != 0) {
162507b65a64Saguzovsk 		return (error);
162607b65a64Saguzovsk 	}
16277c478bd9Sstevel@tonic-gate 	if (save_size) {
162807b65a64Saguzovsk 		addr += size;
16297c478bd9Sstevel@tonic-gate 		size = save_size - size;
163007b65a64Saguzovsk 		szcvec = 0;
16317c478bd9Sstevel@tonic-gate 		goto again;
16327c478bd9Sstevel@tonic-gate 	}
16337c478bd9Sstevel@tonic-gate 	return (0);
16347c478bd9Sstevel@tonic-gate }
16357c478bd9Sstevel@tonic-gate 
1636ec25b48fSsusans /*
1637ec25b48fSsusans  * as_map_ansegs: shared or private anonymous memory.  Note that the flags
1638ec25b48fSsusans  * passed to map_pgszvec cannot be MAP_INITDATA, for anon.
1639ec25b48fSsusans  */
164007b65a64Saguzovsk static int
1641ec25b48fSsusans as_map_ansegs(struct as *as, caddr_t addr, size_t size,
1642*284ce987SPatrick Mooney     segcreate_func_t crfp, struct segvn_crargs *vn_a, boolean_t *segcreated)
164307b65a64Saguzovsk {
1644ec25b48fSsusans 	uint_t szcvec;
1645ec25b48fSsusans 	uchar_t type;
1646ec25b48fSsusans 
1647ec25b48fSsusans 	ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE);
1648ec25b48fSsusans 	if (vn_a->type == MAP_SHARED) {
1649ec25b48fSsusans 		type = MAPPGSZC_SHM;
1650ec25b48fSsusans 	} else if (vn_a->type == MAP_PRIVATE) {
1651ec25b48fSsusans 		if (vn_a->szc == AS_MAP_HEAP) {
1652ec25b48fSsusans 			type = MAPPGSZC_HEAP;
1653ec25b48fSsusans 		} else if (vn_a->szc == AS_MAP_STACK) {
1654ec25b48fSsusans 			type = MAPPGSZC_STACK;
1655ec25b48fSsusans 		} else {
1656ec25b48fSsusans 			type = MAPPGSZC_PRIVM;
1657ec25b48fSsusans 		}
1658ec25b48fSsusans 	}
1659ec25b48fSsusans 	szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ?
1660ec25b48fSsusans 	    (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE),
1661ec25b48fSsusans 	    (vn_a->flags & MAP_TEXT), type, 0);
1662dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
166307b65a64Saguzovsk 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
166407b65a64Saguzovsk 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
166507b65a64Saguzovsk 	ASSERT(vn_a->vp == NULL);
166607b65a64Saguzovsk 
166707b65a64Saguzovsk 	return (as_map_segvn_segs(as, addr, size, szcvec,
166807b65a64Saguzovsk 	    crfp, vn_a, segcreated));
166907b65a64Saguzovsk }
167007b65a64Saguzovsk 
16717c478bd9Sstevel@tonic-gate int
1672*284ce987SPatrick Mooney as_map(struct as *as, caddr_t addr, size_t size, segcreate_func_t crfp,
1673*284ce987SPatrick Mooney     void *argsp)
167402ff05a9Svsakar {
1675dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
167602ff05a9Svsakar 	return (as_map_locked(as, addr, size, crfp, argsp));
167702ff05a9Svsakar }
167802ff05a9Svsakar 
167902ff05a9Svsakar int
1680*284ce987SPatrick Mooney as_map_locked(struct as *as, caddr_t addr, size_t size, segcreate_func_t crfp,
16816430daecSBrandon Baker     void *argsp)
16827c478bd9Sstevel@tonic-gate {
16837c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
16847c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
16857c478bd9Sstevel@tonic-gate 	int error;
1686*284ce987SPatrick Mooney 	boolean_t is_hole = B_FALSE;
16876430daecSBrandon Baker 	/*
16886430daecSBrandon Baker 	 * The use of a_proc is preferred to handle the case where curproc is
16896430daecSBrandon Baker 	 * a door_call server and is allocating memory in the client's (a_proc)
16906430daecSBrandon Baker 	 * address space.
16916430daecSBrandon Baker 	 * When creating a shared memory segment a_proc will be NULL so we
16926430daecSBrandon Baker 	 * fallback to curproc in that case.
16936430daecSBrandon Baker 	 */
16946430daecSBrandon Baker 	struct proc *p = (as->a_proc == NULL) ? curproc : as->a_proc;
1695232cfe63Ssusans 	struct segvn_crargs crargs;
16967c478bd9Sstevel@tonic-gate 
16977c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
16987c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1699843e1988Sjohnlev 	    (size_t)raddr;
17007c478bd9Sstevel@tonic-gate 
17017c478bd9Sstevel@tonic-gate 	/*
17027c478bd9Sstevel@tonic-gate 	 * check for wrap around
17037c478bd9Sstevel@tonic-gate 	 */
17047c478bd9Sstevel@tonic-gate 	if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
1705dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
17067c478bd9Sstevel@tonic-gate 		return (ENOMEM);
17077c478bd9Sstevel@tonic-gate 	}
17087c478bd9Sstevel@tonic-gate 
17097c478bd9Sstevel@tonic-gate 	as->a_updatedir = 1;	/* inform /proc */
17107c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
17117c478bd9Sstevel@tonic-gate 
1712*284ce987SPatrick Mooney 	if (as != &kas) {
1713*284ce987SPatrick Mooney 		/*
1714*284ce987SPatrick Mooney 		 * Ensure that the virtual size of the process will not exceed
1715*284ce987SPatrick Mooney 		 * the configured limit.  Since seg_hole segments will later
1716*284ce987SPatrick Mooney 		 * set the S_HOLE flag indicating their status as a hole in the
1717*284ce987SPatrick Mooney 		 * AS, they are excluded from this check.
1718*284ce987SPatrick Mooney 		 */
1719*284ce987SPatrick Mooney 		if (as->a_size + rsize > (size_t)p->p_vmem_ctl &&
1720*284ce987SPatrick Mooney 		    !AS_MAP_CHECK_SEGHOLE(crfp)) {
1721*284ce987SPatrick Mooney 			AS_LOCK_EXIT(as);
17227c478bd9Sstevel@tonic-gate 
1723*284ce987SPatrick Mooney 			(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM],
1724*284ce987SPatrick Mooney 			    p->p_rctls, p, RCA_UNSAFE_ALL);
1725*284ce987SPatrick Mooney 			return (ENOMEM);
1726*284ce987SPatrick Mooney 		}
17277c478bd9Sstevel@tonic-gate 	}
17287c478bd9Sstevel@tonic-gate 
1729ec25b48fSsusans 	if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) {
1730*284ce987SPatrick Mooney 		boolean_t do_unmap = B_FALSE;
1731*284ce987SPatrick Mooney 
1732232cfe63Ssusans 		crargs = *(struct segvn_crargs *)argsp;
1733*284ce987SPatrick Mooney 		error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs,
1734*284ce987SPatrick Mooney 		    &do_unmap);
1735ec25b48fSsusans 		if (error != 0) {
1736dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
1737*284ce987SPatrick Mooney 			if (do_unmap) {
1738ec25b48fSsusans 				(void) as_unmap(as, addr, size);
1739ec25b48fSsusans 			}
1740ec25b48fSsusans 			return (error);
174107b65a64Saguzovsk 		}
1742ec25b48fSsusans 	} else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) {
1743*284ce987SPatrick Mooney 		boolean_t do_unmap = B_FALSE;
1744*284ce987SPatrick Mooney 
1745232cfe63Ssusans 		crargs = *(struct segvn_crargs *)argsp;
1746*284ce987SPatrick Mooney 		error = as_map_ansegs(as, raddr, rsize, crfp, &crargs,
1747*284ce987SPatrick Mooney 		    &do_unmap);
17487c478bd9Sstevel@tonic-gate 		if (error != 0) {
1749dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
1750*284ce987SPatrick Mooney 			if (do_unmap) {
17517c478bd9Sstevel@tonic-gate 				(void) as_unmap(as, addr, size);
17527c478bd9Sstevel@tonic-gate 			}
17537c478bd9Sstevel@tonic-gate 			return (error);
17547c478bd9Sstevel@tonic-gate 		}
17557c478bd9Sstevel@tonic-gate 	} else {
1756*284ce987SPatrick Mooney 		struct seg *seg, *segref;
1757*284ce987SPatrick Mooney 
1758*284ce987SPatrick Mooney 		seg = segref = seg_alloc(as, addr, size);
17597c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
1760dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
17617c478bd9Sstevel@tonic-gate 			return (ENOMEM);
17627c478bd9Sstevel@tonic-gate 		}
17637c478bd9Sstevel@tonic-gate 
1764*284ce987SPatrick Mooney 		/*
1765*284ce987SPatrick Mooney 		 * It is possible that the segment creation routine will free
1766*284ce987SPatrick Mooney 		 * 'seg' as part of a more advanced operation, such as when
1767*284ce987SPatrick Mooney 		 * segvn concatenates adjacent segments together.  When this
1768*284ce987SPatrick Mooney 		 * occurs, the seg*_create routine must communicate the
1769*284ce987SPatrick Mooney 		 * resulting segment out via the 'struct seg **' parameter.
1770*284ce987SPatrick Mooney 		 *
1771*284ce987SPatrick Mooney 		 * If segment creation fails, it must not free the passed-in
1772*284ce987SPatrick Mooney 		 * segment, nor alter the argument pointer.
1773*284ce987SPatrick Mooney 		 */
1774*284ce987SPatrick Mooney 		error = (*crfp)(&seg, argsp);
17757c478bd9Sstevel@tonic-gate 		if (error != 0) {
1776*284ce987SPatrick Mooney 			VERIFY3P(seg, ==, segref);
17777c478bd9Sstevel@tonic-gate 			seg_free(seg);
1778dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
17797c478bd9Sstevel@tonic-gate 			return (error);
17807c478bd9Sstevel@tonic-gate 		}
1781*284ce987SPatrick Mooney 
17823a30c6acSsusans 		/*
1783*284ce987SPatrick Mooney 		 * Check if the resulting segment represents a hole in the
1784*284ce987SPatrick Mooney 		 * address space, rather than contributing to the AS size.
17853a30c6acSsusans 		 */
1786*284ce987SPatrick Mooney 		is_hole = ((seg->s_flags & S_HOLE) != 0);
1787*284ce987SPatrick Mooney 
1788*284ce987SPatrick Mooney 		/* Add size now so as_unmap will work if as_ctl fails. */
1789*284ce987SPatrick Mooney 		if (!is_hole) {
1790*284ce987SPatrick Mooney 			as->a_size += rsize;
1791*284ce987SPatrick Mooney 			as->a_resvsize += rsize;
1792*284ce987SPatrick Mooney 		}
17937c478bd9Sstevel@tonic-gate 	}
17947c478bd9Sstevel@tonic-gate 
17957c478bd9Sstevel@tonic-gate 	as_setwatch(as);
17967c478bd9Sstevel@tonic-gate 
17977c478bd9Sstevel@tonic-gate 	/*
1798*284ce987SPatrick Mooney 	 * Establish memory locks for the segment if the address space is
1799*284ce987SPatrick Mooney 	 * locked, provided it's not an explicit hole in the AS.
18007c478bd9Sstevel@tonic-gate 	 */
18017c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
1802*284ce987SPatrick Mooney 	if (AS_ISPGLCK(as) && !is_hole) {
18037c478bd9Sstevel@tonic-gate 		mutex_exit(&as->a_contents);
1804dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
18057c478bd9Sstevel@tonic-gate 		error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
18067c478bd9Sstevel@tonic-gate 		if (error != 0)
18077c478bd9Sstevel@tonic-gate 			(void) as_unmap(as, addr, size);
18087c478bd9Sstevel@tonic-gate 	} else {
18097c478bd9Sstevel@tonic-gate 		mutex_exit(&as->a_contents);
1810dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
18117c478bd9Sstevel@tonic-gate 	}
18127c478bd9Sstevel@tonic-gate 	return (error);
18137c478bd9Sstevel@tonic-gate }
18147c478bd9Sstevel@tonic-gate 
18157c478bd9Sstevel@tonic-gate 
18167c478bd9Sstevel@tonic-gate /*
18177c478bd9Sstevel@tonic-gate  * Delete all segments in the address space marked with S_PURGE.
18187c478bd9Sstevel@tonic-gate  * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c).
18197c478bd9Sstevel@tonic-gate  * These segments are deleted as a first step before calls to as_gap(), so
18207c478bd9Sstevel@tonic-gate  * that they don't affect mmap() or shmat().
18217c478bd9Sstevel@tonic-gate  */
18227c478bd9Sstevel@tonic-gate void
18237c478bd9Sstevel@tonic-gate as_purge(struct as *as)
18247c478bd9Sstevel@tonic-gate {
18257c478bd9Sstevel@tonic-gate 	struct seg *seg;
18267c478bd9Sstevel@tonic-gate 	struct seg *next_seg;
18277c478bd9Sstevel@tonic-gate 
18287c478bd9Sstevel@tonic-gate 	/*
18297c478bd9Sstevel@tonic-gate 	 * the setting of NEEDSPURGE is protect by as_rangelock(), so
18307c478bd9Sstevel@tonic-gate 	 * no need to grab a_contents mutex for this check
18317c478bd9Sstevel@tonic-gate 	 */
18327c478bd9Sstevel@tonic-gate 	if ((as->a_flags & AS_NEEDSPURGE) == 0)
18337c478bd9Sstevel@tonic-gate 		return;
18347c478bd9Sstevel@tonic-gate 
1835dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
18367c478bd9Sstevel@tonic-gate 	next_seg = NULL;
18377c478bd9Sstevel@tonic-gate 	seg = AS_SEGFIRST(as);
18387c478bd9Sstevel@tonic-gate 	while (seg != NULL) {
18397c478bd9Sstevel@tonic-gate 		next_seg = AS_SEGNEXT(as, seg);
18407c478bd9Sstevel@tonic-gate 		if (seg->s_flags & S_PURGE)
18417c478bd9Sstevel@tonic-gate 			SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
18427c478bd9Sstevel@tonic-gate 		seg = next_seg;
18437c478bd9Sstevel@tonic-gate 	}
1844dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
18457c478bd9Sstevel@tonic-gate 
18467c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
18477c478bd9Sstevel@tonic-gate 	as->a_flags &= ~AS_NEEDSPURGE;
18487c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
18497c478bd9Sstevel@tonic-gate }
18507c478bd9Sstevel@tonic-gate 
18517c478bd9Sstevel@tonic-gate /*
185246ab9534Smec  * Find a hole within [*basep, *basep + *lenp), which contains a mappable
185346ab9534Smec  * range of addresses at least "minlen" long, where the base of the range is
185446ab9534Smec  * at "off" phase from an "align" boundary and there is space for a
185546ab9534Smec  * "redzone"-sized redzone on eithe rside of the range.  Thus,
185646ab9534Smec  * if align was 4M and off was 16k, the user wants a hole which will start
185746ab9534Smec  * 16k into a 4M page.
18587c478bd9Sstevel@tonic-gate  *
18597c478bd9Sstevel@tonic-gate  * If flags specifies AH_HI, the hole will have the highest possible address
18607c478bd9Sstevel@tonic-gate  * in the range.  We use the as->a_lastgap field to figure out where to
18617c478bd9Sstevel@tonic-gate  * start looking for a gap.
18627c478bd9Sstevel@tonic-gate  *
18637c478bd9Sstevel@tonic-gate  * Otherwise, the gap will have the lowest possible address.
18647c478bd9Sstevel@tonic-gate  *
18657c478bd9Sstevel@tonic-gate  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
18667c478bd9Sstevel@tonic-gate  *
186746ab9534Smec  * If an adequate hole is found, *basep and *lenp are set to reflect the part of
186846ab9534Smec  * the hole that is within range, and 0 is returned. On failure, -1 is returned.
18697c478bd9Sstevel@tonic-gate  *
18707c478bd9Sstevel@tonic-gate  * NOTE: This routine is not correct when base+len overflows caddr_t.
18717c478bd9Sstevel@tonic-gate  */
18727c478bd9Sstevel@tonic-gate int
187346ab9534Smec as_gap_aligned(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp,
187446ab9534Smec     uint_t flags, caddr_t addr, size_t align, size_t redzone, size_t off)
18757c478bd9Sstevel@tonic-gate {
18767c478bd9Sstevel@tonic-gate 	caddr_t lobound = *basep;
18777c478bd9Sstevel@tonic-gate 	caddr_t hibound = lobound + *lenp;
18787c478bd9Sstevel@tonic-gate 	struct seg *lseg, *hseg;
18797c478bd9Sstevel@tonic-gate 	caddr_t lo, hi;
18807c478bd9Sstevel@tonic-gate 	int forward;
18817c478bd9Sstevel@tonic-gate 	caddr_t save_base;
18827c478bd9Sstevel@tonic-gate 	size_t save_len;
18837d87bd0eSmec 	size_t save_minlen;
18847d87bd0eSmec 	size_t save_redzone;
18857d87bd0eSmec 	int fast_path = 1;
18867c478bd9Sstevel@tonic-gate 
18877c478bd9Sstevel@tonic-gate 	save_base = *basep;
18887c478bd9Sstevel@tonic-gate 	save_len = *lenp;
18897d87bd0eSmec 	save_minlen = minlen;
18907d87bd0eSmec 	save_redzone = redzone;
18917d87bd0eSmec 
18927d87bd0eSmec 	/*
18937d87bd0eSmec 	 * For the first pass/fast_path, just add align and redzone into
18947d87bd0eSmec 	 * minlen since if we get an allocation, we can guarantee that it
18957d87bd0eSmec 	 * will fit the alignment and redzone requested.
18967d87bd0eSmec 	 * This increases the chance that hibound will be adjusted to
18977d87bd0eSmec 	 * a_lastgap->s_base which will likely allow us to find an
18987d87bd0eSmec 	 * acceptable hole in the address space quicker.
18997d87bd0eSmec 	 * If we can't find a hole with this fast_path, then we look for
19007d87bd0eSmec 	 * smaller holes in which the alignment and offset may allow
19017d87bd0eSmec 	 * the allocation to fit.
19027d87bd0eSmec 	 */
19037d87bd0eSmec 	minlen += align;
19047d87bd0eSmec 	minlen += 2 * redzone;
19057d87bd0eSmec 	redzone = 0;
19067d87bd0eSmec 
1907dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
19087c478bd9Sstevel@tonic-gate 	if (AS_SEGFIRST(as) == NULL) {
190946ab9534Smec 		if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR,
191046ab9534Smec 		    align, redzone, off)) {
1911dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
19127c478bd9Sstevel@tonic-gate 			return (0);
19137c478bd9Sstevel@tonic-gate 		} else {
1914dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
19157c478bd9Sstevel@tonic-gate 			*basep = save_base;
19167c478bd9Sstevel@tonic-gate 			*lenp = save_len;
19177c478bd9Sstevel@tonic-gate 			return (-1);
19187c478bd9Sstevel@tonic-gate 		}
19197c478bd9Sstevel@tonic-gate 	}
19207c478bd9Sstevel@tonic-gate 
19217d87bd0eSmec retry:
19227c478bd9Sstevel@tonic-gate 	/*
19237c478bd9Sstevel@tonic-gate 	 * Set up to iterate over all the inter-segment holes in the given
19247c478bd9Sstevel@tonic-gate 	 * direction.  lseg is NULL for the lowest-addressed hole and hseg is
19257c478bd9Sstevel@tonic-gate 	 * NULL for the highest-addressed hole.  If moving backwards, we reset
19267c478bd9Sstevel@tonic-gate 	 * sseg to denote the highest-addressed segment.
19277c478bd9Sstevel@tonic-gate 	 */
19287c478bd9Sstevel@tonic-gate 	forward = (flags & AH_DIR) == AH_LO;
19297c478bd9Sstevel@tonic-gate 	if (forward) {
19307c478bd9Sstevel@tonic-gate 		hseg = as_findseg(as, lobound, 1);
19317c478bd9Sstevel@tonic-gate 		lseg = AS_SEGPREV(as, hseg);
19327c478bd9Sstevel@tonic-gate 	} else {
19337c478bd9Sstevel@tonic-gate 
19347c478bd9Sstevel@tonic-gate 		/*
19357c478bd9Sstevel@tonic-gate 		 * If allocating at least as much as the last allocation,
19367c478bd9Sstevel@tonic-gate 		 * use a_lastgap's base as a better estimate of hibound.
19377c478bd9Sstevel@tonic-gate 		 */
19387c478bd9Sstevel@tonic-gate 		if (as->a_lastgap &&
19397c478bd9Sstevel@tonic-gate 		    minlen >= as->a_lastgap->s_size &&
19407c478bd9Sstevel@tonic-gate 		    hibound >= as->a_lastgap->s_base)
19417c478bd9Sstevel@tonic-gate 			hibound = as->a_lastgap->s_base;
19427c478bd9Sstevel@tonic-gate 
19437c478bd9Sstevel@tonic-gate 		hseg = as_findseg(as, hibound, 1);
19447c478bd9Sstevel@tonic-gate 		if (hseg->s_base + hseg->s_size < hibound) {
19457c478bd9Sstevel@tonic-gate 			lseg = hseg;
19467c478bd9Sstevel@tonic-gate 			hseg = NULL;
19477c478bd9Sstevel@tonic-gate 		} else {
19487c478bd9Sstevel@tonic-gate 			lseg = AS_SEGPREV(as, hseg);
19497c478bd9Sstevel@tonic-gate 		}
19507c478bd9Sstevel@tonic-gate 	}
19517c478bd9Sstevel@tonic-gate 
19527c478bd9Sstevel@tonic-gate 	for (;;) {
19537c478bd9Sstevel@tonic-gate 		/*
19547c478bd9Sstevel@tonic-gate 		 * Set lo and hi to the hole's boundaries.  (We should really
19557c478bd9Sstevel@tonic-gate 		 * use MAXADDR in place of hibound in the expression below,
19567c478bd9Sstevel@tonic-gate 		 * but can't express it easily; using hibound in its place is
19577c478bd9Sstevel@tonic-gate 		 * harmless.)
19587c478bd9Sstevel@tonic-gate 		 */
19597c478bd9Sstevel@tonic-gate 		lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
19607c478bd9Sstevel@tonic-gate 		hi = (hseg == NULL) ? hibound : hseg->s_base;
19617c478bd9Sstevel@tonic-gate 		/*
19627c478bd9Sstevel@tonic-gate 		 * If the iteration has moved past the interval from lobound
19637c478bd9Sstevel@tonic-gate 		 * to hibound it's pointless to continue.
19647c478bd9Sstevel@tonic-gate 		 */
19657c478bd9Sstevel@tonic-gate 		if ((forward && lo > hibound) || (!forward && hi < lobound))
19667c478bd9Sstevel@tonic-gate 			break;
19677c478bd9Sstevel@tonic-gate 		else if (lo > hibound || hi < lobound)
19687c478bd9Sstevel@tonic-gate 			goto cont;
19697c478bd9Sstevel@tonic-gate 		/*
19707c478bd9Sstevel@tonic-gate 		 * Candidate hole lies at least partially within the allowable
19717c478bd9Sstevel@tonic-gate 		 * range.  Restrict it to fall completely within that range,
19727c478bd9Sstevel@tonic-gate 		 * i.e., to [max(lo, lobound), min(hi, hibound)].
19737c478bd9Sstevel@tonic-gate 		 */
19747c478bd9Sstevel@tonic-gate 		if (lo < lobound)
19757c478bd9Sstevel@tonic-gate 			lo = lobound;
19767c478bd9Sstevel@tonic-gate 		if (hi > hibound)
19777c478bd9Sstevel@tonic-gate 			hi = hibound;
19787c478bd9Sstevel@tonic-gate 		/*
19797c478bd9Sstevel@tonic-gate 		 * Verify that the candidate hole is big enough and meets
19807d87bd0eSmec 		 * hardware constraints.  If the hole is too small, no need
19817d87bd0eSmec 		 * to do the further checks since they will fail.
19827c478bd9Sstevel@tonic-gate 		 */
19837c478bd9Sstevel@tonic-gate 		*basep = lo;
19847c478bd9Sstevel@tonic-gate 		*lenp = hi - lo;
19857d87bd0eSmec 		if (*lenp >= minlen && valid_va_range_aligned(basep, lenp,
19867d87bd0eSmec 		    minlen, forward ? AH_LO : AH_HI, align, redzone, off) &&
19877c478bd9Sstevel@tonic-gate 		    ((flags & AH_CONTAIN) == 0 ||
19887c478bd9Sstevel@tonic-gate 		    (*basep <= addr && *basep + *lenp > addr))) {
19897c478bd9Sstevel@tonic-gate 			if (!forward)
19907c478bd9Sstevel@tonic-gate 				as->a_lastgap = hseg;
19917c478bd9Sstevel@tonic-gate 			if (hseg != NULL)
19927c478bd9Sstevel@tonic-gate 				as->a_lastgaphl = hseg;
19937c478bd9Sstevel@tonic-gate 			else
19947c478bd9Sstevel@tonic-gate 				as->a_lastgaphl = lseg;
1995dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
19967c478bd9Sstevel@tonic-gate 			return (0);
19977c478bd9Sstevel@tonic-gate 		}
19987c478bd9Sstevel@tonic-gate 	cont:
19997c478bd9Sstevel@tonic-gate 		/*
20007c478bd9Sstevel@tonic-gate 		 * Move to the next hole.
20017c478bd9Sstevel@tonic-gate 		 */
20027c478bd9Sstevel@tonic-gate 		if (forward) {
20037c478bd9Sstevel@tonic-gate 			lseg = hseg;
20047c478bd9Sstevel@tonic-gate 			if (lseg == NULL)
20057c478bd9Sstevel@tonic-gate 				break;
20067c478bd9Sstevel@tonic-gate 			hseg = AS_SEGNEXT(as, hseg);
20077c478bd9Sstevel@tonic-gate 		} else {
20087c478bd9Sstevel@tonic-gate 			hseg = lseg;
20097c478bd9Sstevel@tonic-gate 			if (hseg == NULL)
20107c478bd9Sstevel@tonic-gate 				break;
20117c478bd9Sstevel@tonic-gate 			lseg = AS_SEGPREV(as, lseg);
20127c478bd9Sstevel@tonic-gate 		}
20137c478bd9Sstevel@tonic-gate 	}
20147d87bd0eSmec 	if (fast_path && (align != 0 || save_redzone != 0)) {
20157d87bd0eSmec 		fast_path = 0;
20167d87bd0eSmec 		minlen = save_minlen;
20177d87bd0eSmec 		redzone = save_redzone;
20187d87bd0eSmec 		goto retry;
20197d87bd0eSmec 	}
20207c478bd9Sstevel@tonic-gate 	*basep = save_base;
20217c478bd9Sstevel@tonic-gate 	*lenp = save_len;
2022dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
20237c478bd9Sstevel@tonic-gate 	return (-1);
20247c478bd9Sstevel@tonic-gate }
20257c478bd9Sstevel@tonic-gate 
202646ab9534Smec /*
202746ab9534Smec  * Find a hole of at least size minlen within [*basep, *basep + *lenp).
202846ab9534Smec  *
202946ab9534Smec  * If flags specifies AH_HI, the hole will have the highest possible address
203046ab9534Smec  * in the range.  We use the as->a_lastgap field to figure out where to
203146ab9534Smec  * start looking for a gap.
203246ab9534Smec  *
203346ab9534Smec  * Otherwise, the gap will have the lowest possible address.
203446ab9534Smec  *
203546ab9534Smec  * If flags specifies AH_CONTAIN, the hole will contain the address addr.
203646ab9534Smec  *
203746ab9534Smec  * If an adequate hole is found, base and len are set to reflect the part of
203846ab9534Smec  * the hole that is within range, and 0 is returned, otherwise,
203946ab9534Smec  * -1 is returned.
204046ab9534Smec  *
204146ab9534Smec  * NOTE: This routine is not correct when base+len overflows caddr_t.
204246ab9534Smec  */
204346ab9534Smec int
204446ab9534Smec as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags,
204546ab9534Smec     caddr_t addr)
204646ab9534Smec {
204746ab9534Smec 
204846ab9534Smec 	return (as_gap_aligned(as, minlen, basep, lenp, flags, addr, 0, 0, 0));
204946ab9534Smec }
205046ab9534Smec 
20517c478bd9Sstevel@tonic-gate /*
20527c478bd9Sstevel@tonic-gate  * Return the next range within [base, base + len) that is backed
20537c478bd9Sstevel@tonic-gate  * with "real memory".  Skip holes and non-seg_vn segments.
20547c478bd9Sstevel@tonic-gate  * We're lazy and only return one segment at a time.
20557c478bd9Sstevel@tonic-gate  */
20567c478bd9Sstevel@tonic-gate int
20577c478bd9Sstevel@tonic-gate as_memory(struct as *as, caddr_t *basep, size_t *lenp)
20587c478bd9Sstevel@tonic-gate {
20597c478bd9Sstevel@tonic-gate 	extern struct seg_ops segspt_shmops;	/* needs a header file */
20607c478bd9Sstevel@tonic-gate 	struct seg *seg;
20617c478bd9Sstevel@tonic-gate 	caddr_t addr, eaddr;
20627c478bd9Sstevel@tonic-gate 	caddr_t segend;
20637c478bd9Sstevel@tonic-gate 
2064dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
20657c478bd9Sstevel@tonic-gate 
20667c478bd9Sstevel@tonic-gate 	addr = *basep;
20677c478bd9Sstevel@tonic-gate 	eaddr = addr + *lenp;
20687c478bd9Sstevel@tonic-gate 
20697c478bd9Sstevel@tonic-gate 	seg = as_findseg(as, addr, 0);
20707c478bd9Sstevel@tonic-gate 	if (seg != NULL)
20717c478bd9Sstevel@tonic-gate 		addr = MAX(seg->s_base, addr);
20727c478bd9Sstevel@tonic-gate 
20737c478bd9Sstevel@tonic-gate 	for (;;) {
20747c478bd9Sstevel@tonic-gate 		if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
2075dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
20767c478bd9Sstevel@tonic-gate 			return (EINVAL);
20777c478bd9Sstevel@tonic-gate 		}
20787c478bd9Sstevel@tonic-gate 
20797c478bd9Sstevel@tonic-gate 		if (seg->s_ops == &segvn_ops) {
20807c478bd9Sstevel@tonic-gate 			segend = seg->s_base + seg->s_size;
20817c478bd9Sstevel@tonic-gate 			break;
20827c478bd9Sstevel@tonic-gate 		}
20837c478bd9Sstevel@tonic-gate 
20847c478bd9Sstevel@tonic-gate 		/*
20857c478bd9Sstevel@tonic-gate 		 * We do ISM by looking into the private data
20867c478bd9Sstevel@tonic-gate 		 * to determine the real size of the segment.
20877c478bd9Sstevel@tonic-gate 		 */
20887c478bd9Sstevel@tonic-gate 		if (seg->s_ops == &segspt_shmops) {
20897c478bd9Sstevel@tonic-gate 			segend = seg->s_base + spt_realsize(seg);
20907c478bd9Sstevel@tonic-gate 			if (addr < segend)
20917c478bd9Sstevel@tonic-gate 				break;
20927c478bd9Sstevel@tonic-gate 		}
20937c478bd9Sstevel@tonic-gate 
20947c478bd9Sstevel@tonic-gate 		seg = AS_SEGNEXT(as, seg);
20957c478bd9Sstevel@tonic-gate 
20967c478bd9Sstevel@tonic-gate 		if (seg != NULL)
20977c478bd9Sstevel@tonic-gate 			addr = seg->s_base;
20987c478bd9Sstevel@tonic-gate 	}
20997c478bd9Sstevel@tonic-gate 
21007c478bd9Sstevel@tonic-gate 	*basep = addr;
21017c478bd9Sstevel@tonic-gate 
21027c478bd9Sstevel@tonic-gate 	if (segend > eaddr)
21037c478bd9Sstevel@tonic-gate 		*lenp = eaddr - addr;
21047c478bd9Sstevel@tonic-gate 	else
21057c478bd9Sstevel@tonic-gate 		*lenp = segend - addr;
21067c478bd9Sstevel@tonic-gate 
2107dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
21087c478bd9Sstevel@tonic-gate 	return (0);
21097c478bd9Sstevel@tonic-gate }
21107c478bd9Sstevel@tonic-gate 
21117c478bd9Sstevel@tonic-gate /*
21127c478bd9Sstevel@tonic-gate  * Swap the pages associated with the address space as out to
21137c478bd9Sstevel@tonic-gate  * secondary storage, returning the number of bytes actually
21147c478bd9Sstevel@tonic-gate  * swapped.
21157c478bd9Sstevel@tonic-gate  *
21167c478bd9Sstevel@tonic-gate  * The value returned is intended to correlate well with the process's
21177c478bd9Sstevel@tonic-gate  * memory requirements.  Its usefulness for this purpose depends on
21187c478bd9Sstevel@tonic-gate  * how well the segment-level routines do at returning accurate
21197c478bd9Sstevel@tonic-gate  * information.
21207c478bd9Sstevel@tonic-gate  */
21217c478bd9Sstevel@tonic-gate size_t
21227c478bd9Sstevel@tonic-gate as_swapout(struct as *as)
21237c478bd9Sstevel@tonic-gate {
21247c478bd9Sstevel@tonic-gate 	struct seg *seg;
21257c478bd9Sstevel@tonic-gate 	size_t swpcnt = 0;
21267c478bd9Sstevel@tonic-gate 
21277c478bd9Sstevel@tonic-gate 	/*
21287c478bd9Sstevel@tonic-gate 	 * Kernel-only processes have given up their address
21297c478bd9Sstevel@tonic-gate 	 * spaces.  Of course, we shouldn't be attempting to
21307c478bd9Sstevel@tonic-gate 	 * swap out such processes in the first place...
21317c478bd9Sstevel@tonic-gate 	 */
21327c478bd9Sstevel@tonic-gate 	if (as == NULL)
21337c478bd9Sstevel@tonic-gate 		return (0);
21347c478bd9Sstevel@tonic-gate 
2135dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
21367c478bd9Sstevel@tonic-gate 
21377c478bd9Sstevel@tonic-gate 	/*
21387c478bd9Sstevel@tonic-gate 	 * Free all mapping resources associated with the address
21397c478bd9Sstevel@tonic-gate 	 * space.  The segment-level swapout routines capitalize
21407c478bd9Sstevel@tonic-gate 	 * on this unmapping by scavanging pages that have become
21417c478bd9Sstevel@tonic-gate 	 * unmapped here.
21427c478bd9Sstevel@tonic-gate 	 */
21437c478bd9Sstevel@tonic-gate 	hat_swapout(as->a_hat);
21447c478bd9Sstevel@tonic-gate 
21457c478bd9Sstevel@tonic-gate 	/*
21467c478bd9Sstevel@tonic-gate 	 * Call the swapout routines of all segments in the address
21477c478bd9Sstevel@tonic-gate 	 * space to do the actual work, accumulating the amount of
21487c478bd9Sstevel@tonic-gate 	 * space reclaimed.
21497c478bd9Sstevel@tonic-gate 	 */
21507c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
21517c478bd9Sstevel@tonic-gate 		struct seg_ops *ov = seg->s_ops;
21527c478bd9Sstevel@tonic-gate 
21537c478bd9Sstevel@tonic-gate 		/*
21547c478bd9Sstevel@tonic-gate 		 * We have to check to see if the seg has
21557c478bd9Sstevel@tonic-gate 		 * an ops vector because the seg may have
21567c478bd9Sstevel@tonic-gate 		 * been in the middle of being set up when
21577c478bd9Sstevel@tonic-gate 		 * the process was picked for swapout.
21587c478bd9Sstevel@tonic-gate 		 */
21597c478bd9Sstevel@tonic-gate 		if ((ov != NULL) && (ov->swapout != NULL))
21607c478bd9Sstevel@tonic-gate 			swpcnt += SEGOP_SWAPOUT(seg);
21617c478bd9Sstevel@tonic-gate 	}
2162dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
21637c478bd9Sstevel@tonic-gate 	return (swpcnt);
21647c478bd9Sstevel@tonic-gate }
21657c478bd9Sstevel@tonic-gate 
21667c478bd9Sstevel@tonic-gate /*
21677c478bd9Sstevel@tonic-gate  * Determine whether data from the mappings in interval [addr, addr + size)
21687c478bd9Sstevel@tonic-gate  * are in the primary memory (core) cache.
21697c478bd9Sstevel@tonic-gate  */
21707c478bd9Sstevel@tonic-gate int
21717c478bd9Sstevel@tonic-gate as_incore(struct as *as, caddr_t addr,
21727c478bd9Sstevel@tonic-gate     size_t size, char *vec, size_t *sizep)
21737c478bd9Sstevel@tonic-gate {
21747c478bd9Sstevel@tonic-gate 	struct seg *seg;
21757c478bd9Sstevel@tonic-gate 	size_t ssize;
21767c478bd9Sstevel@tonic-gate 	caddr_t raddr;		/* rounded down addr */
21777c478bd9Sstevel@tonic-gate 	size_t rsize;		/* rounded up size */
21787c478bd9Sstevel@tonic-gate 	size_t isize;			/* iteration size */
21797c478bd9Sstevel@tonic-gate 	int error = 0;		/* result, assume success */
21807c478bd9Sstevel@tonic-gate 
21817c478bd9Sstevel@tonic-gate 	*sizep = 0;
21827c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
21837c478bd9Sstevel@tonic-gate 	rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
2184843e1988Sjohnlev 	    (size_t)raddr;
21857c478bd9Sstevel@tonic-gate 
21867c478bd9Sstevel@tonic-gate 	if (raddr + rsize < raddr)		/* check for wraparound */
21877c478bd9Sstevel@tonic-gate 		return (ENOMEM);
21887c478bd9Sstevel@tonic-gate 
2189dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
21907c478bd9Sstevel@tonic-gate 	seg = as_segat(as, raddr);
21917c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
2192dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
21937c478bd9Sstevel@tonic-gate 		return (-1);
21947c478bd9Sstevel@tonic-gate 	}
21957c478bd9Sstevel@tonic-gate 
21967c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
21977c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
21987c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
21997c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
22007c478bd9Sstevel@tonic-gate 				error = -1;
22017c478bd9Sstevel@tonic-gate 				break;
22027c478bd9Sstevel@tonic-gate 			}
22037c478bd9Sstevel@tonic-gate 		}
22047c478bd9Sstevel@tonic-gate 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
22057c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
22067c478bd9Sstevel@tonic-gate 		else
22077c478bd9Sstevel@tonic-gate 			ssize = rsize;
22087c478bd9Sstevel@tonic-gate 		*sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
22097c478bd9Sstevel@tonic-gate 		if (isize != ssize) {
22107c478bd9Sstevel@tonic-gate 			error = -1;
22117c478bd9Sstevel@tonic-gate 			break;
22127c478bd9Sstevel@tonic-gate 		}
22137c478bd9Sstevel@tonic-gate 		vec += btopr(ssize);
22147c478bd9Sstevel@tonic-gate 	}
2215dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
22167c478bd9Sstevel@tonic-gate 	return (error);
22177c478bd9Sstevel@tonic-gate }
22187c478bd9Sstevel@tonic-gate 
22197c478bd9Sstevel@tonic-gate static void
22207c478bd9Sstevel@tonic-gate as_segunlock(struct seg *seg, caddr_t addr, int attr,
22216430daecSBrandon Baker     ulong_t *bitmap, size_t position, size_t npages)
22227c478bd9Sstevel@tonic-gate {
22237c478bd9Sstevel@tonic-gate 	caddr_t	range_start;
22247c478bd9Sstevel@tonic-gate 	size_t	pos1 = position;
22257c478bd9Sstevel@tonic-gate 	size_t	pos2;
22267c478bd9Sstevel@tonic-gate 	size_t	size;
22277c478bd9Sstevel@tonic-gate 	size_t  end_pos = npages + position;
22287c478bd9Sstevel@tonic-gate 
22297c478bd9Sstevel@tonic-gate 	while (bt_range(bitmap, &pos1, &pos2, end_pos)) {
22307c478bd9Sstevel@tonic-gate 		size = ptob((pos2 - pos1));
22317c478bd9Sstevel@tonic-gate 		range_start = (caddr_t)((uintptr_t)addr +
2232843e1988Sjohnlev 		    ptob(pos1 - position));
22337c478bd9Sstevel@tonic-gate 
22347c478bd9Sstevel@tonic-gate 		(void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK,
2235843e1988Sjohnlev 		    (ulong_t *)NULL, (size_t)NULL);
22367c478bd9Sstevel@tonic-gate 		pos1 = pos2;
22377c478bd9Sstevel@tonic-gate 	}
22387c478bd9Sstevel@tonic-gate }
22397c478bd9Sstevel@tonic-gate 
22407c478bd9Sstevel@tonic-gate static void
22417c478bd9Sstevel@tonic-gate as_unlockerr(struct as *as, int attr, ulong_t *mlock_map,
22426430daecSBrandon Baker     caddr_t raddr, size_t rsize)
22437c478bd9Sstevel@tonic-gate {
22447c478bd9Sstevel@tonic-gate 	struct seg *seg = as_segat(as, raddr);
22457c478bd9Sstevel@tonic-gate 	size_t ssize;
22467c478bd9Sstevel@tonic-gate 
22477c478bd9Sstevel@tonic-gate 	while (rsize != 0) {
22487c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size)
22497c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
22507c478bd9Sstevel@tonic-gate 
22517c478bd9Sstevel@tonic-gate 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
22527c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
22537c478bd9Sstevel@tonic-gate 		else
22547c478bd9Sstevel@tonic-gate 			ssize = rsize;
22557c478bd9Sstevel@tonic-gate 
22567c478bd9Sstevel@tonic-gate 		as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize));
22577c478bd9Sstevel@tonic-gate 
22587c478bd9Sstevel@tonic-gate 		rsize -= ssize;
22597c478bd9Sstevel@tonic-gate 		raddr += ssize;
22607c478bd9Sstevel@tonic-gate 	}
22617c478bd9Sstevel@tonic-gate }
22627c478bd9Sstevel@tonic-gate 
22637c478bd9Sstevel@tonic-gate /*
22647c478bd9Sstevel@tonic-gate  * Cache control operations over the interval [addr, addr + size) in
22657c478bd9Sstevel@tonic-gate  * address space "as".
22667c478bd9Sstevel@tonic-gate  */
22677c478bd9Sstevel@tonic-gate /*ARGSUSED*/
22687c478bd9Sstevel@tonic-gate int
22697c478bd9Sstevel@tonic-gate as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
22707c478bd9Sstevel@tonic-gate     uintptr_t arg, ulong_t *lock_map, size_t pos)
22717c478bd9Sstevel@tonic-gate {
22727c478bd9Sstevel@tonic-gate 	struct seg *seg;	/* working segment */
22737c478bd9Sstevel@tonic-gate 	caddr_t raddr;		/* rounded down addr */
22747c478bd9Sstevel@tonic-gate 	caddr_t initraddr;	/* saved initial rounded down addr */
22757c478bd9Sstevel@tonic-gate 	size_t rsize;		/* rounded up size */
22767c478bd9Sstevel@tonic-gate 	size_t initrsize;	/* saved initial rounded up size */
22777c478bd9Sstevel@tonic-gate 	size_t ssize;		/* size of seg */
22787c478bd9Sstevel@tonic-gate 	int error = 0;			/* result */
22797c478bd9Sstevel@tonic-gate 	size_t mlock_size;	/* size of bitmap */
22807c478bd9Sstevel@tonic-gate 	ulong_t *mlock_map;	/* pointer to bitmap used */
22817c478bd9Sstevel@tonic-gate 				/* to represent the locked */
22827c478bd9Sstevel@tonic-gate 				/* pages. */
22837c478bd9Sstevel@tonic-gate retry:
22847c478bd9Sstevel@tonic-gate 	if (error == IE_RETRY)
2285dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_WRITER);
22867c478bd9Sstevel@tonic-gate 	else
2287dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_READER);
22887c478bd9Sstevel@tonic-gate 
22897c478bd9Sstevel@tonic-gate 	/*
22907c478bd9Sstevel@tonic-gate 	 * If these are address space lock/unlock operations, loop over
22917c478bd9Sstevel@tonic-gate 	 * all segments in the address space, as appropriate.
22927c478bd9Sstevel@tonic-gate 	 */
22937c478bd9Sstevel@tonic-gate 	if (func == MC_LOCKAS) {
22947c478bd9Sstevel@tonic-gate 		size_t npages, idx;
22957c478bd9Sstevel@tonic-gate 		size_t rlen = 0;	/* rounded as length */
22967c478bd9Sstevel@tonic-gate 
22977c478bd9Sstevel@tonic-gate 		idx = pos;
22987c478bd9Sstevel@tonic-gate 
22997c478bd9Sstevel@tonic-gate 		if (arg & MCL_FUTURE) {
23007c478bd9Sstevel@tonic-gate 			mutex_enter(&as->a_contents);
23017c478bd9Sstevel@tonic-gate 			AS_SETPGLCK(as);
23027c478bd9Sstevel@tonic-gate 			mutex_exit(&as->a_contents);
23037c478bd9Sstevel@tonic-gate 		}
23047c478bd9Sstevel@tonic-gate 		if ((arg & MCL_CURRENT) == 0) {
2305dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
23067c478bd9Sstevel@tonic-gate 			return (0);
23077c478bd9Sstevel@tonic-gate 		}
23087c478bd9Sstevel@tonic-gate 
23097c478bd9Sstevel@tonic-gate 		seg = AS_SEGFIRST(as);
23107c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
2311dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
23127c478bd9Sstevel@tonic-gate 			return (0);
23137c478bd9Sstevel@tonic-gate 		}
23147c478bd9Sstevel@tonic-gate 
23157c478bd9Sstevel@tonic-gate 		do {
23167c478bd9Sstevel@tonic-gate 			raddr = (caddr_t)((uintptr_t)seg->s_base &
23177c478bd9Sstevel@tonic-gate 			    (uintptr_t)PAGEMASK);
23187c478bd9Sstevel@tonic-gate 			rlen += (((uintptr_t)(seg->s_base + seg->s_size) +
2319843e1988Sjohnlev 			    PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr;
23207c478bd9Sstevel@tonic-gate 		} while ((seg = AS_SEGNEXT(as, seg)) != NULL);
23217c478bd9Sstevel@tonic-gate 
23227c478bd9Sstevel@tonic-gate 		mlock_size = BT_BITOUL(btopr(rlen));
23237c478bd9Sstevel@tonic-gate 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2324843e1988Sjohnlev 		    sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2325dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
23267c478bd9Sstevel@tonic-gate 				return (EAGAIN);
23277c478bd9Sstevel@tonic-gate 		}
23287c478bd9Sstevel@tonic-gate 
23297c478bd9Sstevel@tonic-gate 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2330*284ce987SPatrick Mooney 			if ((seg->s_flags & S_HOLE) != 0) {
2331*284ce987SPatrick Mooney 				continue;
2332*284ce987SPatrick Mooney 			}
23337c478bd9Sstevel@tonic-gate 			error = SEGOP_LOCKOP(seg, seg->s_base,
23347c478bd9Sstevel@tonic-gate 			    seg->s_size, attr, MC_LOCK, mlock_map, pos);
23357c478bd9Sstevel@tonic-gate 			if (error != 0)
23367c478bd9Sstevel@tonic-gate 				break;
23377c478bd9Sstevel@tonic-gate 			pos += seg_pages(seg);
23387c478bd9Sstevel@tonic-gate 		}
23397c478bd9Sstevel@tonic-gate 
23407c478bd9Sstevel@tonic-gate 		if (error) {
23417c478bd9Sstevel@tonic-gate 			for (seg = AS_SEGFIRST(as); seg != NULL;
2342843e1988Sjohnlev 			    seg = AS_SEGNEXT(as, seg)) {
23437c478bd9Sstevel@tonic-gate 
23447c478bd9Sstevel@tonic-gate 				raddr = (caddr_t)((uintptr_t)seg->s_base &
2345843e1988Sjohnlev 				    (uintptr_t)PAGEMASK);
23467c478bd9Sstevel@tonic-gate 				npages = seg_pages(seg);
23477c478bd9Sstevel@tonic-gate 				as_segunlock(seg, raddr, attr, mlock_map,
2348843e1988Sjohnlev 				    idx, npages);
23497c478bd9Sstevel@tonic-gate 				idx += npages;
23507c478bd9Sstevel@tonic-gate 			}
23517c478bd9Sstevel@tonic-gate 		}
23527c478bd9Sstevel@tonic-gate 
23537c478bd9Sstevel@tonic-gate 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2354dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
23557c478bd9Sstevel@tonic-gate 		goto lockerr;
23567c478bd9Sstevel@tonic-gate 	} else if (func == MC_UNLOCKAS) {
23577c478bd9Sstevel@tonic-gate 		mutex_enter(&as->a_contents);
23587c478bd9Sstevel@tonic-gate 		AS_CLRPGLCK(as);
23597c478bd9Sstevel@tonic-gate 		mutex_exit(&as->a_contents);
23607c478bd9Sstevel@tonic-gate 
23617c478bd9Sstevel@tonic-gate 		for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) {
2362*284ce987SPatrick Mooney 			if ((seg->s_flags & S_HOLE) != 0) {
2363*284ce987SPatrick Mooney 				continue;
2364*284ce987SPatrick Mooney 			}
23657c478bd9Sstevel@tonic-gate 			error = SEGOP_LOCKOP(seg, seg->s_base,
23667c478bd9Sstevel@tonic-gate 			    seg->s_size, attr, MC_UNLOCK, NULL, 0);
23677c478bd9Sstevel@tonic-gate 			if (error != 0)
23687c478bd9Sstevel@tonic-gate 				break;
23697c478bd9Sstevel@tonic-gate 		}
23707c478bd9Sstevel@tonic-gate 
2371dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
23727c478bd9Sstevel@tonic-gate 		goto lockerr;
23737c478bd9Sstevel@tonic-gate 	}
23747c478bd9Sstevel@tonic-gate 
23757c478bd9Sstevel@tonic-gate 	/*
23767c478bd9Sstevel@tonic-gate 	 * Normalize addresses and sizes.
23777c478bd9Sstevel@tonic-gate 	 */
23787c478bd9Sstevel@tonic-gate 	initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
23797c478bd9Sstevel@tonic-gate 	initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2380843e1988Sjohnlev 	    (size_t)raddr;
23817c478bd9Sstevel@tonic-gate 
23827c478bd9Sstevel@tonic-gate 	if (raddr + rsize < raddr) {		/* check for wraparound */
2383dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
23847c478bd9Sstevel@tonic-gate 		return (ENOMEM);
23857c478bd9Sstevel@tonic-gate 	}
23867c478bd9Sstevel@tonic-gate 
23877c478bd9Sstevel@tonic-gate 	/*
23887c478bd9Sstevel@tonic-gate 	 * Get initial segment.
23897c478bd9Sstevel@tonic-gate 	 */
23907c478bd9Sstevel@tonic-gate 	if ((seg = as_segat(as, raddr)) == NULL) {
2391dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
23927c478bd9Sstevel@tonic-gate 		return (ENOMEM);
23937c478bd9Sstevel@tonic-gate 	}
23947c478bd9Sstevel@tonic-gate 
23957c478bd9Sstevel@tonic-gate 	if (func == MC_LOCK) {
23967c478bd9Sstevel@tonic-gate 		mlock_size = BT_BITOUL(btopr(rsize));
23977c478bd9Sstevel@tonic-gate 		if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size *
2398843e1988Sjohnlev 		    sizeof (ulong_t), KM_NOSLEEP)) == NULL) {
2399dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
24007c478bd9Sstevel@tonic-gate 				return (EAGAIN);
24017c478bd9Sstevel@tonic-gate 		}
24027c478bd9Sstevel@tonic-gate 	}
24037c478bd9Sstevel@tonic-gate 
24047c478bd9Sstevel@tonic-gate 	/*
24057c478bd9Sstevel@tonic-gate 	 * Loop over all segments.  If a hole in the address range is
24067c478bd9Sstevel@tonic-gate 	 * discovered, then fail.  For each segment, perform the appropriate
24077c478bd9Sstevel@tonic-gate 	 * control operation.
24087c478bd9Sstevel@tonic-gate 	 */
24097c478bd9Sstevel@tonic-gate 	while (rsize != 0) {
24107c478bd9Sstevel@tonic-gate 
24117c478bd9Sstevel@tonic-gate 		/*
24127c478bd9Sstevel@tonic-gate 		 * Make sure there's no hole, calculate the portion
24137c478bd9Sstevel@tonic-gate 		 * of the next segment to be operated over.
24147c478bd9Sstevel@tonic-gate 		 */
24157c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
24167c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
24177c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
24187c478bd9Sstevel@tonic-gate 				if (func == MC_LOCK) {
24197c478bd9Sstevel@tonic-gate 					as_unlockerr(as, attr, mlock_map,
2420843e1988Sjohnlev 					    initraddr, initrsize - rsize);
24217c478bd9Sstevel@tonic-gate 					kmem_free(mlock_map,
2422843e1988Sjohnlev 					    mlock_size * sizeof (ulong_t));
24237c478bd9Sstevel@tonic-gate 				}
2424dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
24257c478bd9Sstevel@tonic-gate 				return (ENOMEM);
24267c478bd9Sstevel@tonic-gate 			}
24277c478bd9Sstevel@tonic-gate 		}
24287c478bd9Sstevel@tonic-gate 		if ((raddr + rsize) > (seg->s_base + seg->s_size))
24297c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
24307c478bd9Sstevel@tonic-gate 		else
24317c478bd9Sstevel@tonic-gate 			ssize = rsize;
24327c478bd9Sstevel@tonic-gate 
24337c478bd9Sstevel@tonic-gate 		/*
24347c478bd9Sstevel@tonic-gate 		 * Dispatch on specific function.
24357c478bd9Sstevel@tonic-gate 		 */
24367c478bd9Sstevel@tonic-gate 		switch (func) {
24377c478bd9Sstevel@tonic-gate 
24387c478bd9Sstevel@tonic-gate 		/*
24397c478bd9Sstevel@tonic-gate 		 * Synchronize cached data from mappings with backing
24407c478bd9Sstevel@tonic-gate 		 * objects.
24417c478bd9Sstevel@tonic-gate 		 */
24427c478bd9Sstevel@tonic-gate 		case MC_SYNC:
24437c478bd9Sstevel@tonic-gate 			if (error = SEGOP_SYNC(seg, raddr, ssize,
24447c478bd9Sstevel@tonic-gate 			    attr, (uint_t)arg)) {
2445dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
24467c478bd9Sstevel@tonic-gate 				return (error);
24477c478bd9Sstevel@tonic-gate 			}
24487c478bd9Sstevel@tonic-gate 			break;
24497c478bd9Sstevel@tonic-gate 
24507c478bd9Sstevel@tonic-gate 		/*
24517c478bd9Sstevel@tonic-gate 		 * Lock pages in memory.
24527c478bd9Sstevel@tonic-gate 		 */
24537c478bd9Sstevel@tonic-gate 		case MC_LOCK:
24547c478bd9Sstevel@tonic-gate 			if (error = SEGOP_LOCKOP(seg, raddr, ssize,
2455843e1988Sjohnlev 			    attr, func, mlock_map, pos)) {
24567c478bd9Sstevel@tonic-gate 				as_unlockerr(as, attr, mlock_map, initraddr,
2457843e1988Sjohnlev 				    initrsize - rsize + ssize);
24587c478bd9Sstevel@tonic-gate 				kmem_free(mlock_map, mlock_size *
2459843e1988Sjohnlev 				    sizeof (ulong_t));
2460dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
24617c478bd9Sstevel@tonic-gate 				goto lockerr;
24627c478bd9Sstevel@tonic-gate 			}
24637c478bd9Sstevel@tonic-gate 			break;
24647c478bd9Sstevel@tonic-gate 
24657c478bd9Sstevel@tonic-gate 		/*
24667c478bd9Sstevel@tonic-gate 		 * Unlock mapped pages.
24677c478bd9Sstevel@tonic-gate 		 */
24687c478bd9Sstevel@tonic-gate 		case MC_UNLOCK:
24697c478bd9Sstevel@tonic-gate 			(void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
2470843e1988Sjohnlev 			    (ulong_t *)NULL, (size_t)NULL);
24717c478bd9Sstevel@tonic-gate 			break;
24727c478bd9Sstevel@tonic-gate 
24737c478bd9Sstevel@tonic-gate 		/*
24747c478bd9Sstevel@tonic-gate 		 * Store VM advise for mapped pages in segment layer.
24757c478bd9Sstevel@tonic-gate 		 */
24767c478bd9Sstevel@tonic-gate 		case MC_ADVISE:
24777c478bd9Sstevel@tonic-gate 			error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg);
24787c478bd9Sstevel@tonic-gate 
24797c478bd9Sstevel@tonic-gate 			/*
24807c478bd9Sstevel@tonic-gate 			 * Check for regular errors and special retry error
24817c478bd9Sstevel@tonic-gate 			 */
24827c478bd9Sstevel@tonic-gate 			if (error) {
24837c478bd9Sstevel@tonic-gate 				if (error == IE_RETRY) {
24847c478bd9Sstevel@tonic-gate 					/*
24857c478bd9Sstevel@tonic-gate 					 * Need to acquire writers lock, so
24867c478bd9Sstevel@tonic-gate 					 * have to drop readers lock and start
24877c478bd9Sstevel@tonic-gate 					 * all over again
24887c478bd9Sstevel@tonic-gate 					 */
2489dc32d872SJosef 'Jeff' Sipek 					AS_LOCK_EXIT(as);
24907c478bd9Sstevel@tonic-gate 					goto retry;
24917c478bd9Sstevel@tonic-gate 				} else if (error == IE_REATTACH) {
24927c478bd9Sstevel@tonic-gate 					/*
24937c478bd9Sstevel@tonic-gate 					 * Find segment for current address
24947c478bd9Sstevel@tonic-gate 					 * because current segment just got
24957c478bd9Sstevel@tonic-gate 					 * split or concatenated
24967c478bd9Sstevel@tonic-gate 					 */
24977c478bd9Sstevel@tonic-gate 					seg = as_segat(as, raddr);
24987c478bd9Sstevel@tonic-gate 					if (seg == NULL) {
2499dc32d872SJosef 'Jeff' Sipek 						AS_LOCK_EXIT(as);
25007c478bd9Sstevel@tonic-gate 						return (ENOMEM);
25017c478bd9Sstevel@tonic-gate 					}
25027c478bd9Sstevel@tonic-gate 				} else {
25037c478bd9Sstevel@tonic-gate 					/*
25047c478bd9Sstevel@tonic-gate 					 * Regular error
25057c478bd9Sstevel@tonic-gate 					 */
2506dc32d872SJosef 'Jeff' Sipek 					AS_LOCK_EXIT(as);
25077c478bd9Sstevel@tonic-gate 					return (error);
25087c478bd9Sstevel@tonic-gate 				}
25097c478bd9Sstevel@tonic-gate 			}
25107c478bd9Sstevel@tonic-gate 			break;
25117c478bd9Sstevel@tonic-gate 
25129d12795fSRobert Mustacchi 		case MC_INHERIT_ZERO:
25139d12795fSRobert Mustacchi 			if (seg->s_ops->inherit == NULL) {
25149d12795fSRobert Mustacchi 				error = ENOTSUP;
25159d12795fSRobert Mustacchi 			} else {
25169d12795fSRobert Mustacchi 				error = SEGOP_INHERIT(seg, raddr, ssize,
25179d12795fSRobert Mustacchi 				    SEGP_INH_ZERO);
25189d12795fSRobert Mustacchi 			}
25199d12795fSRobert Mustacchi 			if (error != 0) {
2520dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
25219d12795fSRobert Mustacchi 				return (error);
25229d12795fSRobert Mustacchi 			}
25239d12795fSRobert Mustacchi 			break;
25249d12795fSRobert Mustacchi 
25257c478bd9Sstevel@tonic-gate 		/*
25267c478bd9Sstevel@tonic-gate 		 * Can't happen.
25277c478bd9Sstevel@tonic-gate 		 */
25287c478bd9Sstevel@tonic-gate 		default:
25297c478bd9Sstevel@tonic-gate 			panic("as_ctl: bad operation %d", func);
25307c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
25317c478bd9Sstevel@tonic-gate 		}
25327c478bd9Sstevel@tonic-gate 
25337c478bd9Sstevel@tonic-gate 		rsize -= ssize;
25347c478bd9Sstevel@tonic-gate 		raddr += ssize;
25357c478bd9Sstevel@tonic-gate 	}
25367c478bd9Sstevel@tonic-gate 
25377c478bd9Sstevel@tonic-gate 	if (func == MC_LOCK)
25387c478bd9Sstevel@tonic-gate 		kmem_free(mlock_map, mlock_size * sizeof (ulong_t));
2539dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
25407c478bd9Sstevel@tonic-gate 	return (0);
25417c478bd9Sstevel@tonic-gate lockerr:
25427c478bd9Sstevel@tonic-gate 
25437c478bd9Sstevel@tonic-gate 	/*
25447c478bd9Sstevel@tonic-gate 	 * If the lower levels returned EDEADLK for a segment lockop,
25457c478bd9Sstevel@tonic-gate 	 * it means that we should retry the operation.  Let's wait
25467c478bd9Sstevel@tonic-gate 	 * a bit also to let the deadlock causing condition clear.
25477c478bd9Sstevel@tonic-gate 	 * This is part of a gross hack to work around a design flaw
25487c478bd9Sstevel@tonic-gate 	 * in the ufs/sds logging code and should go away when the
25497c478bd9Sstevel@tonic-gate 	 * logging code is re-designed to fix the problem. See bug
25507c478bd9Sstevel@tonic-gate 	 * 4125102 for details of the problem.
25517c478bd9Sstevel@tonic-gate 	 */
25527c478bd9Sstevel@tonic-gate 	if (error == EDEADLK) {
25537c478bd9Sstevel@tonic-gate 		delay(deadlk_wait);
25547c478bd9Sstevel@tonic-gate 		error = 0;
25557c478bd9Sstevel@tonic-gate 		goto retry;
25567c478bd9Sstevel@tonic-gate 	}
25577c478bd9Sstevel@tonic-gate 	return (error);
25587c478bd9Sstevel@tonic-gate }
25597c478bd9Sstevel@tonic-gate 
2560843e1988Sjohnlev int
2561843e1988Sjohnlev fc_decode(faultcode_t fault_err)
25627c478bd9Sstevel@tonic-gate {
25637c478bd9Sstevel@tonic-gate 	int error = 0;
25647c478bd9Sstevel@tonic-gate 
25657c478bd9Sstevel@tonic-gate 	switch (FC_CODE(fault_err)) {
25667c478bd9Sstevel@tonic-gate 	case FC_OBJERR:
25677c478bd9Sstevel@tonic-gate 		error = FC_ERRNO(fault_err);
25687c478bd9Sstevel@tonic-gate 		break;
25697c478bd9Sstevel@tonic-gate 	case FC_PROT:
25707c478bd9Sstevel@tonic-gate 		error = EACCES;
25717c478bd9Sstevel@tonic-gate 		break;
25727c478bd9Sstevel@tonic-gate 	default:
25737c478bd9Sstevel@tonic-gate 		error = EFAULT;
25747c478bd9Sstevel@tonic-gate 		break;
25757c478bd9Sstevel@tonic-gate 	}
25767c478bd9Sstevel@tonic-gate 	return (error);
25777c478bd9Sstevel@tonic-gate }
25787c478bd9Sstevel@tonic-gate 
2579a98e9dbfSaguzovsk /*
2580a98e9dbfSaguzovsk  * Pagelock pages from a range that spans more than 1 segment.  Obtain shadow
2581a98e9dbfSaguzovsk  * lists from each segment and copy them to one contiguous shadow list (plist)
2582a98e9dbfSaguzovsk  * as expected by the caller.  Save pointers to per segment shadow lists at
2583a98e9dbfSaguzovsk  * the tail of plist so that they can be used during as_pageunlock().
2584a98e9dbfSaguzovsk  */
2585a98e9dbfSaguzovsk static int
2586a98e9dbfSaguzovsk as_pagelock_segs(struct as *as, struct seg *seg, struct page ***ppp,
2587a98e9dbfSaguzovsk     caddr_t addr, size_t size, enum seg_rw rw)
2588a98e9dbfSaguzovsk {
2589a98e9dbfSaguzovsk 	caddr_t sv_addr = addr;
2590a98e9dbfSaguzovsk 	size_t sv_size = size;
2591a98e9dbfSaguzovsk 	struct seg *sv_seg = seg;
2592a98e9dbfSaguzovsk 	ulong_t segcnt = 1;
2593a98e9dbfSaguzovsk 	ulong_t cnt;
2594a98e9dbfSaguzovsk 	size_t ssize;
2595a98e9dbfSaguzovsk 	pgcnt_t npages = btop(size);
2596a98e9dbfSaguzovsk 	page_t **plist;
2597a98e9dbfSaguzovsk 	page_t **pl;
2598a98e9dbfSaguzovsk 	int error;
2599a98e9dbfSaguzovsk 	caddr_t eaddr;
2600a98e9dbfSaguzovsk 	faultcode_t fault_err = 0;
2601a98e9dbfSaguzovsk 	pgcnt_t pl_off;
2602a98e9dbfSaguzovsk 	extern struct seg_ops segspt_shmops;
2603a98e9dbfSaguzovsk 
2604dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_LOCK_HELD(as));
2605a98e9dbfSaguzovsk 	ASSERT(seg != NULL);
2606a98e9dbfSaguzovsk 	ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size);
2607a98e9dbfSaguzovsk 	ASSERT(addr + size > seg->s_base + seg->s_size);
2608a98e9dbfSaguzovsk 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2609a98e9dbfSaguzovsk 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2610a98e9dbfSaguzovsk 
2611a98e9dbfSaguzovsk 	/*
2612a98e9dbfSaguzovsk 	 * Count the number of segments covered by the range we are about to
2613a98e9dbfSaguzovsk 	 * lock. The segment count is used to size the shadow list we return
2614a98e9dbfSaguzovsk 	 * back to the caller.
2615a98e9dbfSaguzovsk 	 */
2616a98e9dbfSaguzovsk 	for (; size != 0; size -= ssize, addr += ssize) {
2617a98e9dbfSaguzovsk 		if (addr >= seg->s_base + seg->s_size) {
2618a98e9dbfSaguzovsk 
2619a98e9dbfSaguzovsk 			seg = AS_SEGNEXT(as, seg);
2620a98e9dbfSaguzovsk 			if (seg == NULL || addr != seg->s_base) {
2621dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
2622a98e9dbfSaguzovsk 				return (EFAULT);
2623a98e9dbfSaguzovsk 			}
2624a98e9dbfSaguzovsk 			/*
2625a98e9dbfSaguzovsk 			 * Do a quick check if subsequent segments
2626a98e9dbfSaguzovsk 			 * will most likely support pagelock.
2627a98e9dbfSaguzovsk 			 */
2628a98e9dbfSaguzovsk 			if (seg->s_ops == &segvn_ops) {
2629a98e9dbfSaguzovsk 				vnode_t *vp;
2630a98e9dbfSaguzovsk 
2631a98e9dbfSaguzovsk 				if (SEGOP_GETVP(seg, addr, &vp) != 0 ||
2632a98e9dbfSaguzovsk 				    vp != NULL) {
2633dc32d872SJosef 'Jeff' Sipek 					AS_LOCK_EXIT(as);
2634a98e9dbfSaguzovsk 					goto slow;
2635a98e9dbfSaguzovsk 				}
2636a98e9dbfSaguzovsk 			} else if (seg->s_ops != &segspt_shmops) {
2637dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
2638a98e9dbfSaguzovsk 				goto slow;
2639a98e9dbfSaguzovsk 			}
2640a98e9dbfSaguzovsk 			segcnt++;
2641a98e9dbfSaguzovsk 		}
2642a98e9dbfSaguzovsk 		if (addr + size > seg->s_base + seg->s_size) {
2643a98e9dbfSaguzovsk 			ssize = seg->s_base + seg->s_size - addr;
2644a98e9dbfSaguzovsk 		} else {
2645a98e9dbfSaguzovsk 			ssize = size;
2646a98e9dbfSaguzovsk 		}
2647a98e9dbfSaguzovsk 	}
2648a98e9dbfSaguzovsk 	ASSERT(segcnt > 1);
2649a98e9dbfSaguzovsk 
2650a98e9dbfSaguzovsk 	plist = kmem_zalloc((npages + segcnt) * sizeof (page_t *), KM_SLEEP);
2651a98e9dbfSaguzovsk 
2652a98e9dbfSaguzovsk 	addr = sv_addr;
2653a98e9dbfSaguzovsk 	size = sv_size;
2654a98e9dbfSaguzovsk 	seg = sv_seg;
2655a98e9dbfSaguzovsk 
2656a98e9dbfSaguzovsk 	for (cnt = 0, pl_off = 0; size != 0; size -= ssize, addr += ssize) {
2657a98e9dbfSaguzovsk 		if (addr >= seg->s_base + seg->s_size) {
2658a98e9dbfSaguzovsk 			seg = AS_SEGNEXT(as, seg);
2659a98e9dbfSaguzovsk 			ASSERT(seg != NULL && addr == seg->s_base);
2660a98e9dbfSaguzovsk 			cnt++;
2661a98e9dbfSaguzovsk 			ASSERT(cnt < segcnt);
2662a98e9dbfSaguzovsk 		}
2663a98e9dbfSaguzovsk 		if (addr + size > seg->s_base + seg->s_size) {
2664a98e9dbfSaguzovsk 			ssize = seg->s_base + seg->s_size - addr;
2665a98e9dbfSaguzovsk 		} else {
2666a98e9dbfSaguzovsk 			ssize = size;
2667a98e9dbfSaguzovsk 		}
2668a98e9dbfSaguzovsk 		pl = &plist[npages + cnt];
2669a98e9dbfSaguzovsk 		error = SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2670a98e9dbfSaguzovsk 		    L_PAGELOCK, rw);
2671a98e9dbfSaguzovsk 		if (error) {
2672a98e9dbfSaguzovsk 			break;
2673a98e9dbfSaguzovsk 		}
2674a98e9dbfSaguzovsk 		ASSERT(plist[npages + cnt] != NULL);
2675a98e9dbfSaguzovsk 		ASSERT(pl_off + btop(ssize) <= npages);
2676a98e9dbfSaguzovsk 		bcopy(plist[npages + cnt], &plist[pl_off],
2677a98e9dbfSaguzovsk 		    btop(ssize) * sizeof (page_t *));
2678a98e9dbfSaguzovsk 		pl_off += btop(ssize);
2679a98e9dbfSaguzovsk 	}
2680a98e9dbfSaguzovsk 
2681a98e9dbfSaguzovsk 	if (size == 0) {
2682dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
2683a98e9dbfSaguzovsk 		ASSERT(cnt == segcnt - 1);
2684a98e9dbfSaguzovsk 		*ppp = plist;
2685a98e9dbfSaguzovsk 		return (0);
2686a98e9dbfSaguzovsk 	}
2687a98e9dbfSaguzovsk 
2688a98e9dbfSaguzovsk 	/*
2689a98e9dbfSaguzovsk 	 * one of pagelock calls failed. The error type is in error variable.
2690a98e9dbfSaguzovsk 	 * Unlock what we've locked so far and retry with F_SOFTLOCK if error
2691a98e9dbfSaguzovsk 	 * type is either EFAULT or ENOTSUP. Otherwise just return the error
2692a98e9dbfSaguzovsk 	 * back to the caller.
2693a98e9dbfSaguzovsk 	 */
2694a98e9dbfSaguzovsk 
2695a98e9dbfSaguzovsk 	eaddr = addr;
2696a98e9dbfSaguzovsk 	seg = sv_seg;
2697a98e9dbfSaguzovsk 
2698a98e9dbfSaguzovsk 	for (cnt = 0, addr = sv_addr; addr < eaddr; addr += ssize) {
2699a98e9dbfSaguzovsk 		if (addr >= seg->s_base + seg->s_size) {
2700a98e9dbfSaguzovsk 			seg = AS_SEGNEXT(as, seg);
2701a98e9dbfSaguzovsk 			ASSERT(seg != NULL && addr == seg->s_base);
2702a98e9dbfSaguzovsk 			cnt++;
2703a98e9dbfSaguzovsk 			ASSERT(cnt < segcnt);
2704a98e9dbfSaguzovsk 		}
2705a98e9dbfSaguzovsk 		if (eaddr > seg->s_base + seg->s_size) {
2706a98e9dbfSaguzovsk 			ssize = seg->s_base + seg->s_size - addr;
2707a98e9dbfSaguzovsk 		} else {
2708a98e9dbfSaguzovsk 			ssize = eaddr - addr;
2709a98e9dbfSaguzovsk 		}
2710a98e9dbfSaguzovsk 		pl = &plist[npages + cnt];
2711a98e9dbfSaguzovsk 		ASSERT(*pl != NULL);
2712a98e9dbfSaguzovsk 		(void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2713a98e9dbfSaguzovsk 		    L_PAGEUNLOCK, rw);
2714a98e9dbfSaguzovsk 	}
2715a98e9dbfSaguzovsk 
2716dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
2717a98e9dbfSaguzovsk 
2718a98e9dbfSaguzovsk 	kmem_free(plist, (npages + segcnt) * sizeof (page_t *));
2719a98e9dbfSaguzovsk 
2720a98e9dbfSaguzovsk 	if (error != ENOTSUP && error != EFAULT) {
2721a98e9dbfSaguzovsk 		return (error);
2722a98e9dbfSaguzovsk 	}
2723a98e9dbfSaguzovsk 
2724a98e9dbfSaguzovsk slow:
2725a98e9dbfSaguzovsk 	/*
2726a98e9dbfSaguzovsk 	 * If we are here because pagelock failed due to the need to cow fault
2727a98e9dbfSaguzovsk 	 * in the pages we want to lock F_SOFTLOCK will do this job and in
2728a98e9dbfSaguzovsk 	 * next as_pagelock() call for this address range pagelock will
2729a98e9dbfSaguzovsk 	 * hopefully succeed.
2730a98e9dbfSaguzovsk 	 */
2731a98e9dbfSaguzovsk 	fault_err = as_fault(as->a_hat, as, sv_addr, sv_size, F_SOFTLOCK, rw);
2732a98e9dbfSaguzovsk 	if (fault_err != 0) {
2733a98e9dbfSaguzovsk 		return (fc_decode(fault_err));
2734a98e9dbfSaguzovsk 	}
2735a98e9dbfSaguzovsk 	*ppp = NULL;
2736a98e9dbfSaguzovsk 
2737a98e9dbfSaguzovsk 	return (0);
2738a98e9dbfSaguzovsk }
2739a98e9dbfSaguzovsk 
27407c478bd9Sstevel@tonic-gate /*
27417c478bd9Sstevel@tonic-gate  * lock pages in a given address space. Return shadow list. If
27427c478bd9Sstevel@tonic-gate  * the list is NULL, the MMU mapping is also locked.
27437c478bd9Sstevel@tonic-gate  */
27447c478bd9Sstevel@tonic-gate int
27457c478bd9Sstevel@tonic-gate as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
27467c478bd9Sstevel@tonic-gate     size_t size, enum seg_rw rw)
27477c478bd9Sstevel@tonic-gate {
27487c478bd9Sstevel@tonic-gate 	size_t rsize;
27497c478bd9Sstevel@tonic-gate 	caddr_t raddr;
27507c478bd9Sstevel@tonic-gate 	faultcode_t fault_err;
27517c478bd9Sstevel@tonic-gate 	struct seg *seg;
2752a98e9dbfSaguzovsk 	int err;
27537c478bd9Sstevel@tonic-gate 
27547c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
27557c478bd9Sstevel@tonic-gate 	    "as_pagelock_start: addr %p size %ld", addr, size);
27567c478bd9Sstevel@tonic-gate 
27577c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
27587c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2759843e1988Sjohnlev 	    (size_t)raddr;
2760a98e9dbfSaguzovsk 
27617c478bd9Sstevel@tonic-gate 	/*
27627c478bd9Sstevel@tonic-gate 	 * if the request crosses two segments let
27637c478bd9Sstevel@tonic-gate 	 * as_fault handle it.
27647c478bd9Sstevel@tonic-gate 	 */
2765dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
2766a98e9dbfSaguzovsk 
2767a98e9dbfSaguzovsk 	seg = as_segat(as, raddr);
2768a98e9dbfSaguzovsk 	if (seg == NULL) {
2769dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
2770a98e9dbfSaguzovsk 		return (EFAULT);
2771a98e9dbfSaguzovsk 	}
2772a98e9dbfSaguzovsk 	ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size);
2773a98e9dbfSaguzovsk 	if (raddr + rsize > seg->s_base + seg->s_size) {
2774a98e9dbfSaguzovsk 		return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw));
2775a98e9dbfSaguzovsk 	}
2776a98e9dbfSaguzovsk 	if (raddr + rsize <= raddr) {
2777dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
2778a98e9dbfSaguzovsk 		return (EFAULT);
27797c478bd9Sstevel@tonic-gate 	}
27807c478bd9Sstevel@tonic-gate 
27817c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
27827c478bd9Sstevel@tonic-gate 	    "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize);
27837c478bd9Sstevel@tonic-gate 
27847c478bd9Sstevel@tonic-gate 	/*
27857c478bd9Sstevel@tonic-gate 	 * try to lock pages and pass back shadow list
27867c478bd9Sstevel@tonic-gate 	 */
2787a98e9dbfSaguzovsk 	err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
27887c478bd9Sstevel@tonic-gate 
27897c478bd9Sstevel@tonic-gate 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
27907c478bd9Sstevel@tonic-gate 
2791dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
27927c478bd9Sstevel@tonic-gate 
2793a98e9dbfSaguzovsk 	if (err == 0 || (err != ENOTSUP && err != EFAULT)) {
2794a98e9dbfSaguzovsk 		return (err);
27957c478bd9Sstevel@tonic-gate 	}
27967c478bd9Sstevel@tonic-gate 
27977c478bd9Sstevel@tonic-gate 	/*
2798a98e9dbfSaguzovsk 	 * Use F_SOFTLOCK to lock the pages because pagelock failed either due
2799a98e9dbfSaguzovsk 	 * to no pagelock support for this segment or pages need to be cow
2800a98e9dbfSaguzovsk 	 * faulted in. If fault is needed F_SOFTLOCK will do this job for
2801a98e9dbfSaguzovsk 	 * this as_pagelock() call and in the next as_pagelock() call for the
2802a98e9dbfSaguzovsk 	 * same address range pagelock call will hopefull succeed.
28037c478bd9Sstevel@tonic-gate 	 */
28047c478bd9Sstevel@tonic-gate 	fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
28057c478bd9Sstevel@tonic-gate 	if (fault_err != 0) {
2806843e1988Sjohnlev 		return (fc_decode(fault_err));
28077c478bd9Sstevel@tonic-gate 	}
28087c478bd9Sstevel@tonic-gate 	*ppp = NULL;
28097c478bd9Sstevel@tonic-gate 
28107c478bd9Sstevel@tonic-gate 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
28117c478bd9Sstevel@tonic-gate 	return (0);
28127c478bd9Sstevel@tonic-gate }
28137c478bd9Sstevel@tonic-gate 
2814a98e9dbfSaguzovsk /*
2815a98e9dbfSaguzovsk  * unlock pages locked by as_pagelock_segs().  Retrieve per segment shadow
2816a98e9dbfSaguzovsk  * lists from the end of plist and call pageunlock interface for each segment.
2817a98e9dbfSaguzovsk  * Drop as lock and free plist.
2818a98e9dbfSaguzovsk  */
2819a98e9dbfSaguzovsk static void
2820a98e9dbfSaguzovsk as_pageunlock_segs(struct as *as, struct seg *seg, caddr_t addr, size_t size,
2821a98e9dbfSaguzovsk     struct page **plist, enum seg_rw rw)
2822a98e9dbfSaguzovsk {
2823a98e9dbfSaguzovsk 	ulong_t cnt;
2824a98e9dbfSaguzovsk 	caddr_t eaddr = addr + size;
2825a98e9dbfSaguzovsk 	pgcnt_t npages = btop(size);
2826a98e9dbfSaguzovsk 	size_t ssize;
2827a98e9dbfSaguzovsk 	page_t **pl;
2828a98e9dbfSaguzovsk 
2829dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_LOCK_HELD(as));
2830a98e9dbfSaguzovsk 	ASSERT(seg != NULL);
2831a98e9dbfSaguzovsk 	ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size);
2832a98e9dbfSaguzovsk 	ASSERT(addr + size > seg->s_base + seg->s_size);
2833a98e9dbfSaguzovsk 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
2834a98e9dbfSaguzovsk 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
2835a98e9dbfSaguzovsk 	ASSERT(plist != NULL);
2836a98e9dbfSaguzovsk 
2837a98e9dbfSaguzovsk 	for (cnt = 0; addr < eaddr; addr += ssize) {
2838a98e9dbfSaguzovsk 		if (addr >= seg->s_base + seg->s_size) {
2839a98e9dbfSaguzovsk 			seg = AS_SEGNEXT(as, seg);
2840a98e9dbfSaguzovsk 			ASSERT(seg != NULL && addr == seg->s_base);
2841a98e9dbfSaguzovsk 			cnt++;
2842a98e9dbfSaguzovsk 		}
2843a98e9dbfSaguzovsk 		if (eaddr > seg->s_base + seg->s_size) {
2844a98e9dbfSaguzovsk 			ssize = seg->s_base + seg->s_size - addr;
2845a98e9dbfSaguzovsk 		} else {
2846a98e9dbfSaguzovsk 			ssize = eaddr - addr;
2847a98e9dbfSaguzovsk 		}
2848a98e9dbfSaguzovsk 		pl = &plist[npages + cnt];
2849a98e9dbfSaguzovsk 		ASSERT(*pl != NULL);
2850a98e9dbfSaguzovsk 		(void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl,
2851a98e9dbfSaguzovsk 		    L_PAGEUNLOCK, rw);
2852a98e9dbfSaguzovsk 	}
2853a98e9dbfSaguzovsk 	ASSERT(cnt > 0);
2854dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
2855a98e9dbfSaguzovsk 
2856a98e9dbfSaguzovsk 	cnt++;
2857a98e9dbfSaguzovsk 	kmem_free(plist, (npages + cnt) * sizeof (page_t *));
2858a98e9dbfSaguzovsk }
2859a98e9dbfSaguzovsk 
28607c478bd9Sstevel@tonic-gate /*
28617c478bd9Sstevel@tonic-gate  * unlock pages in a given address range
28627c478bd9Sstevel@tonic-gate  */
28637c478bd9Sstevel@tonic-gate void
28647c478bd9Sstevel@tonic-gate as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
28657c478bd9Sstevel@tonic-gate     enum seg_rw rw)
28667c478bd9Sstevel@tonic-gate {
28677c478bd9Sstevel@tonic-gate 	struct seg *seg;
28687c478bd9Sstevel@tonic-gate 	size_t rsize;
28697c478bd9Sstevel@tonic-gate 	caddr_t raddr;
28707c478bd9Sstevel@tonic-gate 
28717c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
28727c478bd9Sstevel@tonic-gate 	    "as_pageunlock_start: addr %p size %ld", addr, size);
28737c478bd9Sstevel@tonic-gate 
28747c478bd9Sstevel@tonic-gate 	/*
28757c478bd9Sstevel@tonic-gate 	 * if the shadow list is NULL, as_pagelock was
28767c478bd9Sstevel@tonic-gate 	 * falling back to as_fault
28777c478bd9Sstevel@tonic-gate 	 */
28787c478bd9Sstevel@tonic-gate 	if (pp == NULL) {
28797c478bd9Sstevel@tonic-gate 		(void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
28807c478bd9Sstevel@tonic-gate 		return;
28817c478bd9Sstevel@tonic-gate 	}
2882a98e9dbfSaguzovsk 
28837c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
28847c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
2885843e1988Sjohnlev 	    (size_t)raddr;
2886a98e9dbfSaguzovsk 
2887dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
2888a98e9dbfSaguzovsk 	seg = as_segat(as, raddr);
2889a98e9dbfSaguzovsk 	ASSERT(seg != NULL);
2890a98e9dbfSaguzovsk 
28917c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
28927c478bd9Sstevel@tonic-gate 	    "seg_unlock_start: raddr %p rsize %ld", raddr, rsize);
2893a98e9dbfSaguzovsk 
2894a98e9dbfSaguzovsk 	ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size);
2895a98e9dbfSaguzovsk 	if (raddr + rsize <= seg->s_base + seg->s_size) {
2896a98e9dbfSaguzovsk 		SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
2897a98e9dbfSaguzovsk 	} else {
2898a98e9dbfSaguzovsk 		as_pageunlock_segs(as, seg, raddr, rsize, pp, rw);
2899a98e9dbfSaguzovsk 		return;
2900a98e9dbfSaguzovsk 	}
2901dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
29027c478bd9Sstevel@tonic-gate 	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
29037c478bd9Sstevel@tonic-gate }
29047c478bd9Sstevel@tonic-gate 
29057c478bd9Sstevel@tonic-gate int
29067c478bd9Sstevel@tonic-gate as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc,
29077c478bd9Sstevel@tonic-gate     boolean_t wait)
29087c478bd9Sstevel@tonic-gate {
29097c478bd9Sstevel@tonic-gate 	struct seg *seg;
29107c478bd9Sstevel@tonic-gate 	size_t ssize;
29117c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
29127c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
29137c478bd9Sstevel@tonic-gate 	int error = 0;
29147c478bd9Sstevel@tonic-gate 	size_t pgsz = page_get_pagesize(szc);
29157c478bd9Sstevel@tonic-gate 
29167c478bd9Sstevel@tonic-gate setpgsz_top:
29177c478bd9Sstevel@tonic-gate 	if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) {
29187c478bd9Sstevel@tonic-gate 		return (EINVAL);
29197c478bd9Sstevel@tonic-gate 	}
29207c478bd9Sstevel@tonic-gate 
29217c478bd9Sstevel@tonic-gate 	raddr = addr;
29227c478bd9Sstevel@tonic-gate 	rsize = size;
29237c478bd9Sstevel@tonic-gate 
29247c478bd9Sstevel@tonic-gate 	if (raddr + rsize < raddr)		/* check for wraparound */
29257c478bd9Sstevel@tonic-gate 		return (ENOMEM);
29267c478bd9Sstevel@tonic-gate 
2927dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
29287c478bd9Sstevel@tonic-gate 	as_clearwatchprot(as, raddr, rsize);
29297c478bd9Sstevel@tonic-gate 	seg = as_segat(as, raddr);
29307c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
29317c478bd9Sstevel@tonic-gate 		as_setwatch(as);
2932dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
29337c478bd9Sstevel@tonic-gate 		return (ENOMEM);
29347c478bd9Sstevel@tonic-gate 	}
29357c478bd9Sstevel@tonic-gate 
29367c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
29377c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
29387c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
29397c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
29407c478bd9Sstevel@tonic-gate 				error = ENOMEM;
29417c478bd9Sstevel@tonic-gate 				break;
29427c478bd9Sstevel@tonic-gate 			}
29437c478bd9Sstevel@tonic-gate 		}
29447c478bd9Sstevel@tonic-gate 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
29457c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
29467c478bd9Sstevel@tonic-gate 		} else {
29477c478bd9Sstevel@tonic-gate 			ssize = rsize;
29487c478bd9Sstevel@tonic-gate 		}
29497c478bd9Sstevel@tonic-gate 
2950a98e9dbfSaguzovsk retry:
29517c478bd9Sstevel@tonic-gate 		error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
29527c478bd9Sstevel@tonic-gate 
29537c478bd9Sstevel@tonic-gate 		if (error == IE_NOMEM) {
29547c478bd9Sstevel@tonic-gate 			error = EAGAIN;
29557c478bd9Sstevel@tonic-gate 			break;
29567c478bd9Sstevel@tonic-gate 		}
29577c478bd9Sstevel@tonic-gate 
29587c478bd9Sstevel@tonic-gate 		if (error == IE_RETRY) {
2959dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
29607c478bd9Sstevel@tonic-gate 			goto setpgsz_top;
29617c478bd9Sstevel@tonic-gate 		}
29627c478bd9Sstevel@tonic-gate 
29637c478bd9Sstevel@tonic-gate 		if (error == ENOTSUP) {
29647c478bd9Sstevel@tonic-gate 			error = EINVAL;
29657c478bd9Sstevel@tonic-gate 			break;
29667c478bd9Sstevel@tonic-gate 		}
29677c478bd9Sstevel@tonic-gate 
29687c478bd9Sstevel@tonic-gate 		if (wait && (error == EAGAIN)) {
29697c478bd9Sstevel@tonic-gate 			/*
29707c478bd9Sstevel@tonic-gate 			 * Memory is currently locked.  It must be unlocked
29717c478bd9Sstevel@tonic-gate 			 * before this operation can succeed through a retry.
29727c478bd9Sstevel@tonic-gate 			 * The possible reasons for locked memory and
29737c478bd9Sstevel@tonic-gate 			 * corresponding strategies for unlocking are:
29747c478bd9Sstevel@tonic-gate 			 * (1) Normal I/O
29757c478bd9Sstevel@tonic-gate 			 *	wait for a signal that the I/O operation
29767c478bd9Sstevel@tonic-gate 			 *	has completed and the memory is unlocked.
29777c478bd9Sstevel@tonic-gate 			 * (2) Asynchronous I/O
29787c478bd9Sstevel@tonic-gate 			 *	The aio subsystem does not unlock pages when
29797c478bd9Sstevel@tonic-gate 			 *	the I/O is completed. Those pages are unlocked
29807c478bd9Sstevel@tonic-gate 			 *	when the application calls aiowait/aioerror.
29817c478bd9Sstevel@tonic-gate 			 *	So, to prevent blocking forever, cv_broadcast()
29827c478bd9Sstevel@tonic-gate 			 *	is done to wake up aio_cleanup_thread.
29837c478bd9Sstevel@tonic-gate 			 *	Subsequently, segvn_reclaim will be called, and
29847c478bd9Sstevel@tonic-gate 			 *	that will do AS_CLRUNMAPWAIT() and wake us up.
29857c478bd9Sstevel@tonic-gate 			 * (3) Long term page locking:
29867c478bd9Sstevel@tonic-gate 			 *	This is not relevant for as_setpagesize()
29877c478bd9Sstevel@tonic-gate 			 *	because we cannot change the page size for
29887c478bd9Sstevel@tonic-gate 			 *	driver memory. The attempt to do so will
29897c478bd9Sstevel@tonic-gate 			 *	fail with a different error than EAGAIN so
29907c478bd9Sstevel@tonic-gate 			 *	there's no need to trigger as callbacks like
29917c478bd9Sstevel@tonic-gate 			 *	as_unmap, as_setprot or as_free would do.
29927c478bd9Sstevel@tonic-gate 			 */
29937c478bd9Sstevel@tonic-gate 			mutex_enter(&as->a_contents);
2994a98e9dbfSaguzovsk 			if (!AS_ISNOUNMAPWAIT(as)) {
2995a98e9dbfSaguzovsk 				if (AS_ISUNMAPWAIT(as) == 0) {
2996a98e9dbfSaguzovsk 					cv_broadcast(&as->a_cv);
2997a98e9dbfSaguzovsk 				}
2998a98e9dbfSaguzovsk 				AS_SETUNMAPWAIT(as);
2999dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
3000a98e9dbfSaguzovsk 				while (AS_ISUNMAPWAIT(as)) {
3001a98e9dbfSaguzovsk 					cv_wait(&as->a_cv, &as->a_contents);
3002a98e9dbfSaguzovsk 				}
3003a98e9dbfSaguzovsk 			} else {
3004a98e9dbfSaguzovsk 				/*
3005a98e9dbfSaguzovsk 				 * We may have raced with
3006a98e9dbfSaguzovsk 				 * segvn_reclaim()/segspt_reclaim(). In this
3007a98e9dbfSaguzovsk 				 * case clean nounmapwait flag and retry since
3008a98e9dbfSaguzovsk 				 * softlockcnt in this segment may be already
3009a98e9dbfSaguzovsk 				 * 0.  We don't drop as writer lock so our
3010a98e9dbfSaguzovsk 				 * number of retries without sleeping should
3011a98e9dbfSaguzovsk 				 * be very small. See segvn_reclaim() for
3012a98e9dbfSaguzovsk 				 * more comments.
3013a98e9dbfSaguzovsk 				 */
3014a98e9dbfSaguzovsk 				AS_CLRNOUNMAPWAIT(as);
3015a98e9dbfSaguzovsk 				mutex_exit(&as->a_contents);
3016a98e9dbfSaguzovsk 				goto retry;
30177c478bd9Sstevel@tonic-gate 			}
30187c478bd9Sstevel@tonic-gate 			mutex_exit(&as->a_contents);
30197c478bd9Sstevel@tonic-gate 			goto setpgsz_top;
30207c478bd9Sstevel@tonic-gate 		} else if (error != 0) {
30217c478bd9Sstevel@tonic-gate 			break;
30227c478bd9Sstevel@tonic-gate 		}
30237c478bd9Sstevel@tonic-gate 	}
30247c478bd9Sstevel@tonic-gate 	as_setwatch(as);
3025dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
30267c478bd9Sstevel@tonic-gate 	return (error);
30277c478bd9Sstevel@tonic-gate }
30287c478bd9Sstevel@tonic-gate 
3029ec25b48fSsusans /*
3030ec25b48fSsusans  * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments
3031ec25b48fSsusans  * in its chunk where s_szc is less than the szc we want to set.
3032ec25b48fSsusans  */
3033ec25b48fSsusans static int
3034ec25b48fSsusans as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
3035ec25b48fSsusans     int *retry)
3036ec25b48fSsusans {
3037ec25b48fSsusans 	struct seg *seg;
3038ec25b48fSsusans 	size_t ssize;
3039ec25b48fSsusans 	int error;
3040ec25b48fSsusans 
3041dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
3042a98e9dbfSaguzovsk 
3043ec25b48fSsusans 	seg = as_segat(as, raddr);
3044ec25b48fSsusans 	if (seg == NULL) {
3045ec25b48fSsusans 		panic("as_iset3_default_lpsize: no seg");
3046ec25b48fSsusans 	}
3047ec25b48fSsusans 
3048ec25b48fSsusans 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
3049ec25b48fSsusans 		if (raddr >= seg->s_base + seg->s_size) {
3050ec25b48fSsusans 			seg = AS_SEGNEXT(as, seg);
3051ec25b48fSsusans 			if (seg == NULL || raddr != seg->s_base) {
3052ec25b48fSsusans 				panic("as_iset3_default_lpsize: as changed");
3053ec25b48fSsusans 			}
3054ec25b48fSsusans 		}
3055ec25b48fSsusans 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3056ec25b48fSsusans 			ssize = seg->s_base + seg->s_size - raddr;
3057ec25b48fSsusans 		} else {
3058ec25b48fSsusans 			ssize = rsize;
3059ec25b48fSsusans 		}
3060ec25b48fSsusans 
3061ec25b48fSsusans 		if (szc > seg->s_szc) {
3062ec25b48fSsusans 			error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc);
3063ec25b48fSsusans 			/* Only retry on EINVAL segments that have no vnode. */
3064ec25b48fSsusans 			if (error == EINVAL) {
3065ec25b48fSsusans 				vnode_t *vp = NULL;
3066ec25b48fSsusans 				if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) &&
3067ec25b48fSsusans 				    (SEGOP_GETVP(seg, raddr, &vp) != 0 ||
3068ec25b48fSsusans 				    vp == NULL)) {
3069ec25b48fSsusans 					*retry = 1;
3070ec25b48fSsusans 				} else {
3071ec25b48fSsusans 					*retry = 0;
3072ec25b48fSsusans 				}
3073ec25b48fSsusans 			}
3074ec25b48fSsusans 			if (error) {
3075ec25b48fSsusans 				return (error);
3076ec25b48fSsusans 			}
3077ec25b48fSsusans 		}
3078ec25b48fSsusans 	}
3079ec25b48fSsusans 	return (0);
3080ec25b48fSsusans }
3081ec25b48fSsusans 
3082ec25b48fSsusans /*
3083ec25b48fSsusans  * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the
3084ec25b48fSsusans  * pagesize on each segment in its range, but if any fails with EINVAL,
3085ec25b48fSsusans  * then it reduces the pagesizes to the next size in the bitmap and
3086ec25b48fSsusans  * retries as_iset3_default_lpsize(). The reason why the code retries
3087ec25b48fSsusans  * smaller allowed sizes on EINVAL is because (a) the anon offset may not
3088ec25b48fSsusans  * match the bigger sizes, and (b) it's hard to get this offset (to begin
3089ec25b48fSsusans  * with) to pass to map_pgszcvec().
3090ec25b48fSsusans  */
3091ec25b48fSsusans static int
3092ec25b48fSsusans as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc,
3093ec25b48fSsusans     uint_t szcvec)
3094ec25b48fSsusans {
3095ec25b48fSsusans 	int error;
3096ec25b48fSsusans 	int retry;
3097ec25b48fSsusans 
3098dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
3099a98e9dbfSaguzovsk 
3100ec25b48fSsusans 	for (;;) {
3101ec25b48fSsusans 		error = as_iset3_default_lpsize(as, addr, size, szc, &retry);
3102ec25b48fSsusans 		if (error == EINVAL && retry) {
3103ec25b48fSsusans 			szcvec &= ~(1 << szc);
3104ec25b48fSsusans 			if (szcvec <= 1) {
3105ec25b48fSsusans 				return (EINVAL);
3106ec25b48fSsusans 			}
3107ec25b48fSsusans 			szc = highbit(szcvec) - 1;
3108ec25b48fSsusans 		} else {
3109ec25b48fSsusans 			return (error);
3110ec25b48fSsusans 		}
3111ec25b48fSsusans 	}
3112ec25b48fSsusans }
3113ec25b48fSsusans 
3114ec25b48fSsusans /*
3115ec25b48fSsusans  * as_iset1_default_lpsize() breaks its chunk into areas where existing
3116ec25b48fSsusans  * segments have a smaller szc than we want to set. For each such area,
3117ec25b48fSsusans  * it calls as_iset2_default_lpsize()
3118ec25b48fSsusans  */
3119ec25b48fSsusans static int
3120ec25b48fSsusans as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc,
3121ec25b48fSsusans     uint_t szcvec)
3122ec25b48fSsusans {
3123ec25b48fSsusans 	struct seg *seg;
3124ec25b48fSsusans 	size_t ssize;
3125ec25b48fSsusans 	caddr_t setaddr = raddr;
3126ec25b48fSsusans 	size_t setsize = 0;
3127ec25b48fSsusans 	int set;
3128ec25b48fSsusans 	int error;
3129ec25b48fSsusans 
3130dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
3131ec25b48fSsusans 
3132ec25b48fSsusans 	seg = as_segat(as, raddr);
3133ec25b48fSsusans 	if (seg == NULL) {
3134ec25b48fSsusans 		panic("as_iset1_default_lpsize: no seg");
3135ec25b48fSsusans 	}
3136ec25b48fSsusans 	if (seg->s_szc < szc) {
3137ec25b48fSsusans 		set = 1;
3138ec25b48fSsusans 	} else {
3139ec25b48fSsusans 		set = 0;
3140ec25b48fSsusans 	}
3141ec25b48fSsusans 
3142ec25b48fSsusans 	for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3143ec25b48fSsusans 		if (raddr >= seg->s_base + seg->s_size) {
3144ec25b48fSsusans 			seg = AS_SEGNEXT(as, seg);
3145ec25b48fSsusans 			if (seg == NULL || raddr != seg->s_base) {
3146ec25b48fSsusans 				panic("as_iset1_default_lpsize: as changed");
3147ec25b48fSsusans 			}
3148ec25b48fSsusans 			if (seg->s_szc >= szc && set) {
3149ec25b48fSsusans 				ASSERT(setsize != 0);
3150ec25b48fSsusans 				error = as_iset2_default_lpsize(as,
3151ec25b48fSsusans 				    setaddr, setsize, szc, szcvec);
3152ec25b48fSsusans 				if (error) {
3153ec25b48fSsusans 					return (error);
3154ec25b48fSsusans 				}
3155ec25b48fSsusans 				set = 0;
3156ec25b48fSsusans 			} else if (seg->s_szc < szc && !set) {
3157ec25b48fSsusans 				setaddr = raddr;
3158ec25b48fSsusans 				setsize = 0;
3159ec25b48fSsusans 				set = 1;
3160ec25b48fSsusans 			}
3161ec25b48fSsusans 		}
3162ec25b48fSsusans 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3163ec25b48fSsusans 			ssize = seg->s_base + seg->s_size - raddr;
3164ec25b48fSsusans 		} else {
3165ec25b48fSsusans 			ssize = rsize;
3166ec25b48fSsusans 		}
3167ec25b48fSsusans 	}
3168ec25b48fSsusans 	error = 0;
3169ec25b48fSsusans 	if (set) {
3170ec25b48fSsusans 		ASSERT(setsize != 0);
3171ec25b48fSsusans 		error = as_iset2_default_lpsize(as, setaddr, setsize,
3172ec25b48fSsusans 		    szc, szcvec);
3173ec25b48fSsusans 	}
3174ec25b48fSsusans 	return (error);
3175ec25b48fSsusans }
3176ec25b48fSsusans 
3177ec25b48fSsusans /*
3178ec25b48fSsusans  * as_iset_default_lpsize() breaks its chunk according to the size code bitmap
3179ec25b48fSsusans  * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each
3180ec25b48fSsusans  * chunk to as_iset1_default_lpsize().
3181ec25b48fSsusans  */
3182ec25b48fSsusans static int
3183ec25b48fSsusans as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags,
3184ec25b48fSsusans     int type)
3185ec25b48fSsusans {
3186ec25b48fSsusans 	int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM;
3187ec25b48fSsusans 	uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr,
3188843e1988Sjohnlev 	    flags, rtype, 1);
3189ec25b48fSsusans 	uint_t szc;
3190ec25b48fSsusans 	uint_t nszc;
3191ec25b48fSsusans 	int error;
3192ec25b48fSsusans 	caddr_t a;
3193ec25b48fSsusans 	caddr_t eaddr;
3194ec25b48fSsusans 	size_t segsize;
3195ec25b48fSsusans 	size_t pgsz;
3196ec25b48fSsusans 	uint_t save_szcvec;
3197ec25b48fSsusans 
3198dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
3199ec25b48fSsusans 	ASSERT(IS_P2ALIGNED(addr, PAGESIZE));
3200ec25b48fSsusans 	ASSERT(IS_P2ALIGNED(size, PAGESIZE));
3201ec25b48fSsusans 
3202ec25b48fSsusans 	szcvec &= ~1;
3203ec25b48fSsusans 	if (szcvec <= 1) {	/* skip if base page size */
3204ec25b48fSsusans 		return (0);
3205ec25b48fSsusans 	}
3206ec25b48fSsusans 
3207ec25b48fSsusans 	/* Get the pagesize of the first larger page size. */
3208ec25b48fSsusans 	szc = lowbit(szcvec) - 1;
3209ec25b48fSsusans 	pgsz = page_get_pagesize(szc);
3210ec25b48fSsusans 	eaddr = addr + size;
3211ec25b48fSsusans 	addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
3212ec25b48fSsusans 	eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
3213ec25b48fSsusans 
3214ec25b48fSsusans 	save_szcvec = szcvec;
3215ec25b48fSsusans 	szcvec >>= (szc + 1);
3216ec25b48fSsusans 	nszc = szc;
3217ec25b48fSsusans 	while (szcvec) {
3218ec25b48fSsusans 		if ((szcvec & 0x1) == 0) {
3219ec25b48fSsusans 			nszc++;
3220ec25b48fSsusans 			szcvec >>= 1;
3221ec25b48fSsusans 			continue;
3222ec25b48fSsusans 		}
3223ec25b48fSsusans 		nszc++;
3224ec25b48fSsusans 		pgsz = page_get_pagesize(nszc);
3225ec25b48fSsusans 		a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz);
3226ec25b48fSsusans 		if (a != addr) {
3227ec25b48fSsusans 			ASSERT(szc > 0);
3228ec25b48fSsusans 			ASSERT(a < eaddr);
3229ec25b48fSsusans 			segsize = a - addr;
3230ec25b48fSsusans 			error = as_iset1_default_lpsize(as, addr, segsize, szc,
3231ec25b48fSsusans 			    save_szcvec);
3232ec25b48fSsusans 			if (error) {
3233ec25b48fSsusans 				return (error);
3234ec25b48fSsusans 			}
3235ec25b48fSsusans 			addr = a;
3236ec25b48fSsusans 		}
3237ec25b48fSsusans 		szc = nszc;
3238ec25b48fSsusans 		szcvec >>= 1;
3239ec25b48fSsusans 	}
3240ec25b48fSsusans 
3241ec25b48fSsusans 	ASSERT(addr < eaddr);
3242ec25b48fSsusans 	szcvec = save_szcvec;
3243ec25b48fSsusans 	while (szcvec) {
3244ec25b48fSsusans 		a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz);
3245ec25b48fSsusans 		ASSERT(a >= addr);
3246ec25b48fSsusans 		if (a != addr) {
3247ec25b48fSsusans 			ASSERT(szc > 0);
3248ec25b48fSsusans 			segsize = a - addr;
3249ec25b48fSsusans 			error = as_iset1_default_lpsize(as, addr, segsize, szc,
3250ec25b48fSsusans 			    save_szcvec);
3251ec25b48fSsusans 			if (error) {
3252ec25b48fSsusans 				return (error);
3253ec25b48fSsusans 			}
3254ec25b48fSsusans 			addr = a;
3255ec25b48fSsusans 		}
3256ec25b48fSsusans 		szcvec &= ~(1 << szc);
3257ec25b48fSsusans 		if (szcvec) {
3258ec25b48fSsusans 			szc = highbit(szcvec) - 1;
3259ec25b48fSsusans 			pgsz = page_get_pagesize(szc);
3260ec25b48fSsusans 		}
3261ec25b48fSsusans 	}
3262ec25b48fSsusans 	ASSERT(addr == eaddr);
3263ec25b48fSsusans 
3264ec25b48fSsusans 	return (0);
3265ec25b48fSsusans }
3266ec25b48fSsusans 
3267ec25b48fSsusans /*
3268ec25b48fSsusans  * Set the default large page size for the range. Called via memcntl with
3269ec25b48fSsusans  * page size set to 0. as_set_default_lpsize breaks the range down into
3270ec25b48fSsusans  * chunks with the same type/flags, ignores-non segvn segments, and passes
3271ec25b48fSsusans  * each chunk to as_iset_default_lpsize().
3272ec25b48fSsusans  */
3273ec25b48fSsusans int
3274ec25b48fSsusans as_set_default_lpsize(struct as *as, caddr_t addr, size_t size)
3275ec25b48fSsusans {
3276ec25b48fSsusans 	struct seg *seg;
3277ec25b48fSsusans 	caddr_t raddr;
3278ec25b48fSsusans 	size_t rsize;
3279ec25b48fSsusans 	size_t ssize;
3280ec25b48fSsusans 	int rtype, rflags;
3281ec25b48fSsusans 	int stype, sflags;
3282ec25b48fSsusans 	int error;
3283ec25b48fSsusans 	caddr_t	setaddr;
3284ec25b48fSsusans 	size_t setsize;
3285ec25b48fSsusans 	int segvn;
3286ec25b48fSsusans 
3287ec25b48fSsusans 	if (size == 0)
3288ec25b48fSsusans 		return (0);
3289ec25b48fSsusans 
3290dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
3291ec25b48fSsusans again:
3292ec25b48fSsusans 	error = 0;
3293ec25b48fSsusans 
3294ec25b48fSsusans 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
3295ec25b48fSsusans 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
3296ec25b48fSsusans 	    (size_t)raddr;
3297ec25b48fSsusans 
3298ec25b48fSsusans 	if (raddr + rsize < raddr) {		/* check for wraparound */
3299dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
3300ec25b48fSsusans 		return (ENOMEM);
3301ec25b48fSsusans 	}
3302ec25b48fSsusans 	as_clearwatchprot(as, raddr, rsize);
3303ec25b48fSsusans 	seg = as_segat(as, raddr);
3304ec25b48fSsusans 	if (seg == NULL) {
3305ec25b48fSsusans 		as_setwatch(as);
3306dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
3307ec25b48fSsusans 		return (ENOMEM);
3308ec25b48fSsusans 	}
3309ec25b48fSsusans 	if (seg->s_ops == &segvn_ops) {
3310ec25b48fSsusans 		rtype = SEGOP_GETTYPE(seg, addr);
3311ec25b48fSsusans 		rflags = rtype & (MAP_TEXT | MAP_INITDATA);
3312ec25b48fSsusans 		rtype = rtype & (MAP_SHARED | MAP_PRIVATE);
3313ec25b48fSsusans 		segvn = 1;
3314ec25b48fSsusans 	} else {
3315ec25b48fSsusans 		segvn = 0;
3316ec25b48fSsusans 	}
3317ec25b48fSsusans 	setaddr = raddr;
3318ec25b48fSsusans 	setsize = 0;
3319ec25b48fSsusans 
3320ec25b48fSsusans 	for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) {
3321ec25b48fSsusans 		if (raddr >= (seg->s_base + seg->s_size)) {
3322ec25b48fSsusans 			seg = AS_SEGNEXT(as, seg);
3323ec25b48fSsusans 			if (seg == NULL || raddr != seg->s_base) {
3324ec25b48fSsusans 				error = ENOMEM;
3325ec25b48fSsusans 				break;
3326ec25b48fSsusans 			}
3327ec25b48fSsusans 			if (seg->s_ops == &segvn_ops) {
3328ec25b48fSsusans 				stype = SEGOP_GETTYPE(seg, raddr);
3329ec25b48fSsusans 				sflags = stype & (MAP_TEXT | MAP_INITDATA);
3330ec25b48fSsusans 				stype &= (MAP_SHARED | MAP_PRIVATE);
3331ec25b48fSsusans 				if (segvn && (rflags != sflags ||
3332ec25b48fSsusans 				    rtype != stype)) {
3333ec25b48fSsusans 					/*
3334ec25b48fSsusans 					 * The next segment is also segvn but
3335ec25b48fSsusans 					 * has different flags and/or type.
3336ec25b48fSsusans 					 */
3337ec25b48fSsusans 					ASSERT(setsize != 0);
3338ec25b48fSsusans 					error = as_iset_default_lpsize(as,
3339ec25b48fSsusans 					    setaddr, setsize, rflags, rtype);
3340ec25b48fSsusans 					if (error) {
3341ec25b48fSsusans 						break;
3342ec25b48fSsusans 					}
3343ec25b48fSsusans 					rflags = sflags;
3344ec25b48fSsusans 					rtype = stype;
3345ec25b48fSsusans 					setaddr = raddr;
3346ec25b48fSsusans 					setsize = 0;
3347ec25b48fSsusans 				} else if (!segvn) {
3348ec25b48fSsusans 					rflags = sflags;
3349ec25b48fSsusans 					rtype = stype;
3350ec25b48fSsusans 					setaddr = raddr;
3351ec25b48fSsusans 					setsize = 0;
3352ec25b48fSsusans 					segvn = 1;
3353ec25b48fSsusans 				}
3354ec25b48fSsusans 			} else if (segvn) {
3355ec25b48fSsusans 				/* The next segment is not segvn. */
3356ec25b48fSsusans 				ASSERT(setsize != 0);
3357ec25b48fSsusans 				error = as_iset_default_lpsize(as,
3358ec25b48fSsusans 				    setaddr, setsize, rflags, rtype);
3359ec25b48fSsusans 				if (error) {
3360ec25b48fSsusans 					break;
3361ec25b48fSsusans 				}
3362ec25b48fSsusans 				segvn = 0;
3363ec25b48fSsusans 			}
3364ec25b48fSsusans 		}
3365ec25b48fSsusans 		if ((raddr + rsize) > (seg->s_base + seg->s_size)) {
3366ec25b48fSsusans 			ssize = seg->s_base + seg->s_size - raddr;
3367ec25b48fSsusans 		} else {
3368ec25b48fSsusans 			ssize = rsize;
3369ec25b48fSsusans 		}
3370ec25b48fSsusans 	}
3371ec25b48fSsusans 	if (error == 0 && segvn) {
3372ec25b48fSsusans 		/* The last chunk when rsize == 0. */
3373ec25b48fSsusans 		ASSERT(setsize != 0);
3374ec25b48fSsusans 		error = as_iset_default_lpsize(as, setaddr, setsize,
3375ec25b48fSsusans 		    rflags, rtype);
3376ec25b48fSsusans 	}
3377ec25b48fSsusans 
3378ec25b48fSsusans 	if (error == IE_RETRY) {
3379ec25b48fSsusans 		goto again;
3380ec25b48fSsusans 	} else if (error == IE_NOMEM) {
3381ec25b48fSsusans 		error = EAGAIN;
3382ec25b48fSsusans 	} else if (error == ENOTSUP) {
3383ec25b48fSsusans 		error = EINVAL;
3384ec25b48fSsusans 	} else if (error == EAGAIN) {
3385ec25b48fSsusans 		mutex_enter(&as->a_contents);
3386a98e9dbfSaguzovsk 		if (!AS_ISNOUNMAPWAIT(as)) {
3387a98e9dbfSaguzovsk 			if (AS_ISUNMAPWAIT(as) == 0) {
3388a98e9dbfSaguzovsk 				cv_broadcast(&as->a_cv);
3389a98e9dbfSaguzovsk 			}
3390a98e9dbfSaguzovsk 			AS_SETUNMAPWAIT(as);
3391dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
3392a98e9dbfSaguzovsk 			while (AS_ISUNMAPWAIT(as)) {
3393a98e9dbfSaguzovsk 				cv_wait(&as->a_cv, &as->a_contents);
3394a98e9dbfSaguzovsk 			}
3395a98e9dbfSaguzovsk 			mutex_exit(&as->a_contents);
3396dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_ENTER(as, RW_WRITER);
3397a98e9dbfSaguzovsk 		} else {
3398a98e9dbfSaguzovsk 			/*
3399a98e9dbfSaguzovsk 			 * We may have raced with
3400a98e9dbfSaguzovsk 			 * segvn_reclaim()/segspt_reclaim(). In this case
3401a98e9dbfSaguzovsk 			 * clean nounmapwait flag and retry since softlockcnt
3402a98e9dbfSaguzovsk 			 * in this segment may be already 0.  We don't drop as
3403a98e9dbfSaguzovsk 			 * writer lock so our number of retries without
3404a98e9dbfSaguzovsk 			 * sleeping should be very small. See segvn_reclaim()
3405a98e9dbfSaguzovsk 			 * for more comments.
3406a98e9dbfSaguzovsk 			 */
3407a98e9dbfSaguzovsk 			AS_CLRNOUNMAPWAIT(as);
3408a98e9dbfSaguzovsk 			mutex_exit(&as->a_contents);
3409ec25b48fSsusans 		}
3410ec25b48fSsusans 		goto again;
3411ec25b48fSsusans 	}
3412ec25b48fSsusans 
3413ec25b48fSsusans 	as_setwatch(as);
3414dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
3415ec25b48fSsusans 	return (error);
3416ec25b48fSsusans }
3417ec25b48fSsusans 
34187c478bd9Sstevel@tonic-gate /*
34197c478bd9Sstevel@tonic-gate  * Setup all of the uninitialized watched pages that we can.
34207c478bd9Sstevel@tonic-gate  */
34217c478bd9Sstevel@tonic-gate void
34227c478bd9Sstevel@tonic-gate as_setwatch(struct as *as)
34237c478bd9Sstevel@tonic-gate {
34247c478bd9Sstevel@tonic-gate 	struct watched_page *pwp;
34257c478bd9Sstevel@tonic-gate 	struct seg *seg;
34267c478bd9Sstevel@tonic-gate 	caddr_t vaddr;
34277c478bd9Sstevel@tonic-gate 	uint_t prot;
34287c478bd9Sstevel@tonic-gate 	int  err, retrycnt;
34297c478bd9Sstevel@tonic-gate 
34307c478bd9Sstevel@tonic-gate 	if (avl_numnodes(&as->a_wpage) == 0)
34317c478bd9Sstevel@tonic-gate 		return;
34327c478bd9Sstevel@tonic-gate 
3433dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
34347c478bd9Sstevel@tonic-gate 
34357c478bd9Sstevel@tonic-gate 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
34367c478bd9Sstevel@tonic-gate 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
34377c478bd9Sstevel@tonic-gate 		retrycnt = 0;
34387c478bd9Sstevel@tonic-gate 	retry:
34397c478bd9Sstevel@tonic-gate 		vaddr = pwp->wp_vaddr;
34407c478bd9Sstevel@tonic-gate 		if (pwp->wp_oprot != 0 ||	/* already set up */
34417c478bd9Sstevel@tonic-gate 		    (seg = as_segat(as, vaddr)) == NULL ||
34427c478bd9Sstevel@tonic-gate 		    SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
34437c478bd9Sstevel@tonic-gate 			continue;
34447c478bd9Sstevel@tonic-gate 
34457c478bd9Sstevel@tonic-gate 		pwp->wp_oprot = prot;
34467c478bd9Sstevel@tonic-gate 		if (pwp->wp_read)
34477c478bd9Sstevel@tonic-gate 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
34487c478bd9Sstevel@tonic-gate 		if (pwp->wp_write)
34497c478bd9Sstevel@tonic-gate 			prot &= ~PROT_WRITE;
34507c478bd9Sstevel@tonic-gate 		if (pwp->wp_exec)
34517c478bd9Sstevel@tonic-gate 			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
34527c478bd9Sstevel@tonic-gate 		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) {
34537c478bd9Sstevel@tonic-gate 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
34547c478bd9Sstevel@tonic-gate 			if (err == IE_RETRY) {
34557c478bd9Sstevel@tonic-gate 				pwp->wp_oprot = 0;
34567c478bd9Sstevel@tonic-gate 				ASSERT(retrycnt == 0);
34577c478bd9Sstevel@tonic-gate 				retrycnt++;
34587c478bd9Sstevel@tonic-gate 				goto retry;
34597c478bd9Sstevel@tonic-gate 			}
34607c478bd9Sstevel@tonic-gate 		}
34617c478bd9Sstevel@tonic-gate 		pwp->wp_prot = prot;
34627c478bd9Sstevel@tonic-gate 	}
34637c478bd9Sstevel@tonic-gate }
34647c478bd9Sstevel@tonic-gate 
34657c478bd9Sstevel@tonic-gate /*
34667c478bd9Sstevel@tonic-gate  * Clear all of the watched pages in the address space.
34677c478bd9Sstevel@tonic-gate  */
34687c478bd9Sstevel@tonic-gate void
34697c478bd9Sstevel@tonic-gate as_clearwatch(struct as *as)
34707c478bd9Sstevel@tonic-gate {
34717c478bd9Sstevel@tonic-gate 	struct watched_page *pwp;
34727c478bd9Sstevel@tonic-gate 	struct seg *seg;
34737c478bd9Sstevel@tonic-gate 	caddr_t vaddr;
34747c478bd9Sstevel@tonic-gate 	uint_t prot;
34757c478bd9Sstevel@tonic-gate 	int err, retrycnt;
34767c478bd9Sstevel@tonic-gate 
34777c478bd9Sstevel@tonic-gate 	if (avl_numnodes(&as->a_wpage) == 0)
34787c478bd9Sstevel@tonic-gate 		return;
34797c478bd9Sstevel@tonic-gate 
3480dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
34817c478bd9Sstevel@tonic-gate 
34827c478bd9Sstevel@tonic-gate 	for (pwp = avl_first(&as->a_wpage); pwp != NULL;
34837c478bd9Sstevel@tonic-gate 	    pwp = AVL_NEXT(&as->a_wpage, pwp)) {
34847c478bd9Sstevel@tonic-gate 		retrycnt = 0;
34857c478bd9Sstevel@tonic-gate 	retry:
34867c478bd9Sstevel@tonic-gate 		vaddr = pwp->wp_vaddr;
34877c478bd9Sstevel@tonic-gate 		if (pwp->wp_oprot == 0 ||	/* not set up */
34887c478bd9Sstevel@tonic-gate 		    (seg = as_segat(as, vaddr)) == NULL)
34897c478bd9Sstevel@tonic-gate 			continue;
34907c478bd9Sstevel@tonic-gate 
34917c478bd9Sstevel@tonic-gate 		if ((prot = pwp->wp_oprot) != pwp->wp_prot) {
34927c478bd9Sstevel@tonic-gate 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
34937c478bd9Sstevel@tonic-gate 			if (err == IE_RETRY) {
34947c478bd9Sstevel@tonic-gate 				ASSERT(retrycnt == 0);
34957c478bd9Sstevel@tonic-gate 				retrycnt++;
34967c478bd9Sstevel@tonic-gate 				goto retry;
34977c478bd9Sstevel@tonic-gate 			}
34987c478bd9Sstevel@tonic-gate 		}
34997c478bd9Sstevel@tonic-gate 		pwp->wp_oprot = 0;
35007c478bd9Sstevel@tonic-gate 		pwp->wp_prot = 0;
35017c478bd9Sstevel@tonic-gate 	}
35027c478bd9Sstevel@tonic-gate }
35037c478bd9Sstevel@tonic-gate 
35047c478bd9Sstevel@tonic-gate /*
35057c478bd9Sstevel@tonic-gate  * Force a new setup for all the watched pages in the range.
35067c478bd9Sstevel@tonic-gate  */
35077c478bd9Sstevel@tonic-gate static void
35087c478bd9Sstevel@tonic-gate as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
35097c478bd9Sstevel@tonic-gate {
35107c478bd9Sstevel@tonic-gate 	struct watched_page *pwp;
35117c478bd9Sstevel@tonic-gate 	struct watched_page tpw;
35127c478bd9Sstevel@tonic-gate 	caddr_t eaddr = addr + size;
35137c478bd9Sstevel@tonic-gate 	caddr_t vaddr;
35147c478bd9Sstevel@tonic-gate 	struct seg *seg;
35157c478bd9Sstevel@tonic-gate 	int err, retrycnt;
35167c478bd9Sstevel@tonic-gate 	uint_t	wprot;
35177c478bd9Sstevel@tonic-gate 	avl_index_t where;
35187c478bd9Sstevel@tonic-gate 
35197c478bd9Sstevel@tonic-gate 	if (avl_numnodes(&as->a_wpage) == 0)
35207c478bd9Sstevel@tonic-gate 		return;
35217c478bd9Sstevel@tonic-gate 
3522dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
35237c478bd9Sstevel@tonic-gate 
35247c478bd9Sstevel@tonic-gate 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
35257c478bd9Sstevel@tonic-gate 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
35267c478bd9Sstevel@tonic-gate 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
35277c478bd9Sstevel@tonic-gate 
35287c478bd9Sstevel@tonic-gate 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
35297c478bd9Sstevel@tonic-gate 		retrycnt = 0;
35307c478bd9Sstevel@tonic-gate 		vaddr = pwp->wp_vaddr;
35317c478bd9Sstevel@tonic-gate 
35327c478bd9Sstevel@tonic-gate 		wprot = prot;
35337c478bd9Sstevel@tonic-gate 		if (pwp->wp_read)
35347c478bd9Sstevel@tonic-gate 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
35357c478bd9Sstevel@tonic-gate 		if (pwp->wp_write)
35367c478bd9Sstevel@tonic-gate 			wprot &= ~PROT_WRITE;
35377c478bd9Sstevel@tonic-gate 		if (pwp->wp_exec)
35387c478bd9Sstevel@tonic-gate 			wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
35397c478bd9Sstevel@tonic-gate 		if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) {
35407c478bd9Sstevel@tonic-gate 		retry:
35417c478bd9Sstevel@tonic-gate 			seg = as_segat(as, vaddr);
35427c478bd9Sstevel@tonic-gate 			if (seg == NULL) {
35437c478bd9Sstevel@tonic-gate 				panic("as_setwatchprot: no seg");
35447c478bd9Sstevel@tonic-gate 				/*NOTREACHED*/
35457c478bd9Sstevel@tonic-gate 			}
35467c478bd9Sstevel@tonic-gate 			err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot);
35477c478bd9Sstevel@tonic-gate 			if (err == IE_RETRY) {
35487c478bd9Sstevel@tonic-gate 				ASSERT(retrycnt == 0);
35497c478bd9Sstevel@tonic-gate 				retrycnt++;
35507c478bd9Sstevel@tonic-gate 				goto retry;
35517c478bd9Sstevel@tonic-gate 			}
35527c478bd9Sstevel@tonic-gate 		}
35537c478bd9Sstevel@tonic-gate 		pwp->wp_oprot = prot;
35547c478bd9Sstevel@tonic-gate 		pwp->wp_prot = wprot;
35557c478bd9Sstevel@tonic-gate 
35567c478bd9Sstevel@tonic-gate 		pwp = AVL_NEXT(&as->a_wpage, pwp);
35577c478bd9Sstevel@tonic-gate 	}
35587c478bd9Sstevel@tonic-gate }
35597c478bd9Sstevel@tonic-gate 
35607c478bd9Sstevel@tonic-gate /*
35617c478bd9Sstevel@tonic-gate  * Clear all of the watched pages in the range.
35627c478bd9Sstevel@tonic-gate  */
35637c478bd9Sstevel@tonic-gate static void
35647c478bd9Sstevel@tonic-gate as_clearwatchprot(struct as *as, caddr_t addr, size_t size)
35657c478bd9Sstevel@tonic-gate {
35667c478bd9Sstevel@tonic-gate 	caddr_t eaddr = addr + size;
35677c478bd9Sstevel@tonic-gate 	struct watched_page *pwp;
35687c478bd9Sstevel@tonic-gate 	struct watched_page tpw;
35697c478bd9Sstevel@tonic-gate 	uint_t prot;
35707c478bd9Sstevel@tonic-gate 	struct seg *seg;
35717c478bd9Sstevel@tonic-gate 	int err, retrycnt;
35727c478bd9Sstevel@tonic-gate 	avl_index_t where;
35737c478bd9Sstevel@tonic-gate 
35747c478bd9Sstevel@tonic-gate 	if (avl_numnodes(&as->a_wpage) == 0)
35757c478bd9Sstevel@tonic-gate 		return;
35767c478bd9Sstevel@tonic-gate 
35777c478bd9Sstevel@tonic-gate 	tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
35787c478bd9Sstevel@tonic-gate 	if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL)
35797c478bd9Sstevel@tonic-gate 		pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER);
35807c478bd9Sstevel@tonic-gate 
3581dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
35827c478bd9Sstevel@tonic-gate 
35837c478bd9Sstevel@tonic-gate 	while (pwp != NULL && pwp->wp_vaddr < eaddr) {
35847c478bd9Sstevel@tonic-gate 
35857c478bd9Sstevel@tonic-gate 		if ((prot = pwp->wp_oprot) != 0) {
35867c478bd9Sstevel@tonic-gate 			retrycnt = 0;
35877c478bd9Sstevel@tonic-gate 
35887c478bd9Sstevel@tonic-gate 			if (prot != pwp->wp_prot) {
35897c478bd9Sstevel@tonic-gate 			retry:
35907c478bd9Sstevel@tonic-gate 				seg = as_segat(as, pwp->wp_vaddr);
35917c478bd9Sstevel@tonic-gate 				if (seg == NULL)
35927c478bd9Sstevel@tonic-gate 					continue;
35937c478bd9Sstevel@tonic-gate 				err = SEGOP_SETPROT(seg, pwp->wp_vaddr,
35947c478bd9Sstevel@tonic-gate 				    PAGESIZE, prot);
35957c478bd9Sstevel@tonic-gate 				if (err == IE_RETRY) {
35967c478bd9Sstevel@tonic-gate 					ASSERT(retrycnt == 0);
35977c478bd9Sstevel@tonic-gate 					retrycnt++;
35987c478bd9Sstevel@tonic-gate 					goto retry;
35997c478bd9Sstevel@tonic-gate 
36007c478bd9Sstevel@tonic-gate 				}
36017c478bd9Sstevel@tonic-gate 			}
36027c478bd9Sstevel@tonic-gate 			pwp->wp_oprot = 0;
36037c478bd9Sstevel@tonic-gate 			pwp->wp_prot = 0;
36047c478bd9Sstevel@tonic-gate 		}
36057c478bd9Sstevel@tonic-gate 
36067c478bd9Sstevel@tonic-gate 		pwp = AVL_NEXT(&as->a_wpage, pwp);
36077c478bd9Sstevel@tonic-gate 	}
36087c478bd9Sstevel@tonic-gate }
36097c478bd9Sstevel@tonic-gate 
36107c478bd9Sstevel@tonic-gate void
36117c478bd9Sstevel@tonic-gate as_signal_proc(struct as *as, k_siginfo_t *siginfo)
36127c478bd9Sstevel@tonic-gate {
36137c478bd9Sstevel@tonic-gate 	struct proc *p;
36147c478bd9Sstevel@tonic-gate 
36157c478bd9Sstevel@tonic-gate 	mutex_enter(&pidlock);
36167c478bd9Sstevel@tonic-gate 	for (p = practive; p; p = p->p_next) {
36177c478bd9Sstevel@tonic-gate 		if (p->p_as == as) {
36187c478bd9Sstevel@tonic-gate 			mutex_enter(&p->p_lock);
36197c478bd9Sstevel@tonic-gate 			if (p->p_as == as)
36207c478bd9Sstevel@tonic-gate 				sigaddq(p, NULL, siginfo, KM_NOSLEEP);
36217c478bd9Sstevel@tonic-gate 			mutex_exit(&p->p_lock);
36227c478bd9Sstevel@tonic-gate 		}
36237c478bd9Sstevel@tonic-gate 	}
36247c478bd9Sstevel@tonic-gate 	mutex_exit(&pidlock);
36257c478bd9Sstevel@tonic-gate }
36267c478bd9Sstevel@tonic-gate 
36277c478bd9Sstevel@tonic-gate /*
36287c478bd9Sstevel@tonic-gate  * return memory object ID
36297c478bd9Sstevel@tonic-gate  */
36307c478bd9Sstevel@tonic-gate int
36317c478bd9Sstevel@tonic-gate as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
36327c478bd9Sstevel@tonic-gate {
36337c478bd9Sstevel@tonic-gate 	struct seg	*seg;
36347c478bd9Sstevel@tonic-gate 	int		sts;
36357c478bd9Sstevel@tonic-gate 
3636dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
36377c478bd9Sstevel@tonic-gate 	seg = as_segat(as, addr);
36387c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
3639dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
36407c478bd9Sstevel@tonic-gate 		return (EFAULT);
36417c478bd9Sstevel@tonic-gate 	}
36427c478bd9Sstevel@tonic-gate 	/*
36437c478bd9Sstevel@tonic-gate 	 * catch old drivers which may not support getmemid
36447c478bd9Sstevel@tonic-gate 	 */
36457c478bd9Sstevel@tonic-gate 	if (seg->s_ops->getmemid == NULL) {
3646dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
36477c478bd9Sstevel@tonic-gate 		return (ENODEV);
36487c478bd9Sstevel@tonic-gate 	}
36497c478bd9Sstevel@tonic-gate 
36507c478bd9Sstevel@tonic-gate 	sts = SEGOP_GETMEMID(seg, addr, memidp);
36517c478bd9Sstevel@tonic-gate 
3652dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
36537c478bd9Sstevel@tonic-gate 	return (sts);
36547c478bd9Sstevel@tonic-gate }
3655