xref: /illumos-gate/usr/src/uts/common/vm/vm_as.c (revision c6f039c7)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
502ff05a9Svsakar  * Common Development and Distribution License (the "License").
602ff05a9Svsakar  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2217965fd8SKrishnendu Sadhukhan - Sun Microsystems  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
24284ce987SPatrick Mooney  * Copyright 2018 Joyent, Inc.
256430daecSBrandon Baker  * Copyright (c) 2016 by Delphix. All rights reserved.
267c478bd9Sstevel@tonic-gate  */
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
297c478bd9Sstevel@tonic-gate /*	  All Rights Reserved  	*/
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate /*
327c478bd9Sstevel@tonic-gate  * University Copyright- Copyright (c) 1982, 1986, 1988
337c478bd9Sstevel@tonic-gate  * The Regents of the University of California
347c478bd9Sstevel@tonic-gate  * All Rights Reserved
357c478bd9Sstevel@tonic-gate  *
367c478bd9Sstevel@tonic-gate  * University Acknowledgment- Portions of this document are derived from
377c478bd9Sstevel@tonic-gate  * software developed by the University of California, Berkeley, and its
387c478bd9Sstevel@tonic-gate  * contributors.
397c478bd9Sstevel@tonic-gate  */
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate /*
427c478bd9Sstevel@tonic-gate  * VM - address spaces.
437c478bd9Sstevel@tonic-gate  */
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate #include <sys/types.h>
467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
477c478bd9Sstevel@tonic-gate #include <sys/param.h>
487c478bd9Sstevel@tonic-gate #include <sys/errno.h>
497c478bd9Sstevel@tonic-gate #include <sys/systm.h>
507c478bd9Sstevel@tonic-gate #include <sys/mman.h>
517c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
527c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
537c478bd9Sstevel@tonic-gate #include <sys/sysinfo.h>
547c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
557c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
567c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
577c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
587c478bd9Sstevel@tonic-gate #include <sys/debug.h>
597c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h>
607c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
617c478bd9Sstevel@tonic-gate 
627c478bd9Sstevel@tonic-gate #include <vm/hat.h>
637c478bd9Sstevel@tonic-gate #include <vm/as.h>
647c478bd9Sstevel@tonic-gate #include <vm/seg.h>
657c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h>
667c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h>
677c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
687c478bd9Sstevel@tonic-gate #include <vm/seg_map.h>
697c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h>
70284ce987SPatrick Mooney #include <vm/seg_hole.h>
717c478bd9Sstevel@tonic-gate #include <vm/page.h>
727c478bd9Sstevel@tonic-gate 
737c478bd9Sstevel@tonic-gate clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate static struct kmem_cache *as_cache;
767c478bd9Sstevel@tonic-gate 
777c478bd9Sstevel@tonic-gate static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t);
787c478bd9Sstevel@tonic-gate static void as_clearwatchprot(struct as *, caddr_t, size_t);
797c478bd9Sstevel@tonic-gate 
807c478bd9Sstevel@tonic-gate 
817c478bd9Sstevel@tonic-gate /*
827c478bd9Sstevel@tonic-gate  * Verifying the segment lists is very time-consuming; it may not be
837c478bd9Sstevel@tonic-gate  * desirable always to define VERIFY_SEGLIST when DEBUG is set.
847c478bd9Sstevel@tonic-gate  */
857c478bd9Sstevel@tonic-gate #ifdef DEBUG
867c478bd9Sstevel@tonic-gate #define	VERIFY_SEGLIST
877c478bd9Sstevel@tonic-gate int do_as_verify = 0;
887c478bd9Sstevel@tonic-gate #endif
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate /*
917c478bd9Sstevel@tonic-gate  * Allocate a new callback data structure entry and fill in the events of
927c478bd9Sstevel@tonic-gate  * interest, the address range of interest, and the callback argument.
937c478bd9Sstevel@tonic-gate  * Link the entry on the as->a_callbacks list. A callback entry for the
947c478bd9Sstevel@tonic-gate  * entire address space may be specified with vaddr = 0 and size = -1.
957c478bd9Sstevel@tonic-gate  *
967c478bd9Sstevel@tonic-gate  * CALLERS RESPONSIBILITY: If not calling from within the process context for
977c478bd9Sstevel@tonic-gate  * the specified as, the caller must guarantee persistence of the specified as
987c478bd9Sstevel@tonic-gate  * for the duration of this function (eg. pages being locked within the as
997c478bd9Sstevel@tonic-gate  * will guarantee persistence).
1007c478bd9Sstevel@tonic-gate  */
1017c478bd9Sstevel@tonic-gate int
as_add_callback(struct as * as,void (* cb_func)(),void * arg,uint_t events,caddr_t vaddr,size_t size,int sleepflag)1027c478bd9Sstevel@tonic-gate as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events,
1036430daecSBrandon Baker     caddr_t vaddr, size_t size, int sleepflag)
1047c478bd9Sstevel@tonic-gate {
1057c478bd9Sstevel@tonic-gate 	struct as_callback 	*current_head, *cb;
1067c478bd9Sstevel@tonic-gate 	caddr_t 		saddr;
1077c478bd9Sstevel@tonic-gate 	size_t 			rsize;
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 	/* callback function and an event are mandatory */
1107c478bd9Sstevel@tonic-gate 	if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0))
1117c478bd9Sstevel@tonic-gate 		return (EINVAL);
1127c478bd9Sstevel@tonic-gate 
1137c478bd9Sstevel@tonic-gate 	/* Adding a callback after as_free has been called is not allowed */
1147c478bd9Sstevel@tonic-gate 	if (as == &kas)
1157c478bd9Sstevel@tonic-gate 		return (ENOMEM);
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate 	/*
1187c478bd9Sstevel@tonic-gate 	 * vaddr = 0 and size = -1 is used to indicate that the callback range
1197c478bd9Sstevel@tonic-gate 	 * is the entire address space so no rounding is done in that case.
1207c478bd9Sstevel@tonic-gate 	 */
1217c478bd9Sstevel@tonic-gate 	if (size != -1) {
1227c478bd9Sstevel@tonic-gate 		saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
1237c478bd9Sstevel@tonic-gate 		rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) -
124843e1988Sjohnlev 		    (size_t)saddr;
1257c478bd9Sstevel@tonic-gate 		/* check for wraparound */
1267c478bd9Sstevel@tonic-gate 		if (saddr + rsize < saddr)
1277c478bd9Sstevel@tonic-gate 			return (ENOMEM);
1287c478bd9Sstevel@tonic-gate 	} else {
1297c478bd9Sstevel@tonic-gate 		if (vaddr != 0)
1307c478bd9Sstevel@tonic-gate 			return (EINVAL);
1317c478bd9Sstevel@tonic-gate 		saddr = vaddr;
1327c478bd9Sstevel@tonic-gate 		rsize = size;
1337c478bd9Sstevel@tonic-gate 	}
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate 	/* Allocate and initialize a callback entry */
1367c478bd9Sstevel@tonic-gate 	cb = kmem_zalloc(sizeof (struct as_callback), sleepflag);
1377c478bd9Sstevel@tonic-gate 	if (cb == NULL)
1387c478bd9Sstevel@tonic-gate 		return (EAGAIN);
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate 	cb->ascb_func = cb_func;
1417c478bd9Sstevel@tonic-gate 	cb->ascb_arg = arg;
1427c478bd9Sstevel@tonic-gate 	cb->ascb_events = events;
1437c478bd9Sstevel@tonic-gate 	cb->ascb_saddr = saddr;
1447c478bd9Sstevel@tonic-gate 	cb->ascb_len = rsize;
1457c478bd9Sstevel@tonic-gate 
1467c478bd9Sstevel@tonic-gate 	/* Add the entry to the list */
1477c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
1487c478bd9Sstevel@tonic-gate 	current_head = as->a_callbacks;
1497c478bd9Sstevel@tonic-gate 	as->a_callbacks = cb;
1507c478bd9Sstevel@tonic-gate 	cb->ascb_next = current_head;
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate 	/*
1537c478bd9Sstevel@tonic-gate 	 * The call to this function may lose in a race with
1547c478bd9Sstevel@tonic-gate 	 * a pertinent event - eg. a thread does long term memory locking
1557c478bd9Sstevel@tonic-gate 	 * but before the callback is added another thread executes as_unmap.
1567c478bd9Sstevel@tonic-gate 	 * A broadcast here resolves that.
1577c478bd9Sstevel@tonic-gate 	 */
1587c478bd9Sstevel@tonic-gate 	if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) {
1597c478bd9Sstevel@tonic-gate 		AS_CLRUNMAPWAIT(as);
1607c478bd9Sstevel@tonic-gate 		cv_broadcast(&as->a_cv);
1617c478bd9Sstevel@tonic-gate 	}
1627c478bd9Sstevel@tonic-gate 
1637c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
1647c478bd9Sstevel@tonic-gate 	return (0);
1657c478bd9Sstevel@tonic-gate }
1667c478bd9Sstevel@tonic-gate 
1677c478bd9Sstevel@tonic-gate /*
1687c478bd9Sstevel@tonic-gate  * Search the callback list for an entry which pertains to arg.
1697c478bd9Sstevel@tonic-gate  *
1707c478bd9Sstevel@tonic-gate  * This is called from within the client upon completion of the callback.
1717c478bd9Sstevel@tonic-gate  * RETURN VALUES:
1727c478bd9Sstevel@tonic-gate  *	AS_CALLBACK_DELETED  (callback entry found and deleted)
1737c478bd9Sstevel@tonic-gate  *	AS_CALLBACK_NOTFOUND (no callback entry found - this is ok)
1747c478bd9Sstevel@tonic-gate  *	AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this
1757c478bd9Sstevel@tonic-gate  *			entry will be made in as_do_callbacks)
1767c478bd9Sstevel@tonic-gate  *
1777c478bd9Sstevel@tonic-gate  * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED
1787c478bd9Sstevel@tonic-gate  * set, it indicates that as_do_callbacks is processing this entry.  The
1797c478bd9Sstevel@tonic-gate  * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made
1807c478bd9Sstevel@tonic-gate  * to unblock as_do_callbacks, in case it is blocked.
1817c478bd9Sstevel@tonic-gate  *
1827c478bd9Sstevel@tonic-gate  * CALLERS RESPONSIBILITY: If not calling from within the process context for
1837c478bd9Sstevel@tonic-gate  * the specified as, the caller must guarantee persistence of the specified as
1847c478bd9Sstevel@tonic-gate  * for the duration of this function (eg. pages being locked within the as
1857c478bd9Sstevel@tonic-gate  * will guarantee persistence).
1867c478bd9Sstevel@tonic-gate  */
1877c478bd9Sstevel@tonic-gate uint_t
as_delete_callback(struct as * as,void * arg)1887c478bd9Sstevel@tonic-gate as_delete_callback(struct as *as, void *arg)
1897c478bd9Sstevel@tonic-gate {
1907c478bd9Sstevel@tonic-gate 	struct as_callback **prevcb = &as->a_callbacks;
1917c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
1927c478bd9Sstevel@tonic-gate 	uint_t rc = AS_CALLBACK_NOTFOUND;
1937c478bd9Sstevel@tonic-gate 
1947c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
1957c478bd9Sstevel@tonic-gate 	for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) {
1967c478bd9Sstevel@tonic-gate 		if (cb->ascb_arg != arg)
1977c478bd9Sstevel@tonic-gate 			continue;
1987c478bd9Sstevel@tonic-gate 
1997c478bd9Sstevel@tonic-gate 		/*
2007c478bd9Sstevel@tonic-gate 		 * If the events indicate AS_CALLBACK_CALLED, just clear
2017c478bd9Sstevel@tonic-gate 		 * AS_ALL_EVENT in the events field and wakeup the thread
2027c478bd9Sstevel@tonic-gate 		 * that may be waiting in as_do_callbacks.  as_do_callbacks
2037c478bd9Sstevel@tonic-gate 		 * will take care of removing this entry from the list.  In
2047c478bd9Sstevel@tonic-gate 		 * that case, return AS_CALLBACK_DELETE_DEFERRED.  Otherwise
2057c478bd9Sstevel@tonic-gate 		 * (AS_CALLBACK_CALLED not set), just remove it from the
2067c478bd9Sstevel@tonic-gate 		 * list, return the memory and return AS_CALLBACK_DELETED.
2077c478bd9Sstevel@tonic-gate 		 */
2087c478bd9Sstevel@tonic-gate 		if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) {
2097c478bd9Sstevel@tonic-gate 			/* leave AS_CALLBACK_CALLED */
2107c478bd9Sstevel@tonic-gate 			cb->ascb_events &= ~AS_ALL_EVENT;
2117c478bd9Sstevel@tonic-gate 			rc = AS_CALLBACK_DELETE_DEFERRED;
2127c478bd9Sstevel@tonic-gate 			cv_broadcast(&as->a_cv);
2137c478bd9Sstevel@tonic-gate 		} else {
2147c478bd9Sstevel@tonic-gate 			*prevcb = cb->ascb_next;
2157c478bd9Sstevel@tonic-gate 			kmem_free(cb, sizeof (struct as_callback));
2167c478bd9Sstevel@tonic-gate 			rc = AS_CALLBACK_DELETED;
2177c478bd9Sstevel@tonic-gate 		}
2187c478bd9Sstevel@tonic-gate 		break;
2197c478bd9Sstevel@tonic-gate 	}
2207c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
2217c478bd9Sstevel@tonic-gate 	return (rc);
2227c478bd9Sstevel@tonic-gate }
2237c478bd9Sstevel@tonic-gate 
2247c478bd9Sstevel@tonic-gate /*
2257c478bd9Sstevel@tonic-gate  * Searches the as callback list for a matching entry.
2267c478bd9Sstevel@tonic-gate  * Returns a pointer to the first matching callback, or NULL if
2277c478bd9Sstevel@tonic-gate  * nothing is found.
2287c478bd9Sstevel@tonic-gate  * This function never sleeps so it is ok to call it with more
2297c478bd9Sstevel@tonic-gate  * locks held but the (required) a_contents mutex.
2307c478bd9Sstevel@tonic-gate  *
2317c478bd9Sstevel@tonic-gate  * See also comment on as_do_callbacks below.
2327c478bd9Sstevel@tonic-gate  */
2337c478bd9Sstevel@tonic-gate static struct as_callback *
as_find_callback(struct as * as,uint_t events,caddr_t event_addr,size_t event_len)2347c478bd9Sstevel@tonic-gate as_find_callback(struct as *as, uint_t events, caddr_t event_addr,
2356430daecSBrandon Baker     size_t event_len)
2367c478bd9Sstevel@tonic-gate {
2377c478bd9Sstevel@tonic-gate 	struct as_callback	*cb;
2387c478bd9Sstevel@tonic-gate 
2397c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&as->a_contents));
2407c478bd9Sstevel@tonic-gate 	for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) {
2417c478bd9Sstevel@tonic-gate 		/*
2427c478bd9Sstevel@tonic-gate 		 * If the callback has not already been called, then
2437c478bd9Sstevel@tonic-gate 		 * check if events or address range pertains.  An event_len
2447c478bd9Sstevel@tonic-gate 		 * of zero means do an unconditional callback.
2457c478bd9Sstevel@tonic-gate 		 */
2467c478bd9Sstevel@tonic-gate 		if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) ||
2477c478bd9Sstevel@tonic-gate 		    ((event_len != 0) && (((cb->ascb_events & events) == 0) ||
2487c478bd9Sstevel@tonic-gate 		    (event_addr + event_len < cb->ascb_saddr) ||
2497c478bd9Sstevel@tonic-gate 		    (event_addr > (cb->ascb_saddr + cb->ascb_len))))) {
2507c478bd9Sstevel@tonic-gate 			continue;
2517c478bd9Sstevel@tonic-gate 		}
2527c478bd9Sstevel@tonic-gate 		break;
2537c478bd9Sstevel@tonic-gate 	}
2547c478bd9Sstevel@tonic-gate 	return (cb);
2557c478bd9Sstevel@tonic-gate }
2567c478bd9Sstevel@tonic-gate 
2577c478bd9Sstevel@tonic-gate /*
2587c478bd9Sstevel@tonic-gate  * Executes a given callback and removes it from the callback list for
2597c478bd9Sstevel@tonic-gate  * this address space.
2607c478bd9Sstevel@tonic-gate  * This function may sleep so the caller must drop all locks except
2617c478bd9Sstevel@tonic-gate  * a_contents before calling this func.
2627c478bd9Sstevel@tonic-gate  *
2637c478bd9Sstevel@tonic-gate  * See also comments on as_do_callbacks below.
2647c478bd9Sstevel@tonic-gate  */
2657c478bd9Sstevel@tonic-gate static void
as_execute_callback(struct as * as,struct as_callback * cb,uint_t events)2667c478bd9Sstevel@tonic-gate as_execute_callback(struct as *as, struct as_callback *cb,
2676430daecSBrandon Baker     uint_t events)
2687c478bd9Sstevel@tonic-gate {
2697c478bd9Sstevel@tonic-gate 	struct as_callback **prevcb;
2707c478bd9Sstevel@tonic-gate 	void	*cb_arg;
2717c478bd9Sstevel@tonic-gate 
2727c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events));
2737c478bd9Sstevel@tonic-gate 	cb->ascb_events |= AS_CALLBACK_CALLED;
2747c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
2757c478bd9Sstevel@tonic-gate 	(*cb->ascb_func)(as, cb->ascb_arg, events);
2767c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
2777c478bd9Sstevel@tonic-gate 	/*
2787c478bd9Sstevel@tonic-gate 	 * the callback function is required to delete the callback
2797c478bd9Sstevel@tonic-gate 	 * when the callback function determines it is OK for
2807c478bd9Sstevel@tonic-gate 	 * this thread to continue. as_delete_callback will clear
2817c478bd9Sstevel@tonic-gate 	 * the AS_ALL_EVENT in the events field when it is deleted.
2827c478bd9Sstevel@tonic-gate 	 * If the callback function called as_delete_callback,
2837c478bd9Sstevel@tonic-gate 	 * events will already be cleared and there will be no blocking.
2847c478bd9Sstevel@tonic-gate 	 */
2857c478bd9Sstevel@tonic-gate 	while ((cb->ascb_events & events) != 0) {
2867c478bd9Sstevel@tonic-gate 		cv_wait(&as->a_cv, &as->a_contents);
2877c478bd9Sstevel@tonic-gate 	}
2887c478bd9Sstevel@tonic-gate 	/*
2897c478bd9Sstevel@tonic-gate 	 * This entry needs to be taken off the list. Normally, the
2907c478bd9Sstevel@tonic-gate 	 * callback func itself does that, but unfortunately the list
2917c478bd9Sstevel@tonic-gate 	 * may have changed while the callback was running because the
2927c478bd9Sstevel@tonic-gate 	 * a_contents mutex was dropped and someone else other than the
2937c478bd9Sstevel@tonic-gate 	 * callback func itself could have called as_delete_callback,
2947c478bd9Sstevel@tonic-gate 	 * so we have to search to find this entry again.  The entry
2957c478bd9Sstevel@tonic-gate 	 * must have AS_CALLBACK_CALLED, and have the same 'arg'.
2967c478bd9Sstevel@tonic-gate 	 */
2977c478bd9Sstevel@tonic-gate 	cb_arg = cb->ascb_arg;
2987c478bd9Sstevel@tonic-gate 	prevcb = &as->a_callbacks;
2997c478bd9Sstevel@tonic-gate 	for (cb = as->a_callbacks; cb != NULL;
3007c478bd9Sstevel@tonic-gate 	    prevcb = &cb->ascb_next, cb = *prevcb) {
3017c478bd9Sstevel@tonic-gate 		if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) ||
3027c478bd9Sstevel@tonic-gate 		    (cb_arg != cb->ascb_arg)) {
3037c478bd9Sstevel@tonic-gate 			continue;
3047c478bd9Sstevel@tonic-gate 		}
3057c478bd9Sstevel@tonic-gate 		*prevcb = cb->ascb_next;
3067c478bd9Sstevel@tonic-gate 		kmem_free(cb, sizeof (struct as_callback));
3077c478bd9Sstevel@tonic-gate 		break;
3087c478bd9Sstevel@tonic-gate 	}
3097c478bd9Sstevel@tonic-gate }
3107c478bd9Sstevel@tonic-gate 
3117c478bd9Sstevel@tonic-gate /*
3127c478bd9Sstevel@tonic-gate  * Check the callback list for a matching event and intersection of
3137c478bd9Sstevel@tonic-gate  * address range. If there is a match invoke the callback.  Skip an entry if:
3147c478bd9Sstevel@tonic-gate  *    - a callback is already in progress for this entry (AS_CALLBACK_CALLED)
3157c478bd9Sstevel@tonic-gate  *    - not event of interest
3167c478bd9Sstevel@tonic-gate  *    - not address range of interest
3177c478bd9Sstevel@tonic-gate  *
3187c478bd9Sstevel@tonic-gate  * An event_len of zero indicates a request for an unconditional callback
3197c478bd9Sstevel@tonic-gate  * (regardless of event), only the AS_CALLBACK_CALLED is checked.  The
3207c478bd9Sstevel@tonic-gate  * a_contents lock must be dropped before a callback, so only one callback
3217c478bd9Sstevel@tonic-gate  * can be done before returning. Return -1 (true) if a callback was
3227c478bd9Sstevel@tonic-gate  * executed and removed from the list, else return 0 (false).
3237c478bd9Sstevel@tonic-gate  *
3247c478bd9Sstevel@tonic-gate  * The logically separate parts, i.e. finding a matching callback and
3257c478bd9Sstevel@tonic-gate  * executing a given callback have been separated into two functions
3267c478bd9Sstevel@tonic-gate  * so that they can be called with different sets of locks held beyond
3277c478bd9Sstevel@tonic-gate  * the always-required a_contents. as_find_callback does not sleep so
3287c478bd9Sstevel@tonic-gate  * it is ok to call it if more locks than a_contents (i.e. the a_lock
3297c478bd9Sstevel@tonic-gate  * rwlock) are held. as_execute_callback on the other hand may sleep
3307c478bd9Sstevel@tonic-gate  * so all locks beyond a_contents must be dropped by the caller if one
3317c478bd9Sstevel@tonic-gate  * does not want to end comatose.
3327c478bd9Sstevel@tonic-gate  */
3337c478bd9Sstevel@tonic-gate static int
as_do_callbacks(struct as * as,uint_t events,caddr_t event_addr,size_t event_len)3347c478bd9Sstevel@tonic-gate as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr,
3356430daecSBrandon Baker     size_t event_len)
3367c478bd9Sstevel@tonic-gate {
3377c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
3387c478bd9Sstevel@tonic-gate 
3397c478bd9Sstevel@tonic-gate 	if ((cb = as_find_callback(as, events, event_addr, event_len))) {
3407c478bd9Sstevel@tonic-gate 		as_execute_callback(as, cb, events);
3417c478bd9Sstevel@tonic-gate 		return (-1);
3427c478bd9Sstevel@tonic-gate 	}
3437c478bd9Sstevel@tonic-gate 	return (0);
3447c478bd9Sstevel@tonic-gate }
3457c478bd9Sstevel@tonic-gate 
3467c478bd9Sstevel@tonic-gate /*
3477c478bd9Sstevel@tonic-gate  * Search for the segment containing addr. If a segment containing addr
3487c478bd9Sstevel@tonic-gate  * exists, that segment is returned.  If no such segment exists, and
3497c478bd9Sstevel@tonic-gate  * the list spans addresses greater than addr, then the first segment
3507c478bd9Sstevel@tonic-gate  * whose base is greater than addr is returned; otherwise, NULL is
3517c478bd9Sstevel@tonic-gate  * returned unless tail is true, in which case the last element of the
3527c478bd9Sstevel@tonic-gate  * list is returned.
3537c478bd9Sstevel@tonic-gate  *
3547c478bd9Sstevel@tonic-gate  * a_seglast is used to cache the last found segment for repeated
3557c478bd9Sstevel@tonic-gate  * searches to the same addr (which happens frequently).
3567c478bd9Sstevel@tonic-gate  */
3577c478bd9Sstevel@tonic-gate struct seg *
as_findseg(struct as * as,caddr_t addr,int tail)3587c478bd9Sstevel@tonic-gate as_findseg(struct as *as, caddr_t addr, int tail)
3597c478bd9Sstevel@tonic-gate {
3607c478bd9Sstevel@tonic-gate 	struct seg *seg = as->a_seglast;
3617c478bd9Sstevel@tonic-gate 	avl_index_t where;
3627c478bd9Sstevel@tonic-gate 
363dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_LOCK_HELD(as));
3647c478bd9Sstevel@tonic-gate 
3657c478bd9Sstevel@tonic-gate 	if (seg != NULL &&
3667c478bd9Sstevel@tonic-gate 	    seg->s_base <= addr &&
3677c478bd9Sstevel@tonic-gate 	    addr < seg->s_base + seg->s_size)
3687c478bd9Sstevel@tonic-gate 		return (seg);
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate 	seg = avl_find(&as->a_segtree, &addr, &where);
3717c478bd9Sstevel@tonic-gate 	if (seg != NULL)
3727c478bd9Sstevel@tonic-gate 		return (as->a_seglast = seg);
3737c478bd9Sstevel@tonic-gate 
3747c478bd9Sstevel@tonic-gate 	seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
3757c478bd9Sstevel@tonic-gate 	if (seg == NULL && tail)
3767c478bd9Sstevel@tonic-gate 		seg = avl_last(&as->a_segtree);
3777c478bd9Sstevel@tonic-gate 	return (as->a_seglast = seg);
3787c478bd9Sstevel@tonic-gate }
3797c478bd9Sstevel@tonic-gate 
3807c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST
3817c478bd9Sstevel@tonic-gate /*
3827c478bd9Sstevel@tonic-gate  * verify that the linked list is coherent
3837c478bd9Sstevel@tonic-gate  */
3847c478bd9Sstevel@tonic-gate static void
as_verify(struct as * as)3857c478bd9Sstevel@tonic-gate as_verify(struct as *as)
3867c478bd9Sstevel@tonic-gate {
3877c478bd9Sstevel@tonic-gate 	struct seg *seg, *seglast, *p, *n;
3887c478bd9Sstevel@tonic-gate 	uint_t nsegs = 0;
3897c478bd9Sstevel@tonic-gate 
3907c478bd9Sstevel@tonic-gate 	if (do_as_verify == 0)
3917c478bd9Sstevel@tonic-gate 		return;
3927c478bd9Sstevel@tonic-gate 
3937c478bd9Sstevel@tonic-gate 	seglast = as->a_seglast;
3947c478bd9Sstevel@tonic-gate 
3957c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
3967c478bd9Sstevel@tonic-gate 		ASSERT(seg->s_as == as);
3977c478bd9Sstevel@tonic-gate 		p = AS_SEGPREV(as, seg);
3987c478bd9Sstevel@tonic-gate 		n = AS_SEGNEXT(as, seg);
3997c478bd9Sstevel@tonic-gate 		ASSERT(p == NULL || p->s_as == as);
4007c478bd9Sstevel@tonic-gate 		ASSERT(p == NULL || p->s_base < seg->s_base);
4017c478bd9Sstevel@tonic-gate 		ASSERT(n == NULL || n->s_base > seg->s_base);
4027c478bd9Sstevel@tonic-gate 		ASSERT(n != NULL || seg == avl_last(&as->a_segtree));
4037c478bd9Sstevel@tonic-gate 		if (seg == seglast)
4047c478bd9Sstevel@tonic-gate 			seglast = NULL;
4057c478bd9Sstevel@tonic-gate 		nsegs++;
4067c478bd9Sstevel@tonic-gate 	}
4077c478bd9Sstevel@tonic-gate 	ASSERT(seglast == NULL);
4087c478bd9Sstevel@tonic-gate 	ASSERT(avl_numnodes(&as->a_segtree) == nsegs);
4097c478bd9Sstevel@tonic-gate }
4107c478bd9Sstevel@tonic-gate #endif /* VERIFY_SEGLIST */
4117c478bd9Sstevel@tonic-gate 
4127c478bd9Sstevel@tonic-gate /*
4137c478bd9Sstevel@tonic-gate  * Add a new segment to the address space. The avl_find()
4147c478bd9Sstevel@tonic-gate  * may be expensive so we attempt to use last segment accessed
4157c478bd9Sstevel@tonic-gate  * in as_gap() as an insertion point.
4167c478bd9Sstevel@tonic-gate  */
4177c478bd9Sstevel@tonic-gate int
as_addseg(struct as * as,struct seg * newseg)4187c478bd9Sstevel@tonic-gate as_addseg(struct as  *as, struct seg *newseg)
4197c478bd9Sstevel@tonic-gate {
4207c478bd9Sstevel@tonic-gate 	struct seg *seg;
4217c478bd9Sstevel@tonic-gate 	caddr_t addr;
4227c478bd9Sstevel@tonic-gate 	caddr_t eaddr;
4237c478bd9Sstevel@tonic-gate 	avl_index_t where;
4247c478bd9Sstevel@tonic-gate 
425dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate 	as->a_updatedir = 1;	/* inform /proc */
4287c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate 	if (as->a_lastgaphl != NULL) {
4317c478bd9Sstevel@tonic-gate 		struct seg *hseg = NULL;
4327c478bd9Sstevel@tonic-gate 		struct seg *lseg = NULL;
4337c478bd9Sstevel@tonic-gate 
4347c478bd9Sstevel@tonic-gate 		if (as->a_lastgaphl->s_base > newseg->s_base) {
4357c478bd9Sstevel@tonic-gate 			hseg = as->a_lastgaphl;
4367c478bd9Sstevel@tonic-gate 			lseg = AVL_PREV(&as->a_segtree, hseg);
4377c478bd9Sstevel@tonic-gate 		} else {
4387c478bd9Sstevel@tonic-gate 			lseg = as->a_lastgaphl;
4397c478bd9Sstevel@tonic-gate 			hseg = AVL_NEXT(&as->a_segtree, lseg);
4407c478bd9Sstevel@tonic-gate 		}
4417c478bd9Sstevel@tonic-gate 
4427c478bd9Sstevel@tonic-gate 		if (hseg && lseg && lseg->s_base < newseg->s_base &&
4437c478bd9Sstevel@tonic-gate 		    hseg->s_base > newseg->s_base) {
4447c478bd9Sstevel@tonic-gate 			avl_insert_here(&as->a_segtree, newseg, lseg,
4457c478bd9Sstevel@tonic-gate 			    AVL_AFTER);
4467c478bd9Sstevel@tonic-gate 			as->a_lastgaphl = NULL;
4477c478bd9Sstevel@tonic-gate 			as->a_seglast = newseg;
4487c478bd9Sstevel@tonic-gate 			return (0);
4497c478bd9Sstevel@tonic-gate 		}
4507c478bd9Sstevel@tonic-gate 		as->a_lastgaphl = NULL;
4517c478bd9Sstevel@tonic-gate 	}
4527c478bd9Sstevel@tonic-gate 
4537c478bd9Sstevel@tonic-gate 	addr = newseg->s_base;
4547c478bd9Sstevel@tonic-gate 	eaddr = addr + newseg->s_size;
4557c478bd9Sstevel@tonic-gate again:
4567c478bd9Sstevel@tonic-gate 
4577c478bd9Sstevel@tonic-gate 	seg = avl_find(&as->a_segtree, &addr, &where);
4587c478bd9Sstevel@tonic-gate 
4597c478bd9Sstevel@tonic-gate 	if (seg == NULL)
4607c478bd9Sstevel@tonic-gate 		seg = avl_nearest(&as->a_segtree, where, AVL_AFTER);
4617c478bd9Sstevel@tonic-gate 
4627c478bd9Sstevel@tonic-gate 	if (seg == NULL)
4637c478bd9Sstevel@tonic-gate 		seg = avl_last(&as->a_segtree);
4647c478bd9Sstevel@tonic-gate 
4657c478bd9Sstevel@tonic-gate 	if (seg != NULL) {
4667c478bd9Sstevel@tonic-gate 		caddr_t base = seg->s_base;
4677c478bd9Sstevel@tonic-gate 
4687c478bd9Sstevel@tonic-gate 		/*
4697c478bd9Sstevel@tonic-gate 		 * If top of seg is below the requested address, then
4707c478bd9Sstevel@tonic-gate 		 * the insertion point is at the end of the linked list,
4717c478bd9Sstevel@tonic-gate 		 * and seg points to the tail of the list.  Otherwise,
4727c478bd9Sstevel@tonic-gate 		 * the insertion point is immediately before seg.
4737c478bd9Sstevel@tonic-gate 		 */
4747c478bd9Sstevel@tonic-gate 		if (base + seg->s_size > addr) {
4757c478bd9Sstevel@tonic-gate 			if (addr >= base || eaddr > base) {
4767c478bd9Sstevel@tonic-gate #ifdef __sparc
4777c478bd9Sstevel@tonic-gate 				extern struct seg_ops segnf_ops;
4787c478bd9Sstevel@tonic-gate 
4797c478bd9Sstevel@tonic-gate 				/*
4807c478bd9Sstevel@tonic-gate 				 * no-fault segs must disappear if overlaid.
4817c478bd9Sstevel@tonic-gate 				 * XXX need new segment type so
4827c478bd9Sstevel@tonic-gate 				 * we don't have to check s_ops
4837c478bd9Sstevel@tonic-gate 				 */
4847c478bd9Sstevel@tonic-gate 				if (seg->s_ops == &segnf_ops) {
4857c478bd9Sstevel@tonic-gate 					seg_unmap(seg);
4867c478bd9Sstevel@tonic-gate 					goto again;
4877c478bd9Sstevel@tonic-gate 				}
4887c478bd9Sstevel@tonic-gate #endif
4897c478bd9Sstevel@tonic-gate 				return (-1);	/* overlapping segment */
4907c478bd9Sstevel@tonic-gate 			}
4917c478bd9Sstevel@tonic-gate 		}
4927c478bd9Sstevel@tonic-gate 	}
4937c478bd9Sstevel@tonic-gate 	as->a_seglast = newseg;
4947c478bd9Sstevel@tonic-gate 	avl_insert(&as->a_segtree, newseg, where);
4957c478bd9Sstevel@tonic-gate 
4967c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST
4977c478bd9Sstevel@tonic-gate 	as_verify(as);
4987c478bd9Sstevel@tonic-gate #endif
4997c478bd9Sstevel@tonic-gate 	return (0);
5007c478bd9Sstevel@tonic-gate }
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate struct seg *
as_removeseg(struct as * as,struct seg * seg)5037c478bd9Sstevel@tonic-gate as_removeseg(struct as *as, struct seg *seg)
5047c478bd9Sstevel@tonic-gate {
5057c478bd9Sstevel@tonic-gate 	avl_tree_t *t;
5067c478bd9Sstevel@tonic-gate 
507dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_WRITE_HELD(as));
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate 	as->a_updatedir = 1;	/* inform /proc */
5107c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
5117c478bd9Sstevel@tonic-gate 
5127c478bd9Sstevel@tonic-gate 	if (seg == NULL)
5137c478bd9Sstevel@tonic-gate 		return (NULL);
5147c478bd9Sstevel@tonic-gate 
5157c478bd9Sstevel@tonic-gate 	t = &as->a_segtree;
5167c478bd9Sstevel@tonic-gate 	if (as->a_seglast == seg)
5177c478bd9Sstevel@tonic-gate 		as->a_seglast = NULL;
5187c478bd9Sstevel@tonic-gate 	as->a_lastgaphl = NULL;
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate 	/*
5217c478bd9Sstevel@tonic-gate 	 * if this segment is at an address higher than
5227c478bd9Sstevel@tonic-gate 	 * a_lastgap, set a_lastgap to the next segment (NULL if last segment)
5237c478bd9Sstevel@tonic-gate 	 */
5247c478bd9Sstevel@tonic-gate 	if (as->a_lastgap &&
5257c478bd9Sstevel@tonic-gate 	    (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base))
5267c478bd9Sstevel@tonic-gate 		as->a_lastgap = AVL_NEXT(t, seg);
5277c478bd9Sstevel@tonic-gate 
5287c478bd9Sstevel@tonic-gate 	/*
5297c478bd9Sstevel@tonic-gate 	 * remove the segment from the seg tree
5307c478bd9Sstevel@tonic-gate 	 */
5317c478bd9Sstevel@tonic-gate 	avl_remove(t, seg);
5327c478bd9Sstevel@tonic-gate 
5337c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST
5347c478bd9Sstevel@tonic-gate 	as_verify(as);
5357c478bd9Sstevel@tonic-gate #endif
5367c478bd9Sstevel@tonic-gate 	return (seg);
5377c478bd9Sstevel@tonic-gate }
5387c478bd9Sstevel@tonic-gate 
5397c478bd9Sstevel@tonic-gate /*
5407c478bd9Sstevel@tonic-gate  * Find a segment containing addr.
5417c478bd9Sstevel@tonic-gate  */
5427c478bd9Sstevel@tonic-gate struct seg *
as_segat(struct as * as,caddr_t addr)5437c478bd9Sstevel@tonic-gate as_segat(struct as *as, caddr_t addr)
5447c478bd9Sstevel@tonic-gate {
5457c478bd9Sstevel@tonic-gate 	struct seg *seg = as->a_seglast;
5467c478bd9Sstevel@tonic-gate 
547dc32d872SJosef 'Jeff' Sipek 	ASSERT(AS_LOCK_HELD(as));
5487c478bd9Sstevel@tonic-gate 
5497c478bd9Sstevel@tonic-gate 	if (seg != NULL && seg->s_base <= addr &&
5507c478bd9Sstevel@tonic-gate 	    addr < seg->s_base + seg->s_size)
5517c478bd9Sstevel@tonic-gate 		return (seg);
5527c478bd9Sstevel@tonic-gate 
5537c478bd9Sstevel@tonic-gate 	seg = avl_find(&as->a_segtree, &addr, NULL);
5547c478bd9Sstevel@tonic-gate 	return (seg);
5557c478bd9Sstevel@tonic-gate }
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate /*
5587c478bd9Sstevel@tonic-gate  * Serialize all searches for holes in an address space to
5597c478bd9Sstevel@tonic-gate  * prevent two or more threads from allocating the same virtual
5607c478bd9Sstevel@tonic-gate  * address range.  The address space must not be "read/write"
5617c478bd9Sstevel@tonic-gate  * locked by the caller since we may block.
5627c478bd9Sstevel@tonic-gate  */
5637c478bd9Sstevel@tonic-gate void
as_rangelock(struct as * as)5647c478bd9Sstevel@tonic-gate as_rangelock(struct as *as)
5657c478bd9Sstevel@tonic-gate {
5667c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
5677c478bd9Sstevel@tonic-gate 	while (AS_ISCLAIMGAP(as))
5687c478bd9Sstevel@tonic-gate 		cv_wait(&as->a_cv, &as->a_contents);
5697c478bd9Sstevel@tonic-gate 	AS_SETCLAIMGAP(as);
5707c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
5717c478bd9Sstevel@tonic-gate }
5727c478bd9Sstevel@tonic-gate 
5737c478bd9Sstevel@tonic-gate /*
5747c478bd9Sstevel@tonic-gate  * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
5757c478bd9Sstevel@tonic-gate  */
5767c478bd9Sstevel@tonic-gate void
as_rangeunlock(struct as * as)5777c478bd9Sstevel@tonic-gate as_rangeunlock(struct as *as)
5787c478bd9Sstevel@tonic-gate {
5797c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
5807c478bd9Sstevel@tonic-gate 	AS_CLRCLAIMGAP(as);
5817c478bd9Sstevel@tonic-gate 	cv_signal(&as->a_cv);
5827c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
5837c478bd9Sstevel@tonic-gate }
5847c478bd9Sstevel@tonic-gate 
5857c478bd9Sstevel@tonic-gate /*
5867c478bd9Sstevel@tonic-gate  * compar segments (or just an address) by segment address range
5877c478bd9Sstevel@tonic-gate  */
5887c478bd9Sstevel@tonic-gate static int
as_segcompar(const void * x,const void * y)5897c478bd9Sstevel@tonic-gate as_segcompar(const void *x, const void *y)
5907c478bd9Sstevel@tonic-gate {
5917c478bd9Sstevel@tonic-gate 	struct seg *a = (struct seg *)x;
5927c478bd9Sstevel@tonic-gate 	struct seg *b = (struct seg *)y;
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 	if (a->s_base < b->s_base)
5957c478bd9Sstevel@tonic-gate 		return (-1);
5967c478bd9Sstevel@tonic-gate 	if (a->s_base >= b->s_base + b->s_size)
5977c478bd9Sstevel@tonic-gate 		return (1);
5987c478bd9Sstevel@tonic-gate 	return (0);
5997c478bd9Sstevel@tonic-gate }
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate 
6027c478bd9Sstevel@tonic-gate void
as_avlinit(struct as * as)6037c478bd9Sstevel@tonic-gate as_avlinit(struct as *as)
6047c478bd9Sstevel@tonic-gate {
6057c478bd9Sstevel@tonic-gate 	avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg),
6067c478bd9Sstevel@tonic-gate 	    offsetof(struct seg, s_tree));
6077c478bd9Sstevel@tonic-gate 	avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page),
6087c478bd9Sstevel@tonic-gate 	    offsetof(struct watched_page, wp_link));
6097c478bd9Sstevel@tonic-gate }
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6127c478bd9Sstevel@tonic-gate static int
as_constructor(void * buf,void * cdrarg,int kmflags)6137c478bd9Sstevel@tonic-gate as_constructor(void *buf, void *cdrarg, int kmflags)
6147c478bd9Sstevel@tonic-gate {
6157c478bd9Sstevel@tonic-gate 	struct as *as = buf;
6167c478bd9Sstevel@tonic-gate 
6177c478bd9Sstevel@tonic-gate 	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
6187c478bd9Sstevel@tonic-gate 	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
6197c478bd9Sstevel@tonic-gate 	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
6207c478bd9Sstevel@tonic-gate 	as_avlinit(as);
6217c478bd9Sstevel@tonic-gate 	return (0);
6227c478bd9Sstevel@tonic-gate }
6237c478bd9Sstevel@tonic-gate 
6247c478bd9Sstevel@tonic-gate /*ARGSUSED1*/
6257c478bd9Sstevel@tonic-gate static void
as_destructor(void * buf,void * cdrarg)6267c478bd9Sstevel@tonic-gate as_destructor(void *buf, void *cdrarg)
6277c478bd9Sstevel@tonic-gate {
6287c478bd9Sstevel@tonic-gate 	struct as *as = buf;
6297c478bd9Sstevel@tonic-gate 
6307c478bd9Sstevel@tonic-gate 	avl_destroy(&as->a_segtree);
6317c478bd9Sstevel@tonic-gate 	mutex_destroy(&as->a_contents);
6327c478bd9Sstevel@tonic-gate 	cv_destroy(&as->a_cv);
6337c478bd9Sstevel@tonic-gate 	rw_destroy(&as->a_lock);
6347c478bd9Sstevel@tonic-gate }
6357c478bd9Sstevel@tonic-gate 
6367c478bd9Sstevel@tonic-gate void
as_init(void)6377c478bd9Sstevel@tonic-gate as_init(void)
6387c478bd9Sstevel@tonic-gate {
6397c478bd9Sstevel@tonic-gate 	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
640843e1988Sjohnlev 	    as_constructor, as_destructor, NULL, NULL, NULL, 0);
6417c478bd9Sstevel@tonic-gate }
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate /*
6447c478bd9Sstevel@tonic-gate  * Allocate and initialize an address space data structure.
6457c478bd9Sstevel@tonic-gate  * We call hat_alloc to allow any machine dependent
6467c478bd9Sstevel@tonic-gate  * information in the hat structure to be initialized.
6477c478bd9Sstevel@tonic-gate  */
6487c478bd9Sstevel@tonic-gate struct as *
as_alloc(void)6497c478bd9Sstevel@tonic-gate as_alloc(void)
6507c478bd9Sstevel@tonic-gate {
6517c478bd9Sstevel@tonic-gate 	struct as *as;
6527c478bd9Sstevel@tonic-gate 
6537c478bd9Sstevel@tonic-gate 	as = kmem_cache_alloc(as_cache, KM_SLEEP);
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 	as->a_flags		= 0;
6567c478bd9Sstevel@tonic-gate 	as->a_vbits		= 0;
6577c478bd9Sstevel@tonic-gate 	as->a_hrm		= NULL;
6587c478bd9Sstevel@tonic-gate 	as->a_seglast		= NULL;
6597c478bd9Sstevel@tonic-gate 	as->a_size		= 0;
66040688216SSudheer A 	as->a_resvsize		= 0;
6617c478bd9Sstevel@tonic-gate 	as->a_updatedir		= 0;
6627c478bd9Sstevel@tonic-gate 	gethrestime(&as->a_updatetime);
6637c478bd9Sstevel@tonic-gate 	as->a_objectdir		= NULL;
6647c478bd9Sstevel@tonic-gate 	as->a_sizedir		= 0;
6657c478bd9Sstevel@tonic-gate 	as->a_userlimit		= (caddr_t)USERLIMIT;
6667c478bd9Sstevel@tonic-gate 	as->a_lastgap		= NULL;
6677c478bd9Sstevel@tonic-gate 	as->a_lastgaphl		= NULL;
6687c478bd9Sstevel@tonic-gate 	as->a_callbacks		= NULL;
6696430daecSBrandon Baker 	as->a_proc		= NULL;
6707c478bd9Sstevel@tonic-gate 
671dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
6727c478bd9Sstevel@tonic-gate 	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
673dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
6747c478bd9Sstevel@tonic-gate 
6757c478bd9Sstevel@tonic-gate 	return (as);
6767c478bd9Sstevel@tonic-gate }
6777c478bd9Sstevel@tonic-gate 
6787c478bd9Sstevel@tonic-gate /*
6797c478bd9Sstevel@tonic-gate  * Free an address space data structure.
6807c478bd9Sstevel@tonic-gate  * Need to free the hat first and then
6817c478bd9Sstevel@tonic-gate  * all the segments on this as and finally
6827c478bd9Sstevel@tonic-gate  * the space for the as struct itself.
6837c478bd9Sstevel@tonic-gate  */
6847c478bd9Sstevel@tonic-gate void
as_free(struct as * as)6857c478bd9Sstevel@tonic-gate as_free(struct as *as)
6867c478bd9Sstevel@tonic-gate {
6877c478bd9Sstevel@tonic-gate 	struct hat *hat = as->a_hat;
6887c478bd9Sstevel@tonic-gate 	struct seg *seg, *next;
6890d5ae8c1SJosef 'Jeff' Sipek 	boolean_t free_started = B_FALSE;
6907c478bd9Sstevel@tonic-gate 
6917c478bd9Sstevel@tonic-gate top:
6927c478bd9Sstevel@tonic-gate 	/*
6937c478bd9Sstevel@tonic-gate 	 * Invoke ALL callbacks. as_do_callbacks will do one callback
6947c478bd9Sstevel@tonic-gate 	 * per call, and not return (-1) until the callback has completed.
6957c478bd9Sstevel@tonic-gate 	 * When as_do_callbacks returns zero, all callbacks have completed.
6967c478bd9Sstevel@tonic-gate 	 */
6977c478bd9Sstevel@tonic-gate 	mutex_enter(&as->a_contents);
698843e1988Sjohnlev 	while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0))
699843e1988Sjohnlev 		;
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	mutex_exit(&as->a_contents);
702dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
7037c478bd9Sstevel@tonic-gate 
7040d5ae8c1SJosef 'Jeff' Sipek 	if (!free_started) {
7050d5ae8c1SJosef 'Jeff' Sipek 		free_started = B_TRUE;
7067c478bd9Sstevel@tonic-gate 		hat_free_start(hat);
7077c478bd9Sstevel@tonic-gate 	}
7087c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) {
7097c478bd9Sstevel@tonic-gate 		int err;
7107c478bd9Sstevel@tonic-gate 
7117c478bd9Sstevel@tonic-gate 		next = AS_SEGNEXT(as, seg);
712a98e9dbfSaguzovsk retry:
7137c478bd9Sstevel@tonic-gate 		err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size);
7147c478bd9Sstevel@tonic-gate 		if (err == EAGAIN) {
7157c478bd9Sstevel@tonic-gate 			mutex_enter(&as->a_contents);
7167c478bd9Sstevel@tonic-gate 			if (as->a_callbacks) {
717dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
718a98e9dbfSaguzovsk 			} else if (!AS_ISNOUNMAPWAIT(as)) {
7197c478bd9Sstevel@tonic-gate 				/*
7207c478bd9Sstevel@tonic-gate 				 * Memory is currently locked. Wait for a
7217c478bd9Sstevel@tonic-gate 				 * cv_signal that it has been unlocked, then
7227c478bd9Sstevel@tonic-gate 				 * try the operation again.
7237c478bd9Sstevel@tonic-gate 				 */
7247c478bd9Sstevel@tonic-gate 				if (AS_ISUNMAPWAIT(as) == 0)
7257c478bd9Sstevel@tonic-gate 					cv_broadcast(&as->a_cv);
7267c478bd9Sstevel@tonic-gate 				AS_SETUNMAPWAIT(as);
727dc32d872SJosef 'Jeff' Sipek 				AS_LOCK_EXIT(as);
7287c478bd9Sstevel@tonic-gate 				while (AS_ISUNMAPWAIT(as))
7297c478bd9Sstevel@tonic-gate 					cv_wait(&as->a_cv, &as->a_contents);
730a98e9dbfSaguzovsk 			} else {
731a98e9dbfSaguzovsk 				/*
732a98e9dbfSaguzovsk 				 * We may have raced with
733a98e9dbfSaguzovsk 				 * segvn_reclaim()/segspt_reclaim(). In this
734a98e9dbfSaguzovsk 				 * case clean nounmapwait flag and retry since
735a98e9dbfSaguzovsk 				 * softlockcnt in this segment may be already
736a98e9dbfSaguzovsk 				 * 0.  We don't drop as writer lock so our
737a98e9dbfSaguzovsk 				 * number of retries without sleeping should
738a98e9dbfSaguzovsk 				 * be very small. See segvn_reclaim() for
739a98e9dbfSaguzovsk 				 * more comments.
740a98e9dbfSaguzovsk 				 */
741a98e9dbfSaguzovsk 				AS_CLRNOUNMAPWAIT(as);
742a98e9dbfSaguzovsk 				mutex_exit(&as->a_contents);
743a98e9dbfSaguzovsk 				goto retry;
7447c478bd9Sstevel@tonic-gate 			}
7457c478bd9Sstevel@tonic-gate 			mutex_exit(&as->a_contents);
7467c478bd9Sstevel@tonic-gate 			goto top;
7477c478bd9Sstevel@tonic-gate 		} else {
7487c478bd9Sstevel@tonic-gate 			/*
7497c478bd9Sstevel@tonic-gate 			 * We do not expect any other error return at this
7507c478bd9Sstevel@tonic-gate 			 * time. This is similar to an ASSERT in seg_unmap()
7517c478bd9Sstevel@tonic-gate 			 */
7527c478bd9Sstevel@tonic-gate 			ASSERT(err == 0);
7537c478bd9Sstevel@tonic-gate 		}
7547c478bd9Sstevel@tonic-gate 	}
7557c478bd9Sstevel@tonic-gate 	hat_free_end(hat);
756dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
7577c478bd9Sstevel@tonic-gate 
7587c478bd9Sstevel@tonic-gate 	/* /proc stuff */
7597c478bd9Sstevel@tonic-gate 	ASSERT(avl_numnodes(&as->a_wpage) == 0);
7607c478bd9Sstevel@tonic-gate 	if (as->a_objectdir) {
7617c478bd9Sstevel@tonic-gate 		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
7627c478bd9Sstevel@tonic-gate 		as->a_objectdir = NULL;
7637c478bd9Sstevel@tonic-gate 		as->a_sizedir = 0;
7647c478bd9Sstevel@tonic-gate 	}
7657c478bd9Sstevel@tonic-gate 
7667c478bd9Sstevel@tonic-gate 	/*
7677c478bd9Sstevel@tonic-gate 	 * Free the struct as back to kmem.  Assert it has no segments.
7687c478bd9Sstevel@tonic-gate 	 */
7697c478bd9Sstevel@tonic-gate 	ASSERT(avl_numnodes(&as->a_segtree) == 0);
7707c478bd9Sstevel@tonic-gate 	kmem_cache_free(as_cache, as);
7717c478bd9Sstevel@tonic-gate }
7727c478bd9Sstevel@tonic-gate 
7737c478bd9Sstevel@tonic-gate int
as_dup(struct as * as,struct proc * forkedproc)774bb5ca623SVamsi Nagineni as_dup(struct as *as, struct proc *forkedproc)
7757c478bd9Sstevel@tonic-gate {
7767c478bd9Sstevel@tonic-gate 	struct as *newas;
7777c478bd9Sstevel@tonic-gate 	struct seg *seg, *newseg;
77840688216SSudheer A 	size_t	purgesize = 0;
7797c478bd9Sstevel@tonic-gate 	int error;
7807c478bd9Sstevel@tonic-gate 
781dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_WRITER);
7827c478bd9Sstevel@tonic-gate 	as_clearwatch(as);
7837c478bd9Sstevel@tonic-gate 	newas = as_alloc();
7847c478bd9Sstevel@tonic-gate 	newas->a_userlimit = as->a_userlimit;
785bb5ca623SVamsi Nagineni 	newas->a_proc = forkedproc;
786250f6acdSvk 
787dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(newas, RW_WRITER);
7887c478bd9Sstevel@tonic-gate 
78905d3dc4bSpaulsan 	(void) hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_SRD);
7907c478bd9Sstevel@tonic-gate 
7917c478bd9Sstevel@tonic-gate 	for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) {
7927c478bd9Sstevel@tonic-gate 
79340688216SSudheer A 		if (seg->s_flags & S_PURGE) {
79440688216SSudheer A 			purgesize += seg->s_size;
7957c478bd9Sstevel@tonic-gate 			continue;
79640688216SSudheer A 		}
7977c478bd9Sstevel@tonic-gate 
7987c478bd9Sstevel@tonic-gate 		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
7997c478bd9Sstevel@tonic-gate 		if (newseg == NULL) {
800dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(newas);
8017c478bd9Sstevel@tonic-gate 			as_setwatch(as);
802dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
8037c478bd9Sstevel@tonic-gate 			as_free(newas);
8047c478bd9Sstevel@tonic-gate 			return (-1);
8057c478bd9Sstevel@tonic-gate 		}
8067c478bd9Sstevel@tonic-gate 		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
8077c478bd9Sstevel@tonic-gate 			/*
8087c478bd9Sstevel@tonic-gate 			 * We call seg_free() on the new seg
8097c478bd9Sstevel@tonic-gate 			 * because the segment is not set up
8107c478bd9Sstevel@tonic-gate 			 * completely; i.e. it has no ops.
8117c478bd9Sstevel@tonic-gate 			 */
8127c478bd9Sstevel@tonic-gate 			as_setwatch(as);
813dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
8147c478bd9Sstevel@tonic-gate 			seg_free(newseg);
815dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(newas);
8167c478bd9Sstevel@tonic-gate 			as_free(newas);
8177c478bd9Sstevel@tonic-gate 			return (error);
8187c478bd9Sstevel@tonic-gate 		}
819284ce987SPatrick Mooney 		if ((newseg->s_flags & S_HOLE) == 0) {
820284ce987SPatrick Mooney 			newas->a_size += seg->s_size;
821284ce987SPatrick Mooney 		}
8227c478bd9Sstevel@tonic-gate 	}
82340688216SSudheer A 	newas->a_resvsize = as->a_resvsize - purgesize;
8247c478bd9Sstevel@tonic-gate 
8257c478bd9Sstevel@tonic-gate 	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
8267c478bd9Sstevel@tonic-gate 
827dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(newas);
8287c478bd9Sstevel@tonic-gate 
8297c478bd9Sstevel@tonic-gate 	as_setwatch(as);
830dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
8317c478bd9Sstevel@tonic-gate 	if (error != 0) {
8327c478bd9Sstevel@tonic-gate 		as_free(newas);
8337c478bd9Sstevel@tonic-gate 		return (error);
8347c478bd9Sstevel@tonic-gate 	}
835bb5ca623SVamsi Nagineni 	forkedproc->p_as = newas;
8367c478bd9Sstevel@tonic-gate 	return (0);
8377c478bd9Sstevel@tonic-gate }
8387c478bd9Sstevel@tonic-gate 
8397c478bd9Sstevel@tonic-gate /*
8407c478bd9Sstevel@tonic-gate  * Handle a ``fault'' at addr for size bytes.
8417c478bd9Sstevel@tonic-gate  */
8427c478bd9Sstevel@tonic-gate faultcode_t
as_fault(struct hat * hat,struct as * as,caddr_t addr,size_t size,enum fault_type type,enum seg_rw rw)8437c478bd9Sstevel@tonic-gate as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size,
8446430daecSBrandon Baker     enum fault_type type, enum seg_rw rw)
8457c478bd9Sstevel@tonic-gate {
8467c478bd9Sstevel@tonic-gate 	struct seg *seg;
8477c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
8487c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
8497c478bd9Sstevel@tonic-gate 	size_t ssize;
8507c478bd9Sstevel@tonic-gate 	faultcode_t res = 0;
8517c478bd9Sstevel@tonic-gate 	caddr_t addrsav;
8527c478bd9Sstevel@tonic-gate 	struct seg *segsav;
8537c478bd9Sstevel@tonic-gate 	int as_lock_held;
8547c478bd9Sstevel@tonic-gate 	klwp_t *lwp = ttolwp(curthread);
8557c478bd9Sstevel@tonic-gate 
8567c478bd9Sstevel@tonic-gate 
8577c478bd9Sstevel@tonic-gate 
8587c478bd9Sstevel@tonic-gate retry:
8590d5ae8c1SJosef 'Jeff' Sipek 	/*
8600d5ae8c1SJosef 'Jeff' Sipek 	 * Indicate that the lwp is not to be stopped while waiting for a
8610d5ae8c1SJosef 'Jeff' Sipek 	 * pagefault.  This is to avoid deadlock while debugging a process
8620d5ae8c1SJosef 'Jeff' Sipek 	 * via /proc over NFS (in particular).
8630d5ae8c1SJosef 'Jeff' Sipek 	 */
8640d5ae8c1SJosef 'Jeff' Sipek 	if (lwp != NULL)
8650d5ae8c1SJosef 'Jeff' Sipek 		lwp->lwp_nostop++;
8667c478bd9Sstevel@tonic-gate 
8670d5ae8c1SJosef 'Jeff' Sipek 	/*
8680d5ae8c1SJosef 'Jeff' Sipek 	 * same length must be used when we softlock and softunlock.  We
8690d5ae8c1SJosef 'Jeff' Sipek 	 * don't support softunlocking lengths less than the original length
8700d5ae8c1SJosef 'Jeff' Sipek 	 * when there is largepage support.  See seg_dev.c for more
8710d5ae8c1SJosef 'Jeff' Sipek 	 * comments.
8720d5ae8c1SJosef 'Jeff' Sipek 	 */
8730d5ae8c1SJosef 'Jeff' Sipek 	switch (type) {
8747c478bd9Sstevel@tonic-gate 
8750d5ae8c1SJosef 'Jeff' Sipek 	case F_SOFTLOCK:
8760d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ADD_K(vm, softlock, 1);
8770d5ae8c1SJosef 'Jeff' Sipek 		break;
8787c478bd9Sstevel@tonic-gate 
8790d5ae8c1SJosef 'Jeff' Sipek 	case F_SOFTUNLOCK:
8800d5ae8c1SJosef 'Jeff' Sipek 		break;
8817c478bd9Sstevel@tonic-gate 
8820d5ae8c1SJosef 'Jeff' Sipek 	case F_PROT:
8830d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ADD_K(vm, prot_fault, 1);
8840d5ae8c1SJosef 'Jeff' Sipek 		break;
8857c478bd9Sstevel@tonic-gate 
8860d5ae8c1SJosef 'Jeff' Sipek 	case F_INVAL:
8870d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ENTER_K();
8880d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_ADDQ(CPU, vm, as_fault, 1);
8890d5ae8c1SJosef 'Jeff' Sipek 		if (as == &kas)
8900d5ae8c1SJosef 'Jeff' Sipek 			CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1);
8910d5ae8c1SJosef 'Jeff' Sipek 		CPU_STATS_EXIT_K();
8920d5ae8c1SJosef 'Jeff' Sipek 		break;
8937c478bd9Sstevel@tonic-gate 	}
8947c478bd9Sstevel@tonic-gate 
8957c478bd9Sstevel@tonic-gate 	/* Kernel probe */
8967c478bd9Sstevel@tonic-gate 	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
897843e1988Sjohnlev 	    tnf_opaque,	address,	addr,
898843e1988Sjohnlev 	    tnf_fault_type,	fault_type,	type,
899843e1988Sjohnlev 	    tnf_seg_access,	access,		rw);
9007c478bd9Sstevel@tonic-gate 
9017c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
9027c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
903843e1988Sjohnlev 	    (size_t)raddr;
9047c478bd9Sstevel@tonic-gate 
9057c478bd9Sstevel@tonic-gate 	/*
9067c478bd9Sstevel@tonic-gate 	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
9077c478bd9Sstevel@tonic-gate 	 * correctness, but then we could be stuck holding this lock for
9087c478bd9Sstevel@tonic-gate 	 * a LONG time if the fault needs to be resolved on a slow
9097c478bd9Sstevel@tonic-gate 	 * filesystem, and then no-one will be able to exec new commands,
9107c478bd9Sstevel@tonic-gate 	 * as exec'ing requires the write lock on the as.
9117c478bd9Sstevel@tonic-gate 	 */
9127c478bd9Sstevel@tonic-gate 	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
9137c478bd9Sstevel@tonic-gate 	    raddr + size < segkmap->s_base + segkmap->s_size) {
9147c478bd9Sstevel@tonic-gate 		seg = segkmap;
9157c478bd9Sstevel@tonic-gate 		as_lock_held = 0;
9167c478bd9Sstevel@tonic-gate 	} else {
917dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_ENTER(as, RW_READER);
9187c478bd9Sstevel@tonic-gate 
9197c478bd9Sstevel@tonic-gate 		seg = as_segat(as, raddr);
9207c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
921dc32d872SJosef 'Jeff' Sipek 			AS_LOCK_EXIT(as);
9220d5ae8c1SJosef 'Jeff' Sipek 			if (lwp != NULL)
9237c478bd9Sstevel@tonic-gate 				lwp->lwp_nostop--;
9247c478bd9Sstevel@tonic-gate 			return (FC_NOMAP);
9257c478bd9Sstevel@tonic-gate 		}
9267c478bd9Sstevel@tonic-gate 
9277c478bd9Sstevel@tonic-gate 		as_lock_held = 1;
9287c478bd9Sstevel@tonic-gate 	}
9297c478bd9Sstevel@tonic-gate 
9307c478bd9Sstevel@tonic-gate 	addrsav = raddr;
9317c478bd9Sstevel@tonic-gate 	segsav = seg;
9327c478bd9Sstevel@tonic-gate 
9337c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
9347c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
9357c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
9367c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
9377c478bd9Sstevel@tonic-gate 				res = FC_NOMAP;
9387c478bd9Sstevel@tonic-gate 				break;
9397c478bd9Sstevel@tonic-gate 			}
9407c478bd9Sstevel@tonic-gate 		}
9417c478bd9Sstevel@tonic-gate 		if (raddr + rsize > seg->s_base + seg->s_size)
9427c478bd9Sstevel@tonic-gate 			ssize = seg->s_base + seg->s_size - raddr;
9437c478bd9Sstevel@tonic-gate 		else
9447c478bd9Sstevel@tonic-gate 			ssize = rsize;
9457c478bd9Sstevel@tonic-gate 
9460d5ae8c1SJosef 'Jeff' Sipek 		res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
9470d5ae8c1SJosef 'Jeff' Sipek 		if (res != 0)
9487c478bd9Sstevel@tonic-gate 			break;
9497c478bd9Sstevel@tonic-gate 	}
9507c478bd9Sstevel@tonic-gate 
9517c478bd9Sstevel@tonic-gate 	/*
9527c478bd9Sstevel@tonic-gate 	 * If we were SOFTLOCKing and encountered a failure,
9537c478bd9Sstevel@tonic-gate 	 * we must SOFTUNLOCK the range we already did. (Maybe we
9547c478bd9Sstevel@tonic-gate 	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
9557c478bd9Sstevel@tonic-gate 	 * right here...)
9567c478bd9Sstevel@tonic-gate 	 */
9577c478bd9Sstevel@tonic-gate 	if (res != 0 && type == F_SOFTLOCK) {
9587c478bd9Sstevel@tonic-gate 		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
9597c478bd9Sstevel@tonic-gate 			if (addrsav >= seg->s_base + seg->s_size)
9607c478bd9Sstevel@tonic-gate 				seg = AS_SEGNEXT(as, seg);
9617c478bd9Sstevel@tonic-gate 			ASSERT(seg != NULL);
9627c478bd9Sstevel@tonic-gate 			/*
9637c478bd9Sstevel@tonic-gate 			 * Now call the fault routine again to perform the
9647c478bd9Sstevel@tonic-gate 			 * unlock using S_OTHER instead of the rw variable
9657c478bd9Sstevel@tonic-gate 			 * since we never got a chance to touch the pages.
9667c478bd9Sstevel@tonic-gate 			 */
9677c478bd9Sstevel@tonic-gate 			if (raddr > seg->s_base + seg->s_size)
9687c478bd9Sstevel@tonic-gate 				ssize = seg->s_base + seg->s_size - addrsav;
9697c478bd9Sstevel@tonic-gate 			else
9707c478bd9Sstevel@tonic-gate 				ssize = raddr - addrsav;
9717c478bd9Sstevel@tonic-gate 			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
9727c478bd9Sstevel@tonic-gate 			    F_SOFTUNLOCK, S_OTHER);
9737c478bd9Sstevel@tonic-gate 		}
9747c478bd9Sstevel@tonic-gate 	}
9757c478bd9Sstevel@tonic-gate 	if (as_lock_held)
976dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
9770d5ae8c1SJosef 'Jeff' Sipek 	if (lwp != NULL)
9787c478bd9Sstevel@tonic-gate 		lwp->lwp_nostop--;
9798548bf79Snr 
9807c478bd9Sstevel@tonic-gate 	/*
9817c478bd9Sstevel@tonic-gate 	 * If the lower levels returned EDEADLK for a fault,
9827c478bd9Sstevel@tonic-gate 	 * It means that we should retry the fault.  Let's wait
9837c478bd9Sstevel@tonic-gate 	 * a bit also to let the deadlock causing condition clear.
9847c478bd9Sstevel@tonic-gate 	 * This is part of a gross hack to work around a design flaw
9857c478bd9Sstevel@tonic-gate 	 * in the ufs/sds logging code and should go away when the
9867c478bd9Sstevel@tonic-gate 	 * logging code is re-designed to fix the problem. See bug
9877c478bd9Sstevel@tonic-gate 	 * 4125102 for details of the problem.
9887c478bd9Sstevel@tonic-gate 	 */
9897c478bd9Sstevel@tonic-gate 	if (FC_ERRNO(res) == EDEADLK) {
9907c478bd9Sstevel@tonic-gate 		delay(deadlk_wait);
9917c478bd9Sstevel@tonic-gate 		res = 0;
9927c478bd9Sstevel@tonic-gate 		goto retry;
9937c478bd9Sstevel@tonic-gate 	}
9947c478bd9Sstevel@tonic-gate 	return (res);
9957c478bd9Sstevel@tonic-gate }
9967c478bd9Sstevel@tonic-gate 
9977c478bd9Sstevel@tonic-gate 
9987c478bd9Sstevel@tonic-gate 
9997c478bd9Sstevel@tonic-gate /*
10007c478bd9Sstevel@tonic-gate  * Asynchronous ``fault'' at addr for size bytes.
10017c478bd9Sstevel@tonic-gate  */
10027c478bd9Sstevel@tonic-gate faultcode_t
as_faulta(struct as * as,caddr_t addr,size_t size)10037c478bd9Sstevel@tonic-gate as_faulta(struct as *as, caddr_t addr, size_t size)
10047c478bd9Sstevel@tonic-gate {
10057c478bd9Sstevel@tonic-gate 	struct seg *seg;
10067c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
10077c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
10087c478bd9Sstevel@tonic-gate 	faultcode_t res = 0;
10097c478bd9Sstevel@tonic-gate 	klwp_t *lwp = ttolwp(curthread);
10107c478bd9Sstevel@tonic-gate 
10117c478bd9Sstevel@tonic-gate retry:
10127c478bd9Sstevel@tonic-gate 	/*
10137c478bd9Sstevel@tonic-gate 	 * Indicate that the lwp is not to be stopped while waiting
10147c478bd9Sstevel@tonic-gate 	 * for a pagefault.  This is to avoid deadlock while debugging
10157c478bd9Sstevel@tonic-gate 	 * a process via /proc over NFS (in particular).
10167c478bd9Sstevel@tonic-gate 	 */
10178548bf79Snr 	if (lwp != NULL)
10187c478bd9Sstevel@tonic-gate 		lwp->lwp_nostop++;
10197c478bd9Sstevel@tonic-gate 
10207c478bd9Sstevel@tonic-gate 	raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
10217c478bd9Sstevel@tonic-gate 	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
1022843e1988Sjohnlev 	    (size_t)raddr;
10237c478bd9Sstevel@tonic-gate 
1024dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_ENTER(as, RW_READER);
10257c478bd9Sstevel@tonic-gate 	seg = as_segat(as, raddr);
10267c478bd9Sstevel@tonic-gate 	if (seg == NULL) {
1027dc32d872SJosef 'Jeff' Sipek 		AS_LOCK_EXIT(as);
10288548bf79Snr 		if (lwp != NULL)
10297c478bd9Sstevel@tonic-gate 			lwp->lwp_nostop--;
10307c478bd9Sstevel@tonic-gate 		return (FC_NOMAP);
10317c478bd9Sstevel@tonic-gate 	}
10327c478bd9Sstevel@tonic-gate 
10337c478bd9Sstevel@tonic-gate 	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
10347c478bd9Sstevel@tonic-gate 		if (raddr >= seg->s_base + seg->s_size) {
10357c478bd9Sstevel@tonic-gate 			seg = AS_SEGNEXT(as, seg);
10367c478bd9Sstevel@tonic-gate 			if (seg == NULL || raddr != seg->s_base) {
10377c478bd9Sstevel@tonic-gate 				res = FC_NOMAP;
10387c478bd9Sstevel@tonic-gate 				break;
10397c478bd9Sstevel@tonic-gate 			}
10407c478bd9Sstevel@tonic-gate 		}
10417c478bd9Sstevel@tonic-gate 		res = SEGOP_FAULTA(seg, raddr);
10427c478bd9Sstevel@tonic-gate 		if (res != 0)
10437c478bd9Sstevel@tonic-gate 			break;
10447c478bd9Sstevel@tonic-gate 	}
1045dc32d872SJosef 'Jeff' Sipek 	AS_LOCK_EXIT(as);
10468548bf79Snr 	if (lwp != NULL)
10477c478bd9Sstevel@tonic-gate 		lwp->lwp_nostop--;
10487c478bd9Sstevel@tonic-gate 	/*
10497c478bd9Sstevel@tonic-gate 	 * If the lower levels returned EDEADLK for a fault,
10507c478bd9Sstevel@tonic-gate 	 * It means that we should retry the fault.  Let's wait
10517c478bd9Sstevel@tonic-gate 	 * a bit also to let the deadlock causing condition clear.
10527c478bd9Sstevel@tonic-gate 	 * This is part of a gross hack to work around a design flaw
10537c478bd9Sstevel@tonic-gate 	 * in the ufs/sds logging code and should go away when the
10547c478bd9Sstevel@tonic-gate 	 * logging code is re-designed to fix the problem. See bug
10557c478bd9Sstevel@tonic-gate 	 * 4125102 for details of the problem.
10567c478bd9Sstevel@tonic-gate 	 */
10577c478bd9Sstevel@tonic-gate 	if (FC_ERRNO(res) == EDEADLK) {
10587c478bd9Sstevel@tonic-gate 		delay(deadlk_wait);
10597c478bd9Sstevel@tonic-gate 		res = 0;
10607c478bd9Sstevel@tonic-gate 		goto retry;
10617c478bd9Sstevel@tonic-gate 	}
10627c478bd9Sstevel@tonic-gate 	return (res);
10637c478bd9Sstevel@tonic-gate }
10647c478bd9Sstevel@tonic-gate 
10657c478bd9Sstevel@tonic-gate /*
10667c478bd9Sstevel@tonic-gate  * Set the virtual mapping for the interval from [addr : addr + size)
10677c478bd9Sstevel@tonic-gate  * in address space `as' to have the specified protection.
10687c478bd9Sstevel@tonic-gate  * It is ok for the range to cross over several segments,
10697c478bd9Sstevel@tonic-gate  * as long as they are contiguous.
10707c478bd9Sstevel@tonic-gate  */
10717c478bd9Sstevel@tonic-gate int
as_setprot(struct as * as,caddr_t addr,size_t size,uint_t prot)10727c478bd9Sstevel@tonic-gate as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
10737c478bd9Sstevel@tonic-gate {
10747c478bd9Sstevel@tonic-gate 	struct seg *seg;
10757c478bd9Sstevel@tonic-gate 	struct as_callback *cb;
10767c478bd9Sstevel@tonic-gate 	size_t ssize;
10777c478bd9Sstevel@tonic-gate 	caddr_t raddr;			/* rounded down addr */
10787c478bd9Sstevel@tonic-gate 	size_t rsize;			/* rounded up size */
10797c478bd9Sstevel@tonic-gate 	int error = 0, writer = 0;
10807c478bd9Sstevel@tonic-gate 	caddr_t saveraddr;
10817c478bd9Sstevel@tonic-gate 	size_t