17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*02ff05a9Svsakar * Common Development and Distribution License (the "License"). 6*02ff05a9Svsakar * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*02ff05a9Svsakar * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * University Copyright- Copyright (c) 1982, 1986, 1988 317c478bd9Sstevel@tonic-gate * The Regents of the University of California 327c478bd9Sstevel@tonic-gate * All Rights Reserved 337c478bd9Sstevel@tonic-gate * 347c478bd9Sstevel@tonic-gate * University Acknowledgment- Portions of this document are derived from 357c478bd9Sstevel@tonic-gate * software developed by the University of California, Berkeley, and its 367c478bd9Sstevel@tonic-gate * contributors. 377c478bd9Sstevel@tonic-gate */ 387c478bd9Sstevel@tonic-gate 397c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate /* 427c478bd9Sstevel@tonic-gate * VM - address spaces. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate 457c478bd9Sstevel@tonic-gate #include <sys/types.h> 467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 477c478bd9Sstevel@tonic-gate #include <sys/param.h> 487c478bd9Sstevel@tonic-gate #include <sys/errno.h> 497c478bd9Sstevel@tonic-gate #include <sys/systm.h> 507c478bd9Sstevel@tonic-gate #include <sys/mman.h> 517c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 527c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 537c478bd9Sstevel@tonic-gate #include <sys/sysinfo.h> 547c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 557c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 567c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 577c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 587c478bd9Sstevel@tonic-gate #include <sys/debug.h> 597c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h> 607c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 617c478bd9Sstevel@tonic-gate 627c478bd9Sstevel@tonic-gate #include <vm/hat.h> 637c478bd9Sstevel@tonic-gate #include <vm/xhat.h> 647c478bd9Sstevel@tonic-gate #include <vm/as.h> 657c478bd9Sstevel@tonic-gate #include <vm/seg.h> 667c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 677c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h> 687c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 697c478bd9Sstevel@tonic-gate #include <vm/seg_map.h> 707c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 717c478bd9Sstevel@tonic-gate #include <vm/page.h> 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */ 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate static struct kmem_cache *as_cache; 767c478bd9Sstevel@tonic-gate 777c478bd9Sstevel@tonic-gate static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t); 787c478bd9Sstevel@tonic-gate static void as_clearwatchprot(struct as *, caddr_t, size_t); 79*02ff05a9Svsakar int as_map_locked(struct as *, caddr_t, size_t, int ((*)()), void *); 807c478bd9Sstevel@tonic-gate 817c478bd9Sstevel@tonic-gate 827c478bd9Sstevel@tonic-gate /* 837c478bd9Sstevel@tonic-gate * Verifying the segment lists is very time-consuming; it may not be 847c478bd9Sstevel@tonic-gate * desirable always to define VERIFY_SEGLIST when DEBUG is set. 857c478bd9Sstevel@tonic-gate */ 867c478bd9Sstevel@tonic-gate #ifdef DEBUG 877c478bd9Sstevel@tonic-gate #define VERIFY_SEGLIST 887c478bd9Sstevel@tonic-gate int do_as_verify = 0; 897c478bd9Sstevel@tonic-gate #endif 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate /* 927c478bd9Sstevel@tonic-gate * Allocate a new callback data structure entry and fill in the events of 937c478bd9Sstevel@tonic-gate * interest, the address range of interest, and the callback argument. 947c478bd9Sstevel@tonic-gate * Link the entry on the as->a_callbacks list. A callback entry for the 957c478bd9Sstevel@tonic-gate * entire address space may be specified with vaddr = 0 and size = -1. 967c478bd9Sstevel@tonic-gate * 977c478bd9Sstevel@tonic-gate * CALLERS RESPONSIBILITY: If not calling from within the process context for 987c478bd9Sstevel@tonic-gate * the specified as, the caller must guarantee persistence of the specified as 997c478bd9Sstevel@tonic-gate * for the duration of this function (eg. pages being locked within the as 1007c478bd9Sstevel@tonic-gate * will guarantee persistence). 1017c478bd9Sstevel@tonic-gate */ 1027c478bd9Sstevel@tonic-gate int 1037c478bd9Sstevel@tonic-gate as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events, 1047c478bd9Sstevel@tonic-gate caddr_t vaddr, size_t size, int sleepflag) 1057c478bd9Sstevel@tonic-gate { 1067c478bd9Sstevel@tonic-gate struct as_callback *current_head, *cb; 1077c478bd9Sstevel@tonic-gate caddr_t saddr; 1087c478bd9Sstevel@tonic-gate size_t rsize; 1097c478bd9Sstevel@tonic-gate 1107c478bd9Sstevel@tonic-gate /* callback function and an event are mandatory */ 1117c478bd9Sstevel@tonic-gate if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0)) 1127c478bd9Sstevel@tonic-gate return (EINVAL); 1137c478bd9Sstevel@tonic-gate 1147c478bd9Sstevel@tonic-gate /* Adding a callback after as_free has been called is not allowed */ 1157c478bd9Sstevel@tonic-gate if (as == &kas) 1167c478bd9Sstevel@tonic-gate return (ENOMEM); 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate /* 1197c478bd9Sstevel@tonic-gate * vaddr = 0 and size = -1 is used to indicate that the callback range 1207c478bd9Sstevel@tonic-gate * is the entire address space so no rounding is done in that case. 1217c478bd9Sstevel@tonic-gate */ 1227c478bd9Sstevel@tonic-gate if (size != -1) { 1237c478bd9Sstevel@tonic-gate saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 1247c478bd9Sstevel@tonic-gate rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) - 1257c478bd9Sstevel@tonic-gate (size_t)saddr; 1267c478bd9Sstevel@tonic-gate /* check for wraparound */ 1277c478bd9Sstevel@tonic-gate if (saddr + rsize < saddr) 1287c478bd9Sstevel@tonic-gate return (ENOMEM); 1297c478bd9Sstevel@tonic-gate } else { 1307c478bd9Sstevel@tonic-gate if (vaddr != 0) 1317c478bd9Sstevel@tonic-gate return (EINVAL); 1327c478bd9Sstevel@tonic-gate saddr = vaddr; 1337c478bd9Sstevel@tonic-gate rsize = size; 1347c478bd9Sstevel@tonic-gate } 1357c478bd9Sstevel@tonic-gate 1367c478bd9Sstevel@tonic-gate /* Allocate and initialize a callback entry */ 1377c478bd9Sstevel@tonic-gate cb = kmem_zalloc(sizeof (struct as_callback), sleepflag); 1387c478bd9Sstevel@tonic-gate if (cb == NULL) 1397c478bd9Sstevel@tonic-gate return (EAGAIN); 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate cb->ascb_func = cb_func; 1427c478bd9Sstevel@tonic-gate cb->ascb_arg = arg; 1437c478bd9Sstevel@tonic-gate cb->ascb_events = events; 1447c478bd9Sstevel@tonic-gate cb->ascb_saddr = saddr; 1457c478bd9Sstevel@tonic-gate cb->ascb_len = rsize; 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate /* Add the entry to the list */ 1487c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 1497c478bd9Sstevel@tonic-gate current_head = as->a_callbacks; 1507c478bd9Sstevel@tonic-gate as->a_callbacks = cb; 1517c478bd9Sstevel@tonic-gate cb->ascb_next = current_head; 1527c478bd9Sstevel@tonic-gate 1537c478bd9Sstevel@tonic-gate /* 1547c478bd9Sstevel@tonic-gate * The call to this function may lose in a race with 1557c478bd9Sstevel@tonic-gate * a pertinent event - eg. a thread does long term memory locking 1567c478bd9Sstevel@tonic-gate * but before the callback is added another thread executes as_unmap. 1577c478bd9Sstevel@tonic-gate * A broadcast here resolves that. 1587c478bd9Sstevel@tonic-gate */ 1597c478bd9Sstevel@tonic-gate if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) { 1607c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(as); 1617c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 1627c478bd9Sstevel@tonic-gate } 1637c478bd9Sstevel@tonic-gate 1647c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 1657c478bd9Sstevel@tonic-gate return (0); 1667c478bd9Sstevel@tonic-gate } 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate /* 1697c478bd9Sstevel@tonic-gate * Search the callback list for an entry which pertains to arg. 1707c478bd9Sstevel@tonic-gate * 1717c478bd9Sstevel@tonic-gate * This is called from within the client upon completion of the callback. 1727c478bd9Sstevel@tonic-gate * RETURN VALUES: 1737c478bd9Sstevel@tonic-gate * AS_CALLBACK_DELETED (callback entry found and deleted) 1747c478bd9Sstevel@tonic-gate * AS_CALLBACK_NOTFOUND (no callback entry found - this is ok) 1757c478bd9Sstevel@tonic-gate * AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this 1767c478bd9Sstevel@tonic-gate * entry will be made in as_do_callbacks) 1777c478bd9Sstevel@tonic-gate * 1787c478bd9Sstevel@tonic-gate * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED 1797c478bd9Sstevel@tonic-gate * set, it indicates that as_do_callbacks is processing this entry. The 1807c478bd9Sstevel@tonic-gate * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made 1817c478bd9Sstevel@tonic-gate * to unblock as_do_callbacks, in case it is blocked. 1827c478bd9Sstevel@tonic-gate * 1837c478bd9Sstevel@tonic-gate * CALLERS RESPONSIBILITY: If not calling from within the process context for 1847c478bd9Sstevel@tonic-gate * the specified as, the caller must guarantee persistence of the specified as 1857c478bd9Sstevel@tonic-gate * for the duration of this function (eg. pages being locked within the as 1867c478bd9Sstevel@tonic-gate * will guarantee persistence). 1877c478bd9Sstevel@tonic-gate */ 1887c478bd9Sstevel@tonic-gate uint_t 1897c478bd9Sstevel@tonic-gate as_delete_callback(struct as *as, void *arg) 1907c478bd9Sstevel@tonic-gate { 1917c478bd9Sstevel@tonic-gate struct as_callback **prevcb = &as->a_callbacks; 1927c478bd9Sstevel@tonic-gate struct as_callback *cb; 1937c478bd9Sstevel@tonic-gate uint_t rc = AS_CALLBACK_NOTFOUND; 1947c478bd9Sstevel@tonic-gate 1957c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 1967c478bd9Sstevel@tonic-gate for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) { 1977c478bd9Sstevel@tonic-gate if (cb->ascb_arg != arg) 1987c478bd9Sstevel@tonic-gate continue; 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * If the events indicate AS_CALLBACK_CALLED, just clear 2027c478bd9Sstevel@tonic-gate * AS_ALL_EVENT in the events field and wakeup the thread 2037c478bd9Sstevel@tonic-gate * that may be waiting in as_do_callbacks. as_do_callbacks 2047c478bd9Sstevel@tonic-gate * will take care of removing this entry from the list. In 2057c478bd9Sstevel@tonic-gate * that case, return AS_CALLBACK_DELETE_DEFERRED. Otherwise 2067c478bd9Sstevel@tonic-gate * (AS_CALLBACK_CALLED not set), just remove it from the 2077c478bd9Sstevel@tonic-gate * list, return the memory and return AS_CALLBACK_DELETED. 2087c478bd9Sstevel@tonic-gate */ 2097c478bd9Sstevel@tonic-gate if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) { 2107c478bd9Sstevel@tonic-gate /* leave AS_CALLBACK_CALLED */ 2117c478bd9Sstevel@tonic-gate cb->ascb_events &= ~AS_ALL_EVENT; 2127c478bd9Sstevel@tonic-gate rc = AS_CALLBACK_DELETE_DEFERRED; 2137c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 2147c478bd9Sstevel@tonic-gate } else { 2157c478bd9Sstevel@tonic-gate *prevcb = cb->ascb_next; 2167c478bd9Sstevel@tonic-gate kmem_free(cb, sizeof (struct as_callback)); 2177c478bd9Sstevel@tonic-gate rc = AS_CALLBACK_DELETED; 2187c478bd9Sstevel@tonic-gate } 2197c478bd9Sstevel@tonic-gate break; 2207c478bd9Sstevel@tonic-gate } 2217c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 2227c478bd9Sstevel@tonic-gate return (rc); 2237c478bd9Sstevel@tonic-gate } 2247c478bd9Sstevel@tonic-gate 2257c478bd9Sstevel@tonic-gate /* 2267c478bd9Sstevel@tonic-gate * Searches the as callback list for a matching entry. 2277c478bd9Sstevel@tonic-gate * Returns a pointer to the first matching callback, or NULL if 2287c478bd9Sstevel@tonic-gate * nothing is found. 2297c478bd9Sstevel@tonic-gate * This function never sleeps so it is ok to call it with more 2307c478bd9Sstevel@tonic-gate * locks held but the (required) a_contents mutex. 2317c478bd9Sstevel@tonic-gate * 2327c478bd9Sstevel@tonic-gate * See also comment on as_do_callbacks below. 2337c478bd9Sstevel@tonic-gate */ 2347c478bd9Sstevel@tonic-gate static struct as_callback * 2357c478bd9Sstevel@tonic-gate as_find_callback(struct as *as, uint_t events, caddr_t event_addr, 2367c478bd9Sstevel@tonic-gate size_t event_len) 2377c478bd9Sstevel@tonic-gate { 2387c478bd9Sstevel@tonic-gate struct as_callback *cb; 2397c478bd9Sstevel@tonic-gate 2407c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&as->a_contents)); 2417c478bd9Sstevel@tonic-gate for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) { 2427c478bd9Sstevel@tonic-gate /* 2437c478bd9Sstevel@tonic-gate * If the callback has not already been called, then 2447c478bd9Sstevel@tonic-gate * check if events or address range pertains. An event_len 2457c478bd9Sstevel@tonic-gate * of zero means do an unconditional callback. 2467c478bd9Sstevel@tonic-gate */ 2477c478bd9Sstevel@tonic-gate if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) || 2487c478bd9Sstevel@tonic-gate ((event_len != 0) && (((cb->ascb_events & events) == 0) || 2497c478bd9Sstevel@tonic-gate (event_addr + event_len < cb->ascb_saddr) || 2507c478bd9Sstevel@tonic-gate (event_addr > (cb->ascb_saddr + cb->ascb_len))))) { 2517c478bd9Sstevel@tonic-gate continue; 2527c478bd9Sstevel@tonic-gate } 2537c478bd9Sstevel@tonic-gate break; 2547c478bd9Sstevel@tonic-gate } 2557c478bd9Sstevel@tonic-gate return (cb); 2567c478bd9Sstevel@tonic-gate } 2577c478bd9Sstevel@tonic-gate 2587c478bd9Sstevel@tonic-gate /* 2597c478bd9Sstevel@tonic-gate * Executes a given callback and removes it from the callback list for 2607c478bd9Sstevel@tonic-gate * this address space. 2617c478bd9Sstevel@tonic-gate * This function may sleep so the caller must drop all locks except 2627c478bd9Sstevel@tonic-gate * a_contents before calling this func. 2637c478bd9Sstevel@tonic-gate * 2647c478bd9Sstevel@tonic-gate * See also comments on as_do_callbacks below. 2657c478bd9Sstevel@tonic-gate */ 2667c478bd9Sstevel@tonic-gate static void 2677c478bd9Sstevel@tonic-gate as_execute_callback(struct as *as, struct as_callback *cb, 2687c478bd9Sstevel@tonic-gate uint_t events) 2697c478bd9Sstevel@tonic-gate { 2707c478bd9Sstevel@tonic-gate struct as_callback **prevcb; 2717c478bd9Sstevel@tonic-gate void *cb_arg; 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events)); 2747c478bd9Sstevel@tonic-gate cb->ascb_events |= AS_CALLBACK_CALLED; 2757c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 2767c478bd9Sstevel@tonic-gate (*cb->ascb_func)(as, cb->ascb_arg, events); 2777c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 2787c478bd9Sstevel@tonic-gate /* 2797c478bd9Sstevel@tonic-gate * the callback function is required to delete the callback 2807c478bd9Sstevel@tonic-gate * when the callback function determines it is OK for 2817c478bd9Sstevel@tonic-gate * this thread to continue. as_delete_callback will clear 2827c478bd9Sstevel@tonic-gate * the AS_ALL_EVENT in the events field when it is deleted. 2837c478bd9Sstevel@tonic-gate * If the callback function called as_delete_callback, 2847c478bd9Sstevel@tonic-gate * events will already be cleared and there will be no blocking. 2857c478bd9Sstevel@tonic-gate */ 2867c478bd9Sstevel@tonic-gate while ((cb->ascb_events & events) != 0) { 2877c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 2887c478bd9Sstevel@tonic-gate } 2897c478bd9Sstevel@tonic-gate /* 2907c478bd9Sstevel@tonic-gate * This entry needs to be taken off the list. Normally, the 2917c478bd9Sstevel@tonic-gate * callback func itself does that, but unfortunately the list 2927c478bd9Sstevel@tonic-gate * may have changed while the callback was running because the 2937c478bd9Sstevel@tonic-gate * a_contents mutex was dropped and someone else other than the 2947c478bd9Sstevel@tonic-gate * callback func itself could have called as_delete_callback, 2957c478bd9Sstevel@tonic-gate * so we have to search to find this entry again. The entry 2967c478bd9Sstevel@tonic-gate * must have AS_CALLBACK_CALLED, and have the same 'arg'. 2977c478bd9Sstevel@tonic-gate */ 2987c478bd9Sstevel@tonic-gate cb_arg = cb->ascb_arg; 2997c478bd9Sstevel@tonic-gate prevcb = &as->a_callbacks; 3007c478bd9Sstevel@tonic-gate for (cb = as->a_callbacks; cb != NULL; 3017c478bd9Sstevel@tonic-gate prevcb = &cb->ascb_next, cb = *prevcb) { 3027c478bd9Sstevel@tonic-gate if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) || 3037c478bd9Sstevel@tonic-gate (cb_arg != cb->ascb_arg)) { 3047c478bd9Sstevel@tonic-gate continue; 3057c478bd9Sstevel@tonic-gate } 3067c478bd9Sstevel@tonic-gate *prevcb = cb->ascb_next; 3077c478bd9Sstevel@tonic-gate kmem_free(cb, sizeof (struct as_callback)); 3087c478bd9Sstevel@tonic-gate break; 3097c478bd9Sstevel@tonic-gate } 3107c478bd9Sstevel@tonic-gate } 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate /* 3137c478bd9Sstevel@tonic-gate * Check the callback list for a matching event and intersection of 3147c478bd9Sstevel@tonic-gate * address range. If there is a match invoke the callback. Skip an entry if: 3157c478bd9Sstevel@tonic-gate * - a callback is already in progress for this entry (AS_CALLBACK_CALLED) 3167c478bd9Sstevel@tonic-gate * - not event of interest 3177c478bd9Sstevel@tonic-gate * - not address range of interest 3187c478bd9Sstevel@tonic-gate * 3197c478bd9Sstevel@tonic-gate * An event_len of zero indicates a request for an unconditional callback 3207c478bd9Sstevel@tonic-gate * (regardless of event), only the AS_CALLBACK_CALLED is checked. The 3217c478bd9Sstevel@tonic-gate * a_contents lock must be dropped before a callback, so only one callback 3227c478bd9Sstevel@tonic-gate * can be done before returning. Return -1 (true) if a callback was 3237c478bd9Sstevel@tonic-gate * executed and removed from the list, else return 0 (false). 3247c478bd9Sstevel@tonic-gate * 3257c478bd9Sstevel@tonic-gate * The logically separate parts, i.e. finding a matching callback and 3267c478bd9Sstevel@tonic-gate * executing a given callback have been separated into two functions 3277c478bd9Sstevel@tonic-gate * so that they can be called with different sets of locks held beyond 3287c478bd9Sstevel@tonic-gate * the always-required a_contents. as_find_callback does not sleep so 3297c478bd9Sstevel@tonic-gate * it is ok to call it if more locks than a_contents (i.e. the a_lock 3307c478bd9Sstevel@tonic-gate * rwlock) are held. as_execute_callback on the other hand may sleep 3317c478bd9Sstevel@tonic-gate * so all locks beyond a_contents must be dropped by the caller if one 3327c478bd9Sstevel@tonic-gate * does not want to end comatose. 3337c478bd9Sstevel@tonic-gate */ 3347c478bd9Sstevel@tonic-gate static int 3357c478bd9Sstevel@tonic-gate as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr, 3367c478bd9Sstevel@tonic-gate size_t event_len) 3377c478bd9Sstevel@tonic-gate { 3387c478bd9Sstevel@tonic-gate struct as_callback *cb; 3397c478bd9Sstevel@tonic-gate 3407c478bd9Sstevel@tonic-gate if ((cb = as_find_callback(as, events, event_addr, event_len))) { 3417c478bd9Sstevel@tonic-gate as_execute_callback(as, cb, events); 3427c478bd9Sstevel@tonic-gate return (-1); 3437c478bd9Sstevel@tonic-gate } 3447c478bd9Sstevel@tonic-gate return (0); 3457c478bd9Sstevel@tonic-gate } 3467c478bd9Sstevel@tonic-gate 3477c478bd9Sstevel@tonic-gate /* 3487c478bd9Sstevel@tonic-gate * Search for the segment containing addr. If a segment containing addr 3497c478bd9Sstevel@tonic-gate * exists, that segment is returned. If no such segment exists, and 3507c478bd9Sstevel@tonic-gate * the list spans addresses greater than addr, then the first segment 3517c478bd9Sstevel@tonic-gate * whose base is greater than addr is returned; otherwise, NULL is 3527c478bd9Sstevel@tonic-gate * returned unless tail is true, in which case the last element of the 3537c478bd9Sstevel@tonic-gate * list is returned. 3547c478bd9Sstevel@tonic-gate * 3557c478bd9Sstevel@tonic-gate * a_seglast is used to cache the last found segment for repeated 3567c478bd9Sstevel@tonic-gate * searches to the same addr (which happens frequently). 3577c478bd9Sstevel@tonic-gate */ 3587c478bd9Sstevel@tonic-gate struct seg * 3597c478bd9Sstevel@tonic-gate as_findseg(struct as *as, caddr_t addr, int tail) 3607c478bd9Sstevel@tonic-gate { 3617c478bd9Sstevel@tonic-gate struct seg *seg = as->a_seglast; 3627c478bd9Sstevel@tonic-gate avl_index_t where; 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate if (seg != NULL && 3677c478bd9Sstevel@tonic-gate seg->s_base <= addr && 3687c478bd9Sstevel@tonic-gate addr < seg->s_base + seg->s_size) 3697c478bd9Sstevel@tonic-gate return (seg); 3707c478bd9Sstevel@tonic-gate 3717c478bd9Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, &where); 3727c478bd9Sstevel@tonic-gate if (seg != NULL) 3737c478bd9Sstevel@tonic-gate return (as->a_seglast = seg); 3747c478bd9Sstevel@tonic-gate 3757c478bd9Sstevel@tonic-gate seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 3767c478bd9Sstevel@tonic-gate if (seg == NULL && tail) 3777c478bd9Sstevel@tonic-gate seg = avl_last(&as->a_segtree); 3787c478bd9Sstevel@tonic-gate return (as->a_seglast = seg); 3797c478bd9Sstevel@tonic-gate } 3807c478bd9Sstevel@tonic-gate 3817c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 3827c478bd9Sstevel@tonic-gate /* 3837c478bd9Sstevel@tonic-gate * verify that the linked list is coherent 3847c478bd9Sstevel@tonic-gate */ 3857c478bd9Sstevel@tonic-gate static void 3867c478bd9Sstevel@tonic-gate as_verify(struct as *as) 3877c478bd9Sstevel@tonic-gate { 3887c478bd9Sstevel@tonic-gate struct seg *seg, *seglast, *p, *n; 3897c478bd9Sstevel@tonic-gate uint_t nsegs = 0; 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate if (do_as_verify == 0) 3927c478bd9Sstevel@tonic-gate return; 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate seglast = as->a_seglast; 3957c478bd9Sstevel@tonic-gate 3967c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 3977c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == as); 3987c478bd9Sstevel@tonic-gate p = AS_SEGPREV(as, seg); 3997c478bd9Sstevel@tonic-gate n = AS_SEGNEXT(as, seg); 4007c478bd9Sstevel@tonic-gate ASSERT(p == NULL || p->s_as == as); 4017c478bd9Sstevel@tonic-gate ASSERT(p == NULL || p->s_base < seg->s_base); 4027c478bd9Sstevel@tonic-gate ASSERT(n == NULL || n->s_base > seg->s_base); 4037c478bd9Sstevel@tonic-gate ASSERT(n != NULL || seg == avl_last(&as->a_segtree)); 4047c478bd9Sstevel@tonic-gate if (seg == seglast) 4057c478bd9Sstevel@tonic-gate seglast = NULL; 4067c478bd9Sstevel@tonic-gate nsegs++; 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate ASSERT(seglast == NULL); 4097c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_segtree) == nsegs); 4107c478bd9Sstevel@tonic-gate } 4117c478bd9Sstevel@tonic-gate #endif /* VERIFY_SEGLIST */ 4127c478bd9Sstevel@tonic-gate 4137c478bd9Sstevel@tonic-gate /* 4147c478bd9Sstevel@tonic-gate * Add a new segment to the address space. The avl_find() 4157c478bd9Sstevel@tonic-gate * may be expensive so we attempt to use last segment accessed 4167c478bd9Sstevel@tonic-gate * in as_gap() as an insertion point. 4177c478bd9Sstevel@tonic-gate */ 4187c478bd9Sstevel@tonic-gate int 4197c478bd9Sstevel@tonic-gate as_addseg(struct as *as, struct seg *newseg) 4207c478bd9Sstevel@tonic-gate { 4217c478bd9Sstevel@tonic-gate struct seg *seg; 4227c478bd9Sstevel@tonic-gate caddr_t addr; 4237c478bd9Sstevel@tonic-gate caddr_t eaddr; 4247c478bd9Sstevel@tonic-gate avl_index_t where; 4257c478bd9Sstevel@tonic-gate 4267c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 4277c478bd9Sstevel@tonic-gate 4287c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 4297c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 4307c478bd9Sstevel@tonic-gate 4317c478bd9Sstevel@tonic-gate if (as->a_lastgaphl != NULL) { 4327c478bd9Sstevel@tonic-gate struct seg *hseg = NULL; 4337c478bd9Sstevel@tonic-gate struct seg *lseg = NULL; 4347c478bd9Sstevel@tonic-gate 4357c478bd9Sstevel@tonic-gate if (as->a_lastgaphl->s_base > newseg->s_base) { 4367c478bd9Sstevel@tonic-gate hseg = as->a_lastgaphl; 4377c478bd9Sstevel@tonic-gate lseg = AVL_PREV(&as->a_segtree, hseg); 4387c478bd9Sstevel@tonic-gate } else { 4397c478bd9Sstevel@tonic-gate lseg = as->a_lastgaphl; 4407c478bd9Sstevel@tonic-gate hseg = AVL_NEXT(&as->a_segtree, lseg); 4417c478bd9Sstevel@tonic-gate } 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate if (hseg && lseg && lseg->s_base < newseg->s_base && 4447c478bd9Sstevel@tonic-gate hseg->s_base > newseg->s_base) { 4457c478bd9Sstevel@tonic-gate avl_insert_here(&as->a_segtree, newseg, lseg, 4467c478bd9Sstevel@tonic-gate AVL_AFTER); 4477c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 4487c478bd9Sstevel@tonic-gate as->a_seglast = newseg; 4497c478bd9Sstevel@tonic-gate return (0); 4507c478bd9Sstevel@tonic-gate } 4517c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 4527c478bd9Sstevel@tonic-gate } 4537c478bd9Sstevel@tonic-gate 4547c478bd9Sstevel@tonic-gate addr = newseg->s_base; 4557c478bd9Sstevel@tonic-gate eaddr = addr + newseg->s_size; 4567c478bd9Sstevel@tonic-gate again: 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, &where); 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate if (seg == NULL) 4617c478bd9Sstevel@tonic-gate seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate if (seg == NULL) 4647c478bd9Sstevel@tonic-gate seg = avl_last(&as->a_segtree); 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate if (seg != NULL) { 4677c478bd9Sstevel@tonic-gate caddr_t base = seg->s_base; 4687c478bd9Sstevel@tonic-gate 4697c478bd9Sstevel@tonic-gate /* 4707c478bd9Sstevel@tonic-gate * If top of seg is below the requested address, then 4717c478bd9Sstevel@tonic-gate * the insertion point is at the end of the linked list, 4727c478bd9Sstevel@tonic-gate * and seg points to the tail of the list. Otherwise, 4737c478bd9Sstevel@tonic-gate * the insertion point is immediately before seg. 4747c478bd9Sstevel@tonic-gate */ 4757c478bd9Sstevel@tonic-gate if (base + seg->s_size > addr) { 4767c478bd9Sstevel@tonic-gate if (addr >= base || eaddr > base) { 4777c478bd9Sstevel@tonic-gate #ifdef __sparc 4787c478bd9Sstevel@tonic-gate extern struct seg_ops segnf_ops; 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate /* 4817c478bd9Sstevel@tonic-gate * no-fault segs must disappear if overlaid. 4827c478bd9Sstevel@tonic-gate * XXX need new segment type so 4837c478bd9Sstevel@tonic-gate * we don't have to check s_ops 4847c478bd9Sstevel@tonic-gate */ 4857c478bd9Sstevel@tonic-gate if (seg->s_ops == &segnf_ops) { 4867c478bd9Sstevel@tonic-gate seg_unmap(seg); 4877c478bd9Sstevel@tonic-gate goto again; 4887c478bd9Sstevel@tonic-gate } 4897c478bd9Sstevel@tonic-gate #endif 4907c478bd9Sstevel@tonic-gate return (-1); /* overlapping segment */ 4917c478bd9Sstevel@tonic-gate } 4927c478bd9Sstevel@tonic-gate } 4937c478bd9Sstevel@tonic-gate } 4947c478bd9Sstevel@tonic-gate as->a_seglast = newseg; 4957c478bd9Sstevel@tonic-gate avl_insert(&as->a_segtree, newseg, where); 4967c478bd9Sstevel@tonic-gate 4977c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 4987c478bd9Sstevel@tonic-gate as_verify(as); 4997c478bd9Sstevel@tonic-gate #endif 5007c478bd9Sstevel@tonic-gate return (0); 5017c478bd9Sstevel@tonic-gate } 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate struct seg * 5047c478bd9Sstevel@tonic-gate as_removeseg(struct as *as, struct seg *seg) 5057c478bd9Sstevel@tonic-gate { 5067c478bd9Sstevel@tonic-gate avl_tree_t *t; 5077c478bd9Sstevel@tonic-gate 5087c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 5097c478bd9Sstevel@tonic-gate 5107c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 5117c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 5127c478bd9Sstevel@tonic-gate 5137c478bd9Sstevel@tonic-gate if (seg == NULL) 5147c478bd9Sstevel@tonic-gate return (NULL); 5157c478bd9Sstevel@tonic-gate 5167c478bd9Sstevel@tonic-gate t = &as->a_segtree; 5177c478bd9Sstevel@tonic-gate if (as->a_seglast == seg) 5187c478bd9Sstevel@tonic-gate as->a_seglast = NULL; 5197c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 5207c478bd9Sstevel@tonic-gate 5217c478bd9Sstevel@tonic-gate /* 5227c478bd9Sstevel@tonic-gate * if this segment is at an address higher than 5237c478bd9Sstevel@tonic-gate * a_lastgap, set a_lastgap to the next segment (NULL if last segment) 5247c478bd9Sstevel@tonic-gate */ 5257c478bd9Sstevel@tonic-gate if (as->a_lastgap && 5267c478bd9Sstevel@tonic-gate (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base)) 5277c478bd9Sstevel@tonic-gate as->a_lastgap = AVL_NEXT(t, seg); 5287c478bd9Sstevel@tonic-gate 5297c478bd9Sstevel@tonic-gate /* 5307c478bd9Sstevel@tonic-gate * remove the segment from the seg tree 5317c478bd9Sstevel@tonic-gate */ 5327c478bd9Sstevel@tonic-gate avl_remove(t, seg); 5337c478bd9Sstevel@tonic-gate 5347c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 5357c478bd9Sstevel@tonic-gate as_verify(as); 5367c478bd9Sstevel@tonic-gate #endif 5377c478bd9Sstevel@tonic-gate return (seg); 5387c478bd9Sstevel@tonic-gate } 5397c478bd9Sstevel@tonic-gate 5407c478bd9Sstevel@tonic-gate /* 5417c478bd9Sstevel@tonic-gate * Find a segment containing addr. 5427c478bd9Sstevel@tonic-gate */ 5437c478bd9Sstevel@tonic-gate struct seg * 5447c478bd9Sstevel@tonic-gate as_segat(struct as *as, caddr_t addr) 5457c478bd9Sstevel@tonic-gate { 5467c478bd9Sstevel@tonic-gate struct seg *seg = as->a_seglast; 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate ASSERT(AS_LOCK_HELD(as, &as->a_lock)); 5497c478bd9Sstevel@tonic-gate 5507c478bd9Sstevel@tonic-gate if (seg != NULL && seg->s_base <= addr && 5517c478bd9Sstevel@tonic-gate addr < seg->s_base + seg->s_size) 5527c478bd9Sstevel@tonic-gate return (seg); 5537c478bd9Sstevel@tonic-gate 5547c478bd9Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, NULL); 5557c478bd9Sstevel@tonic-gate return (seg); 5567c478bd9Sstevel@tonic-gate } 5577c478bd9Sstevel@tonic-gate 5587c478bd9Sstevel@tonic-gate /* 5597c478bd9Sstevel@tonic-gate * Serialize all searches for holes in an address space to 5607c478bd9Sstevel@tonic-gate * prevent two or more threads from allocating the same virtual 5617c478bd9Sstevel@tonic-gate * address range. The address space must not be "read/write" 5627c478bd9Sstevel@tonic-gate * locked by the caller since we may block. 5637c478bd9Sstevel@tonic-gate */ 5647c478bd9Sstevel@tonic-gate void 5657c478bd9Sstevel@tonic-gate as_rangelock(struct as *as) 5667c478bd9Sstevel@tonic-gate { 5677c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 5687c478bd9Sstevel@tonic-gate while (AS_ISCLAIMGAP(as)) 5697c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 5707c478bd9Sstevel@tonic-gate AS_SETCLAIMGAP(as); 5717c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 5727c478bd9Sstevel@tonic-gate } 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate /* 5757c478bd9Sstevel@tonic-gate * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads. 5767c478bd9Sstevel@tonic-gate */ 5777c478bd9Sstevel@tonic-gate void 5787c478bd9Sstevel@tonic-gate as_rangeunlock(struct as *as) 5797c478bd9Sstevel@tonic-gate { 5807c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 5817c478bd9Sstevel@tonic-gate AS_CLRCLAIMGAP(as); 5827c478bd9Sstevel@tonic-gate cv_signal(&as->a_cv); 5837c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 5847c478bd9Sstevel@tonic-gate } 5857c478bd9Sstevel@tonic-gate 5867c478bd9Sstevel@tonic-gate /* 5877c478bd9Sstevel@tonic-gate * compar segments (or just an address) by segment address range 5887c478bd9Sstevel@tonic-gate */ 5897c478bd9Sstevel@tonic-gate static int 5907c478bd9Sstevel@tonic-gate as_segcompar(const void *x, const void *y) 5917c478bd9Sstevel@tonic-gate { 5927c478bd9Sstevel@tonic-gate struct seg *a = (struct seg *)x; 5937c478bd9Sstevel@tonic-gate struct seg *b = (struct seg *)y; 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate if (a->s_base < b->s_base) 5967c478bd9Sstevel@tonic-gate return (-1); 5977c478bd9Sstevel@tonic-gate if (a->s_base >= b->s_base + b->s_size) 5987c478bd9Sstevel@tonic-gate return (1); 5997c478bd9Sstevel@tonic-gate return (0); 6007c478bd9Sstevel@tonic-gate } 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate 6037c478bd9Sstevel@tonic-gate void 6047c478bd9Sstevel@tonic-gate as_avlinit(struct as *as) 6057c478bd9Sstevel@tonic-gate { 6067c478bd9Sstevel@tonic-gate avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg), 6077c478bd9Sstevel@tonic-gate offsetof(struct seg, s_tree)); 6087c478bd9Sstevel@tonic-gate avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page), 6097c478bd9Sstevel@tonic-gate offsetof(struct watched_page, wp_link)); 6107c478bd9Sstevel@tonic-gate } 6117c478bd9Sstevel@tonic-gate 6127c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6137c478bd9Sstevel@tonic-gate static int 6147c478bd9Sstevel@tonic-gate as_constructor(void *buf, void *cdrarg, int kmflags) 6157c478bd9Sstevel@tonic-gate { 6167c478bd9Sstevel@tonic-gate struct as *as = buf; 6177c478bd9Sstevel@tonic-gate 6187c478bd9Sstevel@tonic-gate mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL); 6197c478bd9Sstevel@tonic-gate cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL); 6207c478bd9Sstevel@tonic-gate rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL); 6217c478bd9Sstevel@tonic-gate as_avlinit(as); 6227c478bd9Sstevel@tonic-gate return (0); 6237c478bd9Sstevel@tonic-gate } 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate /*ARGSUSED1*/ 6267c478bd9Sstevel@tonic-gate static void 6277c478bd9Sstevel@tonic-gate as_destructor(void *buf, void *cdrarg) 6287c478bd9Sstevel@tonic-gate { 6297c478bd9Sstevel@tonic-gate struct as *as = buf; 6307c478bd9Sstevel@tonic-gate 6317c478bd9Sstevel@tonic-gate avl_destroy(&as->a_segtree); 6327c478bd9Sstevel@tonic-gate mutex_destroy(&as->a_contents); 6337c478bd9Sstevel@tonic-gate cv_destroy(&as->a_cv); 6347c478bd9Sstevel@tonic-gate rw_destroy(&as->a_lock); 6357c478bd9Sstevel@tonic-gate } 6367c478bd9Sstevel@tonic-gate 6377c478bd9Sstevel@tonic-gate void 6387c478bd9Sstevel@tonic-gate as_init(void) 6397c478bd9Sstevel@tonic-gate { 6407c478bd9Sstevel@tonic-gate as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0, 6417c478bd9Sstevel@tonic-gate as_constructor, as_destructor, NULL, NULL, NULL, 0); 6427c478bd9Sstevel@tonic-gate } 6437c478bd9Sstevel@tonic-gate 6447c478bd9Sstevel@tonic-gate /* 6457c478bd9Sstevel@tonic-gate * Allocate and initialize an address space data structure. 6467c478bd9Sstevel@tonic-gate * We call hat_alloc to allow any machine dependent 6477c478bd9Sstevel@tonic-gate * information in the hat structure to be initialized. 6487c478bd9Sstevel@tonic-gate */ 6497c478bd9Sstevel@tonic-gate struct as * 6507c478bd9Sstevel@tonic-gate as_alloc(void) 6517c478bd9Sstevel@tonic-gate { 6527c478bd9Sstevel@tonic-gate struct as *as; 6537c478bd9Sstevel@tonic-gate 6547c478bd9Sstevel@tonic-gate as = kmem_cache_alloc(as_cache, KM_SLEEP); 6557c478bd9Sstevel@tonic-gate 6567c478bd9Sstevel@tonic-gate as->a_flags = 0; 6577c478bd9Sstevel@tonic-gate as->a_vbits = 0; 6587c478bd9Sstevel@tonic-gate as->a_hrm = NULL; 6597c478bd9Sstevel@tonic-gate as->a_seglast = NULL; 6607c478bd9Sstevel@tonic-gate as->a_size = 0; 6617c478bd9Sstevel@tonic-gate as->a_updatedir = 0; 6627c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 6637c478bd9Sstevel@tonic-gate as->a_objectdir = NULL; 6647c478bd9Sstevel@tonic-gate as->a_sizedir = 0; 6657c478bd9Sstevel@tonic-gate as->a_userlimit = (caddr_t)USERLIMIT; 6667c478bd9Sstevel@tonic-gate as->a_lastgap = NULL; 6677c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 6687c478bd9Sstevel@tonic-gate as->a_callbacks = NULL; 6697c478bd9Sstevel@tonic-gate 6707c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 6717c478bd9Sstevel@tonic-gate as->a_hat = hat_alloc(as); /* create hat for default system mmu */ 6727c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 6737c478bd9Sstevel@tonic-gate 6747c478bd9Sstevel@tonic-gate as->a_xhat = NULL; 6757c478bd9Sstevel@tonic-gate 6767c478bd9Sstevel@tonic-gate return (as); 6777c478bd9Sstevel@tonic-gate } 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate /* 6807c478bd9Sstevel@tonic-gate * Free an address space data structure. 6817c478bd9Sstevel@tonic-gate * Need to free the hat first and then 6827c478bd9Sstevel@tonic-gate * all the segments on this as and finally 6837c478bd9Sstevel@tonic-gate * the space for the as struct itself. 6847c478bd9Sstevel@tonic-gate */ 6857c478bd9Sstevel@tonic-gate void 6867c478bd9Sstevel@tonic-gate as_free(struct as *as) 6877c478bd9Sstevel@tonic-gate { 6887c478bd9Sstevel@tonic-gate struct hat *hat = as->a_hat; 6897c478bd9Sstevel@tonic-gate struct seg *seg, *next; 6907c478bd9Sstevel@tonic-gate int called = 0; 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate top: 6937c478bd9Sstevel@tonic-gate /* 6947c478bd9Sstevel@tonic-gate * Invoke ALL callbacks. as_do_callbacks will do one callback 6957c478bd9Sstevel@tonic-gate * per call, and not return (-1) until the callback has completed. 6967c478bd9Sstevel@tonic-gate * When as_do_callbacks returns zero, all callbacks have completed. 6977c478bd9Sstevel@tonic-gate */ 6987c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 6997c478bd9Sstevel@tonic-gate while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0)); 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate /* This will prevent new XHATs from attaching to as */ 7027c478bd9Sstevel@tonic-gate if (!called) 7037c478bd9Sstevel@tonic-gate AS_SETBUSY(as); 7047c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 7057c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 7067c478bd9Sstevel@tonic-gate 7077c478bd9Sstevel@tonic-gate if (!called) { 7087c478bd9Sstevel@tonic-gate called = 1; 7097c478bd9Sstevel@tonic-gate hat_free_start(hat); 7107c478bd9Sstevel@tonic-gate if (as->a_xhat != NULL) 7117c478bd9Sstevel@tonic-gate xhat_free_start_all(as); 7127c478bd9Sstevel@tonic-gate } 7137c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) { 7147c478bd9Sstevel@tonic-gate int err; 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate next = AS_SEGNEXT(as, seg); 7177c478bd9Sstevel@tonic-gate err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 7187c478bd9Sstevel@tonic-gate if (err == EAGAIN) { 7197c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 7207c478bd9Sstevel@tonic-gate if (as->a_callbacks) { 7217c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 7227c478bd9Sstevel@tonic-gate } else { 7237c478bd9Sstevel@tonic-gate /* 7247c478bd9Sstevel@tonic-gate * Memory is currently locked. Wait for a 7257c478bd9Sstevel@tonic-gate * cv_signal that it has been unlocked, then 7267c478bd9Sstevel@tonic-gate * try the operation again. 7277c478bd9Sstevel@tonic-gate */ 7287c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 7297c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 7307c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 7317c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 7327c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 7337c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 7347c478bd9Sstevel@tonic-gate } 7357c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 7367c478bd9Sstevel@tonic-gate goto top; 7377c478bd9Sstevel@tonic-gate } else { 7387c478bd9Sstevel@tonic-gate /* 7397c478bd9Sstevel@tonic-gate * We do not expect any other error return at this 7407c478bd9Sstevel@tonic-gate * time. This is similar to an ASSERT in seg_unmap() 7417c478bd9Sstevel@tonic-gate */ 7427c478bd9Sstevel@tonic-gate ASSERT(err == 0); 7437c478bd9Sstevel@tonic-gate } 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate hat_free_end(hat); 7467c478bd9Sstevel@tonic-gate if (as->a_xhat != NULL) 7477c478bd9Sstevel@tonic-gate xhat_free_end_all(as); 7487c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 7497c478bd9Sstevel@tonic-gate 7507c478bd9Sstevel@tonic-gate /* /proc stuff */ 7517c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_wpage) == 0); 7527c478bd9Sstevel@tonic-gate if (as->a_objectdir) { 7537c478bd9Sstevel@tonic-gate kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *)); 7547c478bd9Sstevel@tonic-gate as->a_objectdir = NULL; 7557c478bd9Sstevel@tonic-gate as->a_sizedir = 0; 7567c478bd9Sstevel@tonic-gate } 7577c478bd9Sstevel@tonic-gate 7587c478bd9Sstevel@tonic-gate /* 7597c478bd9Sstevel@tonic-gate * Free the struct as back to kmem. Assert it has no segments. 7607c478bd9Sstevel@tonic-gate */ 7617c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_segtree) == 0); 7627c478bd9Sstevel@tonic-gate kmem_cache_free(as_cache, as); 7637c478bd9Sstevel@tonic-gate } 7647c478bd9Sstevel@tonic-gate 7657c478bd9Sstevel@tonic-gate int 7667c478bd9Sstevel@tonic-gate as_dup(struct as *as, struct as **outas) 7677c478bd9Sstevel@tonic-gate { 7687c478bd9Sstevel@tonic-gate struct as *newas; 7697c478bd9Sstevel@tonic-gate struct seg *seg, *newseg; 7707c478bd9Sstevel@tonic-gate int error; 7717c478bd9Sstevel@tonic-gate 7727c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 7737c478bd9Sstevel@tonic-gate as_clearwatch(as); 7747c478bd9Sstevel@tonic-gate newas = as_alloc(); 7757c478bd9Sstevel@tonic-gate newas->a_userlimit = as->a_userlimit; 7767c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER); 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate /* This will prevent new XHATs from attaching */ 7797c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 7807c478bd9Sstevel@tonic-gate AS_SETBUSY(as); 7817c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 7827c478bd9Sstevel@tonic-gate mutex_enter(&newas->a_contents); 7837c478bd9Sstevel@tonic-gate AS_SETBUSY(newas); 7847c478bd9Sstevel@tonic-gate mutex_exit(&newas->a_contents); 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate 7877c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 7887c478bd9Sstevel@tonic-gate 7897c478bd9Sstevel@tonic-gate if (seg->s_flags & S_PURGE) 7907c478bd9Sstevel@tonic-gate continue; 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate newseg = seg_alloc(newas, seg->s_base, seg->s_size); 7937c478bd9Sstevel@tonic-gate if (newseg == NULL) { 7947c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(newas, &newas->a_lock); 7957c478bd9Sstevel@tonic-gate as_setwatch(as); 7967c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 7977c478bd9Sstevel@tonic-gate AS_CLRBUSY(as); 7987c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 7997c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 8007c478bd9Sstevel@tonic-gate as_free(newas); 8017c478bd9Sstevel@tonic-gate return (-1); 8027c478bd9Sstevel@tonic-gate } 8037c478bd9Sstevel@tonic-gate if ((error = SEGOP_DUP(seg, newseg)) != 0) { 8047c478bd9Sstevel@tonic-gate /* 8057c478bd9Sstevel@tonic-gate * We call seg_free() on the new seg 8067c478bd9Sstevel@tonic-gate * because the segment is not set up 8077c478bd9Sstevel@tonic-gate * completely; i.e. it has no ops. 8087c478bd9Sstevel@tonic-gate */ 8097c478bd9Sstevel@tonic-gate as_setwatch(as); 8107c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 8117c478bd9Sstevel@tonic-gate AS_CLRBUSY(as); 8127c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 8137c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 8147c478bd9Sstevel@tonic-gate seg_free(newseg); 8157c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(newas, &newas->a_lock); 8167c478bd9Sstevel@tonic-gate as_free(newas); 8177c478bd9Sstevel@tonic-gate return (error); 8187c478bd9Sstevel@tonic-gate } 8197c478bd9Sstevel@tonic-gate newas->a_size += seg->s_size; 8207c478bd9Sstevel@tonic-gate } 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL); 8237c478bd9Sstevel@tonic-gate if (as->a_xhat != NULL) 8247c478bd9Sstevel@tonic-gate error |= xhat_dup_all(as, newas, NULL, 0, HAT_DUP_ALL); 8257c478bd9Sstevel@tonic-gate 8267c478bd9Sstevel@tonic-gate mutex_enter(&newas->a_contents); 8277c478bd9Sstevel@tonic-gate AS_CLRBUSY(newas); 8287c478bd9Sstevel@tonic-gate mutex_exit(&newas->a_contents); 8297c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(newas, &newas->a_lock); 8307c478bd9Sstevel@tonic-gate 8317c478bd9Sstevel@tonic-gate as_setwatch(as); 8327c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 8337c478bd9Sstevel@tonic-gate AS_CLRBUSY(as); 8347c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 8357c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 8367c478bd9Sstevel@tonic-gate if (error != 0) { 8377c478bd9Sstevel@tonic-gate as_free(newas); 8387c478bd9Sstevel@tonic-gate return (error); 8397c478bd9Sstevel@tonic-gate } 8407c478bd9Sstevel@tonic-gate *outas = newas; 8417c478bd9Sstevel@tonic-gate return (0); 8427c478bd9Sstevel@tonic-gate } 8437c478bd9Sstevel@tonic-gate 8447c478bd9Sstevel@tonic-gate /* 8457c478bd9Sstevel@tonic-gate * Handle a ``fault'' at addr for size bytes. 8467c478bd9Sstevel@tonic-gate */ 8477c478bd9Sstevel@tonic-gate faultcode_t 8487c478bd9Sstevel@tonic-gate as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size, 8497c478bd9Sstevel@tonic-gate enum fault_type type, enum seg_rw rw) 8507c478bd9Sstevel@tonic-gate { 8517c478bd9Sstevel@tonic-gate struct seg *seg; 8527c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 8537c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 8547c478bd9Sstevel@tonic-gate size_t ssize; 8557c478bd9Sstevel@tonic-gate faultcode_t res = 0; 8567c478bd9Sstevel@tonic-gate caddr_t addrsav; 8577c478bd9Sstevel@tonic-gate struct seg *segsav; 8587c478bd9Sstevel@tonic-gate int as_lock_held; 8597c478bd9Sstevel@tonic-gate klwp_t *lwp = ttolwp(curthread); 8607c478bd9Sstevel@tonic-gate int is_xhat = 0; 8617c478bd9Sstevel@tonic-gate int holding_wpage = 0; 8627c478bd9Sstevel@tonic-gate extern struct seg_ops segdev_ops; 8637c478bd9Sstevel@tonic-gate 8647c478bd9Sstevel@tonic-gate 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate if (as->a_hat != hat) { 8677c478bd9Sstevel@tonic-gate /* This must be an XHAT then */ 8687c478bd9Sstevel@tonic-gate is_xhat = 1; 8697c478bd9Sstevel@tonic-gate 8707c478bd9Sstevel@tonic-gate if ((type != F_INVAL) || (as == &kas)) 8717c478bd9Sstevel@tonic-gate return (FC_NOSUPPORT); 8727c478bd9Sstevel@tonic-gate } 8737c478bd9Sstevel@tonic-gate 8747c478bd9Sstevel@tonic-gate retry: 8757c478bd9Sstevel@tonic-gate if (!is_xhat) { 8767c478bd9Sstevel@tonic-gate /* 8777c478bd9Sstevel@tonic-gate * Indicate that the lwp is not to be stopped while waiting 8787c478bd9Sstevel@tonic-gate * for a pagefault. This is to avoid deadlock while debugging 8797c478bd9Sstevel@tonic-gate * a process via /proc over NFS (in particular). 8807c478bd9Sstevel@tonic-gate */ 88116ade92dScwb if (lwp != NULL) { 8827c478bd9Sstevel@tonic-gate lwp->lwp_nostop++; 88316ade92dScwb lwp->lwp_nostop_r++; 88416ade92dScwb } 8857c478bd9Sstevel@tonic-gate 8867c478bd9Sstevel@tonic-gate /* 8877c478bd9Sstevel@tonic-gate * same length must be used when we softlock and softunlock. 8887c478bd9Sstevel@tonic-gate * We don't support softunlocking lengths less than 8897c478bd9Sstevel@tonic-gate * the original length when there is largepage support. 8907c478bd9Sstevel@tonic-gate * See seg_dev.c for more comments. 8917c478bd9Sstevel@tonic-gate */ 8927c478bd9Sstevel@tonic-gate switch (type) { 8937c478bd9Sstevel@tonic-gate 8947c478bd9Sstevel@tonic-gate case F_SOFTLOCK: 8957c478bd9Sstevel@tonic-gate CPU_STATS_ADD_K(vm, softlock, 1); 8967c478bd9Sstevel@tonic-gate break; 8977c478bd9Sstevel@tonic-gate 8987c478bd9Sstevel@tonic-gate case F_SOFTUNLOCK: 8997c478bd9Sstevel@tonic-gate break; 9007c478bd9Sstevel@tonic-gate 9017c478bd9Sstevel@tonic-gate case F_PROT: 9027c478bd9Sstevel@tonic-gate CPU_STATS_ADD_K(vm, prot_fault, 1); 9037c478bd9Sstevel@tonic-gate break; 9047c478bd9Sstevel@tonic-gate 9057c478bd9Sstevel@tonic-gate case F_INVAL: 9067c478bd9Sstevel@tonic-gate CPU_STATS_ENTER_K(); 9077c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, vm, as_fault, 1); 9087c478bd9Sstevel@tonic-gate if (as == &kas) 9097c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1); 9107c478bd9Sstevel@tonic-gate CPU_STATS_EXIT_K(); 9117c478bd9Sstevel@tonic-gate break; 9127c478bd9Sstevel@tonic-gate } 9137c478bd9Sstevel@tonic-gate } 9147c478bd9Sstevel@tonic-gate 9157c478bd9Sstevel@tonic-gate /* Kernel probe */ 9167c478bd9Sstevel@tonic-gate TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */, 9177c478bd9Sstevel@tonic-gate tnf_opaque, address, addr, 9187c478bd9Sstevel@tonic-gate tnf_fault_type, fault_type, type, 9197c478bd9Sstevel@tonic-gate tnf_seg_access, access, rw); 9207c478bd9Sstevel@tonic-gate 9217c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 9227c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 9237c478bd9Sstevel@tonic-gate (size_t)raddr; 9247c478bd9Sstevel@tonic-gate 9257c478bd9Sstevel@tonic-gate /* 9267c478bd9Sstevel@tonic-gate * XXX -- Don't grab the as lock for segkmap. We should grab it for 9277c478bd9Sstevel@tonic-gate * correctness, but then we could be stuck holding this lock for 9287c478bd9Sstevel@tonic-gate * a LONG time if the fault needs to be resolved on a slow 9297c478bd9Sstevel@tonic-gate * filesystem, and then no-one will be able to exec new commands, 9307c478bd9Sstevel@tonic-gate * as exec'ing requires the write lock on the as. 9317c478bd9Sstevel@tonic-gate */ 9327c478bd9Sstevel@tonic-gate if (as == &kas && segkmap && segkmap->s_base <= raddr && 9337c478bd9Sstevel@tonic-gate raddr + size < segkmap->s_base + segkmap->s_size) { 9347c478bd9Sstevel@tonic-gate /* 9357c478bd9Sstevel@tonic-gate * if (as==&kas), this can't be XHAT: we've already returned 9367c478bd9Sstevel@tonic-gate * FC_NOSUPPORT. 9377c478bd9Sstevel@tonic-gate */ 9387c478bd9Sstevel@tonic-gate seg = segkmap; 9397c478bd9Sstevel@tonic-gate as_lock_held = 0; 9407c478bd9Sstevel@tonic-gate } else { 9417c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 9427c478bd9Sstevel@tonic-gate if (is_xhat && avl_numnodes(&as->a_wpage) != 0) { 9437c478bd9Sstevel@tonic-gate /* 9447c478bd9Sstevel@tonic-gate * Grab and hold the writers' lock on the as 9457c478bd9Sstevel@tonic-gate * if the fault is to a watched page. 9467c478bd9Sstevel@tonic-gate * This will keep CPUs from "peeking" at the 9477c478bd9Sstevel@tonic-gate * address range while we're temporarily boosting 9487c478bd9Sstevel@tonic-gate * the permissions for the XHAT device to 9497c478bd9Sstevel@tonic-gate * resolve the fault in the segment layer. 9507c478bd9Sstevel@tonic-gate * 9517c478bd9Sstevel@tonic-gate * We could check whether faulted address 9527c478bd9Sstevel@tonic-gate * is within a watched page and only then grab 9537c478bd9Sstevel@tonic-gate * the writer lock, but this is simpler. 9547c478bd9Sstevel@tonic-gate */ 9557c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 9567c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 9577c478bd9Sstevel@tonic-gate } 9587c478bd9Sstevel@tonic-gate 9597c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 9607c478bd9Sstevel@tonic-gate if (seg == NULL) { 9617c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 96216ade92dScwb if ((lwp != NULL) && (!is_xhat)) { 9637c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 96416ade92dScwb lwp->lwp_nostop_r--; 96516ade92dScwb } 9667c478bd9Sstevel@tonic-gate return (FC_NOMAP); 9677c478bd9Sstevel@tonic-gate } 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate as_lock_held = 1; 9707c478bd9Sstevel@tonic-gate } 9717c478bd9Sstevel@tonic-gate 9727c478bd9Sstevel@tonic-gate addrsav = raddr; 9737c478bd9Sstevel@tonic-gate segsav = seg; 9747c478bd9Sstevel@tonic-gate 9757c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 9767c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 9777c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 9787c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 9797c478bd9Sstevel@tonic-gate res = FC_NOMAP; 9807c478bd9Sstevel@tonic-gate break; 9817c478bd9Sstevel@tonic-gate } 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate if (raddr + rsize > seg->s_base + seg->s_size) 9847c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 9857c478bd9Sstevel@tonic-gate else 9867c478bd9Sstevel@tonic-gate ssize = rsize; 9877c478bd9Sstevel@tonic-gate 9887c478bd9Sstevel@tonic-gate if (!is_xhat || (seg->s_ops != &segdev_ops)) { 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate if (is_xhat && avl_numnodes(&as->a_wpage) != 0 && 9917c478bd9Sstevel@tonic-gate pr_is_watchpage_as(raddr, rw, as)) { 9927c478bd9Sstevel@tonic-gate /* 9937c478bd9Sstevel@tonic-gate * Handle watch pages. If we're faulting on a 9947c478bd9Sstevel@tonic-gate * watched page from an X-hat, we have to 9957c478bd9Sstevel@tonic-gate * restore the original permissions while we 9967c478bd9Sstevel@tonic-gate * handle the fault. 9977c478bd9Sstevel@tonic-gate */ 9987c478bd9Sstevel@tonic-gate as_clearwatch(as); 9997c478bd9Sstevel@tonic-gate holding_wpage = 1; 10007c478bd9Sstevel@tonic-gate } 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw); 10037c478bd9Sstevel@tonic-gate 10047c478bd9Sstevel@tonic-gate /* Restore watchpoints */ 10057c478bd9Sstevel@tonic-gate if (holding_wpage) { 10067c478bd9Sstevel@tonic-gate as_setwatch(as); 10077c478bd9Sstevel@tonic-gate holding_wpage = 0; 10087c478bd9Sstevel@tonic-gate } 10097c478bd9Sstevel@tonic-gate 10107c478bd9Sstevel@tonic-gate if (res != 0) 10117c478bd9Sstevel@tonic-gate break; 10127c478bd9Sstevel@tonic-gate } else { 10137c478bd9Sstevel@tonic-gate /* XHAT does not support seg_dev */ 10147c478bd9Sstevel@tonic-gate res = FC_NOSUPPORT; 10157c478bd9Sstevel@tonic-gate break; 10167c478bd9Sstevel@tonic-gate } 10177c478bd9Sstevel@tonic-gate } 10187c478bd9Sstevel@tonic-gate 10197c478bd9Sstevel@tonic-gate /* 10207c478bd9Sstevel@tonic-gate * If we were SOFTLOCKing and encountered a failure, 10217c478bd9Sstevel@tonic-gate * we must SOFTUNLOCK the range we already did. (Maybe we 10227c478bd9Sstevel@tonic-gate * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing 10237c478bd9Sstevel@tonic-gate * right here...) 10247c478bd9Sstevel@tonic-gate */ 10257c478bd9Sstevel@tonic-gate if (res != 0 && type == F_SOFTLOCK) { 10267c478bd9Sstevel@tonic-gate for (seg = segsav; addrsav < raddr; addrsav += ssize) { 10277c478bd9Sstevel@tonic-gate if (addrsav >= seg->s_base + seg->s_size) 10287c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 10297c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 10307c478bd9Sstevel@tonic-gate /* 10317c478bd9Sstevel@tonic-gate * Now call the fault routine again to perform the 10327c478bd9Sstevel@tonic-gate * unlock using S_OTHER instead of the rw variable 10337c478bd9Sstevel@tonic-gate * since we never got a chance to touch the pages. 10347c478bd9Sstevel@tonic-gate */ 10357c478bd9Sstevel@tonic-gate if (raddr > seg->s_base + seg->s_size) 10367c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - addrsav; 10377c478bd9Sstevel@tonic-gate else 10387c478bd9Sstevel@tonic-gate ssize = raddr - addrsav; 10397c478bd9Sstevel@tonic-gate (void) SEGOP_FAULT(hat, seg, addrsav, ssize, 10407c478bd9Sstevel@tonic-gate F_SOFTUNLOCK, S_OTHER); 10417c478bd9Sstevel@tonic-gate } 10427c478bd9Sstevel@tonic-gate } 10437c478bd9Sstevel@tonic-gate if (as_lock_held) 10447c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 104516ade92dScwb if ((lwp != NULL) && (!is_xhat)) { 10467c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 104716ade92dScwb lwp->lwp_nostop_r--; 104816ade92dScwb } 10497c478bd9Sstevel@tonic-gate /* 10507c478bd9Sstevel@tonic-gate * If the lower levels returned EDEADLK for a fault, 10517c478bd9Sstevel@tonic-gate * It means that we should retry the fault. Let's wait 10527c478bd9Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 10537c478bd9Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 10547c478bd9Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 10557c478bd9Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 10567c478bd9Sstevel@tonic-gate * 4125102 for details of the problem. 10577c478bd9Sstevel@tonic-gate */ 10587c478bd9Sstevel@tonic-gate if (FC_ERRNO(res) == EDEADLK) { 10597c478bd9Sstevel@tonic-gate delay(deadlk_wait); 10607c478bd9Sstevel@tonic-gate res = 0; 10617c478bd9Sstevel@tonic-gate goto retry; 10627c478bd9Sstevel@tonic-gate } 10637c478bd9Sstevel@tonic-gate return (res); 10647c478bd9Sstevel@tonic-gate } 10657c478bd9Sstevel@tonic-gate 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate /* 10697c478bd9Sstevel@tonic-gate * Asynchronous ``fault'' at addr for size bytes. 10707c478bd9Sstevel@tonic-gate */ 10717c478bd9Sstevel@tonic-gate faultcode_t 10727c478bd9Sstevel@tonic-gate as_faulta(struct as *as, caddr_t addr, size_t size) 10737c478bd9Sstevel@tonic-gate { 10747c478bd9Sstevel@tonic-gate struct seg *seg; 10757c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 10767c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 10777c478bd9Sstevel@tonic-gate faultcode_t res = 0; 10787c478bd9Sstevel@tonic-gate klwp_t *lwp = ttolwp(curthread); 10797c478bd9Sstevel@tonic-gate 10807c478bd9Sstevel@tonic-gate retry: 10817c478bd9Sstevel@tonic-gate /* 10827c478bd9Sstevel@tonic-gate * Indicate that the lwp is not to be stopped while waiting 10837c478bd9Sstevel@tonic-gate * for a pagefault. This is to avoid deadlock while debugging 10847c478bd9Sstevel@tonic-gate * a process via /proc over NFS (in particular). 10857c478bd9Sstevel@tonic-gate */ 108616ade92dScwb if (lwp != NULL) { 10877c478bd9Sstevel@tonic-gate lwp->lwp_nostop++; 108816ade92dScwb lwp->lwp_nostop_r++; 108916ade92dScwb } 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 10927c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 10937c478bd9Sstevel@tonic-gate (size_t)raddr; 10947c478bd9Sstevel@tonic-gate 10957c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 10967c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 10977c478bd9Sstevel@tonic-gate if (seg == NULL) { 10987c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 109916ade92dScwb if (lwp != NULL) { 11007c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 110116ade92dScwb lwp->lwp_nostop_r--; 110216ade92dScwb } 11037c478bd9Sstevel@tonic-gate return (FC_NOMAP); 11047c478bd9Sstevel@tonic-gate } 11057c478bd9Sstevel@tonic-gate 11067c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) { 11077c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 11087c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 11097c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 11107c478bd9Sstevel@tonic-gate res = FC_NOMAP; 11117c478bd9Sstevel@tonic-gate break; 11127c478bd9Sstevel@tonic-gate } 11137c478bd9Sstevel@tonic-gate } 11147c478bd9Sstevel@tonic-gate res = SEGOP_FAULTA(seg, raddr); 11157c478bd9Sstevel@tonic-gate if (res != 0) 11167c478bd9Sstevel@tonic-gate break; 11177c478bd9Sstevel@tonic-gate } 11187c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 111916ade92dScwb if (lwp != NULL) { 11207c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 112116ade92dScwb lwp->lwp_nostop_r--; 112216ade92dScwb } 11237c478bd9Sstevel@tonic-gate /* 11247c478bd9Sstevel@tonic-gate * If the lower levels returned EDEADLK for a fault, 11257c478bd9Sstevel@tonic-gate * It means that we should retry the fault. Let's wait 11267c478bd9Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 11277c478bd9Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 11287c478bd9Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 11297c478bd9Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 11307c478bd9Sstevel@tonic-gate * 4125102 for details of the problem. 11317c478bd9Sstevel@tonic-gate */ 11327c478bd9Sstevel@tonic-gate if (FC_ERRNO(res) == EDEADLK) { 11337c478bd9Sstevel@tonic-gate delay(deadlk_wait); 11347c478bd9Sstevel@tonic-gate res = 0; 11357c478bd9Sstevel@tonic-gate goto retry; 11367c478bd9Sstevel@tonic-gate } 11377c478bd9Sstevel@tonic-gate return (res); 11387c478bd9Sstevel@tonic-gate } 11397c478bd9Sstevel@tonic-gate 11407c478bd9Sstevel@tonic-gate /* 11417c478bd9Sstevel@tonic-gate * Set the virtual mapping for the interval from [addr : addr + size) 11427c478bd9Sstevel@tonic-gate * in address space `as' to have the specified protection. 11437c478bd9Sstevel@tonic-gate * It is ok for the range to cross over several segments, 11447c478bd9Sstevel@tonic-gate * as long as they are contiguous. 11457c478bd9Sstevel@tonic-gate */ 11467c478bd9Sstevel@tonic-gate int 11477c478bd9Sstevel@tonic-gate as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 11487c478bd9Sstevel@tonic-gate { 11497c478bd9Sstevel@tonic-gate struct seg *seg; 11507c478bd9Sstevel@tonic-gate struct as_callback *cb; 11517c478bd9Sstevel@tonic-gate size_t ssize; 11527c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 11537c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 11547c478bd9Sstevel@tonic-gate int error = 0, writer = 0; 11557c478bd9Sstevel@tonic-gate caddr_t saveraddr; 11567c478bd9Sstevel@tonic-gate size_t saversize; 11577c478bd9Sstevel@tonic-gate 11587c478bd9Sstevel@tonic-gate setprot_top: 11597c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 11607c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 11617c478bd9Sstevel@tonic-gate (size_t)raddr; 11627c478bd9Sstevel@tonic-gate 11637c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 11647c478bd9Sstevel@tonic-gate return (ENOMEM); 11657c478bd9Sstevel@tonic-gate 11667c478bd9Sstevel@tonic-gate saveraddr = raddr; 11677c478bd9Sstevel@tonic-gate saversize = rsize; 11687c478bd9Sstevel@tonic-gate 11697c478bd9Sstevel@tonic-gate /* 11707c478bd9Sstevel@tonic-gate * Normally we only lock the as as a reader. But 11717c478bd9Sstevel@tonic-gate * if due to setprot the segment driver needs to split 11727c478bd9Sstevel@tonic-gate * a segment it will return IE_RETRY. Therefore we re-aquire 11737c478bd9Sstevel@tonic-gate * the as lock as a writer so the segment driver can change 11747c478bd9Sstevel@tonic-gate * the seg list. Also the segment driver will return IE_RETRY 11757c478bd9Sstevel@tonic-gate * after it has changed the segment list so we therefore keep 11767c478bd9Sstevel@tonic-gate * locking as a writer. Since these opeartions should be rare 11777c478bd9Sstevel@tonic-gate * want to only lock as a writer when necessary. 11787c478bd9Sstevel@tonic-gate */ 11797c478bd9Sstevel@tonic-gate if (writer || avl_numnodes(&as->a_wpage) != 0) { 11807c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 11817c478bd9Sstevel@tonic-gate } else { 11827c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 11837c478bd9Sstevel@tonic-gate } 11847c478bd9Sstevel@tonic-gate 11857c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 11867c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 11877c478bd9Sstevel@tonic-gate if (seg == NULL) { 11887c478bd9Sstevel@tonic-gate as_setwatch(as); 11897c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 11907c478bd9Sstevel@tonic-gate return (ENOMEM); 11917c478bd9Sstevel@tonic-gate } 11927c478bd9Sstevel@tonic-gate 11937c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 11947c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 11957c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 11967c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 11977c478bd9Sstevel@tonic-gate error = ENOMEM; 11987c478bd9Sstevel@tonic-gate break; 11997c478bd9Sstevel@tonic-gate } 12007c478bd9Sstevel@tonic-gate } 12017c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 12027c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 12037c478bd9Sstevel@tonic-gate else 12047c478bd9Sstevel@tonic-gate ssize = rsize; 12057c478bd9Sstevel@tonic-gate error = SEGOP_SETPROT(seg, raddr, ssize, prot); 12067c478bd9Sstevel@tonic-gate 12077c478bd9Sstevel@tonic-gate if (error == IE_NOMEM) { 12087c478bd9Sstevel@tonic-gate error = EAGAIN; 12097c478bd9Sstevel@tonic-gate break; 12107c478bd9Sstevel@tonic-gate } 12117c478bd9Sstevel@tonic-gate 12127c478bd9Sstevel@tonic-gate if (error == IE_RETRY) { 12137c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12147c478bd9Sstevel@tonic-gate writer = 1; 12157c478bd9Sstevel@tonic-gate goto setprot_top; 12167c478bd9Sstevel@tonic-gate } 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate if (error == EAGAIN) { 12197c478bd9Sstevel@tonic-gate /* 12207c478bd9Sstevel@tonic-gate * Make sure we have a_lock as writer. 12217c478bd9Sstevel@tonic-gate */ 12227c478bd9Sstevel@tonic-gate if (writer == 0) { 12237c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12247c478bd9Sstevel@tonic-gate writer = 1; 12257c478bd9Sstevel@tonic-gate goto setprot_top; 12267c478bd9Sstevel@tonic-gate } 12277c478bd9Sstevel@tonic-gate 12287c478bd9Sstevel@tonic-gate /* 12297c478bd9Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 12307c478bd9Sstevel@tonic-gate * before this operation can succeed through a retry. 12317c478bd9Sstevel@tonic-gate * The possible reasons for locked memory and 12327c478bd9Sstevel@tonic-gate * corresponding strategies for unlocking are: 12337c478bd9Sstevel@tonic-gate * (1) Normal I/O 12347c478bd9Sstevel@tonic-gate * wait for a signal that the I/O operation 12357c478bd9Sstevel@tonic-gate * has completed and the memory is unlocked. 12367c478bd9Sstevel@tonic-gate * (2) Asynchronous I/O 12377c478bd9Sstevel@tonic-gate * The aio subsystem does not unlock pages when 12387c478bd9Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 12397c478bd9Sstevel@tonic-gate * when the application calls aiowait/aioerror. 12407c478bd9Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 12417c478bd9Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 12427c478bd9Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 12437c478bd9Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 12447c478bd9Sstevel@tonic-gate * (3) Long term page locking: 12457c478bd9Sstevel@tonic-gate * Drivers intending to have pages locked for a 12467c478bd9Sstevel@tonic-gate * period considerably longer than for normal I/O 12477c478bd9Sstevel@tonic-gate * (essentially forever) may have registered for a 12487c478bd9Sstevel@tonic-gate * callback so they may unlock these pages on 12497c478bd9Sstevel@tonic-gate * request. This is needed to allow this operation 12507c478bd9Sstevel@tonic-gate * to succeed. Each entry on the callback list is 12517c478bd9Sstevel@tonic-gate * examined. If the event or address range pertains 12527c478bd9Sstevel@tonic-gate * the callback is invoked (unless it already is in 12537c478bd9Sstevel@tonic-gate * progress). The a_contents lock must be dropped 12547c478bd9Sstevel@tonic-gate * before the callback, so only one callback can 12557c478bd9Sstevel@tonic-gate * be done at a time. Go to the top and do more 12567c478bd9Sstevel@tonic-gate * until zero is returned. If zero is returned, 12577c478bd9Sstevel@tonic-gate * either there were no callbacks for this event 12587c478bd9Sstevel@tonic-gate * or they were already in progress. 12597c478bd9Sstevel@tonic-gate */ 12607c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 12617c478bd9Sstevel@tonic-gate if (as->a_callbacks && 12627c478bd9Sstevel@tonic-gate (cb = as_find_callback(as, AS_SETPROT_EVENT, 12637c478bd9Sstevel@tonic-gate seg->s_base, seg->s_size))) { 12647c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12657c478bd9Sstevel@tonic-gate as_execute_callback(as, cb, AS_SETPROT_EVENT); 12667c478bd9Sstevel@tonic-gate } else { 12677c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 12687c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 12697c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 12707c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12717c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 12727c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 12737c478bd9Sstevel@tonic-gate } 12747c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 12757c478bd9Sstevel@tonic-gate goto setprot_top; 12767c478bd9Sstevel@tonic-gate } else if (error != 0) 12777c478bd9Sstevel@tonic-gate break; 12787c478bd9Sstevel@tonic-gate } 12797c478bd9Sstevel@tonic-gate if (error != 0) { 12807c478bd9Sstevel@tonic-gate as_setwatch(as); 12817c478bd9Sstevel@tonic-gate } else { 12827c478bd9Sstevel@tonic-gate as_setwatchprot(as, saveraddr, saversize, prot); 12837c478bd9Sstevel@tonic-gate } 12847c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 12857c478bd9Sstevel@tonic-gate return (error); 12867c478bd9Sstevel@tonic-gate } 12877c478bd9Sstevel@tonic-gate 12887c478bd9Sstevel@tonic-gate /* 12897c478bd9Sstevel@tonic-gate * Check to make sure that the interval [addr, addr + size) 12907c478bd9Sstevel@tonic-gate * in address space `as' has at least the specified protection. 12917c478bd9Sstevel@tonic-gate * It is ok for the range to cross over several segments, as long 12927c478bd9Sstevel@tonic-gate * as they are contiguous. 12937c478bd9Sstevel@tonic-gate */ 12947c478bd9Sstevel@tonic-gate int 12957c478bd9Sstevel@tonic-gate as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 12967c478bd9Sstevel@tonic-gate { 12977c478bd9Sstevel@tonic-gate struct seg *seg; 12987c478bd9Sstevel@tonic-gate size_t ssize; 12997c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 13007c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 13017c478bd9Sstevel@tonic-gate int error = 0; 13027c478bd9Sstevel@tonic-gate 13037c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 13047c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 13057c478bd9Sstevel@tonic-gate (size_t)raddr; 13067c478bd9Sstevel@tonic-gate 13077c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 13087c478bd9Sstevel@tonic-gate return (ENOMEM); 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate /* 13117c478bd9Sstevel@tonic-gate * This is ugly as sin... 13127c478bd9Sstevel@tonic-gate * Normally, we only acquire the address space readers lock. 13137c478bd9Sstevel@tonic-gate * However, if the address space has watchpoints present, 13147c478bd9Sstevel@tonic-gate * we must acquire the writer lock on the address space for 13157c478bd9Sstevel@tonic-gate * the benefit of as_clearwatchprot() and as_setwatchprot(). 13167c478bd9Sstevel@tonic-gate */ 13177c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) != 0) 13187c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 13197c478bd9Sstevel@tonic-gate else 13207c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 13217c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 13227c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 13237c478bd9Sstevel@tonic-gate if (seg == NULL) { 13247c478bd9Sstevel@tonic-gate as_setwatch(as); 13257c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 13267c478bd9Sstevel@tonic-gate return (ENOMEM); 13277c478bd9Sstevel@tonic-gate } 13287c478bd9Sstevel@tonic-gate 13297c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 13307c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 13317c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 13327c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 13337c478bd9Sstevel@tonic-gate error = ENOMEM; 13347c478bd9Sstevel@tonic-gate break; 13357c478bd9Sstevel@tonic-gate } 13367c478bd9Sstevel@tonic-gate } 13377c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 13387c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 13397c478bd9Sstevel@tonic-gate else 13407c478bd9Sstevel@tonic-gate ssize = rsize; 13417c478bd9Sstevel@tonic-gate 13427c478bd9Sstevel@tonic-gate error = SEGOP_CHECKPROT(seg, raddr, ssize, prot); 13437c478bd9Sstevel@tonic-gate if (error != 0) 13447c478bd9Sstevel@tonic-gate break; 13457c478bd9Sstevel@tonic-gate } 13467c478bd9Sstevel@tonic-gate as_setwatch(as); 13477c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 13487c478bd9Sstevel@tonic-gate return (error); 13497c478bd9Sstevel@tonic-gate } 13507c478bd9Sstevel@tonic-gate 13517c478bd9Sstevel@tonic-gate int 13527c478bd9Sstevel@tonic-gate as_unmap(struct as *as, caddr_t addr, size_t size) 13537c478bd9Sstevel@tonic-gate { 13547c478bd9Sstevel@tonic-gate struct seg *seg, *seg_next; 13557c478bd9Sstevel@tonic-gate struct as_callback *cb; 13567c478bd9Sstevel@tonic-gate caddr_t raddr, eaddr; 13577c478bd9Sstevel@tonic-gate size_t ssize; 13587c478bd9Sstevel@tonic-gate int err; 13597c478bd9Sstevel@tonic-gate 13607c478bd9Sstevel@tonic-gate top: 13617c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 13627c478bd9Sstevel@tonic-gate eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & 13637c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK); 13647c478bd9Sstevel@tonic-gate 13657c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 13687c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 13697c478bd9Sstevel@tonic-gate 13707c478bd9Sstevel@tonic-gate /* 13717c478bd9Sstevel@tonic-gate * Use as_findseg to find the first segment in the range, then 13727c478bd9Sstevel@tonic-gate * step through the segments in order, following s_next. 13737c478bd9Sstevel@tonic-gate */ 13747c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, eaddr - raddr); 13757c478bd9Sstevel@tonic-gate 13767c478bd9Sstevel@tonic-gate for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) { 13777c478bd9Sstevel@tonic-gate if (eaddr <= seg->s_base) 13787c478bd9Sstevel@tonic-gate break; /* eaddr was in a gap; all done */ 13797c478bd9Sstevel@tonic-gate 13807c478bd9Sstevel@tonic-gate /* this is implied by the test above */ 13817c478bd9Sstevel@tonic-gate ASSERT(raddr < eaddr); 13827c478bd9Sstevel@tonic-gate 13837c478bd9Sstevel@tonic-gate if (raddr < seg->s_base) 13847c478bd9Sstevel@tonic-gate raddr = seg->s_base; /* raddr was in a gap */ 13857c478bd9Sstevel@tonic-gate 13867c478bd9Sstevel@tonic-gate if (eaddr > (seg->s_base + seg->s_size)) 13877c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 13887c478bd9Sstevel@tonic-gate else 13897c478bd9Sstevel@tonic-gate ssize = eaddr - raddr; 13907c478bd9Sstevel@tonic-gate 13917c478bd9Sstevel@tonic-gate /* 13927c478bd9Sstevel@tonic-gate * Save next segment pointer since seg can be 13937c478bd9Sstevel@tonic-gate * destroyed during the segment unmap operation. 13947c478bd9Sstevel@tonic-gate */ 13957c478bd9Sstevel@tonic-gate seg_next = AS_SEGNEXT(as, seg); 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate err = SEGOP_UNMAP(seg, raddr, ssize); 13987c478bd9Sstevel@tonic-gate if (err == EAGAIN) { 13997c478bd9Sstevel@tonic-gate /* 14007c478bd9Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 14017c478bd9Sstevel@tonic-gate * before this operation can succeed through a retry. 14027c478bd9Sstevel@tonic-gate * The possible reasons for locked memory and 14037c478bd9Sstevel@tonic-gate * corresponding strategies for unlocking are: 14047c478bd9Sstevel@tonic-gate * (1) Normal I/O 14057c478bd9Sstevel@tonic-gate * wait for a signal that the I/O operation 14067c478bd9Sstevel@tonic-gate * has completed and the memory is unlocked. 14077c478bd9Sstevel@tonic-gate * (2) Asynchronous I/O 14087c478bd9Sstevel@tonic-gate * The aio subsystem does not unlock pages when 14097c478bd9Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 14107c478bd9Sstevel@tonic-gate * when the application calls aiowait/aioerror. 14117c478bd9Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 14127c478bd9Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 14137c478bd9Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 14147c478bd9Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 14157c478bd9Sstevel@tonic-gate * (3) Long term page locking: 14167c478bd9Sstevel@tonic-gate * Drivers intending to have pages locked for a 14177c478bd9Sstevel@tonic-gate * period considerably longer than for normal I/O 14187c478bd9Sstevel@tonic-gate * (essentially forever) may have registered for a 14197c478bd9Sstevel@tonic-gate * callback so they may unlock these pages on 14207c478bd9Sstevel@tonic-gate * request. This is needed to allow this operation 14217c478bd9Sstevel@tonic-gate * to succeed. Each entry on the callback list is 14227c478bd9Sstevel@tonic-gate * examined. If the event or address range pertains 14237c478bd9Sstevel@tonic-gate * the callback is invoked (unless it already is in 14247c478bd9Sstevel@tonic-gate * progress). The a_contents lock must be dropped 14257c478bd9Sstevel@tonic-gate * before the callback, so only one callback can 14267c478bd9Sstevel@tonic-gate * be done at a time. Go to the top and do more 14277c478bd9Sstevel@tonic-gate * until zero is returned. If zero is returned, 14287c478bd9Sstevel@tonic-gate * either there were no callbacks for this event 14297c478bd9Sstevel@tonic-gate * or they were already in progress. 14307c478bd9Sstevel@tonic-gate */ 14317c478bd9Sstevel@tonic-gate as_setwatch(as); 14327c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 14337c478bd9Sstevel@tonic-gate if (as->a_callbacks && 14347c478bd9Sstevel@tonic-gate (cb = as_find_callback(as, AS_UNMAP_EVENT, 14357c478bd9Sstevel@tonic-gate seg->s_base, seg->s_size))) { 14367c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14377c478bd9Sstevel@tonic-gate as_execute_callback(as, cb, AS_UNMAP_EVENT); 14387c478bd9Sstevel@tonic-gate } else { 14397c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 14407c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 14417c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 14427c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14437c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 14447c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 14457c478bd9Sstevel@tonic-gate } 14467c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 14477c478bd9Sstevel@tonic-gate goto top; 14487c478bd9Sstevel@tonic-gate } else if (err == IE_RETRY) { 14497c478bd9Sstevel@tonic-gate as_setwatch(as); 14507c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14517c478bd9Sstevel@tonic-gate goto top; 14527c478bd9Sstevel@tonic-gate } else if (err) { 14537c478bd9Sstevel@tonic-gate as_setwatch(as); 14547c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14557c478bd9Sstevel@tonic-gate return (-1); 14567c478bd9Sstevel@tonic-gate } 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate as->a_size -= ssize; 14597c478bd9Sstevel@tonic-gate raddr += ssize; 14607c478bd9Sstevel@tonic-gate } 14617c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 14627c478bd9Sstevel@tonic-gate return (0); 14637c478bd9Sstevel@tonic-gate } 14647c478bd9Sstevel@tonic-gate 14657c478bd9Sstevel@tonic-gate static int 14667c478bd9Sstevel@tonic-gate as_map_vnsegs(struct as *as, caddr_t addr, size_t size, 14677c478bd9Sstevel@tonic-gate int (*crfp)(), struct segvn_crargs *vn_a, int *segcreated) 14687c478bd9Sstevel@tonic-gate { 14697c478bd9Sstevel@tonic-gate int text = vn_a->flags & MAP_TEXT; 14707c478bd9Sstevel@tonic-gate uint_t szcvec = map_execseg_pgszcvec(text, addr, size); 14717c478bd9Sstevel@tonic-gate uint_t szc; 14727c478bd9Sstevel@tonic-gate uint_t nszc; 14737c478bd9Sstevel@tonic-gate int error; 14747c478bd9Sstevel@tonic-gate caddr_t a; 14757c478bd9Sstevel@tonic-gate caddr_t eaddr; 14767c478bd9Sstevel@tonic-gate size_t segsize; 14777c478bd9Sstevel@tonic-gate struct seg *seg; 14787c478bd9Sstevel@tonic-gate uint_t save_szcvec; 14797c478bd9Sstevel@tonic-gate size_t pgsz; 14807c478bd9Sstevel@tonic-gate struct vattr va; 14817c478bd9Sstevel@tonic-gate u_offset_t eoff; 14827c478bd9Sstevel@tonic-gate size_t save_size = 0; 14837c478bd9Sstevel@tonic-gate 14847c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 14857c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 14867c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 14877c478bd9Sstevel@tonic-gate ASSERT(vn_a->vp != NULL); 14887c478bd9Sstevel@tonic-gate ASSERT(vn_a->amp == NULL); 14897c478bd9Sstevel@tonic-gate 14907c478bd9Sstevel@tonic-gate again: 14917c478bd9Sstevel@tonic-gate if (szcvec <= 1) { 14927c478bd9Sstevel@tonic-gate seg = seg_alloc(as, addr, size); 14937c478bd9Sstevel@tonic-gate if (seg == NULL) { 14947c478bd9Sstevel@tonic-gate return (ENOMEM); 14957c478bd9Sstevel@tonic-gate } 14967c478bd9Sstevel@tonic-gate vn_a->szc = 0; 14977c478bd9Sstevel@tonic-gate error = (*crfp)(seg, vn_a); 14987c478bd9Sstevel@tonic-gate if (error != 0) { 14997c478bd9Sstevel@tonic-gate seg_free(seg); 15007c478bd9Sstevel@tonic-gate } 15017c478bd9Sstevel@tonic-gate return (error); 15027c478bd9Sstevel@tonic-gate } 15037c478bd9Sstevel@tonic-gate 15047c478bd9Sstevel@tonic-gate va.va_mask = AT_SIZE; 15057c478bd9Sstevel@tonic-gate if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred) != 0) { 15067c478bd9Sstevel@tonic-gate szcvec = 0; 15077c478bd9Sstevel@tonic-gate goto again; 15087c478bd9Sstevel@tonic-gate } 15097c478bd9Sstevel@tonic-gate eoff = vn_a->offset & PAGEMASK; 15107c478bd9Sstevel@tonic-gate if (eoff >= va.va_size) { 15117c478bd9Sstevel@tonic-gate szcvec = 0; 15127c478bd9Sstevel@tonic-gate goto again; 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate eoff += size; 15157c478bd9Sstevel@tonic-gate if (btopr(va.va_size) < btopr(eoff)) { 15167c478bd9Sstevel@tonic-gate save_size = size; 15177c478bd9Sstevel@tonic-gate size = va.va_size - (vn_a->offset & PAGEMASK); 15187c478bd9Sstevel@tonic-gate size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t); 15197c478bd9Sstevel@tonic-gate szcvec = map_execseg_pgszcvec(text, addr, size); 15207c478bd9Sstevel@tonic-gate if (szcvec <= 1) { 15217c478bd9Sstevel@tonic-gate size = save_size; 15227c478bd9Sstevel@tonic-gate goto again; 15237c478bd9Sstevel@tonic-gate } 15247c478bd9Sstevel@tonic-gate } 15257c478bd9Sstevel@tonic-gate 15267c478bd9Sstevel@tonic-gate eaddr = addr + size; 15277c478bd9Sstevel@tonic-gate save_szcvec = szcvec; 15287c478bd9Sstevel@tonic-gate szcvec >>= 1; 15297c478bd9Sstevel@tonic-gate szc = 0; 15307c478bd9Sstevel@tonic-gate nszc = 0; 15317c478bd9Sstevel@tonic-gate while (szcvec) { 15327c478bd9Sstevel@tonic-gate if ((szcvec & 0x1) == 0) { 15337c478bd9Sstevel@tonic-gate nszc++; 15347c478bd9Sstevel@tonic-gate szcvec >>= 1; 15357c478bd9Sstevel@tonic-gate continue; 15367c478bd9Sstevel@tonic-gate } 15377c478bd9Sstevel@tonic-gate nszc++; 15387c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(nszc); 15397c478bd9Sstevel@tonic-gate a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 15407c478bd9Sstevel@tonic-gate if (a != addr) { 15417c478bd9Sstevel@tonic-gate ASSERT(a < eaddr); 15427c478bd9Sstevel@tonic-gate segsize = a - addr; 15437c478bd9Sstevel@tonic-gate seg = seg_alloc(as, addr, segsize); 15447c478bd9Sstevel@tonic-gate if (seg == NULL) { 15457c478bd9Sstevel@tonic-gate return (ENOMEM); 15467c478bd9Sstevel@tonic-gate } 15477c478bd9Sstevel@tonic-gate vn_a->szc = szc; 15487c478bd9Sstevel@tonic-gate error = (*crfp)(seg, vn_a); 15497c478bd9Sstevel@tonic-gate if (error != 0) { 15507c478bd9Sstevel@tonic-gate seg_free(seg); 15517c478bd9Sstevel@tonic-gate return (error); 15527c478bd9Sstevel@tonic-gate } 15537c478bd9Sstevel@tonic-gate *segcreated = 1; 15547c478bd9Sstevel@tonic-gate vn_a->offset += segsize; 15557c478bd9Sstevel@tonic-gate addr = a; 15567c478bd9Sstevel@tonic-gate } 15577c478bd9Sstevel@tonic-gate szc = nszc; 15587c478bd9Sstevel@tonic-gate szcvec >>= 1; 15597c478bd9Sstevel@tonic-gate } 15607c478bd9Sstevel@tonic-gate 15617c478bd9Sstevel@tonic-gate ASSERT(addr < eaddr); 15627c478bd9Sstevel@tonic-gate szcvec = save_szcvec | 1; /* add 8K pages */ 15637c478bd9Sstevel@tonic-gate while (szcvec) { 15647c478bd9Sstevel@tonic-gate a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 15657c478bd9Sstevel@tonic-gate ASSERT(a >= addr); 15667c478bd9Sstevel@tonic-gate if (a != addr) { 15677c478bd9Sstevel@tonic-gate segsize = a - addr; 15687c478bd9Sstevel@tonic-gate seg = seg_alloc(as, addr, segsize); 15697c478bd9Sstevel@tonic-gate if (seg == NULL) { 15707c478bd9Sstevel@tonic-gate return (ENOMEM); 15717c478bd9Sstevel@tonic-gate } 15727c478bd9Sstevel@tonic-gate vn_a->szc = szc; 15737c478bd9Sstevel@tonic-gate error = (*crfp)(seg, vn_a); 15747c478bd9Sstevel@tonic-gate if (error != 0) { 15757c478bd9Sstevel@tonic-gate seg_free(seg); 15767c478bd9Sstevel@tonic-gate return (error); 15777c478bd9Sstevel@tonic-gate } 15787c478bd9Sstevel@tonic-gate *segcreated = 1; 15797c478bd9Sstevel@tonic-gate vn_a->offset += segsize; 15807c478bd9Sstevel@tonic-gate addr = a; 15817c478bd9Sstevel@tonic-gate } 15827c478bd9Sstevel@tonic-gate szcvec &= ~(1 << szc); 15837c478bd9Sstevel@tonic-gate if (szcvec) { 15847c478bd9Sstevel@tonic-gate szc = highbit(szcvec) - 1; 15857c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(szc); 15867c478bd9Sstevel@tonic-gate } 15877c478bd9Sstevel@tonic-gate } 15887c478bd9Sstevel@tonic-gate ASSERT(addr == eaddr); 15897c478bd9Sstevel@tonic-gate 15907c478bd9Sstevel@tonic-gate if (save_size) { 15917c478bd9Sstevel@tonic-gate size = save_size - size; 15927c478bd9Sstevel@tonic-gate goto again; 15937c478bd9Sstevel@tonic-gate } 15947c478bd9Sstevel@tonic-gate 15957c478bd9Sstevel@tonic-gate return (0); 15967c478bd9Sstevel@tonic-gate } 15977c478bd9Sstevel@tonic-gate 15987c478bd9Sstevel@tonic-gate int 15997c478bd9Sstevel@tonic-gate as_map(struct as *as, caddr_t addr, size_t size, int (*crfp)(), void *argsp) 1600*02ff05a9Svsakar { 1601*02ff05a9Svsakar AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 1602*02ff05a9Svsakar return (as_map_locked(as, addr, size, crfp, argsp)); 1603*02ff05a9Svsakar } 1604*02ff05a9Svsakar 1605*02ff05a9Svsakar int 1606*02ff05a9Svsakar as_map_locked(struct as *as, caddr_t addr, size_t size, int (*crfp)(), 1607*02ff05a9Svsakar void *argsp) 16087c478bd9Sstevel@tonic-gate { 16097c478bd9Sstevel@tonic-gate struct seg *seg = NULL; 16107c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 16117c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 16127c478bd9Sstevel@tonic-gate int error; 16137c478bd9Sstevel@tonic-gate struct proc *p = curproc; 16147c478bd9Sstevel@tonic-gate 16157c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 16167c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 16177c478bd9Sstevel@tonic-gate (size_t)raddr; 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate /* 16207c478bd9Sstevel@tonic-gate * check for wrap around 16217c478bd9Sstevel@tonic-gate */ 16227c478bd9Sstevel@tonic-gate if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) { 16237c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16247c478bd9Sstevel@tonic-gate return (ENOMEM); 16257c478bd9Sstevel@tonic-gate } 16267c478bd9Sstevel@tonic-gate 16277c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 16287c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 16297c478bd9Sstevel@tonic-gate 16307c478bd9Sstevel@tonic-gate if (as != &kas && as->a_size + rsize > (size_t)p->p_vmem_ctl) { 16317c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16327c478bd9Sstevel@tonic-gate 16337c478bd9Sstevel@tonic-gate (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], p->p_rctls, p, 16347c478bd9Sstevel@tonic-gate RCA_UNSAFE_ALL); 16357c478bd9Sstevel@tonic-gate 16367c478bd9Sstevel@tonic-gate return (ENOMEM); 16377c478bd9Sstevel@tonic-gate } 16387c478bd9Sstevel@tonic-gate 16397c478bd9Sstevel@tonic-gate if (AS_MAP_VNSEGS_USELPGS(crfp, argsp)) { 16407c478bd9Sstevel@tonic-gate int unmap = 0; 16417c478bd9Sstevel@tonic-gate error = as_map_vnsegs(as, raddr, rsize, crfp, 16427c478bd9Sstevel@tonic-gate (struct segvn_crargs *)argsp, &unmap); 16437c478bd9Sstevel@tonic-gate if (error != 0) { 16447c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16457c478bd9Sstevel@tonic-gate if (unmap) { 16467c478bd9Sstevel@tonic-gate (void) as_unmap(as, addr, size); 16477c478bd9Sstevel@tonic-gate } 16487c478bd9Sstevel@tonic-gate return (error); 16497c478bd9Sstevel@tonic-gate } 16507c478bd9Sstevel@tonic-gate } else { 16517c478bd9Sstevel@tonic-gate seg = seg_alloc(as, addr, size); 16527c478bd9Sstevel@tonic-gate if (seg == NULL) { 16537c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16547c478bd9Sstevel@tonic-gate return (ENOMEM); 16557c478bd9Sstevel@tonic-gate } 16567c478bd9Sstevel@tonic-gate 16577c478bd9Sstevel@tonic-gate error = (*crfp)(seg, argsp); 16587c478bd9Sstevel@tonic-gate if (error != 0) { 16597c478bd9Sstevel@tonic-gate seg_free(seg); 16607c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16617c478bd9Sstevel@tonic-gate return (error); 16627c478bd9Sstevel@tonic-gate } 16637c478bd9Sstevel@tonic-gate } 16647c478bd9Sstevel@tonic-gate 16657c478bd9Sstevel@tonic-gate /* 16667c478bd9Sstevel@tonic-gate * Add size now so as_unmap will work if as_ctl fails. 16677c478bd9Sstevel@tonic-gate */ 16687c478bd9Sstevel@tonic-gate as->a_size += rsize; 16697c478bd9Sstevel@tonic-gate 16707c478bd9Sstevel@tonic-gate as_setwatch(as); 16717c478bd9Sstevel@tonic-gate 16727c478bd9Sstevel@tonic-gate /* 16737c478bd9Sstevel@tonic-gate * If the address space is locked, 16747c478bd9Sstevel@tonic-gate * establish memory locks for the new segment. 16757c478bd9Sstevel@tonic-gate */ 16767c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 16777c478bd9Sstevel@tonic-gate if (AS_ISPGLCK(as)) { 16787c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 16797c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16807c478bd9Sstevel@tonic-gate error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0); 16817c478bd9Sstevel@tonic-gate if (error != 0) 16827c478bd9Sstevel@tonic-gate (void) as_unmap(as, addr, size); 16837c478bd9Sstevel@tonic-gate } else { 16847c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 16857c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 16867c478bd9Sstevel@tonic-gate } 16877c478bd9Sstevel@tonic-gate return (error); 16887c478bd9Sstevel@tonic-gate } 16897c478bd9Sstevel@tonic-gate 16907c478bd9Sstevel@tonic-gate 16917c478bd9Sstevel@tonic-gate /* 16927c478bd9Sstevel@tonic-gate * Delete all segments in the address space marked with S_PURGE. 16937c478bd9Sstevel@tonic-gate * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c). 16947c478bd9Sstevel@tonic-gate * These segments are deleted as a first step before calls to as_gap(), so 16957c478bd9Sstevel@tonic-gate * that they don't affect mmap() or shmat(). 16967c478bd9Sstevel@tonic-gate */ 16977c478bd9Sstevel@tonic-gate void 16987c478bd9Sstevel@tonic-gate as_purge(struct as *as) 16997c478bd9Sstevel@tonic-gate { 17007c478bd9Sstevel@tonic-gate struct seg *seg; 17017c478bd9Sstevel@tonic-gate struct seg *next_seg; 17027c478bd9Sstevel@tonic-gate 17037c478bd9Sstevel@tonic-gate /* 17047c478bd9Sstevel@tonic-gate * the setting of NEEDSPURGE is protect by as_rangelock(), so 17057c478bd9Sstevel@tonic-gate * no need to grab a_contents mutex for this check 17067c478bd9Sstevel@tonic-gate */ 17077c478bd9Sstevel@tonic-gate if ((as->a_flags & AS_NEEDSPURGE) == 0) 17087c478bd9Sstevel@tonic-gate return; 17097c478bd9Sstevel@tonic-gate 17107c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 17117c478bd9Sstevel@tonic-gate next_seg = NULL; 17127c478bd9Sstevel@tonic-gate seg = AS_SEGFIRST(as); 17137c478bd9Sstevel@tonic-gate while (seg != NULL) { 17147c478bd9Sstevel@tonic-gate next_seg = AS_SEGNEXT(as, seg); 17157c478bd9Sstevel@tonic-gate if (seg->s_flags & S_PURGE) 17167c478bd9Sstevel@tonic-gate SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 17177c478bd9Sstevel@tonic-gate seg = next_seg; 17187c478bd9Sstevel@tonic-gate } 17197c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17207c478bd9Sstevel@tonic-gate 17217c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 17227c478bd9Sstevel@tonic-gate as->a_flags &= ~AS_NEEDSPURGE; 17237c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 17247c478bd9Sstevel@tonic-gate } 17257c478bd9Sstevel@tonic-gate 17267c478bd9Sstevel@tonic-gate /* 17277c478bd9Sstevel@tonic-gate * Find a hole of at least size minlen within [base, base + len). 17287c478bd9Sstevel@tonic-gate * 17297c478bd9Sstevel@tonic-gate * If flags specifies AH_HI, the hole will have the highest possible address 17307c478bd9Sstevel@tonic-gate * in the range. We use the as->a_lastgap field to figure out where to 17317c478bd9Sstevel@tonic-gate * start looking for a gap. 17327c478bd9Sstevel@tonic-gate * 17337c478bd9Sstevel@tonic-gate * Otherwise, the gap will have the lowest possible address. 17347c478bd9Sstevel@tonic-gate * 17357c478bd9Sstevel@tonic-gate * If flags specifies AH_CONTAIN, the hole will contain the address addr. 17367c478bd9Sstevel@tonic-gate * 17377c478bd9Sstevel@tonic-gate * If an adequate hole is found, base and len are set to reflect the part of 17387c478bd9Sstevel@tonic-gate * the hole that is within range, and 0 is returned, otherwise, 17397c478bd9Sstevel@tonic-gate * -1 is returned. 17407c478bd9Sstevel@tonic-gate * 17417c478bd9Sstevel@tonic-gate * NOTE: This routine is not correct when base+len overflows caddr_t. 17427c478bd9Sstevel@tonic-gate */ 17437c478bd9Sstevel@tonic-gate int 17447c478bd9Sstevel@tonic-gate as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags, 17457c478bd9Sstevel@tonic-gate caddr_t addr) 17467c478bd9Sstevel@tonic-gate { 17477c478bd9Sstevel@tonic-gate caddr_t lobound = *basep; 17487c478bd9Sstevel@tonic-gate caddr_t hibound = lobound + *lenp; 17497c478bd9Sstevel@tonic-gate struct seg *lseg, *hseg; 17507c478bd9Sstevel@tonic-gate caddr_t lo, hi; 17517c478bd9Sstevel@tonic-gate int forward; 17527c478bd9Sstevel@tonic-gate caddr_t save_base; 17537c478bd9Sstevel@tonic-gate size_t save_len; 17547c478bd9Sstevel@tonic-gate 17557c478bd9Sstevel@tonic-gate save_base = *basep; 17567c478bd9Sstevel@tonic-gate save_len = *lenp; 17577c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 17587c478bd9Sstevel@tonic-gate if (AS_SEGFIRST(as) == NULL) { 17597c478bd9Sstevel@tonic-gate if (valid_va_range(basep, lenp, minlen, flags & AH_DIR)) { 17607c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17617c478bd9Sstevel@tonic-gate return (0); 17627c478bd9Sstevel@tonic-gate } else { 17637c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 17647c478bd9Sstevel@tonic-gate *basep = save_base; 17657c478bd9Sstevel@tonic-gate *lenp = save_len; 17667c478bd9Sstevel@tonic-gate return (-1); 17677c478bd9Sstevel@tonic-gate } 17687c478bd9Sstevel@tonic-gate } 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate /* 17717c478bd9Sstevel@tonic-gate * Set up to iterate over all the inter-segment holes in the given 17727c478bd9Sstevel@tonic-gate * direction. lseg is NULL for the lowest-addressed hole and hseg is 17737c478bd9Sstevel@tonic-gate * NULL for the highest-addressed hole. If moving backwards, we reset 17747c478bd9Sstevel@tonic-gate * sseg to denote the highest-addressed segment. 17757c478bd9Sstevel@tonic-gate */ 17767c478bd9Sstevel@tonic-gate forward = (flags & AH_DIR) == AH_LO; 17777c478bd9Sstevel@tonic-gate if (forward) { 17787c478bd9Sstevel@tonic-gate hseg = as_findseg(as, lobound, 1); 17797c478bd9Sstevel@tonic-gate lseg = AS_SEGPREV(as, hseg); 17807c478bd9Sstevel@tonic-gate } else { 17817c478bd9Sstevel@tonic-gate 17827c478bd9Sstevel@tonic-gate /* 17837c478bd9Sstevel@tonic-gate * If allocating at least as much as the last allocation, 17847c478bd9Sstevel@tonic-gate * use a_lastgap's base as a better estimate of hibound. 17857c478bd9Sstevel@tonic-gate */ 17867c478bd9Sstevel@tonic-gate if (as->a_lastgap && 17877c478bd9Sstevel@tonic-gate minlen >= as->a_lastgap->s_size && 17887c478bd9Sstevel@tonic-gate hibound >= as->a_lastgap->s_base) 17897c478bd9Sstevel@tonic-gate hibound = as->a_lastgap->s_base; 17907c478bd9Sstevel@tonic-gate 17917c478bd9Sstevel@tonic-gate hseg = as_findseg(as, hibound, 1); 17927c478bd9Sstevel@tonic-gate if (hseg->s_base + hseg->s_size < hibound) { 17937c478bd9Sstevel@tonic-gate lseg = hseg; 17947c478bd9Sstevel@tonic-gate hseg = NULL; 17957c478bd9Sstevel@tonic-gate } else { 17967c478bd9Sstevel@tonic-gate lseg = AS_SEGPREV(as, hseg); 17977c478bd9Sstevel@tonic-gate } 17987c478bd9Sstevel@tonic-gate } 17997c478bd9Sstevel@tonic-gate 18007c478bd9Sstevel@tonic-gate for (;;) { 18017c478bd9Sstevel@tonic-gate /* 18027c478bd9Sstevel@tonic-gate * Set lo and hi to the hole's boundaries. (We should really 18037c478bd9Sstevel@tonic-gate * use MAXADDR in place of hibound in the expression below, 18047c478bd9Sstevel@tonic-gate * but can't express it easily; using hibound in its place is 18057c478bd9Sstevel@tonic-gate * harmless.) 18067c478bd9Sstevel@tonic-gate */ 18077c478bd9Sstevel@tonic-gate lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size; 18087c478bd9Sstevel@tonic-gate hi = (hseg == NULL) ? hibound : hseg->s_base; 18097c478bd9Sstevel@tonic-gate /* 18107c478bd9Sstevel@tonic-gate * If the iteration has moved past the interval from lobound 18117c478bd9Sstevel@tonic-gate * to hibound it's pointless to continue. 18127c478bd9Sstevel@tonic-gate */ 18137c478bd9Sstevel@tonic-gate if ((forward && lo > hibound) || (!forward && hi < lobound)) 18147c478bd9Sstevel@tonic-gate break; 18157c478bd9Sstevel@tonic-gate else if (lo > hibound || hi < lobound) 18167c478bd9Sstevel@tonic-gate goto cont; 18177c478bd9Sstevel@tonic-gate /* 18187c478bd9Sstevel@tonic-gate * Candidate hole lies at least partially within the allowable 18197c478bd9Sstevel@tonic-gate * range. Restrict it to fall completely within that range, 18207c478bd9Sstevel@tonic-gate * i.e., to [max(lo, lobound), min(hi, hibound)]. 18217c478bd9Sstevel@tonic-gate */ 18227c478bd9Sstevel@tonic-gate if (lo < lobound) 18237c478bd9Sstevel@tonic-gate lo = lobound; 18247c478bd9Sstevel@tonic-gate if (hi > hibound) 18257c478bd9Sstevel@tonic-gate hi = hibound; 18267c478bd9Sstevel@tonic-gate /* 18277c478bd9Sstevel@tonic-gate * Verify that the candidate hole is big enough and meets 18287c478bd9Sstevel@tonic-gate * hardware constraints. 18297c478bd9Sstevel@tonic-gate */ 18307c478bd9Sstevel@tonic-gate *basep = lo; 18317c478bd9Sstevel@tonic-gate *lenp = hi - lo; 18327c478bd9Sstevel@tonic-gate if (valid_va_range(basep, lenp, minlen, 18337c478bd9Sstevel@tonic-gate forward ? AH_LO : AH_HI) && 18347c478bd9Sstevel@tonic-gate ((flags & AH_CONTAIN) == 0 || 18357c478bd9Sstevel@tonic-gate (*basep <= addr && *basep + *lenp > addr))) { 18367c478bd9Sstevel@tonic-gate if (!forward) 18377c478bd9Sstevel@tonic-gate as->a_lastgap = hseg; 18387c478bd9Sstevel@tonic-gate if (hseg != NULL) 18397c478bd9Sstevel@tonic-gate as->a_lastgaphl = hseg; 18407c478bd9Sstevel@tonic-gate else 18417c478bd9Sstevel@tonic-gate as->a_lastgaphl = lseg; 18427c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 18437c478bd9Sstevel@tonic-gate return (0); 18447c478bd9Sstevel@tonic-gate } 18457c478bd9Sstevel@tonic-gate cont: 18467c478bd9Sstevel@tonic-gate /* 18477c478bd9Sstevel@tonic-gate * Move to the next hole. 18487c478bd9Sstevel@tonic-gate */ 18497c478bd9Sstevel@tonic-gate if (forward) { 18507c478bd9Sstevel@tonic-gate lseg = hseg; 18517c478bd9Sstevel@tonic-gate if (lseg == NULL) 18527c478bd9Sstevel@tonic-gate break; 18537c478bd9Sstevel@tonic-gate hseg = AS_SEGNEXT(as, hseg); 18547c478bd9Sstevel@tonic-gate } else { 18557c478bd9Sstevel@tonic-gate hseg = lseg; 18567c478bd9Sstevel@tonic-gate if (hseg == NULL) 18577c478bd9Sstevel@tonic-gate break; 18587c478bd9Sstevel@tonic-gate lseg = AS_SEGPREV(as, lseg); 18597c478bd9Sstevel@tonic-gate } 18607c478bd9Sstevel@tonic-gate } 18617c478bd9Sstevel@tonic-gate *basep = save_base; 18627c478bd9Sstevel@tonic-gate *lenp = save_len; 18637c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 18647c478bd9Sstevel@tonic-gate return (-1); 18657c478bd9Sstevel@tonic-gate } 18667c478bd9Sstevel@tonic-gate 18677c478bd9Sstevel@tonic-gate /* 18687c478bd9Sstevel@tonic-gate * Return the next range within [base, base + len) that is backed 18697c478bd9Sstevel@tonic-gate * with "real memory". Skip holes and non-seg_vn segments. 18707c478bd9Sstevel@tonic-gate * We're lazy and only return one segment at a time. 18717c478bd9Sstevel@tonic-gate */ 18727c478bd9Sstevel@tonic-gate int 18737c478bd9Sstevel@tonic-gate as_memory(struct as *as, caddr_t *basep, size_t *lenp) 18747c478bd9Sstevel@tonic-gate { 18757c478bd9Sstevel@tonic-gate extern struct seg_ops segspt_shmops; /* needs a header file */ 18767c478bd9Sstevel@tonic-gate struct seg *seg; 18777c478bd9Sstevel@tonic-gate caddr_t addr, eaddr; 18787c478bd9Sstevel@tonic-gate caddr_t segend; 18797c478bd9Sstevel@tonic-gate 18807c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 18817c478bd9Sstevel@tonic-gate 18827c478bd9Sstevel@tonic-gate addr = *basep; 18837c478bd9Sstevel@tonic-gate eaddr = addr + *lenp; 18847c478bd9Sstevel@tonic-gate 18857c478bd9Sstevel@tonic-gate seg = as_findseg(as, addr, 0); 18867c478bd9Sstevel@tonic-gate if (seg != NULL) 18877c478bd9Sstevel@tonic-gate addr = MAX(seg->s_base, addr); 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate for (;;) { 18907c478bd9Sstevel@tonic-gate if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) { 18917c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 18927c478bd9Sstevel@tonic-gate return (EINVAL); 18937c478bd9Sstevel@tonic-gate } 18947c478bd9Sstevel@tonic-gate 18957c478bd9Sstevel@tonic-gate if (seg->s_ops == &segvn_ops) { 18967c478bd9Sstevel@tonic-gate segend = seg->s_base + seg->s_size; 18977c478bd9Sstevel@tonic-gate break; 18987c478bd9Sstevel@tonic-gate } 18997c478bd9Sstevel@tonic-gate 19007c478bd9Sstevel@tonic-gate /* 19017c478bd9Sstevel@tonic-gate * We do ISM by looking into the private data 19027c478bd9Sstevel@tonic-gate * to determine the real size of the segment. 19037c478bd9Sstevel@tonic-gate */ 19047c478bd9Sstevel@tonic-gate if (seg->s_ops == &segspt_shmops) { 19057c478bd9Sstevel@tonic-gate segend = seg->s_base + spt_realsize(seg); 19067c478bd9Sstevel@tonic-gate if (addr < segend) 19077c478bd9Sstevel@tonic-gate break; 19087c478bd9Sstevel@tonic-gate } 19097c478bd9Sstevel@tonic-gate 19107c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 19117c478bd9Sstevel@tonic-gate 19127c478bd9Sstevel@tonic-gate if (seg != NULL) 19137c478bd9Sstevel@tonic-gate addr = seg->s_base; 19147c478bd9Sstevel@tonic-gate } 19157c478bd9Sstevel@tonic-gate 19167c478bd9Sstevel@tonic-gate *basep = addr; 19177c478bd9Sstevel@tonic-gate 19187c478bd9Sstevel@tonic-gate if (segend > eaddr) 19197c478bd9Sstevel@tonic-gate *lenp = eaddr - addr; 19207c478bd9Sstevel@tonic-gate else 19217c478bd9Sstevel@tonic-gate *lenp = segend - addr; 19227c478bd9Sstevel@tonic-gate 19237c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 19247c478bd9Sstevel@tonic-gate return (0); 19257c478bd9Sstevel@tonic-gate } 19267c478bd9Sstevel@tonic-gate 19277c478bd9Sstevel@tonic-gate /* 19287c478bd9Sstevel@tonic-gate * Swap the pages associated with the address space as out to 19297c478bd9Sstevel@tonic-gate * secondary storage, returning the number of bytes actually 19307c478bd9Sstevel@tonic-gate * swapped. 19317c478bd9Sstevel@tonic-gate * 19327c478bd9Sstevel@tonic-gate * The value returned is intended to correlate well with the process's 19337c478bd9Sstevel@tonic-gate * memory requirements. Its usefulness for this purpose depends on 19347c478bd9Sstevel@tonic-gate * how well the segment-level routines do at returning accurate 19357c478bd9Sstevel@tonic-gate * information. 19367c478bd9Sstevel@tonic-gate */ 19377c478bd9Sstevel@tonic-gate size_t 19387c478bd9Sstevel@tonic-gate as_swapout(struct as *as) 19397c478bd9Sstevel@tonic-gate { 19407c478bd9Sstevel@tonic-gate struct seg *seg; 19417c478bd9Sstevel@tonic-gate size_t swpcnt = 0; 19427c478bd9Sstevel@tonic-gate 19437c478bd9Sstevel@tonic-gate /* 19447c478bd9Sstevel@tonic-gate * Kernel-only processes have given up their address 19457c478bd9Sstevel@tonic-gate * spaces. Of course, we shouldn't be attempting to 19467c478bd9Sstevel@tonic-gate * swap out such processes in the first place... 19477c478bd9Sstevel@tonic-gate */ 19487c478bd9Sstevel@tonic-gate if (as == NULL) 19497c478bd9Sstevel@tonic-gate return (0); 19507c478bd9Sstevel@tonic-gate 19517c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 19527c478bd9Sstevel@tonic-gate 19537c478bd9Sstevel@tonic-gate /* Prevent XHATs from attaching */ 19547c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 19557c478bd9Sstevel@tonic-gate AS_SETBUSY(as); 19567c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 19577c478bd9Sstevel@tonic-gate 19587c478bd9Sstevel@tonic-gate 19597c478bd9Sstevel@tonic-gate /* 19607c478bd9Sstevel@tonic-gate * Free all mapping resources associated with the address 19617c478bd9Sstevel@tonic-gate * space. The segment-level swapout routines capitalize 19627c478bd9Sstevel@tonic-gate * on this unmapping by scavanging pages that have become 19637c478bd9Sstevel@tonic-gate * unmapped here. 19647c478bd9Sstevel@tonic-gate */ 19657c478bd9Sstevel@tonic-gate hat_swapout(as->a_hat); 19667c478bd9Sstevel@tonic-gate if (as->a_xhat != NULL) 19677c478bd9Sstevel@tonic-gate xhat_swapout_all(as); 19687c478bd9Sstevel@tonic-gate 19697c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 19707c478bd9Sstevel@tonic-gate AS_CLRBUSY(as); 19717c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 19727c478bd9Sstevel@tonic-gate 19737c478bd9Sstevel@tonic-gate /* 19747c478bd9Sstevel@tonic-gate * Call the swapout routines of all segments in the address 19757c478bd9Sstevel@tonic-gate * space to do the actual work, accumulating the amount of 19767c478bd9Sstevel@tonic-gate * space reclaimed. 19777c478bd9Sstevel@tonic-gate */ 19787c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 19797c478bd9Sstevel@tonic-gate struct seg_ops *ov = seg->s_ops; 19807c478bd9Sstevel@tonic-gate 19817c478bd9Sstevel@tonic-gate /* 19827c478bd9Sstevel@tonic-gate * We have to check to see if the seg has 19837c478bd9Sstevel@tonic-gate * an ops vector because the seg may have 19847c478bd9Sstevel@tonic-gate * been in the middle of being set up when 19857c478bd9Sstevel@tonic-gate * the process was picked for swapout. 19867c478bd9Sstevel@tonic-gate */ 19877c478bd9Sstevel@tonic-gate if ((ov != NULL) && (ov->swapout != NULL)) 19887c478bd9Sstevel@tonic-gate swpcnt += SEGOP_SWAPOUT(seg); 19897c478bd9Sstevel@tonic-gate } 19907c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 19917c478bd9Sstevel@tonic-gate return (swpcnt); 19927c478bd9Sstevel@tonic-gate } 19937c478bd9Sstevel@tonic-gate 19947c478bd9Sstevel@tonic-gate /* 19957c478bd9Sstevel@tonic-gate * Determine whether data from the mappings in interval [addr, addr + size) 19967c478bd9Sstevel@tonic-gate * are in the primary memory (core) cache. 19977c478bd9Sstevel@tonic-gate */ 19987c478bd9Sstevel@tonic-gate int 19997c478bd9Sstevel@tonic-gate as_incore(struct as *as, caddr_t addr, 20007c478bd9Sstevel@tonic-gate size_t size, char *vec, size_t *sizep) 20017c478bd9Sstevel@tonic-gate { 20027c478bd9Sstevel@tonic-gate struct seg *seg; 20037c478bd9Sstevel@tonic-gate size_t ssize; 20047c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 20057c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 20067c478bd9Sstevel@tonic-gate size_t isize; /* iteration size */ 20077c478bd9Sstevel@tonic-gate int error = 0; /* result, assume success */ 20087c478bd9Sstevel@tonic-gate 20097c478bd9Sstevel@tonic-gate *sizep = 0; 20107c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 20117c478bd9Sstevel@tonic-gate rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) - 20127c478bd9Sstevel@tonic-gate (size_t)raddr; 20137c478bd9Sstevel@tonic-gate 20147c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 20157c478bd9Sstevel@tonic-gate return (ENOMEM); 20167c478bd9Sstevel@tonic-gate 20177c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 20187c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 20197c478bd9Sstevel@tonic-gate if (seg == NULL) { 20207c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 20217c478bd9Sstevel@tonic-gate return (-1); 20227c478bd9Sstevel@tonic-gate } 20237c478bd9Sstevel@tonic-gate 20247c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 20257c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 20267c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 20277c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 20287c478bd9Sstevel@tonic-gate error = -1; 20297c478bd9Sstevel@tonic-gate break; 20307c478bd9Sstevel@tonic-gate } 20317c478bd9Sstevel@tonic-gate } 20327c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 20337c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 20347c478bd9Sstevel@tonic-gate else 20357c478bd9Sstevel@tonic-gate ssize = rsize; 20367c478bd9Sstevel@tonic-gate *sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec); 20377c478bd9Sstevel@tonic-gate if (isize != ssize) { 20387c478bd9Sstevel@tonic-gate error = -1; 20397c478bd9Sstevel@tonic-gate break; 20407c478bd9Sstevel@tonic-gate } 20417c478bd9Sstevel@tonic-gate vec += btopr(ssize); 20427c478bd9Sstevel@tonic-gate } 20437c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 20447c478bd9Sstevel@tonic-gate return (error); 20457c478bd9Sstevel@tonic-gate } 20467c478bd9Sstevel@tonic-gate 20477c478bd9Sstevel@tonic-gate static void 20487c478bd9Sstevel@tonic-gate as_segunlock(struct seg *seg, caddr_t addr, int attr, 20497c478bd9Sstevel@tonic-gate ulong_t *bitmap, size_t position, size_t npages) 20507c478bd9Sstevel@tonic-gate { 20517c478bd9Sstevel@tonic-gate caddr_t range_start; 20527c478bd9Sstevel@tonic-gate size_t pos1 = position; 20537c478bd9Sstevel@tonic-gate size_t pos2; 20547c478bd9Sstevel@tonic-gate size_t size; 20557c478bd9Sstevel@tonic-gate size_t end_pos = npages + position; 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate while (bt_range(bitmap, &pos1, &pos2, end_pos)) { 20587c478bd9Sstevel@tonic-gate size = ptob((pos2 - pos1)); 20597c478bd9Sstevel@tonic-gate range_start = (caddr_t)((uintptr_t)addr + 20607c478bd9Sstevel@tonic-gate ptob(pos1 - position)); 20617c478bd9Sstevel@tonic-gate 20627c478bd9Sstevel@tonic-gate (void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK, 20637c478bd9Sstevel@tonic-gate (ulong_t *)NULL, (size_t)NULL); 20647c478bd9Sstevel@tonic-gate pos1 = pos2; 20657c478bd9Sstevel@tonic-gate } 20667c478bd9Sstevel@tonic-gate } 20677c478bd9Sstevel@tonic-gate 20687c478bd9Sstevel@tonic-gate static void 20697c478bd9Sstevel@tonic-gate as_unlockerr(struct as *as, int attr, ulong_t *mlock_map, 20707c478bd9Sstevel@tonic-gate caddr_t raddr, size_t rsize) 20717c478bd9Sstevel@tonic-gate { 20727c478bd9Sstevel@tonic-gate struct seg *seg = as_segat(as, raddr); 20737c478bd9Sstevel@tonic-gate size_t ssize; 20747c478bd9Sstevel@tonic-gate 20757c478bd9Sstevel@tonic-gate while (rsize != 0) { 20767c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) 20777c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 20787c478bd9Sstevel@tonic-gate 20797c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 20807c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 20817c478bd9Sstevel@tonic-gate else 20827c478bd9Sstevel@tonic-gate ssize = rsize; 20837c478bd9Sstevel@tonic-gate 20847c478bd9Sstevel@tonic-gate as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize)); 20857c478bd9Sstevel@tonic-gate 20867c478bd9Sstevel@tonic-gate rsize -= ssize; 20877c478bd9Sstevel@tonic-gate raddr += ssize; 20887c478bd9Sstevel@tonic-gate } 20897c478bd9Sstevel@tonic-gate } 20907c478bd9Sstevel@tonic-gate 20917c478bd9Sstevel@tonic-gate /* 20927c478bd9Sstevel@tonic-gate * Cache control operations over the interval [addr, addr + size) in 20937c478bd9Sstevel@tonic-gate * address space "as". 20947c478bd9Sstevel@tonic-gate */ 20957c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20967c478bd9Sstevel@tonic-gate int 20977c478bd9Sstevel@tonic-gate as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr, 20987c478bd9Sstevel@tonic-gate uintptr_t arg, ulong_t *lock_map, size_t pos) 20997c478bd9Sstevel@tonic-gate { 21007c478bd9Sstevel@tonic-gate struct seg *seg; /* working segment */ 21017c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 21027c478bd9Sstevel@tonic-gate caddr_t initraddr; /* saved initial rounded down addr */ 21037c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 21047c478bd9Sstevel@tonic-gate size_t initrsize; /* saved initial rounded up size */ 21057c478bd9Sstevel@tonic-gate size_t ssize; /* size of seg */ 21067c478bd9Sstevel@tonic-gate int error = 0; /* result */ 21077c478bd9Sstevel@tonic-gate size_t mlock_size; /* size of bitmap */ 21087c478bd9Sstevel@tonic-gate ulong_t *mlock_map; /* pointer to bitmap used */ 21097c478bd9Sstevel@tonic-gate /* to represent the locked */ 21107c478bd9Sstevel@tonic-gate /* pages. */ 21117c478bd9Sstevel@tonic-gate retry: 21127c478bd9Sstevel@tonic-gate if (error == IE_RETRY) 21137c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 21147c478bd9Sstevel@tonic-gate else 21157c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 21167c478bd9Sstevel@tonic-gate 21177c478bd9Sstevel@tonic-gate /* 21187c478bd9Sstevel@tonic-gate * If these are address space lock/unlock operations, loop over 21197c478bd9Sstevel@tonic-gate * all segments in the address space, as appropriate. 21207c478bd9Sstevel@tonic-gate */ 21217c478bd9Sstevel@tonic-gate if (func == MC_LOCKAS) { 21227c478bd9Sstevel@tonic-gate size_t npages, idx; 21237c478bd9Sstevel@tonic-gate size_t rlen = 0; /* rounded as length */ 21247c478bd9Sstevel@tonic-gate 21257c478bd9Sstevel@tonic-gate idx = pos; 21267c478bd9Sstevel@tonic-gate 21277c478bd9Sstevel@tonic-gate if (arg & MCL_FUTURE) { 21287c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 21297c478bd9Sstevel@tonic-gate AS_SETPGLCK(as); 21307c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 21317c478bd9Sstevel@tonic-gate } 21327c478bd9Sstevel@tonic-gate if ((arg & MCL_CURRENT) == 0) { 21337c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21347c478bd9Sstevel@tonic-gate return (0); 21357c478bd9Sstevel@tonic-gate } 21367c478bd9Sstevel@tonic-gate 21377c478bd9Sstevel@tonic-gate seg = AS_SEGFIRST(as); 21387c478bd9Sstevel@tonic-gate if (seg == NULL) { 21397c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21407c478bd9Sstevel@tonic-gate return (0); 21417c478bd9Sstevel@tonic-gate } 21427c478bd9Sstevel@tonic-gate 21437c478bd9Sstevel@tonic-gate do { 21447c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)seg->s_base & 21457c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK); 21467c478bd9Sstevel@tonic-gate rlen += (((uintptr_t)(seg->s_base + seg->s_size) + 21477c478bd9Sstevel@tonic-gate PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr; 21487c478bd9Sstevel@tonic-gate } while ((seg = AS_SEGNEXT(as, seg)) != NULL); 21497c478bd9Sstevel@tonic-gate 21507c478bd9Sstevel@tonic-gate mlock_size = BT_BITOUL(btopr(rlen)); 21517c478bd9Sstevel@tonic-gate if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 21527c478bd9Sstevel@tonic-gate sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 21537c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21547c478bd9Sstevel@tonic-gate return (EAGAIN); 21557c478bd9Sstevel@tonic-gate } 21567c478bd9Sstevel@tonic-gate 21577c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 21587c478bd9Sstevel@tonic-gate error = SEGOP_LOCKOP(seg, seg->s_base, 21597c478bd9Sstevel@tonic-gate seg->s_size, attr, MC_LOCK, mlock_map, pos); 21607c478bd9Sstevel@tonic-gate if (error != 0) 21617c478bd9Sstevel@tonic-gate break; 21627c478bd9Sstevel@tonic-gate pos += seg_pages(seg); 21637c478bd9Sstevel@tonic-gate } 21647c478bd9Sstevel@tonic-gate 21657c478bd9Sstevel@tonic-gate if (error) { 21667c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; 21677c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg)) { 21687c478bd9Sstevel@tonic-gate 21697c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)seg->s_base & 21707c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK); 21717c478bd9Sstevel@tonic-gate npages = seg_pages(seg); 21727c478bd9Sstevel@tonic-gate as_segunlock(seg, raddr, attr, mlock_map, 21737c478bd9Sstevel@tonic-gate idx, npages); 21747c478bd9Sstevel@tonic-gate idx += npages; 21757c478bd9Sstevel@tonic-gate } 21767c478bd9Sstevel@tonic-gate } 21777c478bd9Sstevel@tonic-gate 21787c478bd9Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 21797c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21807c478bd9Sstevel@tonic-gate goto lockerr; 21817c478bd9Sstevel@tonic-gate } else if (func == MC_UNLOCKAS) { 21827c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 21837c478bd9Sstevel@tonic-gate AS_CLRPGLCK(as); 21847c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 21857c478bd9Sstevel@tonic-gate 21867c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 21877c478bd9Sstevel@tonic-gate error = SEGOP_LOCKOP(seg, seg->s_base, 21887c478bd9Sstevel@tonic-gate seg->s_size, attr, MC_UNLOCK, NULL, 0); 21897c478bd9Sstevel@tonic-gate if (error != 0) 21907c478bd9Sstevel@tonic-gate break; 21917c478bd9Sstevel@tonic-gate } 21927c478bd9Sstevel@tonic-gate 21937c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 21947c478bd9Sstevel@tonic-gate goto lockerr; 21957c478bd9Sstevel@tonic-gate } 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate /* 21987c478bd9Sstevel@tonic-gate * Normalize addresses and sizes. 21997c478bd9Sstevel@tonic-gate */ 22007c478bd9Sstevel@tonic-gate initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 22017c478bd9Sstevel@tonic-gate initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 22027c478bd9Sstevel@tonic-gate (size_t)raddr; 22037c478bd9Sstevel@tonic-gate 22047c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) { /* check for wraparound */ 22057c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22067c478bd9Sstevel@tonic-gate return (ENOMEM); 22077c478bd9Sstevel@tonic-gate } 22087c478bd9Sstevel@tonic-gate 22097c478bd9Sstevel@tonic-gate /* 22107c478bd9Sstevel@tonic-gate * Get initial segment. 22117c478bd9Sstevel@tonic-gate */ 22127c478bd9Sstevel@tonic-gate if ((seg = as_segat(as, raddr)) == NULL) { 22137c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22147c478bd9Sstevel@tonic-gate return (ENOMEM); 22157c478bd9Sstevel@tonic-gate } 22167c478bd9Sstevel@tonic-gate 22177c478bd9Sstevel@tonic-gate if (func == MC_LOCK) { 22187c478bd9Sstevel@tonic-gate mlock_size = BT_BITOUL(btopr(rsize)); 22197c478bd9Sstevel@tonic-gate if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 22207c478bd9Sstevel@tonic-gate sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 22217c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22227c478bd9Sstevel@tonic-gate return (EAGAIN); 22237c478bd9Sstevel@tonic-gate } 22247c478bd9Sstevel@tonic-gate } 22257c478bd9Sstevel@tonic-gate 22267c478bd9Sstevel@tonic-gate /* 22277c478bd9Sstevel@tonic-gate * Loop over all segments. If a hole in the address range is 22287c478bd9Sstevel@tonic-gate * discovered, then fail. For each segment, perform the appropriate 22297c478bd9Sstevel@tonic-gate * control operation. 22307c478bd9Sstevel@tonic-gate */ 22317c478bd9Sstevel@tonic-gate while (rsize != 0) { 22327c478bd9Sstevel@tonic-gate 22337c478bd9Sstevel@tonic-gate /* 22347c478bd9Sstevel@tonic-gate * Make sure there's no hole, calculate the portion 22357c478bd9Sstevel@tonic-gate * of the next segment to be operated over. 22367c478bd9Sstevel@tonic-gate */ 22377c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 22387c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 22397c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 22407c478bd9Sstevel@tonic-gate if (func == MC_LOCK) { 22417c478bd9Sstevel@tonic-gate as_unlockerr(as, attr, mlock_map, 22427c478bd9Sstevel@tonic-gate initraddr, initrsize - rsize); 22437c478bd9Sstevel@tonic-gate kmem_free(mlock_map, 22447c478bd9Sstevel@tonic-gate mlock_size * sizeof (ulong_t)); 22457c478bd9Sstevel@tonic-gate } 22467c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22477c478bd9Sstevel@tonic-gate return (ENOMEM); 22487c478bd9Sstevel@tonic-gate } 22497c478bd9Sstevel@tonic-gate } 22507c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 22517c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 22527c478bd9Sstevel@tonic-gate else 22537c478bd9Sstevel@tonic-gate ssize = rsize; 22547c478bd9Sstevel@tonic-gate 22557c478bd9Sstevel@tonic-gate /* 22567c478bd9Sstevel@tonic-gate * Dispatch on specific function. 22577c478bd9Sstevel@tonic-gate */ 22587c478bd9Sstevel@tonic-gate switch (func) { 22597c478bd9Sstevel@tonic-gate 22607c478bd9Sstevel@tonic-gate /* 22617c478bd9Sstevel@tonic-gate * Synchronize cached data from mappings with backing 22627c478bd9Sstevel@tonic-gate * objects. 22637c478bd9Sstevel@tonic-gate */ 22647c478bd9Sstevel@tonic-gate case MC_SYNC: 22657c478bd9Sstevel@tonic-gate if (error = SEGOP_SYNC(seg, raddr, ssize, 22667c478bd9Sstevel@tonic-gate attr, (uint_t)arg)) { 22677c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22687c478bd9Sstevel@tonic-gate return (error); 22697c478bd9Sstevel@tonic-gate } 22707c478bd9Sstevel@tonic-gate break; 22717c478bd9Sstevel@tonic-gate 22727c478bd9Sstevel@tonic-gate /* 22737c478bd9Sstevel@tonic-gate * Lock pages in memory. 22747c478bd9Sstevel@tonic-gate */ 22757c478bd9Sstevel@tonic-gate case MC_LOCK: 22767c478bd9Sstevel@tonic-gate if (error = SEGOP_LOCKOP(seg, raddr, ssize, 22777c478bd9Sstevel@tonic-gate attr, func, mlock_map, pos)) { 22787c478bd9Sstevel@tonic-gate as_unlockerr(as, attr, mlock_map, initraddr, 22797c478bd9Sstevel@tonic-gate initrsize - rsize + ssize); 22807c478bd9Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * 22817c478bd9Sstevel@tonic-gate sizeof (ulong_t)); 22827c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 22837c478bd9Sstevel@tonic-gate goto lockerr; 22847c478bd9Sstevel@tonic-gate } 22857c478bd9Sstevel@tonic-gate break; 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate /* 22887c478bd9Sstevel@tonic-gate * Unlock mapped pages. 22897c478bd9Sstevel@tonic-gate */ 22907c478bd9Sstevel@tonic-gate case MC_UNLOCK: 22917c478bd9Sstevel@tonic-gate (void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func, 22927c478bd9Sstevel@tonic-gate (ulong_t *)NULL, (size_t)NULL); 22937c478bd9Sstevel@tonic-gate break; 22947c478bd9Sstevel@tonic-gate 22957c478bd9Sstevel@tonic-gate /* 22967c478bd9Sstevel@tonic-gate * Store VM advise for mapped pages in segment layer. 22977c478bd9Sstevel@tonic-gate */ 22987c478bd9Sstevel@tonic-gate case MC_ADVISE: 22997c478bd9Sstevel@tonic-gate error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg); 23007c478bd9Sstevel@tonic-gate 23017c478bd9Sstevel@tonic-gate /* 23027c478bd9Sstevel@tonic-gate * Check for regular errors and special retry error 23037c478bd9Sstevel@tonic-gate */ 23047c478bd9Sstevel@tonic-gate if (error) { 23057c478bd9Sstevel@tonic-gate if (error == IE_RETRY) { 23067c478bd9Sstevel@tonic-gate /* 23077c478bd9Sstevel@tonic-gate * Need to acquire writers lock, so 23087c478bd9Sstevel@tonic-gate * have to drop readers lock and start 23097c478bd9Sstevel@tonic-gate * all over again 23107c478bd9Sstevel@tonic-gate */ 23117c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23127c478bd9Sstevel@tonic-gate goto retry; 23137c478bd9Sstevel@tonic-gate } else if (error == IE_REATTACH) { 23147c478bd9Sstevel@tonic-gate /* 23157c478bd9Sstevel@tonic-gate * Find segment for current address 23167c478bd9Sstevel@tonic-gate * because current segment just got 23177c478bd9Sstevel@tonic-gate * split or concatenated 23187c478bd9Sstevel@tonic-gate */ 23197c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 23207c478bd9Sstevel@tonic-gate if (seg == NULL) { 23217c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23227c478bd9Sstevel@tonic-gate return (ENOMEM); 23237c478bd9Sstevel@tonic-gate } 23247c478bd9Sstevel@tonic-gate } else { 23257c478bd9Sstevel@tonic-gate /* 23267c478bd9Sstevel@tonic-gate * Regular error 23277c478bd9Sstevel@tonic-gate */ 23287c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23297c478bd9Sstevel@tonic-gate return (error); 23307c478bd9Sstevel@tonic-gate } 23317c478bd9Sstevel@tonic-gate } 23327c478bd9Sstevel@tonic-gate break; 23337c478bd9Sstevel@tonic-gate 23347c478bd9Sstevel@tonic-gate /* 23357c478bd9Sstevel@tonic-gate * Can't happen. 23367c478bd9Sstevel@tonic-gate */ 23377c478bd9Sstevel@tonic-gate default: 23387c478bd9Sstevel@tonic-gate panic("as_ctl: bad operation %d", func); 23397c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 23407c478bd9Sstevel@tonic-gate } 23417c478bd9Sstevel@tonic-gate 23427c478bd9Sstevel@tonic-gate rsize -= ssize; 23437c478bd9Sstevel@tonic-gate raddr += ssize; 23447c478bd9Sstevel@tonic-gate } 23457c478bd9Sstevel@tonic-gate 23467c478bd9Sstevel@tonic-gate if (func == MC_LOCK) 23477c478bd9Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 23487c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 23497c478bd9Sstevel@tonic-gate return (0); 23507c478bd9Sstevel@tonic-gate lockerr: 23517c478bd9Sstevel@tonic-gate 23527c478bd9Sstevel@tonic-gate /* 23537c478bd9Sstevel@tonic-gate * If the lower levels returned EDEADLK for a segment lockop, 23547c478bd9Sstevel@tonic-gate * it means that we should retry the operation. Let's wait 23557c478bd9Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 23567c478bd9Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 23577c478bd9Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 23587c478bd9Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 23597c478bd9Sstevel@tonic-gate * 4125102 for details of the problem. 23607c478bd9Sstevel@tonic-gate */ 23617c478bd9Sstevel@tonic-gate if (error == EDEADLK) { 23627c478bd9Sstevel@tonic-gate delay(deadlk_wait); 23637c478bd9Sstevel@tonic-gate error = 0; 23647c478bd9Sstevel@tonic-gate goto retry; 23657c478bd9Sstevel@tonic-gate } 23667c478bd9Sstevel@tonic-gate return (error); 23677c478bd9Sstevel@tonic-gate } 23687c478bd9Sstevel@tonic-gate 23697c478bd9Sstevel@tonic-gate /* 23707c478bd9Sstevel@tonic-gate * Special code for exec to move the stack segment from its interim 23717c478bd9Sstevel@tonic-gate * place in the old address to the right place in the new address space. 23727c478bd9Sstevel@tonic-gate */ 23737c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 23747c478bd9Sstevel@tonic-gate int 23757c478bd9Sstevel@tonic-gate as_exec(struct as *oas, caddr_t ostka, size_t stksz, 23767c478bd9Sstevel@tonic-gate struct as *nas, caddr_t nstka, uint_t hatflag) 23777c478bd9Sstevel@tonic-gate { 23787c478bd9Sstevel@tonic-gate struct seg *stkseg; 23797c478bd9Sstevel@tonic-gate 23807c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER); 23817c478bd9Sstevel@tonic-gate stkseg = as_segat(oas, ostka); 23827c478bd9Sstevel@tonic-gate stkseg = as_removeseg(oas, stkseg); 23837c478bd9Sstevel@tonic-gate ASSERT(stkseg != NULL); 23847c478bd9Sstevel@tonic-gate ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz); 23857c478bd9Sstevel@tonic-gate stkseg->s_as = nas; 23867c478bd9Sstevel@tonic-gate stkseg->s_base = nstka; 23877c478bd9Sstevel@tonic-gate 23887c478bd9Sstevel@tonic-gate /* 23897c478bd9Sstevel@tonic-gate * It's ok to lock the address space we are about to exec to. 23907c478bd9Sstevel@tonic-gate */ 23917c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER); 23927c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&nas->a_wpage) == 0); 23937c478bd9Sstevel@tonic-gate nas->a_size += stkseg->s_size; 23947c478bd9Sstevel@tonic-gate oas->a_size -= stkseg->s_size; 23957c478bd9Sstevel@tonic-gate (void) as_addseg(nas, stkseg); 23967c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(nas, &nas->a_lock); 23977c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(oas, &oas->a_lock); 23987c478bd9Sstevel@tonic-gate return (0); 23997c478bd9Sstevel@tonic-gate } 24007c478bd9Sstevel@tonic-gate 24017c478bd9Sstevel@tonic-gate static int 24027c478bd9Sstevel@tonic-gate f_decode(faultcode_t fault_err) 24037c478bd9Sstevel@tonic-gate { 24047c478bd9Sstevel@tonic-gate int error = 0; 24057c478bd9Sstevel@tonic-gate 24067c478bd9Sstevel@tonic-gate switch (FC_CODE(fault_err)) { 24077c478bd9Sstevel@tonic-gate case FC_OBJERR: 24087c478bd9Sstevel@tonic-gate error = FC_ERRNO(fault_err); 24097c478bd9Sstevel@tonic-gate break; 24107c478bd9Sstevel@tonic-gate case FC_PROT: 24117c478bd9Sstevel@tonic-gate error = EACCES; 24127c478bd9Sstevel@tonic-gate break; 24137c478bd9Sstevel@tonic-gate default: 24147c478bd9Sstevel@tonic-gate error = EFAULT; 24157c478bd9Sstevel@tonic-gate break; 24167c478bd9Sstevel@tonic-gate } 24177c478bd9Sstevel@tonic-gate return (error); 24187c478bd9Sstevel@tonic-gate } 24197c478bd9Sstevel@tonic-gate 24207c478bd9Sstevel@tonic-gate /* 24217c478bd9Sstevel@tonic-gate * lock pages in a given address space. Return shadow list. If 24227c478bd9Sstevel@tonic-gate * the list is NULL, the MMU mapping is also locked. 24237c478bd9Sstevel@tonic-gate */ 24247c478bd9Sstevel@tonic-gate int 24257c478bd9Sstevel@tonic-gate as_pagelock(struct as *as, struct page ***ppp, caddr_t addr, 24267c478bd9Sstevel@tonic-gate size_t size, enum seg_rw rw) 24277c478bd9Sstevel@tonic-gate { 24287c478bd9Sstevel@tonic-gate size_t rsize; 24297c478bd9Sstevel@tonic-gate caddr_t base; 24307c478bd9Sstevel@tonic-gate caddr_t raddr; 24317c478bd9Sstevel@tonic-gate faultcode_t fault_err; 24327c478bd9Sstevel@tonic-gate struct seg *seg; 24337c478bd9Sstevel@tonic-gate int res; 24347c478bd9Sstevel@tonic-gate int prefaulted = 0; 24357c478bd9Sstevel@tonic-gate 24367c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START, 24377c478bd9Sstevel@tonic-gate "as_pagelock_start: addr %p size %ld", addr, size); 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 24407c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 24417c478bd9Sstevel@tonic-gate (size_t)raddr; 24427c478bd9Sstevel@tonic-gate top: 24437c478bd9Sstevel@tonic-gate /* 24447c478bd9Sstevel@tonic-gate * if the request crosses two segments let 24457c478bd9Sstevel@tonic-gate * as_fault handle it. 24467c478bd9Sstevel@tonic-gate */ 24477c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 24487c478bd9Sstevel@tonic-gate seg = as_findseg(as, addr, 0); 24497c478bd9Sstevel@tonic-gate if ((seg == NULL) || ((base = seg->s_base) > addr) || 24507c478bd9Sstevel@tonic-gate (addr + size) > base + seg->s_size) { 24517c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24527c478bd9Sstevel@tonic-gate goto slow; 24537c478bd9Sstevel@tonic-gate } 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START, 24567c478bd9Sstevel@tonic-gate "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize); 24577c478bd9Sstevel@tonic-gate 24587c478bd9Sstevel@tonic-gate /* 24597c478bd9Sstevel@tonic-gate * try to lock pages and pass back shadow list 24607c478bd9Sstevel@tonic-gate */ 24617c478bd9Sstevel@tonic-gate res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw); 24627c478bd9Sstevel@tonic-gate 24637c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end"); 24647c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 24657c478bd9Sstevel@tonic-gate if (res == 0) { 24667c478bd9Sstevel@tonic-gate return (0); 24677c478bd9Sstevel@tonic-gate } else if (res == ENOTSUP || prefaulted) { 24687c478bd9Sstevel@tonic-gate /* 24697c478bd9Sstevel@tonic-gate * (1) segment driver doesn't support PAGELOCK fastpath, or 24707c478bd9Sstevel@tonic-gate * (2) we've already tried fast path unsuccessfully after 24717c478bd9Sstevel@tonic-gate * faulting in the addr range below; system might be 24727c478bd9Sstevel@tonic-gate * thrashing or there may not be enough availrmem. 24737c478bd9Sstevel@tonic-gate */ 24747c478bd9Sstevel@tonic-gate goto slow; 24757c478bd9Sstevel@tonic-gate } 24767c478bd9Sstevel@tonic-gate 24777c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_FAULT_START, 24787c478bd9Sstevel@tonic-gate "as_fault_start: addr %p size %ld", addr, size); 24797c478bd9Sstevel@tonic-gate 24807c478bd9Sstevel@tonic-gate /* 24817c478bd9Sstevel@tonic-gate * we might get here because of some COW fault or non 24827c478bd9Sstevel@tonic-gate * existing page. Let as_fault deal with it. Just load 24837c478bd9Sstevel@tonic-gate * the page, don't lock the MMU mapping. 24847c478bd9Sstevel@tonic-gate */ 24857c478bd9Sstevel@tonic-gate fault_err = as_fault(as->a_hat, as, addr, size, F_INVAL, rw); 24867c478bd9Sstevel@tonic-gate if (fault_err != 0) { 24877c478bd9Sstevel@tonic-gate return (f_decode(fault_err)); 24887c478bd9Sstevel@tonic-gate } 24897c478bd9Sstevel@tonic-gate 24907c478bd9Sstevel@tonic-gate prefaulted = 1; 24917c478bd9Sstevel@tonic-gate 24927c478bd9Sstevel@tonic-gate /* 24937c478bd9Sstevel@tonic-gate * try fast path again; since we've dropped a_lock, 24947c478bd9Sstevel@tonic-gate * we need to try the dance from the start to see if 24957c478bd9Sstevel@tonic-gate * the addr range is still valid. 24967c478bd9Sstevel@tonic-gate */ 24977c478bd9Sstevel@tonic-gate goto top; 24987c478bd9Sstevel@tonic-gate slow: 24997c478bd9Sstevel@tonic-gate /* 25007c478bd9Sstevel@tonic-gate * load the page and lock the MMU mapping. 25017c478bd9Sstevel@tonic-gate */ 25027c478bd9Sstevel@tonic-gate fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw); 25037c478bd9Sstevel@tonic-gate if (fault_err != 0) { 25047c478bd9Sstevel@tonic-gate return (f_decode(fault_err)); 25057c478bd9Sstevel@tonic-gate } 25067c478bd9Sstevel@tonic-gate *ppp = NULL; 25077c478bd9Sstevel@tonic-gate 25087c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end"); 25097c478bd9Sstevel@tonic-gate return (0); 25107c478bd9Sstevel@tonic-gate } 25117c478bd9Sstevel@tonic-gate 25127c478bd9Sstevel@tonic-gate /* 25137c478bd9Sstevel@tonic-gate * unlock pages in a given address range 25147c478bd9Sstevel@tonic-gate */ 25157c478bd9Sstevel@tonic-gate void 25167c478bd9Sstevel@tonic-gate as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size, 25177c478bd9Sstevel@tonic-gate enum seg_rw rw) 25187c478bd9Sstevel@tonic-gate { 25197c478bd9Sstevel@tonic-gate struct seg *seg; 25207c478bd9Sstevel@tonic-gate size_t rsize; 25217c478bd9Sstevel@tonic-gate caddr_t raddr; 25227c478bd9Sstevel@tonic-gate 25237c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START, 25247c478bd9Sstevel@tonic-gate "as_pageunlock_start: addr %p size %ld", addr, size); 25257c478bd9Sstevel@tonic-gate 25267c478bd9Sstevel@tonic-gate /* 25277c478bd9Sstevel@tonic-gate * if the shadow list is NULL, as_pagelock was 25287c478bd9Sstevel@tonic-gate * falling back to as_fault 25297c478bd9Sstevel@tonic-gate */ 25307c478bd9Sstevel@tonic-gate if (pp == NULL) { 25317c478bd9Sstevel@tonic-gate (void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw); 25327c478bd9Sstevel@tonic-gate return; 25337c478bd9Sstevel@tonic-gate } 25347c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 25357c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 25367c478bd9Sstevel@tonic-gate (size_t)raddr; 25377c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 25387c478bd9Sstevel@tonic-gate seg = as_findseg(as, addr, 0); 25397c478bd9Sstevel@tonic-gate ASSERT(seg); 25407c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START, 25417c478bd9Sstevel@tonic-gate "seg_unlock_start: raddr %p rsize %ld", raddr, rsize); 25427c478bd9Sstevel@tonic-gate SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw); 25437c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 25447c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end"); 25457c478bd9Sstevel@tonic-gate } 25467c478bd9Sstevel@tonic-gate 25477c478bd9Sstevel@tonic-gate /* 25487c478bd9Sstevel@tonic-gate * reclaim cached pages in a given address range 25497c478bd9Sstevel@tonic-gate */ 25507c478bd9Sstevel@tonic-gate void 25517c478bd9Sstevel@tonic-gate as_pagereclaim(struct as *as, struct page **pp, caddr_t addr, 25527c478bd9Sstevel@tonic-gate size_t size, enum seg_rw rw) 25537c478bd9Sstevel@tonic-gate { 25547c478bd9Sstevel@tonic-gate struct seg *seg; 25557c478bd9Sstevel@tonic-gate size_t rsize; 25567c478bd9Sstevel@tonic-gate caddr_t raddr; 25577c478bd9Sstevel@tonic-gate 25587c478bd9Sstevel@tonic-gate ASSERT(AS_READ_HELD(as, &as->a_lock)); 25597c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 25607c478bd9Sstevel@tonic-gate 25617c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 25627c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 25637c478bd9Sstevel@tonic-gate (size_t)raddr; 25647c478bd9Sstevel@tonic-gate seg = as_findseg(as, addr, 0); 25657c478bd9Sstevel@tonic-gate ASSERT(seg); 25667c478bd9Sstevel@tonic-gate SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGERECLAIM, rw); 25677c478bd9Sstevel@tonic-gate } 25687c478bd9Sstevel@tonic-gate 25697c478bd9Sstevel@tonic-gate #define MAXPAGEFLIP 4 25707c478bd9Sstevel@tonic-gate #define MAXPAGEFLIPSIZ MAXPAGEFLIP*PAGESIZE 25717c478bd9Sstevel@tonic-gate 25727c478bd9Sstevel@tonic-gate int 25737c478bd9Sstevel@tonic-gate as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc, 25747c478bd9Sstevel@tonic-gate boolean_t wait) 25757c478bd9Sstevel@tonic-gate { 25767c478bd9Sstevel@tonic-gate struct seg *seg; 25777c478bd9Sstevel@tonic-gate size_t ssize; 25787c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 25797c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 25807c478bd9Sstevel@tonic-gate int error = 0; 25817c478bd9Sstevel@tonic-gate size_t pgsz = page_get_pagesize(szc); 25827c478bd9Sstevel@tonic-gate 25837c478bd9Sstevel@tonic-gate setpgsz_top: 25847c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) { 25857c478bd9Sstevel@tonic-gate return (EINVAL); 25867c478bd9Sstevel@tonic-gate } 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate raddr = addr; 25897c478bd9Sstevel@tonic-gate rsize = size; 25907c478bd9Sstevel@tonic-gate 25917c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 25927c478bd9Sstevel@tonic-gate return (ENOMEM); 25937c478bd9Sstevel@tonic-gate 25947c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER); 25957c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 25967c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 25977c478bd9Sstevel@tonic-gate if (seg == NULL) { 25987c478bd9Sstevel@tonic-gate as_setwatch(as); 25997c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 26007c478bd9Sstevel@tonic-gate return (ENOMEM); 26017c478bd9Sstevel@tonic-gate } 26027c478bd9Sstevel@tonic-gate 26037c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 26047c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 26057c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 26067c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 26077c478bd9Sstevel@tonic-gate error = ENOMEM; 26087c478bd9Sstevel@tonic-gate break; 26097c478bd9Sstevel@tonic-gate } 26107c478bd9Sstevel@tonic-gate } 26117c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 26127c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 26137c478bd9Sstevel@tonic-gate } else { 26147c478bd9Sstevel@tonic-gate ssize = rsize; 26157c478bd9Sstevel@tonic-gate } 26167c478bd9Sstevel@tonic-gate 26177c478bd9Sstevel@tonic-gate error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 26187c478bd9Sstevel@tonic-gate 26197c478bd9Sstevel@tonic-gate if (error == IE_NOMEM) { 26207c478bd9Sstevel@tonic-gate error = EAGAIN; 26217c478bd9Sstevel@tonic-gate break; 26227c478bd9Sstevel@tonic-gate } 26237c478bd9Sstevel@tonic-gate 26247c478bd9Sstevel@tonic-gate if (error == IE_RETRY) { 26257c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 26267c478bd9Sstevel@tonic-gate goto setpgsz_top; 26277c478bd9Sstevel@tonic-gate } 26287c478bd9Sstevel@tonic-gate 26297c478bd9Sstevel@tonic-gate if (error == ENOTSUP) { 26307c478bd9Sstevel@tonic-gate error = EINVAL; 26317c478bd9Sstevel@tonic-gate break; 26327c478bd9Sstevel@tonic-gate } 26337c478bd9Sstevel@tonic-gate 26347c478bd9Sstevel@tonic-gate if (wait && (error == EAGAIN)) { 26357c478bd9Sstevel@tonic-gate /* 26367c478bd9Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 26377c478bd9Sstevel@tonic-gate * before this operation can succeed through a retry. 26387c478bd9Sstevel@tonic-gate * The possible reasons for locked memory and 26397c478bd9Sstevel@tonic-gate * corresponding strategies for unlocking are: 26407c478bd9Sstevel@tonic-gate * (1) Normal I/O 26417c478bd9Sstevel@tonic-gate * wait for a signal that the I/O operation 26427c478bd9Sstevel@tonic-gate * has completed and the memory is unlocked. 26437c478bd9Sstevel@tonic-gate * (2) Asynchronous I/O 26447c478bd9Sstevel@tonic-gate * The aio subsystem does not unlock pages when 26457c478bd9Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 26467c478bd9Sstevel@tonic-gate * when the application calls aiowait/aioerror. 26477c478bd9Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 26487c478bd9Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 26497c478bd9Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 26507c478bd9Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 26517c478bd9Sstevel@tonic-gate * (3) Long term page locking: 26527c478bd9Sstevel@tonic-gate * This is not relevant for as_setpagesize() 26537c478bd9Sstevel@tonic-gate * because we cannot change the page size for 26547c478bd9Sstevel@tonic-gate * driver memory. The attempt to do so will 26557c478bd9Sstevel@tonic-gate * fail with a different error than EAGAIN so 26567c478bd9Sstevel@tonic-gate * there's no need to trigger as callbacks like 26577c478bd9Sstevel@tonic-gate * as_unmap, as_setprot or as_free would do. 26587c478bd9Sstevel@tonic-gate */ 26597c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 26607c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) { 26617c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 26627c478bd9Sstevel@tonic-gate } 26637c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 26647c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 26657c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) { 26667c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 26677c478bd9Sstevel@tonic-gate } 26687c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 26697c478bd9Sstevel@tonic-gate goto setpgsz_top; 26707c478bd9Sstevel@tonic-gate } else if (error != 0) { 26717c478bd9Sstevel@tonic-gate break; 26727c478bd9Sstevel@tonic-gate } 26737c478bd9Sstevel@tonic-gate } 26747c478bd9Sstevel@tonic-gate as_setwatch(as); 26757c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 26767c478bd9Sstevel@tonic-gate return (error); 26777c478bd9Sstevel@tonic-gate } 26787c478bd9Sstevel@tonic-gate 26797c478bd9Sstevel@tonic-gate /* 26807c478bd9Sstevel@tonic-gate * Setup all of the uninitialized watched pages that we can. 26817c478bd9Sstevel@tonic-gate */ 26827c478bd9Sstevel@tonic-gate void 26837c478bd9Sstevel@tonic-gate as_setwatch(struct as *as) 26847c478bd9Sstevel@tonic-gate { 26857c478bd9Sstevel@tonic-gate struct watched_page *pwp; 26867c478bd9Sstevel@tonic-gate struct seg *seg; 26877c478bd9Sstevel@tonic-gate caddr_t vaddr; 26887c478bd9Sstevel@tonic-gate uint_t prot; 26897c478bd9Sstevel@tonic-gate int err, retrycnt; 26907c478bd9Sstevel@tonic-gate 26917c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 26927c478bd9Sstevel@tonic-gate return; 26937c478bd9Sstevel@tonic-gate 26947c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 26957c478bd9Sstevel@tonic-gate 26967c478bd9Sstevel@tonic-gate for (pwp = avl_first(&as->a_wpage); pwp != NULL; 26977c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp)) { 26987c478bd9Sstevel@tonic-gate retrycnt = 0; 26997c478bd9Sstevel@tonic-gate retry: 27007c478bd9Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 27017c478bd9Sstevel@tonic-gate if (pwp->wp_oprot != 0 || /* already set up */ 27027c478bd9Sstevel@tonic-gate (seg = as_segat(as, vaddr)) == NULL || 27037c478bd9Sstevel@tonic-gate SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0) 27047c478bd9Sstevel@tonic-gate continue; 27057c478bd9Sstevel@tonic-gate 27067c478bd9Sstevel@tonic-gate pwp->wp_oprot = prot; 27077c478bd9Sstevel@tonic-gate if (pwp->wp_read) 27087c478bd9Sstevel@tonic-gate prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 27097c478bd9Sstevel@tonic-gate if (pwp->wp_write) 27107c478bd9Sstevel@tonic-gate prot &= ~PROT_WRITE; 27117c478bd9Sstevel@tonic-gate if (pwp->wp_exec) 27127c478bd9Sstevel@tonic-gate prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 27137c478bd9Sstevel@tonic-gate if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) { 27147c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 27157c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 27167c478bd9Sstevel@tonic-gate pwp->wp_oprot = 0; 27177c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 27187c478bd9Sstevel@tonic-gate retrycnt++; 27197c478bd9Sstevel@tonic-gate goto retry; 27207c478bd9Sstevel@tonic-gate } 27217c478bd9Sstevel@tonic-gate } 27227c478bd9Sstevel@tonic-gate pwp->wp_prot = prot; 27237c478bd9Sstevel@tonic-gate } 27247c478bd9Sstevel@tonic-gate } 27257c478bd9Sstevel@tonic-gate 27267c478bd9Sstevel@tonic-gate /* 27277c478bd9Sstevel@tonic-gate * Clear all of the watched pages in the address space. 27287c478bd9Sstevel@tonic-gate */ 27297c478bd9Sstevel@tonic-gate void 27307c478bd9Sstevel@tonic-gate as_clearwatch(struct as *as) 27317c478bd9Sstevel@tonic-gate { 27327c478bd9Sstevel@tonic-gate struct watched_page *pwp; 27337c478bd9Sstevel@tonic-gate struct seg *seg; 27347c478bd9Sstevel@tonic-gate caddr_t vaddr; 27357c478bd9Sstevel@tonic-gate uint_t prot; 27367c478bd9Sstevel@tonic-gate int err, retrycnt; 27377c478bd9Sstevel@tonic-gate 27387c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 27397c478bd9Sstevel@tonic-gate return; 27407c478bd9Sstevel@tonic-gate 27417c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 27427c478bd9Sstevel@tonic-gate 27437c478bd9Sstevel@tonic-gate for (pwp = avl_first(&as->a_wpage); pwp != NULL; 27447c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp)) { 27457c478bd9Sstevel@tonic-gate retrycnt = 0; 27467c478bd9Sstevel@tonic-gate retry: 27477c478bd9Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 27487c478bd9Sstevel@tonic-gate if (pwp->wp_oprot == 0 || /* not set up */ 27497c478bd9Sstevel@tonic-gate (seg = as_segat(as, vaddr)) == NULL) 27507c478bd9Sstevel@tonic-gate continue; 27517c478bd9Sstevel@tonic-gate 27527c478bd9Sstevel@tonic-gate if ((prot = pwp->wp_oprot) != pwp->wp_prot) { 27537c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 27547c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 27557c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 27567c478bd9Sstevel@tonic-gate retrycnt++; 27577c478bd9Sstevel@tonic-gate goto retry; 27587c478bd9Sstevel@tonic-gate } 27597c478bd9Sstevel@tonic-gate } 27607c478bd9Sstevel@tonic-gate pwp->wp_oprot = 0; 27617c478bd9Sstevel@tonic-gate pwp->wp_prot = 0; 27627c478bd9Sstevel@tonic-gate } 27637c478bd9Sstevel@tonic-gate } 27647c478bd9Sstevel@tonic-gate 27657c478bd9Sstevel@tonic-gate /* 27667c478bd9Sstevel@tonic-gate * Force a new setup for all the watched pages in the range. 27677c478bd9Sstevel@tonic-gate */ 27687c478bd9Sstevel@tonic-gate static void 27697c478bd9Sstevel@tonic-gate as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 27707c478bd9Sstevel@tonic-gate { 27717c478bd9Sstevel@tonic-gate struct watched_page *pwp; 27727c478bd9Sstevel@tonic-gate struct watched_page tpw; 27737c478bd9Sstevel@tonic-gate caddr_t eaddr = addr + size; 27747c478bd9Sstevel@tonic-gate caddr_t vaddr; 27757c478bd9Sstevel@tonic-gate struct seg *seg; 27767c478bd9Sstevel@tonic-gate int err, retrycnt; 27777c478bd9Sstevel@tonic-gate uint_t wprot; 27787c478bd9Sstevel@tonic-gate avl_index_t where; 27797c478bd9Sstevel@tonic-gate 27807c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 27817c478bd9Sstevel@tonic-gate return; 27827c478bd9Sstevel@tonic-gate 27837c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 27847c478bd9Sstevel@tonic-gate 27857c478bd9Sstevel@tonic-gate tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 27867c478bd9Sstevel@tonic-gate if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 27877c478bd9Sstevel@tonic-gate pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 27887c478bd9Sstevel@tonic-gate 27897c478bd9Sstevel@tonic-gate while (pwp != NULL && pwp->wp_vaddr < eaddr) { 27907c478bd9Sstevel@tonic-gate retrycnt = 0; 27917c478bd9Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 27927c478bd9Sstevel@tonic-gate 27937c478bd9Sstevel@tonic-gate wprot = prot; 27947c478bd9Sstevel@tonic-gate if (pwp->wp_read) 27957c478bd9Sstevel@tonic-gate wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 27967c478bd9Sstevel@tonic-gate if (pwp->wp_write) 27977c478bd9Sstevel@tonic-gate wprot &= ~PROT_WRITE; 27987c478bd9Sstevel@tonic-gate if (pwp->wp_exec) 27997c478bd9Sstevel@tonic-gate wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 28007c478bd9Sstevel@tonic-gate if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) { 28017c478bd9Sstevel@tonic-gate retry: 28027c478bd9Sstevel@tonic-gate seg = as_segat(as, vaddr); 28037c478bd9Sstevel@tonic-gate if (seg == NULL) { 28047c478bd9Sstevel@tonic-gate panic("as_setwatchprot: no seg"); 28057c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 28067c478bd9Sstevel@tonic-gate } 28077c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot); 28087c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 28097c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 28107c478bd9Sstevel@tonic-gate retrycnt++; 28117c478bd9Sstevel@tonic-gate goto retry; 28127c478bd9Sstevel@tonic-gate } 28137c478bd9Sstevel@tonic-gate } 28147c478bd9Sstevel@tonic-gate pwp->wp_oprot = prot; 28157c478bd9Sstevel@tonic-gate pwp->wp_prot = wprot; 28167c478bd9Sstevel@tonic-gate 28177c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp); 28187c478bd9Sstevel@tonic-gate } 28197c478bd9Sstevel@tonic-gate } 28207c478bd9Sstevel@tonic-gate 28217c478bd9Sstevel@tonic-gate /* 28227c478bd9Sstevel@tonic-gate * Clear all of the watched pages in the range. 28237c478bd9Sstevel@tonic-gate */ 28247c478bd9Sstevel@tonic-gate static void 28257c478bd9Sstevel@tonic-gate as_clearwatchprot(struct as *as, caddr_t addr, size_t size) 28267c478bd9Sstevel@tonic-gate { 28277c478bd9Sstevel@tonic-gate caddr_t eaddr = addr + size; 28287c478bd9Sstevel@tonic-gate struct watched_page *pwp; 28297c478bd9Sstevel@tonic-gate struct watched_page tpw; 28307c478bd9Sstevel@tonic-gate uint_t prot; 28317c478bd9Sstevel@tonic-gate struct seg *seg; 28327c478bd9Sstevel@tonic-gate int err, retrycnt; 28337c478bd9Sstevel@tonic-gate avl_index_t where; 28347c478bd9Sstevel@tonic-gate 28357c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 28367c478bd9Sstevel@tonic-gate return; 28377c478bd9Sstevel@tonic-gate 28387c478bd9Sstevel@tonic-gate tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 28397c478bd9Sstevel@tonic-gate if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 28407c478bd9Sstevel@tonic-gate pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 28417c478bd9Sstevel@tonic-gate 28427c478bd9Sstevel@tonic-gate ASSERT(AS_WRITE_HELD(as, &as->a_lock)); 28437c478bd9Sstevel@tonic-gate 28447c478bd9Sstevel@tonic-gate while (pwp != NULL && pwp->wp_vaddr < eaddr) { 28457c478bd9Sstevel@tonic-gate ASSERT(addr >= pwp->wp_vaddr); 28467c478bd9Sstevel@tonic-gate 28477c478bd9Sstevel@tonic-gate if ((prot = pwp->wp_oprot) != 0) { 28487c478bd9Sstevel@tonic-gate retrycnt = 0; 28497c478bd9Sstevel@tonic-gate 28507c478bd9Sstevel@tonic-gate if (prot != pwp->wp_prot) { 28517c478bd9Sstevel@tonic-gate retry: 28527c478bd9Sstevel@tonic-gate seg = as_segat(as, pwp->wp_vaddr); 28537c478bd9Sstevel@tonic-gate if (seg == NULL) 28547c478bd9Sstevel@tonic-gate continue; 28557c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, pwp->wp_vaddr, 28567c478bd9Sstevel@tonic-gate PAGESIZE, prot); 28577c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 28587c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 28597c478bd9Sstevel@tonic-gate retrycnt++; 28607c478bd9Sstevel@tonic-gate goto retry; 28617c478bd9Sstevel@tonic-gate 28627c478bd9Sstevel@tonic-gate } 28637c478bd9Sstevel@tonic-gate } 28647c478bd9Sstevel@tonic-gate pwp->wp_oprot = 0; 28657c478bd9Sstevel@tonic-gate pwp->wp_prot = 0; 28667c478bd9Sstevel@tonic-gate } 28677c478bd9Sstevel@tonic-gate 28687c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp); 28697c478bd9Sstevel@tonic-gate } 28707c478bd9Sstevel@tonic-gate } 28717c478bd9Sstevel@tonic-gate 28727c478bd9Sstevel@tonic-gate void 28737c478bd9Sstevel@tonic-gate as_signal_proc(struct as *as, k_siginfo_t *siginfo) 28747c478bd9Sstevel@tonic-gate { 28757c478bd9Sstevel@tonic-gate struct proc *p; 28767c478bd9Sstevel@tonic-gate 28777c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 28787c478bd9Sstevel@tonic-gate for (p = practive; p; p = p->p_next) { 28797c478bd9Sstevel@tonic-gate if (p->p_as == as) { 28807c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 28817c478bd9Sstevel@tonic-gate if (p->p_as == as) 28827c478bd9Sstevel@tonic-gate sigaddq(p, NULL, siginfo, KM_NOSLEEP); 28837c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 28847c478bd9Sstevel@tonic-gate } 28857c478bd9Sstevel@tonic-gate } 28867c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 28877c478bd9Sstevel@tonic-gate } 28887c478bd9Sstevel@tonic-gate 28897c478bd9Sstevel@tonic-gate /* 28907c478bd9Sstevel@tonic-gate * return memory object ID 28917c478bd9Sstevel@tonic-gate */ 28927c478bd9Sstevel@tonic-gate int 28937c478bd9Sstevel@tonic-gate as_getmemid(struct as *as, caddr_t addr, memid_t *memidp) 28947c478bd9Sstevel@tonic-gate { 28957c478bd9Sstevel@tonic-gate struct seg *seg; 28967c478bd9Sstevel@tonic-gate int sts; 28977c478bd9Sstevel@tonic-gate 28987c478bd9Sstevel@tonic-gate AS_LOCK_ENTER(as, &as->a_lock, RW_READER); 28997c478bd9Sstevel@tonic-gate seg = as_segat(as, addr); 29007c478bd9Sstevel@tonic-gate if (seg == NULL) { 29017c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 29027c478bd9Sstevel@tonic-gate return (EFAULT); 29037c478bd9Sstevel@tonic-gate } 29047c478bd9Sstevel@tonic-gate /* 29057c478bd9Sstevel@tonic-gate * catch old drivers which may not support getmemid 29067c478bd9Sstevel@tonic-gate */ 29077c478bd9Sstevel@tonic-gate if (seg->s_ops->getmemid == NULL) { 29087c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 29097c478bd9Sstevel@tonic-gate return (ENODEV); 29107c478bd9Sstevel@tonic-gate } 29117c478bd9Sstevel@tonic-gate 29127c478bd9Sstevel@tonic-gate sts = SEGOP_GETMEMID(seg, addr, memidp); 29137c478bd9Sstevel@tonic-gate 29147c478bd9Sstevel@tonic-gate AS_LOCK_EXIT(as, &as->a_lock); 29157c478bd9Sstevel@tonic-gate return (sts); 29167c478bd9Sstevel@tonic-gate } 2917