17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 502ff05a9Svsakar * Common Development and Distribution License (the "License"). 602ff05a9Svsakar * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 2217965fd8SKrishnendu Sadhukhan - Sun Microsystems * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 24*284ce987SPatrick Mooney * Copyright 2018 Joyent, Inc. 256430daecSBrandon Baker * Copyright (c) 2016 by Delphix. All rights reserved. 267c478bd9Sstevel@tonic-gate */ 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 297c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 307c478bd9Sstevel@tonic-gate 317c478bd9Sstevel@tonic-gate /* 327c478bd9Sstevel@tonic-gate * University Copyright- Copyright (c) 1982, 1986, 1988 337c478bd9Sstevel@tonic-gate * The Regents of the University of California 347c478bd9Sstevel@tonic-gate * All Rights Reserved 357c478bd9Sstevel@tonic-gate * 367c478bd9Sstevel@tonic-gate * University Acknowledgment- Portions of this document are derived from 377c478bd9Sstevel@tonic-gate * software developed by the University of California, Berkeley, and its 387c478bd9Sstevel@tonic-gate * contributors. 397c478bd9Sstevel@tonic-gate */ 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate /* 427c478bd9Sstevel@tonic-gate * VM - address spaces. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate 457c478bd9Sstevel@tonic-gate #include <sys/types.h> 467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 477c478bd9Sstevel@tonic-gate #include <sys/param.h> 487c478bd9Sstevel@tonic-gate #include <sys/errno.h> 497c478bd9Sstevel@tonic-gate #include <sys/systm.h> 507c478bd9Sstevel@tonic-gate #include <sys/mman.h> 517c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 527c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 537c478bd9Sstevel@tonic-gate #include <sys/sysinfo.h> 547c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 557c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 567c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 577c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 587c478bd9Sstevel@tonic-gate #include <sys/debug.h> 597c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h> 607c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 617c478bd9Sstevel@tonic-gate 627c478bd9Sstevel@tonic-gate #include <vm/hat.h> 637c478bd9Sstevel@tonic-gate #include <vm/as.h> 647c478bd9Sstevel@tonic-gate #include <vm/seg.h> 657c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 667c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h> 677c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 687c478bd9Sstevel@tonic-gate #include <vm/seg_map.h> 697c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 70*284ce987SPatrick Mooney #include <vm/seg_hole.h> 717c478bd9Sstevel@tonic-gate #include <vm/page.h> 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */ 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate static struct kmem_cache *as_cache; 767c478bd9Sstevel@tonic-gate 777c478bd9Sstevel@tonic-gate static void as_setwatchprot(struct as *, caddr_t, size_t, uint_t); 787c478bd9Sstevel@tonic-gate static void as_clearwatchprot(struct as *, caddr_t, size_t); 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate 817c478bd9Sstevel@tonic-gate /* 827c478bd9Sstevel@tonic-gate * Verifying the segment lists is very time-consuming; it may not be 837c478bd9Sstevel@tonic-gate * desirable always to define VERIFY_SEGLIST when DEBUG is set. 847c478bd9Sstevel@tonic-gate */ 857c478bd9Sstevel@tonic-gate #ifdef DEBUG 867c478bd9Sstevel@tonic-gate #define VERIFY_SEGLIST 877c478bd9Sstevel@tonic-gate int do_as_verify = 0; 887c478bd9Sstevel@tonic-gate #endif 897c478bd9Sstevel@tonic-gate 907c478bd9Sstevel@tonic-gate /* 917c478bd9Sstevel@tonic-gate * Allocate a new callback data structure entry and fill in the events of 927c478bd9Sstevel@tonic-gate * interest, the address range of interest, and the callback argument. 937c478bd9Sstevel@tonic-gate * Link the entry on the as->a_callbacks list. A callback entry for the 947c478bd9Sstevel@tonic-gate * entire address space may be specified with vaddr = 0 and size = -1. 957c478bd9Sstevel@tonic-gate * 967c478bd9Sstevel@tonic-gate * CALLERS RESPONSIBILITY: If not calling from within the process context for 977c478bd9Sstevel@tonic-gate * the specified as, the caller must guarantee persistence of the specified as 987c478bd9Sstevel@tonic-gate * for the duration of this function (eg. pages being locked within the as 997c478bd9Sstevel@tonic-gate * will guarantee persistence). 1007c478bd9Sstevel@tonic-gate */ 1017c478bd9Sstevel@tonic-gate int 1027c478bd9Sstevel@tonic-gate as_add_callback(struct as *as, void (*cb_func)(), void *arg, uint_t events, 1036430daecSBrandon Baker caddr_t vaddr, size_t size, int sleepflag) 1047c478bd9Sstevel@tonic-gate { 1057c478bd9Sstevel@tonic-gate struct as_callback *current_head, *cb; 1067c478bd9Sstevel@tonic-gate caddr_t saddr; 1077c478bd9Sstevel@tonic-gate size_t rsize; 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate /* callback function and an event are mandatory */ 1107c478bd9Sstevel@tonic-gate if ((cb_func == NULL) || ((events & AS_ALL_EVENT) == 0)) 1117c478bd9Sstevel@tonic-gate return (EINVAL); 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate /* Adding a callback after as_free has been called is not allowed */ 1147c478bd9Sstevel@tonic-gate if (as == &kas) 1157c478bd9Sstevel@tonic-gate return (ENOMEM); 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate /* 1187c478bd9Sstevel@tonic-gate * vaddr = 0 and size = -1 is used to indicate that the callback range 1197c478bd9Sstevel@tonic-gate * is the entire address space so no rounding is done in that case. 1207c478bd9Sstevel@tonic-gate */ 1217c478bd9Sstevel@tonic-gate if (size != -1) { 1227c478bd9Sstevel@tonic-gate saddr = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 1237c478bd9Sstevel@tonic-gate rsize = (((size_t)(vaddr + size) + PAGEOFFSET) & PAGEMASK) - 124843e1988Sjohnlev (size_t)saddr; 1257c478bd9Sstevel@tonic-gate /* check for wraparound */ 1267c478bd9Sstevel@tonic-gate if (saddr + rsize < saddr) 1277c478bd9Sstevel@tonic-gate return (ENOMEM); 1287c478bd9Sstevel@tonic-gate } else { 1297c478bd9Sstevel@tonic-gate if (vaddr != 0) 1307c478bd9Sstevel@tonic-gate return (EINVAL); 1317c478bd9Sstevel@tonic-gate saddr = vaddr; 1327c478bd9Sstevel@tonic-gate rsize = size; 1337c478bd9Sstevel@tonic-gate } 1347c478bd9Sstevel@tonic-gate 1357c478bd9Sstevel@tonic-gate /* Allocate and initialize a callback entry */ 1367c478bd9Sstevel@tonic-gate cb = kmem_zalloc(sizeof (struct as_callback), sleepflag); 1377c478bd9Sstevel@tonic-gate if (cb == NULL) 1387c478bd9Sstevel@tonic-gate return (EAGAIN); 1397c478bd9Sstevel@tonic-gate 1407c478bd9Sstevel@tonic-gate cb->ascb_func = cb_func; 1417c478bd9Sstevel@tonic-gate cb->ascb_arg = arg; 1427c478bd9Sstevel@tonic-gate cb->ascb_events = events; 1437c478bd9Sstevel@tonic-gate cb->ascb_saddr = saddr; 1447c478bd9Sstevel@tonic-gate cb->ascb_len = rsize; 1457c478bd9Sstevel@tonic-gate 1467c478bd9Sstevel@tonic-gate /* Add the entry to the list */ 1477c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 1487c478bd9Sstevel@tonic-gate current_head = as->a_callbacks; 1497c478bd9Sstevel@tonic-gate as->a_callbacks = cb; 1507c478bd9Sstevel@tonic-gate cb->ascb_next = current_head; 1517c478bd9Sstevel@tonic-gate 1527c478bd9Sstevel@tonic-gate /* 1537c478bd9Sstevel@tonic-gate * The call to this function may lose in a race with 1547c478bd9Sstevel@tonic-gate * a pertinent event - eg. a thread does long term memory locking 1557c478bd9Sstevel@tonic-gate * but before the callback is added another thread executes as_unmap. 1567c478bd9Sstevel@tonic-gate * A broadcast here resolves that. 1577c478bd9Sstevel@tonic-gate */ 1587c478bd9Sstevel@tonic-gate if ((cb->ascb_events & AS_UNMAPWAIT_EVENT) && AS_ISUNMAPWAIT(as)) { 1597c478bd9Sstevel@tonic-gate AS_CLRUNMAPWAIT(as); 1607c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 1617c478bd9Sstevel@tonic-gate } 1627c478bd9Sstevel@tonic-gate 1637c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 1647c478bd9Sstevel@tonic-gate return (0); 1657c478bd9Sstevel@tonic-gate } 1667c478bd9Sstevel@tonic-gate 1677c478bd9Sstevel@tonic-gate /* 1687c478bd9Sstevel@tonic-gate * Search the callback list for an entry which pertains to arg. 1697c478bd9Sstevel@tonic-gate * 1707c478bd9Sstevel@tonic-gate * This is called from within the client upon completion of the callback. 1717c478bd9Sstevel@tonic-gate * RETURN VALUES: 1727c478bd9Sstevel@tonic-gate * AS_CALLBACK_DELETED (callback entry found and deleted) 1737c478bd9Sstevel@tonic-gate * AS_CALLBACK_NOTFOUND (no callback entry found - this is ok) 1747c478bd9Sstevel@tonic-gate * AS_CALLBACK_DELETE_DEFERRED (callback is in process, delete of this 1757c478bd9Sstevel@tonic-gate * entry will be made in as_do_callbacks) 1767c478bd9Sstevel@tonic-gate * 1777c478bd9Sstevel@tonic-gate * If as_delete_callback encounters a matching entry with AS_CALLBACK_CALLED 1787c478bd9Sstevel@tonic-gate * set, it indicates that as_do_callbacks is processing this entry. The 1797c478bd9Sstevel@tonic-gate * AS_ALL_EVENT events are cleared in the entry, and a broadcast is made 1807c478bd9Sstevel@tonic-gate * to unblock as_do_callbacks, in case it is blocked. 1817c478bd9Sstevel@tonic-gate * 1827c478bd9Sstevel@tonic-gate * CALLERS RESPONSIBILITY: If not calling from within the process context for 1837c478bd9Sstevel@tonic-gate * the specified as, the caller must guarantee persistence of the specified as 1847c478bd9Sstevel@tonic-gate * for the duration of this function (eg. pages being locked within the as 1857c478bd9Sstevel@tonic-gate * will guarantee persistence). 1867c478bd9Sstevel@tonic-gate */ 1877c478bd9Sstevel@tonic-gate uint_t 1887c478bd9Sstevel@tonic-gate as_delete_callback(struct as *as, void *arg) 1897c478bd9Sstevel@tonic-gate { 1907c478bd9Sstevel@tonic-gate struct as_callback **prevcb = &as->a_callbacks; 1917c478bd9Sstevel@tonic-gate struct as_callback *cb; 1927c478bd9Sstevel@tonic-gate uint_t rc = AS_CALLBACK_NOTFOUND; 1937c478bd9Sstevel@tonic-gate 1947c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 1957c478bd9Sstevel@tonic-gate for (cb = as->a_callbacks; cb; prevcb = &cb->ascb_next, cb = *prevcb) { 1967c478bd9Sstevel@tonic-gate if (cb->ascb_arg != arg) 1977c478bd9Sstevel@tonic-gate continue; 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate /* 2007c478bd9Sstevel@tonic-gate * If the events indicate AS_CALLBACK_CALLED, just clear 2017c478bd9Sstevel@tonic-gate * AS_ALL_EVENT in the events field and wakeup the thread 2027c478bd9Sstevel@tonic-gate * that may be waiting in as_do_callbacks. as_do_callbacks 2037c478bd9Sstevel@tonic-gate * will take care of removing this entry from the list. In 2047c478bd9Sstevel@tonic-gate * that case, return AS_CALLBACK_DELETE_DEFERRED. Otherwise 2057c478bd9Sstevel@tonic-gate * (AS_CALLBACK_CALLED not set), just remove it from the 2067c478bd9Sstevel@tonic-gate * list, return the memory and return AS_CALLBACK_DELETED. 2077c478bd9Sstevel@tonic-gate */ 2087c478bd9Sstevel@tonic-gate if ((cb->ascb_events & AS_CALLBACK_CALLED) != 0) { 2097c478bd9Sstevel@tonic-gate /* leave AS_CALLBACK_CALLED */ 2107c478bd9Sstevel@tonic-gate cb->ascb_events &= ~AS_ALL_EVENT; 2117c478bd9Sstevel@tonic-gate rc = AS_CALLBACK_DELETE_DEFERRED; 2127c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 2137c478bd9Sstevel@tonic-gate } else { 2147c478bd9Sstevel@tonic-gate *prevcb = cb->ascb_next; 2157c478bd9Sstevel@tonic-gate kmem_free(cb, sizeof (struct as_callback)); 2167c478bd9Sstevel@tonic-gate rc = AS_CALLBACK_DELETED; 2177c478bd9Sstevel@tonic-gate } 2187c478bd9Sstevel@tonic-gate break; 2197c478bd9Sstevel@tonic-gate } 2207c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 2217c478bd9Sstevel@tonic-gate return (rc); 2227c478bd9Sstevel@tonic-gate } 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate /* 2257c478bd9Sstevel@tonic-gate * Searches the as callback list for a matching entry. 2267c478bd9Sstevel@tonic-gate * Returns a pointer to the first matching callback, or NULL if 2277c478bd9Sstevel@tonic-gate * nothing is found. 2287c478bd9Sstevel@tonic-gate * This function never sleeps so it is ok to call it with more 2297c478bd9Sstevel@tonic-gate * locks held but the (required) a_contents mutex. 2307c478bd9Sstevel@tonic-gate * 2317c478bd9Sstevel@tonic-gate * See also comment on as_do_callbacks below. 2327c478bd9Sstevel@tonic-gate */ 2337c478bd9Sstevel@tonic-gate static struct as_callback * 2347c478bd9Sstevel@tonic-gate as_find_callback(struct as *as, uint_t events, caddr_t event_addr, 2356430daecSBrandon Baker size_t event_len) 2367c478bd9Sstevel@tonic-gate { 2377c478bd9Sstevel@tonic-gate struct as_callback *cb; 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&as->a_contents)); 2407c478bd9Sstevel@tonic-gate for (cb = as->a_callbacks; cb != NULL; cb = cb->ascb_next) { 2417c478bd9Sstevel@tonic-gate /* 2427c478bd9Sstevel@tonic-gate * If the callback has not already been called, then 2437c478bd9Sstevel@tonic-gate * check if events or address range pertains. An event_len 2447c478bd9Sstevel@tonic-gate * of zero means do an unconditional callback. 2457c478bd9Sstevel@tonic-gate */ 2467c478bd9Sstevel@tonic-gate if (((cb->ascb_events & AS_CALLBACK_CALLED) != 0) || 2477c478bd9Sstevel@tonic-gate ((event_len != 0) && (((cb->ascb_events & events) == 0) || 2487c478bd9Sstevel@tonic-gate (event_addr + event_len < cb->ascb_saddr) || 2497c478bd9Sstevel@tonic-gate (event_addr > (cb->ascb_saddr + cb->ascb_len))))) { 2507c478bd9Sstevel@tonic-gate continue; 2517c478bd9Sstevel@tonic-gate } 2527c478bd9Sstevel@tonic-gate break; 2537c478bd9Sstevel@tonic-gate } 2547c478bd9Sstevel@tonic-gate return (cb); 2557c478bd9Sstevel@tonic-gate } 2567c478bd9Sstevel@tonic-gate 2577c478bd9Sstevel@tonic-gate /* 2587c478bd9Sstevel@tonic-gate * Executes a given callback and removes it from the callback list for 2597c478bd9Sstevel@tonic-gate * this address space. 2607c478bd9Sstevel@tonic-gate * This function may sleep so the caller must drop all locks except 2617c478bd9Sstevel@tonic-gate * a_contents before calling this func. 2627c478bd9Sstevel@tonic-gate * 2637c478bd9Sstevel@tonic-gate * See also comments on as_do_callbacks below. 2647c478bd9Sstevel@tonic-gate */ 2657c478bd9Sstevel@tonic-gate static void 2667c478bd9Sstevel@tonic-gate as_execute_callback(struct as *as, struct as_callback *cb, 2676430daecSBrandon Baker uint_t events) 2687c478bd9Sstevel@tonic-gate { 2697c478bd9Sstevel@tonic-gate struct as_callback **prevcb; 2707c478bd9Sstevel@tonic-gate void *cb_arg; 2717c478bd9Sstevel@tonic-gate 2727c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&as->a_contents) && (cb->ascb_events & events)); 2737c478bd9Sstevel@tonic-gate cb->ascb_events |= AS_CALLBACK_CALLED; 2747c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 2757c478bd9Sstevel@tonic-gate (*cb->ascb_func)(as, cb->ascb_arg, events); 2767c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 2777c478bd9Sstevel@tonic-gate /* 2787c478bd9Sstevel@tonic-gate * the callback function is required to delete the callback 2797c478bd9Sstevel@tonic-gate * when the callback function determines it is OK for 2807c478bd9Sstevel@tonic-gate * this thread to continue. as_delete_callback will clear 2817c478bd9Sstevel@tonic-gate * the AS_ALL_EVENT in the events field when it is deleted. 2827c478bd9Sstevel@tonic-gate * If the callback function called as_delete_callback, 2837c478bd9Sstevel@tonic-gate * events will already be cleared and there will be no blocking. 2847c478bd9Sstevel@tonic-gate */ 2857c478bd9Sstevel@tonic-gate while ((cb->ascb_events & events) != 0) { 2867c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 2877c478bd9Sstevel@tonic-gate } 2887c478bd9Sstevel@tonic-gate /* 2897c478bd9Sstevel@tonic-gate * This entry needs to be taken off the list. Normally, the 2907c478bd9Sstevel@tonic-gate * callback func itself does that, but unfortunately the list 2917c478bd9Sstevel@tonic-gate * may have changed while the callback was running because the 2927c478bd9Sstevel@tonic-gate * a_contents mutex was dropped and someone else other than the 2937c478bd9Sstevel@tonic-gate * callback func itself could have called as_delete_callback, 2947c478bd9Sstevel@tonic-gate * so we have to search to find this entry again. The entry 2957c478bd9Sstevel@tonic-gate * must have AS_CALLBACK_CALLED, and have the same 'arg'. 2967c478bd9Sstevel@tonic-gate */ 2977c478bd9Sstevel@tonic-gate cb_arg = cb->ascb_arg; 2987c478bd9Sstevel@tonic-gate prevcb = &as->a_callbacks; 2997c478bd9Sstevel@tonic-gate for (cb = as->a_callbacks; cb != NULL; 3007c478bd9Sstevel@tonic-gate prevcb = &cb->ascb_next, cb = *prevcb) { 3017c478bd9Sstevel@tonic-gate if (((cb->ascb_events & AS_CALLBACK_CALLED) == 0) || 3027c478bd9Sstevel@tonic-gate (cb_arg != cb->ascb_arg)) { 3037c478bd9Sstevel@tonic-gate continue; 3047c478bd9Sstevel@tonic-gate } 3057c478bd9Sstevel@tonic-gate *prevcb = cb->ascb_next; 3067c478bd9Sstevel@tonic-gate kmem_free(cb, sizeof (struct as_callback)); 3077c478bd9Sstevel@tonic-gate break; 3087c478bd9Sstevel@tonic-gate } 3097c478bd9Sstevel@tonic-gate } 3107c478bd9Sstevel@tonic-gate 3117c478bd9Sstevel@tonic-gate /* 3127c478bd9Sstevel@tonic-gate * Check the callback list for a matching event and intersection of 3137c478bd9Sstevel@tonic-gate * address range. If there is a match invoke the callback. Skip an entry if: 3147c478bd9Sstevel@tonic-gate * - a callback is already in progress for this entry (AS_CALLBACK_CALLED) 3157c478bd9Sstevel@tonic-gate * - not event of interest 3167c478bd9Sstevel@tonic-gate * - not address range of interest 3177c478bd9Sstevel@tonic-gate * 3187c478bd9Sstevel@tonic-gate * An event_len of zero indicates a request for an unconditional callback 3197c478bd9Sstevel@tonic-gate * (regardless of event), only the AS_CALLBACK_CALLED is checked. The 3207c478bd9Sstevel@tonic-gate * a_contents lock must be dropped before a callback, so only one callback 3217c478bd9Sstevel@tonic-gate * can be done before returning. Return -1 (true) if a callback was 3227c478bd9Sstevel@tonic-gate * executed and removed from the list, else return 0 (false). 3237c478bd9Sstevel@tonic-gate * 3247c478bd9Sstevel@tonic-gate * The logically separate parts, i.e. finding a matching callback and 3257c478bd9Sstevel@tonic-gate * executing a given callback have been separated into two functions 3267c478bd9Sstevel@tonic-gate * so that they can be called with different sets of locks held beyond 3277c478bd9Sstevel@tonic-gate * the always-required a_contents. as_find_callback does not sleep so 3287c478bd9Sstevel@tonic-gate * it is ok to call it if more locks than a_contents (i.e. the a_lock 3297c478bd9Sstevel@tonic-gate * rwlock) are held. as_execute_callback on the other hand may sleep 3307c478bd9Sstevel@tonic-gate * so all locks beyond a_contents must be dropped by the caller if one 3317c478bd9Sstevel@tonic-gate * does not want to end comatose. 3327c478bd9Sstevel@tonic-gate */ 3337c478bd9Sstevel@tonic-gate static int 3347c478bd9Sstevel@tonic-gate as_do_callbacks(struct as *as, uint_t events, caddr_t event_addr, 3356430daecSBrandon Baker size_t event_len) 3367c478bd9Sstevel@tonic-gate { 3377c478bd9Sstevel@tonic-gate struct as_callback *cb; 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate if ((cb = as_find_callback(as, events, event_addr, event_len))) { 3407c478bd9Sstevel@tonic-gate as_execute_callback(as, cb, events); 3417c478bd9Sstevel@tonic-gate return (-1); 3427c478bd9Sstevel@tonic-gate } 3437c478bd9Sstevel@tonic-gate return (0); 3447c478bd9Sstevel@tonic-gate } 3457c478bd9Sstevel@tonic-gate 3467c478bd9Sstevel@tonic-gate /* 3477c478bd9Sstevel@tonic-gate * Search for the segment containing addr. If a segment containing addr 3487c478bd9Sstevel@tonic-gate * exists, that segment is returned. If no such segment exists, and 3497c478bd9Sstevel@tonic-gate * the list spans addresses greater than addr, then the first segment 3507c478bd9Sstevel@tonic-gate * whose base is greater than addr is returned; otherwise, NULL is 3517c478bd9Sstevel@tonic-gate * returned unless tail is true, in which case the last element of the 3527c478bd9Sstevel@tonic-gate * list is returned. 3537c478bd9Sstevel@tonic-gate * 3547c478bd9Sstevel@tonic-gate * a_seglast is used to cache the last found segment for repeated 3557c478bd9Sstevel@tonic-gate * searches to the same addr (which happens frequently). 3567c478bd9Sstevel@tonic-gate */ 3577c478bd9Sstevel@tonic-gate struct seg * 3587c478bd9Sstevel@tonic-gate as_findseg(struct as *as, caddr_t addr, int tail) 3597c478bd9Sstevel@tonic-gate { 3607c478bd9Sstevel@tonic-gate struct seg *seg = as->a_seglast; 3617c478bd9Sstevel@tonic-gate avl_index_t where; 3627c478bd9Sstevel@tonic-gate 363dc32d872SJosef 'Jeff' Sipek ASSERT(AS_LOCK_HELD(as)); 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate if (seg != NULL && 3667c478bd9Sstevel@tonic-gate seg->s_base <= addr && 3677c478bd9Sstevel@tonic-gate addr < seg->s_base + seg->s_size) 3687c478bd9Sstevel@tonic-gate return (seg); 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, &where); 3717c478bd9Sstevel@tonic-gate if (seg != NULL) 3727c478bd9Sstevel@tonic-gate return (as->a_seglast = seg); 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 3757c478bd9Sstevel@tonic-gate if (seg == NULL && tail) 3767c478bd9Sstevel@tonic-gate seg = avl_last(&as->a_segtree); 3777c478bd9Sstevel@tonic-gate return (as->a_seglast = seg); 3787c478bd9Sstevel@tonic-gate } 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 3817c478bd9Sstevel@tonic-gate /* 3827c478bd9Sstevel@tonic-gate * verify that the linked list is coherent 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate static void 3857c478bd9Sstevel@tonic-gate as_verify(struct as *as) 3867c478bd9Sstevel@tonic-gate { 3877c478bd9Sstevel@tonic-gate struct seg *seg, *seglast, *p, *n; 3887c478bd9Sstevel@tonic-gate uint_t nsegs = 0; 3897c478bd9Sstevel@tonic-gate 3907c478bd9Sstevel@tonic-gate if (do_as_verify == 0) 3917c478bd9Sstevel@tonic-gate return; 3927c478bd9Sstevel@tonic-gate 3937c478bd9Sstevel@tonic-gate seglast = as->a_seglast; 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 3967c478bd9Sstevel@tonic-gate ASSERT(seg->s_as == as); 3977c478bd9Sstevel@tonic-gate p = AS_SEGPREV(as, seg); 3987c478bd9Sstevel@tonic-gate n = AS_SEGNEXT(as, seg); 3997c478bd9Sstevel@tonic-gate ASSERT(p == NULL || p->s_as == as); 4007c478bd9Sstevel@tonic-gate ASSERT(p == NULL || p->s_base < seg->s_base); 4017c478bd9Sstevel@tonic-gate ASSERT(n == NULL || n->s_base > seg->s_base); 4027c478bd9Sstevel@tonic-gate ASSERT(n != NULL || seg == avl_last(&as->a_segtree)); 4037c478bd9Sstevel@tonic-gate if (seg == seglast) 4047c478bd9Sstevel@tonic-gate seglast = NULL; 4057c478bd9Sstevel@tonic-gate nsegs++; 4067c478bd9Sstevel@tonic-gate } 4077c478bd9Sstevel@tonic-gate ASSERT(seglast == NULL); 4087c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_segtree) == nsegs); 4097c478bd9Sstevel@tonic-gate } 4107c478bd9Sstevel@tonic-gate #endif /* VERIFY_SEGLIST */ 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate /* 4137c478bd9Sstevel@tonic-gate * Add a new segment to the address space. The avl_find() 4147c478bd9Sstevel@tonic-gate * may be expensive so we attempt to use last segment accessed 4157c478bd9Sstevel@tonic-gate * in as_gap() as an insertion point. 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate int 4187c478bd9Sstevel@tonic-gate as_addseg(struct as *as, struct seg *newseg) 4197c478bd9Sstevel@tonic-gate { 4207c478bd9Sstevel@tonic-gate struct seg *seg; 4217c478bd9Sstevel@tonic-gate caddr_t addr; 4227c478bd9Sstevel@tonic-gate caddr_t eaddr; 4237c478bd9Sstevel@tonic-gate avl_index_t where; 4247c478bd9Sstevel@tonic-gate 425dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 4287c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 4297c478bd9Sstevel@tonic-gate 4307c478bd9Sstevel@tonic-gate if (as->a_lastgaphl != NULL) { 4317c478bd9Sstevel@tonic-gate struct seg *hseg = NULL; 4327c478bd9Sstevel@tonic-gate struct seg *lseg = NULL; 4337c478bd9Sstevel@tonic-gate 4347c478bd9Sstevel@tonic-gate if (as->a_lastgaphl->s_base > newseg->s_base) { 4357c478bd9Sstevel@tonic-gate hseg = as->a_lastgaphl; 4367c478bd9Sstevel@tonic-gate lseg = AVL_PREV(&as->a_segtree, hseg); 4377c478bd9Sstevel@tonic-gate } else { 4387c478bd9Sstevel@tonic-gate lseg = as->a_lastgaphl; 4397c478bd9Sstevel@tonic-gate hseg = AVL_NEXT(&as->a_segtree, lseg); 4407c478bd9Sstevel@tonic-gate } 4417c478bd9Sstevel@tonic-gate 4427c478bd9Sstevel@tonic-gate if (hseg && lseg && lseg->s_base < newseg->s_base && 4437c478bd9Sstevel@tonic-gate hseg->s_base > newseg->s_base) { 4447c478bd9Sstevel@tonic-gate avl_insert_here(&as->a_segtree, newseg, lseg, 4457c478bd9Sstevel@tonic-gate AVL_AFTER); 4467c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 4477c478bd9Sstevel@tonic-gate as->a_seglast = newseg; 4487c478bd9Sstevel@tonic-gate return (0); 4497c478bd9Sstevel@tonic-gate } 4507c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 4517c478bd9Sstevel@tonic-gate } 4527c478bd9Sstevel@tonic-gate 4537c478bd9Sstevel@tonic-gate addr = newseg->s_base; 4547c478bd9Sstevel@tonic-gate eaddr = addr + newseg->s_size; 4557c478bd9Sstevel@tonic-gate again: 4567c478bd9Sstevel@tonic-gate 4577c478bd9Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, &where); 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate if (seg == NULL) 4607c478bd9Sstevel@tonic-gate seg = avl_nearest(&as->a_segtree, where, AVL_AFTER); 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate if (seg == NULL) 4637c478bd9Sstevel@tonic-gate seg = avl_last(&as->a_segtree); 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate if (seg != NULL) { 4667c478bd9Sstevel@tonic-gate caddr_t base = seg->s_base; 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate /* 4697c478bd9Sstevel@tonic-gate * If top of seg is below the requested address, then 4707c478bd9Sstevel@tonic-gate * the insertion point is at the end of the linked list, 4717c478bd9Sstevel@tonic-gate * and seg points to the tail of the list. Otherwise, 4727c478bd9Sstevel@tonic-gate * the insertion point is immediately before seg. 4737c478bd9Sstevel@tonic-gate */ 4747c478bd9Sstevel@tonic-gate if (base + seg->s_size > addr) { 4757c478bd9Sstevel@tonic-gate if (addr >= base || eaddr > base) { 4767c478bd9Sstevel@tonic-gate #ifdef __sparc 4777c478bd9Sstevel@tonic-gate extern struct seg_ops segnf_ops; 4787c478bd9Sstevel@tonic-gate 4797c478bd9Sstevel@tonic-gate /* 4807c478bd9Sstevel@tonic-gate * no-fault segs must disappear if overlaid. 4817c478bd9Sstevel@tonic-gate * XXX need new segment type so 4827c478bd9Sstevel@tonic-gate * we don't have to check s_ops 4837c478bd9Sstevel@tonic-gate */ 4847c478bd9Sstevel@tonic-gate if (seg->s_ops == &segnf_ops) { 4857c478bd9Sstevel@tonic-gate seg_unmap(seg); 4867c478bd9Sstevel@tonic-gate goto again; 4877c478bd9Sstevel@tonic-gate } 4887c478bd9Sstevel@tonic-gate #endif 4897c478bd9Sstevel@tonic-gate return (-1); /* overlapping segment */ 4907c478bd9Sstevel@tonic-gate } 4917c478bd9Sstevel@tonic-gate } 4927c478bd9Sstevel@tonic-gate } 4937c478bd9Sstevel@tonic-gate as->a_seglast = newseg; 4947c478bd9Sstevel@tonic-gate avl_insert(&as->a_segtree, newseg, where); 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 4977c478bd9Sstevel@tonic-gate as_verify(as); 4987c478bd9Sstevel@tonic-gate #endif 4997c478bd9Sstevel@tonic-gate return (0); 5007c478bd9Sstevel@tonic-gate } 5017c478bd9Sstevel@tonic-gate 5027c478bd9Sstevel@tonic-gate struct seg * 5037c478bd9Sstevel@tonic-gate as_removeseg(struct as *as, struct seg *seg) 5047c478bd9Sstevel@tonic-gate { 5057c478bd9Sstevel@tonic-gate avl_tree_t *t; 5067c478bd9Sstevel@tonic-gate 507dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 5107c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate if (seg == NULL) 5137c478bd9Sstevel@tonic-gate return (NULL); 5147c478bd9Sstevel@tonic-gate 5157c478bd9Sstevel@tonic-gate t = &as->a_segtree; 5167c478bd9Sstevel@tonic-gate if (as->a_seglast == seg) 5177c478bd9Sstevel@tonic-gate as->a_seglast = NULL; 5187c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 5197c478bd9Sstevel@tonic-gate 5207c478bd9Sstevel@tonic-gate /* 5217c478bd9Sstevel@tonic-gate * if this segment is at an address higher than 5227c478bd9Sstevel@tonic-gate * a_lastgap, set a_lastgap to the next segment (NULL if last segment) 5237c478bd9Sstevel@tonic-gate */ 5247c478bd9Sstevel@tonic-gate if (as->a_lastgap && 5257c478bd9Sstevel@tonic-gate (seg == as->a_lastgap || seg->s_base > as->a_lastgap->s_base)) 5267c478bd9Sstevel@tonic-gate as->a_lastgap = AVL_NEXT(t, seg); 5277c478bd9Sstevel@tonic-gate 5287c478bd9Sstevel@tonic-gate /* 5297c478bd9Sstevel@tonic-gate * remove the segment from the seg tree 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate avl_remove(t, seg); 5327c478bd9Sstevel@tonic-gate 5337c478bd9Sstevel@tonic-gate #ifdef VERIFY_SEGLIST 5347c478bd9Sstevel@tonic-gate as_verify(as); 5357c478bd9Sstevel@tonic-gate #endif 5367c478bd9Sstevel@tonic-gate return (seg); 5377c478bd9Sstevel@tonic-gate } 5387c478bd9Sstevel@tonic-gate 5397c478bd9Sstevel@tonic-gate /* 5407c478bd9Sstevel@tonic-gate * Find a segment containing addr. 5417c478bd9Sstevel@tonic-gate */ 5427c478bd9Sstevel@tonic-gate struct seg * 5437c478bd9Sstevel@tonic-gate as_segat(struct as *as, caddr_t addr) 5447c478bd9Sstevel@tonic-gate { 5457c478bd9Sstevel@tonic-gate struct seg *seg = as->a_seglast; 5467c478bd9Sstevel@tonic-gate 547dc32d872SJosef 'Jeff' Sipek ASSERT(AS_LOCK_HELD(as)); 5487c478bd9Sstevel@tonic-gate 5497c478bd9Sstevel@tonic-gate if (seg != NULL && seg->s_base <= addr && 5507c478bd9Sstevel@tonic-gate addr < seg->s_base + seg->s_size) 5517c478bd9Sstevel@tonic-gate return (seg); 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate seg = avl_find(&as->a_segtree, &addr, NULL); 5547c478bd9Sstevel@tonic-gate return (seg); 5557c478bd9Sstevel@tonic-gate } 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate /* 5587c478bd9Sstevel@tonic-gate * Serialize all searches for holes in an address space to 5597c478bd9Sstevel@tonic-gate * prevent two or more threads from allocating the same virtual 5607c478bd9Sstevel@tonic-gate * address range. The address space must not be "read/write" 5617c478bd9Sstevel@tonic-gate * locked by the caller since we may block. 5627c478bd9Sstevel@tonic-gate */ 5637c478bd9Sstevel@tonic-gate void 5647c478bd9Sstevel@tonic-gate as_rangelock(struct as *as) 5657c478bd9Sstevel@tonic-gate { 5667c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 5677c478bd9Sstevel@tonic-gate while (AS_ISCLAIMGAP(as)) 5687c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 5697c478bd9Sstevel@tonic-gate AS_SETCLAIMGAP(as); 5707c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 5717c478bd9Sstevel@tonic-gate } 5727c478bd9Sstevel@tonic-gate 5737c478bd9Sstevel@tonic-gate /* 5747c478bd9Sstevel@tonic-gate * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads. 5757c478bd9Sstevel@tonic-gate */ 5767c478bd9Sstevel@tonic-gate void 5777c478bd9Sstevel@tonic-gate as_rangeunlock(struct as *as) 5787c478bd9Sstevel@tonic-gate { 5797c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 5807c478bd9Sstevel@tonic-gate AS_CLRCLAIMGAP(as); 5817c478bd9Sstevel@tonic-gate cv_signal(&as->a_cv); 5827c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 5837c478bd9Sstevel@tonic-gate } 5847c478bd9Sstevel@tonic-gate 5857c478bd9Sstevel@tonic-gate /* 5867c478bd9Sstevel@tonic-gate * compar segments (or just an address) by segment address range 5877c478bd9Sstevel@tonic-gate */ 5887c478bd9Sstevel@tonic-gate static int 5897c478bd9Sstevel@tonic-gate as_segcompar(const void *x, const void *y) 5907c478bd9Sstevel@tonic-gate { 5917c478bd9Sstevel@tonic-gate struct seg *a = (struct seg *)x; 5927c478bd9Sstevel@tonic-gate struct seg *b = (struct seg *)y; 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate if (a->s_base < b->s_base) 5957c478bd9Sstevel@tonic-gate return (-1); 5967c478bd9Sstevel@tonic-gate if (a->s_base >= b->s_base + b->s_size) 5977c478bd9Sstevel@tonic-gate return (1); 5987c478bd9Sstevel@tonic-gate return (0); 5997c478bd9Sstevel@tonic-gate } 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate void 6037c478bd9Sstevel@tonic-gate as_avlinit(struct as *as) 6047c478bd9Sstevel@tonic-gate { 6057c478bd9Sstevel@tonic-gate avl_create(&as->a_segtree, as_segcompar, sizeof (struct seg), 6067c478bd9Sstevel@tonic-gate offsetof(struct seg, s_tree)); 6077c478bd9Sstevel@tonic-gate avl_create(&as->a_wpage, wp_compare, sizeof (struct watched_page), 6087c478bd9Sstevel@tonic-gate offsetof(struct watched_page, wp_link)); 6097c478bd9Sstevel@tonic-gate } 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 6127c478bd9Sstevel@tonic-gate static int 6137c478bd9Sstevel@tonic-gate as_constructor(void *buf, void *cdrarg, int kmflags) 6147c478bd9Sstevel@tonic-gate { 6157c478bd9Sstevel@tonic-gate struct as *as = buf; 6167c478bd9Sstevel@tonic-gate 6177c478bd9Sstevel@tonic-gate mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL); 6187c478bd9Sstevel@tonic-gate cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL); 6197c478bd9Sstevel@tonic-gate rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL); 6207c478bd9Sstevel@tonic-gate as_avlinit(as); 6217c478bd9Sstevel@tonic-gate return (0); 6227c478bd9Sstevel@tonic-gate } 6237c478bd9Sstevel@tonic-gate 6247c478bd9Sstevel@tonic-gate /*ARGSUSED1*/ 6257c478bd9Sstevel@tonic-gate static void 6267c478bd9Sstevel@tonic-gate as_destructor(void *buf, void *cdrarg) 6277c478bd9Sstevel@tonic-gate { 6287c478bd9Sstevel@tonic-gate struct as *as = buf; 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate avl_destroy(&as->a_segtree); 6317c478bd9Sstevel@tonic-gate mutex_destroy(&as->a_contents); 6327c478bd9Sstevel@tonic-gate cv_destroy(&as->a_cv); 6337c478bd9Sstevel@tonic-gate rw_destroy(&as->a_lock); 6347c478bd9Sstevel@tonic-gate } 6357c478bd9Sstevel@tonic-gate 6367c478bd9Sstevel@tonic-gate void 6377c478bd9Sstevel@tonic-gate as_init(void) 6387c478bd9Sstevel@tonic-gate { 6397c478bd9Sstevel@tonic-gate as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0, 640843e1988Sjohnlev as_constructor, as_destructor, NULL, NULL, NULL, 0); 6417c478bd9Sstevel@tonic-gate } 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate /* 6447c478bd9Sstevel@tonic-gate * Allocate and initialize an address space data structure. 6457c478bd9Sstevel@tonic-gate * We call hat_alloc to allow any machine dependent 6467c478bd9Sstevel@tonic-gate * information in the hat structure to be initialized. 6477c478bd9Sstevel@tonic-gate */ 6487c478bd9Sstevel@tonic-gate struct as * 6497c478bd9Sstevel@tonic-gate as_alloc(void) 6507c478bd9Sstevel@tonic-gate { 6517c478bd9Sstevel@tonic-gate struct as *as; 6527c478bd9Sstevel@tonic-gate 6537c478bd9Sstevel@tonic-gate as = kmem_cache_alloc(as_cache, KM_SLEEP); 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate as->a_flags = 0; 6567c478bd9Sstevel@tonic-gate as->a_vbits = 0; 6577c478bd9Sstevel@tonic-gate as->a_hrm = NULL; 6587c478bd9Sstevel@tonic-gate as->a_seglast = NULL; 6597c478bd9Sstevel@tonic-gate as->a_size = 0; 66040688216SSudheer A as->a_resvsize = 0; 6617c478bd9Sstevel@tonic-gate as->a_updatedir = 0; 6627c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 6637c478bd9Sstevel@tonic-gate as->a_objectdir = NULL; 6647c478bd9Sstevel@tonic-gate as->a_sizedir = 0; 6657c478bd9Sstevel@tonic-gate as->a_userlimit = (caddr_t)USERLIMIT; 6667c478bd9Sstevel@tonic-gate as->a_lastgap = NULL; 6677c478bd9Sstevel@tonic-gate as->a_lastgaphl = NULL; 6687c478bd9Sstevel@tonic-gate as->a_callbacks = NULL; 6696430daecSBrandon Baker as->a_proc = NULL; 6707c478bd9Sstevel@tonic-gate 671dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 6727c478bd9Sstevel@tonic-gate as->a_hat = hat_alloc(as); /* create hat for default system mmu */ 673dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 6747c478bd9Sstevel@tonic-gate 6757c478bd9Sstevel@tonic-gate return (as); 6767c478bd9Sstevel@tonic-gate } 6777c478bd9Sstevel@tonic-gate 6787c478bd9Sstevel@tonic-gate /* 6797c478bd9Sstevel@tonic-gate * Free an address space data structure. 6807c478bd9Sstevel@tonic-gate * Need to free the hat first and then 6817c478bd9Sstevel@tonic-gate * all the segments on this as and finally 6827c478bd9Sstevel@tonic-gate * the space for the as struct itself. 6837c478bd9Sstevel@tonic-gate */ 6847c478bd9Sstevel@tonic-gate void 6857c478bd9Sstevel@tonic-gate as_free(struct as *as) 6867c478bd9Sstevel@tonic-gate { 6877c478bd9Sstevel@tonic-gate struct hat *hat = as->a_hat; 6887c478bd9Sstevel@tonic-gate struct seg *seg, *next; 6890d5ae8c1SJosef 'Jeff' Sipek boolean_t free_started = B_FALSE; 6907c478bd9Sstevel@tonic-gate 6917c478bd9Sstevel@tonic-gate top: 6927c478bd9Sstevel@tonic-gate /* 6937c478bd9Sstevel@tonic-gate * Invoke ALL callbacks. as_do_callbacks will do one callback 6947c478bd9Sstevel@tonic-gate * per call, and not return (-1) until the callback has completed. 6957c478bd9Sstevel@tonic-gate * When as_do_callbacks returns zero, all callbacks have completed. 6967c478bd9Sstevel@tonic-gate */ 6977c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 698843e1988Sjohnlev while (as->a_callbacks && as_do_callbacks(as, AS_ALL_EVENT, 0, 0)) 699843e1988Sjohnlev ; 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 702dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 7037c478bd9Sstevel@tonic-gate 7040d5ae8c1SJosef 'Jeff' Sipek if (!free_started) { 7050d5ae8c1SJosef 'Jeff' Sipek free_started = B_TRUE; 7067c478bd9Sstevel@tonic-gate hat_free_start(hat); 7077c478bd9Sstevel@tonic-gate } 7087c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = next) { 7097c478bd9Sstevel@tonic-gate int err; 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate next = AS_SEGNEXT(as, seg); 712a98e9dbfSaguzovsk retry: 7137c478bd9Sstevel@tonic-gate err = SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 7147c478bd9Sstevel@tonic-gate if (err == EAGAIN) { 7157c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 7167c478bd9Sstevel@tonic-gate if (as->a_callbacks) { 717dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 718a98e9dbfSaguzovsk } else if (!AS_ISNOUNMAPWAIT(as)) { 7197c478bd9Sstevel@tonic-gate /* 7207c478bd9Sstevel@tonic-gate * Memory is currently locked. Wait for a 7217c478bd9Sstevel@tonic-gate * cv_signal that it has been unlocked, then 7227c478bd9Sstevel@tonic-gate * try the operation again. 7237c478bd9Sstevel@tonic-gate */ 7247c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 7257c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 7267c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 727dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 7287c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 7297c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 730a98e9dbfSaguzovsk } else { 731a98e9dbfSaguzovsk /* 732a98e9dbfSaguzovsk * We may have raced with 733a98e9dbfSaguzovsk * segvn_reclaim()/segspt_reclaim(). In this 734a98e9dbfSaguzovsk * case clean nounmapwait flag and retry since 735a98e9dbfSaguzovsk * softlockcnt in this segment may be already 736a98e9dbfSaguzovsk * 0. We don't drop as writer lock so our 737a98e9dbfSaguzovsk * number of retries without sleeping should 738a98e9dbfSaguzovsk * be very small. See segvn_reclaim() for 739a98e9dbfSaguzovsk * more comments. 740a98e9dbfSaguzovsk */ 741a98e9dbfSaguzovsk AS_CLRNOUNMAPWAIT(as); 742a98e9dbfSaguzovsk mutex_exit(&as->a_contents); 743a98e9dbfSaguzovsk goto retry; 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 7467c478bd9Sstevel@tonic-gate goto top; 7477c478bd9Sstevel@tonic-gate } else { 7487c478bd9Sstevel@tonic-gate /* 7497c478bd9Sstevel@tonic-gate * We do not expect any other error return at this 7507c478bd9Sstevel@tonic-gate * time. This is similar to an ASSERT in seg_unmap() 7517c478bd9Sstevel@tonic-gate */ 7527c478bd9Sstevel@tonic-gate ASSERT(err == 0); 7537c478bd9Sstevel@tonic-gate } 7547c478bd9Sstevel@tonic-gate } 7557c478bd9Sstevel@tonic-gate hat_free_end(hat); 756dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 7577c478bd9Sstevel@tonic-gate 7587c478bd9Sstevel@tonic-gate /* /proc stuff */ 7597c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_wpage) == 0); 7607c478bd9Sstevel@tonic-gate if (as->a_objectdir) { 7617c478bd9Sstevel@tonic-gate kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *)); 7627c478bd9Sstevel@tonic-gate as->a_objectdir = NULL; 7637c478bd9Sstevel@tonic-gate as->a_sizedir = 0; 7647c478bd9Sstevel@tonic-gate } 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate /* 7677c478bd9Sstevel@tonic-gate * Free the struct as back to kmem. Assert it has no segments. 7687c478bd9Sstevel@tonic-gate */ 7697c478bd9Sstevel@tonic-gate ASSERT(avl_numnodes(&as->a_segtree) == 0); 7707c478bd9Sstevel@tonic-gate kmem_cache_free(as_cache, as); 7717c478bd9Sstevel@tonic-gate } 7727c478bd9Sstevel@tonic-gate 7737c478bd9Sstevel@tonic-gate int 774bb5ca623SVamsi Nagineni as_dup(struct as *as, struct proc *forkedproc) 7757c478bd9Sstevel@tonic-gate { 7767c478bd9Sstevel@tonic-gate struct as *newas; 7777c478bd9Sstevel@tonic-gate struct seg *seg, *newseg; 77840688216SSudheer A size_t purgesize = 0; 7797c478bd9Sstevel@tonic-gate int error; 7807c478bd9Sstevel@tonic-gate 781dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 7827c478bd9Sstevel@tonic-gate as_clearwatch(as); 7837c478bd9Sstevel@tonic-gate newas = as_alloc(); 7847c478bd9Sstevel@tonic-gate newas->a_userlimit = as->a_userlimit; 785bb5ca623SVamsi Nagineni newas->a_proc = forkedproc; 786250f6acdSvk 787dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(newas, RW_WRITER); 7887c478bd9Sstevel@tonic-gate 78905d3dc4bSpaulsan (void) hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_SRD); 7907c478bd9Sstevel@tonic-gate 7917c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 7927c478bd9Sstevel@tonic-gate 79340688216SSudheer A if (seg->s_flags & S_PURGE) { 79440688216SSudheer A purgesize += seg->s_size; 7957c478bd9Sstevel@tonic-gate continue; 79640688216SSudheer A } 7977c478bd9Sstevel@tonic-gate 7987c478bd9Sstevel@tonic-gate newseg = seg_alloc(newas, seg->s_base, seg->s_size); 7997c478bd9Sstevel@tonic-gate if (newseg == NULL) { 800dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(newas); 8017c478bd9Sstevel@tonic-gate as_setwatch(as); 802dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 8037c478bd9Sstevel@tonic-gate as_free(newas); 8047c478bd9Sstevel@tonic-gate return (-1); 8057c478bd9Sstevel@tonic-gate } 8067c478bd9Sstevel@tonic-gate if ((error = SEGOP_DUP(seg, newseg)) != 0) { 8077c478bd9Sstevel@tonic-gate /* 8087c478bd9Sstevel@tonic-gate * We call seg_free() on the new seg 8097c478bd9Sstevel@tonic-gate * because the segment is not set up 8107c478bd9Sstevel@tonic-gate * completely; i.e. it has no ops. 8117c478bd9Sstevel@tonic-gate */ 8127c478bd9Sstevel@tonic-gate as_setwatch(as); 813dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 8147c478bd9Sstevel@tonic-gate seg_free(newseg); 815dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(newas); 8167c478bd9Sstevel@tonic-gate as_free(newas); 8177c478bd9Sstevel@tonic-gate return (error); 8187c478bd9Sstevel@tonic-gate } 819*284ce987SPatrick Mooney if ((newseg->s_flags & S_HOLE) == 0) { 820*284ce987SPatrick Mooney newas->a_size += seg->s_size; 821*284ce987SPatrick Mooney } 8227c478bd9Sstevel@tonic-gate } 82340688216SSudheer A newas->a_resvsize = as->a_resvsize - purgesize; 8247c478bd9Sstevel@tonic-gate 8257c478bd9Sstevel@tonic-gate error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL); 8267c478bd9Sstevel@tonic-gate 827dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(newas); 8287c478bd9Sstevel@tonic-gate 8297c478bd9Sstevel@tonic-gate as_setwatch(as); 830dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 8317c478bd9Sstevel@tonic-gate if (error != 0) { 8327c478bd9Sstevel@tonic-gate as_free(newas); 8337c478bd9Sstevel@tonic-gate return (error); 8347c478bd9Sstevel@tonic-gate } 835bb5ca623SVamsi Nagineni forkedproc->p_as = newas; 8367c478bd9Sstevel@tonic-gate return (0); 8377c478bd9Sstevel@tonic-gate } 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate /* 8407c478bd9Sstevel@tonic-gate * Handle a ``fault'' at addr for size bytes. 8417c478bd9Sstevel@tonic-gate */ 8427c478bd9Sstevel@tonic-gate faultcode_t 8437c478bd9Sstevel@tonic-gate as_fault(struct hat *hat, struct as *as, caddr_t addr, size_t size, 8446430daecSBrandon Baker enum fault_type type, enum seg_rw rw) 8457c478bd9Sstevel@tonic-gate { 8467c478bd9Sstevel@tonic-gate struct seg *seg; 8477c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 8487c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 8497c478bd9Sstevel@tonic-gate size_t ssize; 8507c478bd9Sstevel@tonic-gate faultcode_t res = 0; 8517c478bd9Sstevel@tonic-gate caddr_t addrsav; 8527c478bd9Sstevel@tonic-gate struct seg *segsav; 8537c478bd9Sstevel@tonic-gate int as_lock_held; 8547c478bd9Sstevel@tonic-gate klwp_t *lwp = ttolwp(curthread); 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate 8577c478bd9Sstevel@tonic-gate 8587c478bd9Sstevel@tonic-gate retry: 8590d5ae8c1SJosef 'Jeff' Sipek /* 8600d5ae8c1SJosef 'Jeff' Sipek * Indicate that the lwp is not to be stopped while waiting for a 8610d5ae8c1SJosef 'Jeff' Sipek * pagefault. This is to avoid deadlock while debugging a process 8620d5ae8c1SJosef 'Jeff' Sipek * via /proc over NFS (in particular). 8630d5ae8c1SJosef 'Jeff' Sipek */ 8640d5ae8c1SJosef 'Jeff' Sipek if (lwp != NULL) 8650d5ae8c1SJosef 'Jeff' Sipek lwp->lwp_nostop++; 8667c478bd9Sstevel@tonic-gate 8670d5ae8c1SJosef 'Jeff' Sipek /* 8680d5ae8c1SJosef 'Jeff' Sipek * same length must be used when we softlock and softunlock. We 8690d5ae8c1SJosef 'Jeff' Sipek * don't support softunlocking lengths less than the original length 8700d5ae8c1SJosef 'Jeff' Sipek * when there is largepage support. See seg_dev.c for more 8710d5ae8c1SJosef 'Jeff' Sipek * comments. 8720d5ae8c1SJosef 'Jeff' Sipek */ 8730d5ae8c1SJosef 'Jeff' Sipek switch (type) { 8747c478bd9Sstevel@tonic-gate 8750d5ae8c1SJosef 'Jeff' Sipek case F_SOFTLOCK: 8760d5ae8c1SJosef 'Jeff' Sipek CPU_STATS_ADD_K(vm, softlock, 1); 8770d5ae8c1SJosef 'Jeff' Sipek break; 8787c478bd9Sstevel@tonic-gate 8790d5ae8c1SJosef 'Jeff' Sipek case F_SOFTUNLOCK: 8800d5ae8c1SJosef 'Jeff' Sipek break; 8817c478bd9Sstevel@tonic-gate 8820d5ae8c1SJosef 'Jeff' Sipek case F_PROT: 8830d5ae8c1SJosef 'Jeff' Sipek CPU_STATS_ADD_K(vm, prot_fault, 1); 8840d5ae8c1SJosef 'Jeff' Sipek break; 8857c478bd9Sstevel@tonic-gate 8860d5ae8c1SJosef 'Jeff' Sipek case F_INVAL: 8870d5ae8c1SJosef 'Jeff' Sipek CPU_STATS_ENTER_K(); 8880d5ae8c1SJosef 'Jeff' Sipek CPU_STATS_ADDQ(CPU, vm, as_fault, 1); 8890d5ae8c1SJosef 'Jeff' Sipek if (as == &kas) 8900d5ae8c1SJosef 'Jeff' Sipek CPU_STATS_ADDQ(CPU, vm, kernel_asflt, 1); 8910d5ae8c1SJosef 'Jeff' Sipek CPU_STATS_EXIT_K(); 8920d5ae8c1SJosef 'Jeff' Sipek break; 8937c478bd9Sstevel@tonic-gate } 8947c478bd9Sstevel@tonic-gate 8957c478bd9Sstevel@tonic-gate /* Kernel probe */ 8967c478bd9Sstevel@tonic-gate TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */, 897843e1988Sjohnlev tnf_opaque, address, addr, 898843e1988Sjohnlev tnf_fault_type, fault_type, type, 899843e1988Sjohnlev tnf_seg_access, access, rw); 9007c478bd9Sstevel@tonic-gate 9017c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 9027c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 903843e1988Sjohnlev (size_t)raddr; 9047c478bd9Sstevel@tonic-gate 9057c478bd9Sstevel@tonic-gate /* 9067c478bd9Sstevel@tonic-gate * XXX -- Don't grab the as lock for segkmap. We should grab it for 9077c478bd9Sstevel@tonic-gate * correctness, but then we could be stuck holding this lock for 9087c478bd9Sstevel@tonic-gate * a LONG time if the fault needs to be resolved on a slow 9097c478bd9Sstevel@tonic-gate * filesystem, and then no-one will be able to exec new commands, 9107c478bd9Sstevel@tonic-gate * as exec'ing requires the write lock on the as. 9117c478bd9Sstevel@tonic-gate */ 9127c478bd9Sstevel@tonic-gate if (as == &kas && segkmap && segkmap->s_base <= raddr && 9137c478bd9Sstevel@tonic-gate raddr + size < segkmap->s_base + segkmap->s_size) { 9147c478bd9Sstevel@tonic-gate seg = segkmap; 9157c478bd9Sstevel@tonic-gate as_lock_held = 0; 9167c478bd9Sstevel@tonic-gate } else { 917dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 9187c478bd9Sstevel@tonic-gate 9197c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 9207c478bd9Sstevel@tonic-gate if (seg == NULL) { 921dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 9220d5ae8c1SJosef 'Jeff' Sipek if (lwp != NULL) 9237c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 9247c478bd9Sstevel@tonic-gate return (FC_NOMAP); 9257c478bd9Sstevel@tonic-gate } 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate as_lock_held = 1; 9287c478bd9Sstevel@tonic-gate } 9297c478bd9Sstevel@tonic-gate 9307c478bd9Sstevel@tonic-gate addrsav = raddr; 9317c478bd9Sstevel@tonic-gate segsav = seg; 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 9347c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 9357c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 9367c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 9377c478bd9Sstevel@tonic-gate res = FC_NOMAP; 9387c478bd9Sstevel@tonic-gate break; 9397c478bd9Sstevel@tonic-gate } 9407c478bd9Sstevel@tonic-gate } 9417c478bd9Sstevel@tonic-gate if (raddr + rsize > seg->s_base + seg->s_size) 9427c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 9437c478bd9Sstevel@tonic-gate else 9447c478bd9Sstevel@tonic-gate ssize = rsize; 9457c478bd9Sstevel@tonic-gate 9460d5ae8c1SJosef 'Jeff' Sipek res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw); 9470d5ae8c1SJosef 'Jeff' Sipek if (res != 0) 9487c478bd9Sstevel@tonic-gate break; 9497c478bd9Sstevel@tonic-gate } 9507c478bd9Sstevel@tonic-gate 9517c478bd9Sstevel@tonic-gate /* 9527c478bd9Sstevel@tonic-gate * If we were SOFTLOCKing and encountered a failure, 9537c478bd9Sstevel@tonic-gate * we must SOFTUNLOCK the range we already did. (Maybe we 9547c478bd9Sstevel@tonic-gate * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing 9557c478bd9Sstevel@tonic-gate * right here...) 9567c478bd9Sstevel@tonic-gate */ 9577c478bd9Sstevel@tonic-gate if (res != 0 && type == F_SOFTLOCK) { 9587c478bd9Sstevel@tonic-gate for (seg = segsav; addrsav < raddr; addrsav += ssize) { 9597c478bd9Sstevel@tonic-gate if (addrsav >= seg->s_base + seg->s_size) 9607c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 9617c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 9627c478bd9Sstevel@tonic-gate /* 9637c478bd9Sstevel@tonic-gate * Now call the fault routine again to perform the 9647c478bd9Sstevel@tonic-gate * unlock using S_OTHER instead of the rw variable 9657c478bd9Sstevel@tonic-gate * since we never got a chance to touch the pages. 9667c478bd9Sstevel@tonic-gate */ 9677c478bd9Sstevel@tonic-gate if (raddr > seg->s_base + seg->s_size) 9687c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - addrsav; 9697c478bd9Sstevel@tonic-gate else 9707c478bd9Sstevel@tonic-gate ssize = raddr - addrsav; 9717c478bd9Sstevel@tonic-gate (void) SEGOP_FAULT(hat, seg, addrsav, ssize, 9727c478bd9Sstevel@tonic-gate F_SOFTUNLOCK, S_OTHER); 9737c478bd9Sstevel@tonic-gate } 9747c478bd9Sstevel@tonic-gate } 9757c478bd9Sstevel@tonic-gate if (as_lock_held) 976dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 9770d5ae8c1SJosef 'Jeff' Sipek if (lwp != NULL) 9787c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 9798548bf79Snr 9807c478bd9Sstevel@tonic-gate /* 9817c478bd9Sstevel@tonic-gate * If the lower levels returned EDEADLK for a fault, 9827c478bd9Sstevel@tonic-gate * It means that we should retry the fault. Let's wait 9837c478bd9Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 9847c478bd9Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 9857c478bd9Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 9867c478bd9Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 9877c478bd9Sstevel@tonic-gate * 4125102 for details of the problem. 9887c478bd9Sstevel@tonic-gate */ 9897c478bd9Sstevel@tonic-gate if (FC_ERRNO(res) == EDEADLK) { 9907c478bd9Sstevel@tonic-gate delay(deadlk_wait); 9917c478bd9Sstevel@tonic-gate res = 0; 9927c478bd9Sstevel@tonic-gate goto retry; 9937c478bd9Sstevel@tonic-gate } 9947c478bd9Sstevel@tonic-gate return (res); 9957c478bd9Sstevel@tonic-gate } 9967c478bd9Sstevel@tonic-gate 9977c478bd9Sstevel@tonic-gate 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate /* 10007c478bd9Sstevel@tonic-gate * Asynchronous ``fault'' at addr for size bytes. 10017c478bd9Sstevel@tonic-gate */ 10027c478bd9Sstevel@tonic-gate faultcode_t 10037c478bd9Sstevel@tonic-gate as_faulta(struct as *as, caddr_t addr, size_t size) 10047c478bd9Sstevel@tonic-gate { 10057c478bd9Sstevel@tonic-gate struct seg *seg; 10067c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 10077c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 10087c478bd9Sstevel@tonic-gate faultcode_t res = 0; 10097c478bd9Sstevel@tonic-gate klwp_t *lwp = ttolwp(curthread); 10107c478bd9Sstevel@tonic-gate 10117c478bd9Sstevel@tonic-gate retry: 10127c478bd9Sstevel@tonic-gate /* 10137c478bd9Sstevel@tonic-gate * Indicate that the lwp is not to be stopped while waiting 10147c478bd9Sstevel@tonic-gate * for a pagefault. This is to avoid deadlock while debugging 10157c478bd9Sstevel@tonic-gate * a process via /proc over NFS (in particular). 10167c478bd9Sstevel@tonic-gate */ 10178548bf79Snr if (lwp != NULL) 10187c478bd9Sstevel@tonic-gate lwp->lwp_nostop++; 10197c478bd9Sstevel@tonic-gate 10207c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 10217c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1022843e1988Sjohnlev (size_t)raddr; 10237c478bd9Sstevel@tonic-gate 1024dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 10257c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 10267c478bd9Sstevel@tonic-gate if (seg == NULL) { 1027dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 10288548bf79Snr if (lwp != NULL) 10297c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 10307c478bd9Sstevel@tonic-gate return (FC_NOMAP); 10317c478bd9Sstevel@tonic-gate } 10327c478bd9Sstevel@tonic-gate 10337c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) { 10347c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 10357c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 10367c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 10377c478bd9Sstevel@tonic-gate res = FC_NOMAP; 10387c478bd9Sstevel@tonic-gate break; 10397c478bd9Sstevel@tonic-gate } 10407c478bd9Sstevel@tonic-gate } 10417c478bd9Sstevel@tonic-gate res = SEGOP_FAULTA(seg, raddr); 10427c478bd9Sstevel@tonic-gate if (res != 0) 10437c478bd9Sstevel@tonic-gate break; 10447c478bd9Sstevel@tonic-gate } 1045dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 10468548bf79Snr if (lwp != NULL) 10477c478bd9Sstevel@tonic-gate lwp->lwp_nostop--; 10487c478bd9Sstevel@tonic-gate /* 10497c478bd9Sstevel@tonic-gate * If the lower levels returned EDEADLK for a fault, 10507c478bd9Sstevel@tonic-gate * It means that we should retry the fault. Let's wait 10517c478bd9Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 10527c478bd9Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 10537c478bd9Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 10547c478bd9Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 10557c478bd9Sstevel@tonic-gate * 4125102 for details of the problem. 10567c478bd9Sstevel@tonic-gate */ 10577c478bd9Sstevel@tonic-gate if (FC_ERRNO(res) == EDEADLK) { 10587c478bd9Sstevel@tonic-gate delay(deadlk_wait); 10597c478bd9Sstevel@tonic-gate res = 0; 10607c478bd9Sstevel@tonic-gate goto retry; 10617c478bd9Sstevel@tonic-gate } 10627c478bd9Sstevel@tonic-gate return (res); 10637c478bd9Sstevel@tonic-gate } 10647c478bd9Sstevel@tonic-gate 10657c478bd9Sstevel@tonic-gate /* 10667c478bd9Sstevel@tonic-gate * Set the virtual mapping for the interval from [addr : addr + size) 10677c478bd9Sstevel@tonic-gate * in address space `as' to have the specified protection. 10687c478bd9Sstevel@tonic-gate * It is ok for the range to cross over several segments, 10697c478bd9Sstevel@tonic-gate * as long as they are contiguous. 10707c478bd9Sstevel@tonic-gate */ 10717c478bd9Sstevel@tonic-gate int 10727c478bd9Sstevel@tonic-gate as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 10737c478bd9Sstevel@tonic-gate { 10747c478bd9Sstevel@tonic-gate struct seg *seg; 10757c478bd9Sstevel@tonic-gate struct as_callback *cb; 10767c478bd9Sstevel@tonic-gate size_t ssize; 10777c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 10787c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 10797c478bd9Sstevel@tonic-gate int error = 0, writer = 0; 10807c478bd9Sstevel@tonic-gate caddr_t saveraddr; 10817c478bd9Sstevel@tonic-gate size_t saversize; 10827c478bd9Sstevel@tonic-gate 10837c478bd9Sstevel@tonic-gate setprot_top: 10847c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 10857c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1086843e1988Sjohnlev (size_t)raddr; 10877c478bd9Sstevel@tonic-gate 10887c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 10897c478bd9Sstevel@tonic-gate return (ENOMEM); 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate saveraddr = raddr; 10927c478bd9Sstevel@tonic-gate saversize = rsize; 10937c478bd9Sstevel@tonic-gate 10947c478bd9Sstevel@tonic-gate /* 10957c478bd9Sstevel@tonic-gate * Normally we only lock the as as a reader. But 10967c478bd9Sstevel@tonic-gate * if due to setprot the segment driver needs to split 1097da6c28aaSamw * a segment it will return IE_RETRY. Therefore we re-acquire 10987c478bd9Sstevel@tonic-gate * the as lock as a writer so the segment driver can change 10997c478bd9Sstevel@tonic-gate * the seg list. Also the segment driver will return IE_RETRY 11007c478bd9Sstevel@tonic-gate * after it has changed the segment list so we therefore keep 11017c478bd9Sstevel@tonic-gate * locking as a writer. Since these opeartions should be rare 11027c478bd9Sstevel@tonic-gate * want to only lock as a writer when necessary. 11037c478bd9Sstevel@tonic-gate */ 11047c478bd9Sstevel@tonic-gate if (writer || avl_numnodes(&as->a_wpage) != 0) { 1105dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 11067c478bd9Sstevel@tonic-gate } else { 1107dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 11087c478bd9Sstevel@tonic-gate } 11097c478bd9Sstevel@tonic-gate 11107c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 11117c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 11127c478bd9Sstevel@tonic-gate if (seg == NULL) { 11137c478bd9Sstevel@tonic-gate as_setwatch(as); 1114dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 11157c478bd9Sstevel@tonic-gate return (ENOMEM); 11167c478bd9Sstevel@tonic-gate } 11177c478bd9Sstevel@tonic-gate 11187c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 11197c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 11207c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 11217c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 11227c478bd9Sstevel@tonic-gate error = ENOMEM; 11237c478bd9Sstevel@tonic-gate break; 11247c478bd9Sstevel@tonic-gate } 11257c478bd9Sstevel@tonic-gate } 11267c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 11277c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 11287c478bd9Sstevel@tonic-gate else 11297c478bd9Sstevel@tonic-gate ssize = rsize; 1130a98e9dbfSaguzovsk retry: 11317c478bd9Sstevel@tonic-gate error = SEGOP_SETPROT(seg, raddr, ssize, prot); 11327c478bd9Sstevel@tonic-gate 11337c478bd9Sstevel@tonic-gate if (error == IE_NOMEM) { 11347c478bd9Sstevel@tonic-gate error = EAGAIN; 11357c478bd9Sstevel@tonic-gate break; 11367c478bd9Sstevel@tonic-gate } 11377c478bd9Sstevel@tonic-gate 11387c478bd9Sstevel@tonic-gate if (error == IE_RETRY) { 1139dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 11407c478bd9Sstevel@tonic-gate writer = 1; 11417c478bd9Sstevel@tonic-gate goto setprot_top; 11427c478bd9Sstevel@tonic-gate } 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate if (error == EAGAIN) { 11457c478bd9Sstevel@tonic-gate /* 11467c478bd9Sstevel@tonic-gate * Make sure we have a_lock as writer. 11477c478bd9Sstevel@tonic-gate */ 11487c478bd9Sstevel@tonic-gate if (writer == 0) { 1149dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 11507c478bd9Sstevel@tonic-gate writer = 1; 11517c478bd9Sstevel@tonic-gate goto setprot_top; 11527c478bd9Sstevel@tonic-gate } 11537c478bd9Sstevel@tonic-gate 11547c478bd9Sstevel@tonic-gate /* 11557c478bd9Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 11567c478bd9Sstevel@tonic-gate * before this operation can succeed through a retry. 11577c478bd9Sstevel@tonic-gate * The possible reasons for locked memory and 11587c478bd9Sstevel@tonic-gate * corresponding strategies for unlocking are: 11597c478bd9Sstevel@tonic-gate * (1) Normal I/O 11607c478bd9Sstevel@tonic-gate * wait for a signal that the I/O operation 11617c478bd9Sstevel@tonic-gate * has completed and the memory is unlocked. 11627c478bd9Sstevel@tonic-gate * (2) Asynchronous I/O 11637c478bd9Sstevel@tonic-gate * The aio subsystem does not unlock pages when 11647c478bd9Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 11657c478bd9Sstevel@tonic-gate * when the application calls aiowait/aioerror. 11667c478bd9Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 11677c478bd9Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 11687c478bd9Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 11697c478bd9Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 11707c478bd9Sstevel@tonic-gate * (3) Long term page locking: 11717c478bd9Sstevel@tonic-gate * Drivers intending to have pages locked for a 11727c478bd9Sstevel@tonic-gate * period considerably longer than for normal I/O 11737c478bd9Sstevel@tonic-gate * (essentially forever) may have registered for a 11747c478bd9Sstevel@tonic-gate * callback so they may unlock these pages on 11757c478bd9Sstevel@tonic-gate * request. This is needed to allow this operation 11767c478bd9Sstevel@tonic-gate * to succeed. Each entry on the callback list is 11777c478bd9Sstevel@tonic-gate * examined. If the event or address range pertains 11787c478bd9Sstevel@tonic-gate * the callback is invoked (unless it already is in 11797c478bd9Sstevel@tonic-gate * progress). The a_contents lock must be dropped 11807c478bd9Sstevel@tonic-gate * before the callback, so only one callback can 11817c478bd9Sstevel@tonic-gate * be done at a time. Go to the top and do more 11827c478bd9Sstevel@tonic-gate * until zero is returned. If zero is returned, 11837c478bd9Sstevel@tonic-gate * either there were no callbacks for this event 11847c478bd9Sstevel@tonic-gate * or they were already in progress. 11857c478bd9Sstevel@tonic-gate */ 11867c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 11877c478bd9Sstevel@tonic-gate if (as->a_callbacks && 1188843e1988Sjohnlev (cb = as_find_callback(as, AS_SETPROT_EVENT, 1189843e1988Sjohnlev seg->s_base, seg->s_size))) { 1190dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 11917c478bd9Sstevel@tonic-gate as_execute_callback(as, cb, AS_SETPROT_EVENT); 1192a98e9dbfSaguzovsk } else if (!AS_ISNOUNMAPWAIT(as)) { 11937c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 11947c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 11957c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 1196dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 11977c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 11987c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 1199a98e9dbfSaguzovsk } else { 1200a98e9dbfSaguzovsk /* 1201a98e9dbfSaguzovsk * We may have raced with 1202a98e9dbfSaguzovsk * segvn_reclaim()/segspt_reclaim(). In this 1203a98e9dbfSaguzovsk * case clean nounmapwait flag and retry since 1204a98e9dbfSaguzovsk * softlockcnt in this segment may be already 1205a98e9dbfSaguzovsk * 0. We don't drop as writer lock so our 1206a98e9dbfSaguzovsk * number of retries without sleeping should 1207a98e9dbfSaguzovsk * be very small. See segvn_reclaim() for 1208a98e9dbfSaguzovsk * more comments. 1209a98e9dbfSaguzovsk */ 1210a98e9dbfSaguzovsk AS_CLRNOUNMAPWAIT(as); 1211a98e9dbfSaguzovsk mutex_exit(&as->a_contents); 1212a98e9dbfSaguzovsk goto retry; 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 12157c478bd9Sstevel@tonic-gate goto setprot_top; 12167c478bd9Sstevel@tonic-gate } else if (error != 0) 12177c478bd9Sstevel@tonic-gate break; 12187c478bd9Sstevel@tonic-gate } 12197c478bd9Sstevel@tonic-gate if (error != 0) { 12207c478bd9Sstevel@tonic-gate as_setwatch(as); 12217c478bd9Sstevel@tonic-gate } else { 12227c478bd9Sstevel@tonic-gate as_setwatchprot(as, saveraddr, saversize, prot); 12237c478bd9Sstevel@tonic-gate } 1224dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 12257c478bd9Sstevel@tonic-gate return (error); 12267c478bd9Sstevel@tonic-gate } 12277c478bd9Sstevel@tonic-gate 12287c478bd9Sstevel@tonic-gate /* 12297c478bd9Sstevel@tonic-gate * Check to make sure that the interval [addr, addr + size) 12307c478bd9Sstevel@tonic-gate * in address space `as' has at least the specified protection. 12317c478bd9Sstevel@tonic-gate * It is ok for the range to cross over several segments, as long 12327c478bd9Sstevel@tonic-gate * as they are contiguous. 12337c478bd9Sstevel@tonic-gate */ 12347c478bd9Sstevel@tonic-gate int 12357c478bd9Sstevel@tonic-gate as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 12367c478bd9Sstevel@tonic-gate { 12377c478bd9Sstevel@tonic-gate struct seg *seg; 12387c478bd9Sstevel@tonic-gate size_t ssize; 12397c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 12407c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 12417c478bd9Sstevel@tonic-gate int error = 0; 12427c478bd9Sstevel@tonic-gate 12437c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 12447c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1245843e1988Sjohnlev (size_t)raddr; 12467c478bd9Sstevel@tonic-gate 12477c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 12487c478bd9Sstevel@tonic-gate return (ENOMEM); 12497c478bd9Sstevel@tonic-gate 12507c478bd9Sstevel@tonic-gate /* 12517c478bd9Sstevel@tonic-gate * This is ugly as sin... 12527c478bd9Sstevel@tonic-gate * Normally, we only acquire the address space readers lock. 12537c478bd9Sstevel@tonic-gate * However, if the address space has watchpoints present, 12547c478bd9Sstevel@tonic-gate * we must acquire the writer lock on the address space for 12557c478bd9Sstevel@tonic-gate * the benefit of as_clearwatchprot() and as_setwatchprot(). 12567c478bd9Sstevel@tonic-gate */ 12577c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) != 0) 1258dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 12597c478bd9Sstevel@tonic-gate else 1260dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 12617c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 12627c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 12637c478bd9Sstevel@tonic-gate if (seg == NULL) { 12647c478bd9Sstevel@tonic-gate as_setwatch(as); 1265dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 12667c478bd9Sstevel@tonic-gate return (ENOMEM); 12677c478bd9Sstevel@tonic-gate } 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 12707c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 12717c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 12727c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 12737c478bd9Sstevel@tonic-gate error = ENOMEM; 12747c478bd9Sstevel@tonic-gate break; 12757c478bd9Sstevel@tonic-gate } 12767c478bd9Sstevel@tonic-gate } 12777c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 12787c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 12797c478bd9Sstevel@tonic-gate else 12807c478bd9Sstevel@tonic-gate ssize = rsize; 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate error = SEGOP_CHECKPROT(seg, raddr, ssize, prot); 12837c478bd9Sstevel@tonic-gate if (error != 0) 12847c478bd9Sstevel@tonic-gate break; 12857c478bd9Sstevel@tonic-gate } 12867c478bd9Sstevel@tonic-gate as_setwatch(as); 1287dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 12887c478bd9Sstevel@tonic-gate return (error); 12897c478bd9Sstevel@tonic-gate } 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate int 12927c478bd9Sstevel@tonic-gate as_unmap(struct as *as, caddr_t addr, size_t size) 12937c478bd9Sstevel@tonic-gate { 12947c478bd9Sstevel@tonic-gate struct seg *seg, *seg_next; 12957c478bd9Sstevel@tonic-gate struct as_callback *cb; 12967c478bd9Sstevel@tonic-gate caddr_t raddr, eaddr; 129740688216SSudheer A size_t ssize, rsize = 0; 12987c478bd9Sstevel@tonic-gate int err; 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate top: 13017c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 13027c478bd9Sstevel@tonic-gate eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & 13037c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK); 13047c478bd9Sstevel@tonic-gate 1305dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 13067c478bd9Sstevel@tonic-gate 13077c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 13087c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 13097c478bd9Sstevel@tonic-gate 13107c478bd9Sstevel@tonic-gate /* 13117c478bd9Sstevel@tonic-gate * Use as_findseg to find the first segment in the range, then 13127c478bd9Sstevel@tonic-gate * step through the segments in order, following s_next. 13137c478bd9Sstevel@tonic-gate */ 13147c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, eaddr - raddr); 13157c478bd9Sstevel@tonic-gate 13167c478bd9Sstevel@tonic-gate for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) { 1317*284ce987SPatrick Mooney const boolean_t is_hole = ((seg->s_flags & S_HOLE) != 0); 1318*284ce987SPatrick Mooney 13197c478bd9Sstevel@tonic-gate if (eaddr <= seg->s_base) 13207c478bd9Sstevel@tonic-gate break; /* eaddr was in a gap; all done */ 13217c478bd9Sstevel@tonic-gate 13227c478bd9Sstevel@tonic-gate /* this is implied by the test above */ 13237c478bd9Sstevel@tonic-gate ASSERT(raddr < eaddr); 13247c478bd9Sstevel@tonic-gate 13257c478bd9Sstevel@tonic-gate if (raddr < seg->s_base) 13267c478bd9Sstevel@tonic-gate raddr = seg->s_base; /* raddr was in a gap */ 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate if (eaddr > (seg->s_base + seg->s_size)) 13297c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 13307c478bd9Sstevel@tonic-gate else 13317c478bd9Sstevel@tonic-gate ssize = eaddr - raddr; 13327c478bd9Sstevel@tonic-gate 13337c478bd9Sstevel@tonic-gate /* 13347c478bd9Sstevel@tonic-gate * Save next segment pointer since seg can be 13357c478bd9Sstevel@tonic-gate * destroyed during the segment unmap operation. 13367c478bd9Sstevel@tonic-gate */ 13377c478bd9Sstevel@tonic-gate seg_next = AS_SEGNEXT(as, seg); 13387c478bd9Sstevel@tonic-gate 133940688216SSudheer A /* 134040688216SSudheer A * We didn't count /dev/null mappings, so ignore them here. 134140688216SSudheer A * We'll handle MAP_NORESERVE cases in segvn_unmap(). (Again, 134240688216SSudheer A * we have to do this check here while we have seg.) 134340688216SSudheer A */ 134417965fd8SKrishnendu Sadhukhan - Sun Microsystems rsize = 0; 134540688216SSudheer A if (!SEG_IS_DEVNULL_MAPPING(seg) && 134640688216SSudheer A !SEG_IS_PARTIAL_RESV(seg)) 134740688216SSudheer A rsize = ssize; 134840688216SSudheer A 1349a98e9dbfSaguzovsk retry: 13507c478bd9Sstevel@tonic-gate err = SEGOP_UNMAP(seg, raddr, ssize); 13517c478bd9Sstevel@tonic-gate if (err == EAGAIN) { 13527c478bd9Sstevel@tonic-gate /* 13537c478bd9Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 13547c478bd9Sstevel@tonic-gate * before this operation can succeed through a retry. 13557c478bd9Sstevel@tonic-gate * The possible reasons for locked memory and 13567c478bd9Sstevel@tonic-gate * corresponding strategies for unlocking are: 13577c478bd9Sstevel@tonic-gate * (1) Normal I/O 13587c478bd9Sstevel@tonic-gate * wait for a signal that the I/O operation 13597c478bd9Sstevel@tonic-gate * has completed and the memory is unlocked. 13607c478bd9Sstevel@tonic-gate * (2) Asynchronous I/O 13617c478bd9Sstevel@tonic-gate * The aio subsystem does not unlock pages when 13627c478bd9Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 13637c478bd9Sstevel@tonic-gate * when the application calls aiowait/aioerror. 13647c478bd9Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 13657c478bd9Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 13667c478bd9Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 13677c478bd9Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 13687c478bd9Sstevel@tonic-gate * (3) Long term page locking: 13697c478bd9Sstevel@tonic-gate * Drivers intending to have pages locked for a 13707c478bd9Sstevel@tonic-gate * period considerably longer than for normal I/O 13717c478bd9Sstevel@tonic-gate * (essentially forever) may have registered for a 13727c478bd9Sstevel@tonic-gate * callback so they may unlock these pages on 13737c478bd9Sstevel@tonic-gate * request. This is needed to allow this operation 13747c478bd9Sstevel@tonic-gate * to succeed. Each entry on the callback list is 13757c478bd9Sstevel@tonic-gate * examined. If the event or address range pertains 13767c478bd9Sstevel@tonic-gate * the callback is invoked (unless it already is in 13777c478bd9Sstevel@tonic-gate * progress). The a_contents lock must be dropped 13787c478bd9Sstevel@tonic-gate * before the callback, so only one callback can 13797c478bd9Sstevel@tonic-gate * be done at a time. Go to the top and do more 13807c478bd9Sstevel@tonic-gate * until zero is returned. If zero is returned, 13817c478bd9Sstevel@tonic-gate * either there were no callbacks for this event 13827c478bd9Sstevel@tonic-gate * or they were already in progress. 13837c478bd9Sstevel@tonic-gate */ 13847c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 13857c478bd9Sstevel@tonic-gate if (as->a_callbacks && 1386843e1988Sjohnlev (cb = as_find_callback(as, AS_UNMAP_EVENT, 1387843e1988Sjohnlev seg->s_base, seg->s_size))) { 1388dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 13897c478bd9Sstevel@tonic-gate as_execute_callback(as, cb, AS_UNMAP_EVENT); 1390a98e9dbfSaguzovsk } else if (!AS_ISNOUNMAPWAIT(as)) { 13917c478bd9Sstevel@tonic-gate if (AS_ISUNMAPWAIT(as) == 0) 13927c478bd9Sstevel@tonic-gate cv_broadcast(&as->a_cv); 13937c478bd9Sstevel@tonic-gate AS_SETUNMAPWAIT(as); 1394dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 13957c478bd9Sstevel@tonic-gate while (AS_ISUNMAPWAIT(as)) 13967c478bd9Sstevel@tonic-gate cv_wait(&as->a_cv, &as->a_contents); 1397a98e9dbfSaguzovsk } else { 1398a98e9dbfSaguzovsk /* 1399a98e9dbfSaguzovsk * We may have raced with 1400a98e9dbfSaguzovsk * segvn_reclaim()/segspt_reclaim(). In this 1401a98e9dbfSaguzovsk * case clean nounmapwait flag and retry since 1402a98e9dbfSaguzovsk * softlockcnt in this segment may be already 1403a98e9dbfSaguzovsk * 0. We don't drop as writer lock so our 1404a98e9dbfSaguzovsk * number of retries without sleeping should 1405a98e9dbfSaguzovsk * be very small. See segvn_reclaim() for 1406a98e9dbfSaguzovsk * more comments. 1407a98e9dbfSaguzovsk */ 1408a98e9dbfSaguzovsk AS_CLRNOUNMAPWAIT(as); 1409a98e9dbfSaguzovsk mutex_exit(&as->a_contents); 1410a98e9dbfSaguzovsk goto retry; 14117c478bd9Sstevel@tonic-gate } 14127c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 14137c478bd9Sstevel@tonic-gate goto top; 14147c478bd9Sstevel@tonic-gate } else if (err == IE_RETRY) { 1415dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 14167c478bd9Sstevel@tonic-gate goto top; 14177c478bd9Sstevel@tonic-gate } else if (err) { 14187c478bd9Sstevel@tonic-gate as_setwatch(as); 1419dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 14207c478bd9Sstevel@tonic-gate return (-1); 14217c478bd9Sstevel@tonic-gate } 14227c478bd9Sstevel@tonic-gate 1423*284ce987SPatrick Mooney if (!is_hole) { 1424*284ce987SPatrick Mooney as->a_size -= ssize; 1425*284ce987SPatrick Mooney if (rsize) 1426*284ce987SPatrick Mooney as->a_resvsize -= rsize; 1427*284ce987SPatrick Mooney } 14287c478bd9Sstevel@tonic-gate raddr += ssize; 14297c478bd9Sstevel@tonic-gate } 1430dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 14317c478bd9Sstevel@tonic-gate return (0); 14327c478bd9Sstevel@tonic-gate } 14337c478bd9Sstevel@tonic-gate 14347c478bd9Sstevel@tonic-gate static int 143507b65a64Saguzovsk as_map_segvn_segs(struct as *as, caddr_t addr, size_t size, uint_t szcvec, 1436*284ce987SPatrick Mooney segcreate_func_t crfp, struct segvn_crargs *vn_a, boolean_t *segcreated) 14377c478bd9Sstevel@tonic-gate { 1438*284ce987SPatrick Mooney uint_t szc, nszc, save_szcvec; 14397c478bd9Sstevel@tonic-gate int error; 1440*284ce987SPatrick Mooney caddr_t a, eaddr; 14417c478bd9Sstevel@tonic-gate size_t pgsz; 1442*284ce987SPatrick Mooney const boolean_t do_off = (vn_a->vp != NULL || vn_a->amp != NULL); 14437c478bd9Sstevel@tonic-gate 1444dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 14457c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 14467c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 144707b65a64Saguzovsk ASSERT(vn_a->vp == NULL || vn_a->amp == NULL); 1448*284ce987SPatrick Mooney 144907b65a64Saguzovsk if (!do_off) { 145007b65a64Saguzovsk vn_a->offset = 0; 145107b65a64Saguzovsk } 14527c478bd9Sstevel@tonic-gate 14537c478bd9Sstevel@tonic-gate if (szcvec <= 1) { 1454*284ce987SPatrick Mooney struct seg *seg, *segref; 1455*284ce987SPatrick Mooney 1456*284ce987SPatrick Mooney seg = segref = seg_alloc(as, addr, size); 14577c478bd9Sstevel@tonic-gate if (seg == NULL) { 14587c478bd9Sstevel@tonic-gate return (ENOMEM); 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate vn_a->szc = 0; 1461*284ce987SPatrick Mooney error = (*crfp)(&seg, vn_a); 14627c478bd9Sstevel@tonic-gate if (error != 0) { 1463*284ce987SPatrick Mooney VERIFY3P(seg, ==, segref); 14647c478bd9Sstevel@tonic-gate seg_free(seg); 14653a30c6acSsusans } else { 14663a30c6acSsusans as->a_size += size; 146717965fd8SKrishnendu Sadhukhan - Sun Microsystems as->a_resvsize += size; 14687c478bd9Sstevel@tonic-gate } 14697c478bd9Sstevel@tonic-gate return (error); 14707c478bd9Sstevel@tonic-gate } 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate eaddr = addr + size; 14737c478bd9Sstevel@tonic-gate save_szcvec = szcvec; 14747c478bd9Sstevel@tonic-gate szcvec >>= 1; 14757c478bd9Sstevel@tonic-gate szc = 0; 14767c478bd9Sstevel@tonic-gate nszc = 0; 14777c478bd9Sstevel@tonic-gate while (szcvec) { 14787c478bd9Sstevel@tonic-gate if ((szcvec & 0x1) == 0) { 14797c478bd9Sstevel@tonic-gate nszc++; 14807c478bd9Sstevel@tonic-gate szcvec >>= 1; 14817c478bd9Sstevel@tonic-gate continue; 14827c478bd9Sstevel@tonic-gate } 14837c478bd9Sstevel@tonic-gate nszc++; 14847c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(nszc); 14857c478bd9Sstevel@tonic-gate a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 14867c478bd9Sstevel@tonic-gate if (a != addr) { 1487*284ce987SPatrick Mooney struct seg *seg, *segref; 1488*284ce987SPatrick Mooney size_t segsize; 1489*284ce987SPatrick Mooney 14907c478bd9Sstevel@tonic-gate ASSERT(a < eaddr); 1491*284ce987SPatrick Mooney 14927c478bd9Sstevel@tonic-gate segsize = a - addr; 1493*284ce987SPatrick Mooney seg = segref = seg_alloc(as, addr, segsize); 14947c478bd9Sstevel@tonic-gate if (seg == NULL) { 14957c478bd9Sstevel@tonic-gate return (ENOMEM); 14967c478bd9Sstevel@tonic-gate } 14977c478bd9Sstevel@tonic-gate vn_a->szc = szc; 1498*284ce987SPatrick Mooney error = (*crfp)(&seg, vn_a); 14997c478bd9Sstevel@tonic-gate if (error != 0) { 1500*284ce987SPatrick Mooney VERIFY3P(seg, ==, segref); 15017c478bd9Sstevel@tonic-gate seg_free(seg); 15027c478bd9Sstevel@tonic-gate return (error); 15037c478bd9Sstevel@tonic-gate } 15043a30c6acSsusans as->a_size += segsize; 150517965fd8SKrishnendu Sadhukhan - Sun Microsystems as->a_resvsize += segsize; 1506*284ce987SPatrick Mooney *segcreated = B_TRUE; 150707b65a64Saguzovsk if (do_off) { 150807b65a64Saguzovsk vn_a->offset += segsize; 150907b65a64Saguzovsk } 15107c478bd9Sstevel@tonic-gate addr = a; 15117c478bd9Sstevel@tonic-gate } 15127c478bd9Sstevel@tonic-gate szc = nszc; 15137c478bd9Sstevel@tonic-gate szcvec >>= 1; 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate ASSERT(addr < eaddr); 15177c478bd9Sstevel@tonic-gate szcvec = save_szcvec | 1; /* add 8K pages */ 15187c478bd9Sstevel@tonic-gate while (szcvec) { 15197c478bd9Sstevel@tonic-gate a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 15207c478bd9Sstevel@tonic-gate ASSERT(a >= addr); 15217c478bd9Sstevel@tonic-gate if (a != addr) { 1522*284ce987SPatrick Mooney struct seg *seg, *segref; 1523*284ce987SPatrick Mooney size_t segsize; 1524*284ce987SPatrick Mooney 15257c478bd9Sstevel@tonic-gate segsize = a - addr; 1526*284ce987SPatrick Mooney seg = segref = seg_alloc(as, addr, segsize); 15277c478bd9Sstevel@tonic-gate if (seg == NULL) { 15287c478bd9Sstevel@tonic-gate return (ENOMEM); 15297c478bd9Sstevel@tonic-gate } 15307c478bd9Sstevel@tonic-gate vn_a->szc = szc; 1531*284ce987SPatrick Mooney error = (*crfp)(&seg, vn_a); 15327c478bd9Sstevel@tonic-gate if (error != 0) { 1533*284ce987SPatrick Mooney VERIFY3P(seg, ==, segref); 15347c478bd9Sstevel@tonic-gate seg_free(seg); 15357c478bd9Sstevel@tonic-gate return (error); 15367c478bd9Sstevel@tonic-gate } 15373a30c6acSsusans as->a_size += segsize; 153817965fd8SKrishnendu Sadhukhan - Sun Microsystems as->a_resvsize += segsize; 1539*284ce987SPatrick Mooney *segcreated = B_TRUE; 154007b65a64Saguzovsk if (do_off) { 154107b65a64Saguzovsk vn_a->offset += segsize; 154207b65a64Saguzovsk } 15437c478bd9Sstevel@tonic-gate addr = a; 15447c478bd9Sstevel@tonic-gate } 15457c478bd9Sstevel@tonic-gate szcvec &= ~(1 << szc); 15467c478bd9Sstevel@tonic-gate if (szcvec) { 15477c478bd9Sstevel@tonic-gate szc = highbit(szcvec) - 1; 15487c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(szc); 15497c478bd9Sstevel@tonic-gate } 15507c478bd9Sstevel@tonic-gate } 15517c478bd9Sstevel@tonic-gate ASSERT(addr == eaddr); 15527c478bd9Sstevel@tonic-gate 155307b65a64Saguzovsk return (0); 155407b65a64Saguzovsk } 155507b65a64Saguzovsk 155607b65a64Saguzovsk static int 155707b65a64Saguzovsk as_map_vnsegs(struct as *as, caddr_t addr, size_t size, 1558*284ce987SPatrick Mooney segcreate_func_t crfp, struct segvn_crargs *vn_a, boolean_t *segcreated) 155907b65a64Saguzovsk { 1560ec25b48fSsusans uint_t mapflags = vn_a->flags & (MAP_TEXT | MAP_INITDATA); 1561ec25b48fSsusans int type = (vn_a->type == MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM; 1562ec25b48fSsusans uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags, 1563ec25b48fSsusans type, 0); 156407b65a64Saguzovsk int error; 156507b65a64Saguzovsk struct vattr va; 156607b65a64Saguzovsk u_offset_t eoff; 156707b65a64Saguzovsk size_t save_size = 0; 15682cb27123Saguzovsk extern size_t textrepl_size_thresh; 156907b65a64Saguzovsk 1570dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 157107b65a64Saguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 157207b65a64Saguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 157307b65a64Saguzovsk ASSERT(vn_a->vp != NULL); 157407b65a64Saguzovsk ASSERT(vn_a->amp == NULL); 157507b65a64Saguzovsk 157607b65a64Saguzovsk again: 157707b65a64Saguzovsk if (szcvec <= 1) { 1578*284ce987SPatrick Mooney struct seg *seg, *segref; 1579*284ce987SPatrick Mooney 1580*284ce987SPatrick Mooney seg = segref = seg_alloc(as, addr, size); 158107b65a64Saguzovsk if (seg == NULL) { 158207b65a64Saguzovsk return (ENOMEM); 158307b65a64Saguzovsk } 158407b65a64Saguzovsk vn_a->szc = 0; 1585*284ce987SPatrick Mooney error = (*crfp)(&seg, vn_a); 158607b65a64Saguzovsk if (error != 0) { 1587*284ce987SPatrick Mooney VERIFY3P(seg, ==, segref); 158807b65a64Saguzovsk seg_free(seg); 15893a30c6acSsusans } else { 15903a30c6acSsusans as->a_size += size; 159117965fd8SKrishnendu Sadhukhan - Sun Microsystems as->a_resvsize += size; 159207b65a64Saguzovsk } 159307b65a64Saguzovsk return (error); 159407b65a64Saguzovsk } 159507b65a64Saguzovsk 159607b65a64Saguzovsk va.va_mask = AT_SIZE; 1597da6c28aaSamw if (VOP_GETATTR(vn_a->vp, &va, ATTR_HINT, vn_a->cred, NULL) != 0) { 159807b65a64Saguzovsk szcvec = 0; 159907b65a64Saguzovsk goto again; 160007b65a64Saguzovsk } 160107b65a64Saguzovsk eoff = vn_a->offset & PAGEMASK; 160207b65a64Saguzovsk if (eoff >= va.va_size) { 160307b65a64Saguzovsk szcvec = 0; 160407b65a64Saguzovsk goto again; 160507b65a64Saguzovsk } 160607b65a64Saguzovsk eoff += size; 160707b65a64Saguzovsk if (btopr(va.va_size) < btopr(eoff)) { 160807b65a64Saguzovsk save_size = size; 160907b65a64Saguzovsk size = va.va_size - (vn_a->offset & PAGEMASK); 161007b65a64Saguzovsk size = P2ROUNDUP_TYPED(size, PAGESIZE, size_t); 1611ec25b48fSsusans szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, mapflags, 1612ec25b48fSsusans type, 0); 161307b65a64Saguzovsk if (szcvec <= 1) { 161407b65a64Saguzovsk size = save_size; 161507b65a64Saguzovsk goto again; 161607b65a64Saguzovsk } 161707b65a64Saguzovsk } 161807b65a64Saguzovsk 16192cb27123Saguzovsk if (size > textrepl_size_thresh) { 16202cb27123Saguzovsk vn_a->flags |= _MAP_TEXTREPL; 16212cb27123Saguzovsk } 162207b65a64Saguzovsk error = as_map_segvn_segs(as, addr, size, szcvec, crfp, vn_a, 162307b65a64Saguzovsk segcreated); 162407b65a64Saguzovsk if (error != 0) { 162507b65a64Saguzovsk return (error); 162607b65a64Saguzovsk } 16277c478bd9Sstevel@tonic-gate if (save_size) { 162807b65a64Saguzovsk addr += size; 16297c478bd9Sstevel@tonic-gate size = save_size - size; 163007b65a64Saguzovsk szcvec = 0; 16317c478bd9Sstevel@tonic-gate goto again; 16327c478bd9Sstevel@tonic-gate } 16337c478bd9Sstevel@tonic-gate return (0); 16347c478bd9Sstevel@tonic-gate } 16357c478bd9Sstevel@tonic-gate 1636ec25b48fSsusans /* 1637ec25b48fSsusans * as_map_ansegs: shared or private anonymous memory. Note that the flags 1638ec25b48fSsusans * passed to map_pgszvec cannot be MAP_INITDATA, for anon. 1639ec25b48fSsusans */ 164007b65a64Saguzovsk static int 1641ec25b48fSsusans as_map_ansegs(struct as *as, caddr_t addr, size_t size, 1642*284ce987SPatrick Mooney segcreate_func_t crfp, struct segvn_crargs *vn_a, boolean_t *segcreated) 164307b65a64Saguzovsk { 1644ec25b48fSsusans uint_t szcvec; 1645ec25b48fSsusans uchar_t type; 1646ec25b48fSsusans 1647ec25b48fSsusans ASSERT(vn_a->type == MAP_SHARED || vn_a->type == MAP_PRIVATE); 1648ec25b48fSsusans if (vn_a->type == MAP_SHARED) { 1649ec25b48fSsusans type = MAPPGSZC_SHM; 1650ec25b48fSsusans } else if (vn_a->type == MAP_PRIVATE) { 1651ec25b48fSsusans if (vn_a->szc == AS_MAP_HEAP) { 1652ec25b48fSsusans type = MAPPGSZC_HEAP; 1653ec25b48fSsusans } else if (vn_a->szc == AS_MAP_STACK) { 1654ec25b48fSsusans type = MAPPGSZC_STACK; 1655ec25b48fSsusans } else { 1656ec25b48fSsusans type = MAPPGSZC_PRIVM; 1657ec25b48fSsusans } 1658ec25b48fSsusans } 1659ec25b48fSsusans szcvec = map_pgszcvec(addr, size, vn_a->amp == NULL ? 1660ec25b48fSsusans (uintptr_t)addr : (uintptr_t)P2ROUNDUP(vn_a->offset, PAGESIZE), 1661ec25b48fSsusans (vn_a->flags & MAP_TEXT), type, 0); 1662dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 166307b65a64Saguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 166407b65a64Saguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 166507b65a64Saguzovsk ASSERT(vn_a->vp == NULL); 166607b65a64Saguzovsk 166707b65a64Saguzovsk return (as_map_segvn_segs(as, addr, size, szcvec, 166807b65a64Saguzovsk crfp, vn_a, segcreated)); 166907b65a64Saguzovsk } 167007b65a64Saguzovsk 16717c478bd9Sstevel@tonic-gate int 1672*284ce987SPatrick Mooney as_map(struct as *as, caddr_t addr, size_t size, segcreate_func_t crfp, 1673*284ce987SPatrick Mooney void *argsp) 167402ff05a9Svsakar { 1675dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 167602ff05a9Svsakar return (as_map_locked(as, addr, size, crfp, argsp)); 167702ff05a9Svsakar } 167802ff05a9Svsakar 167902ff05a9Svsakar int 1680*284ce987SPatrick Mooney as_map_locked(struct as *as, caddr_t addr, size_t size, segcreate_func_t crfp, 16816430daecSBrandon Baker void *argsp) 16827c478bd9Sstevel@tonic-gate { 16837c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 16847c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 16857c478bd9Sstevel@tonic-gate int error; 1686*284ce987SPatrick Mooney boolean_t is_hole = B_FALSE; 16876430daecSBrandon Baker /* 16886430daecSBrandon Baker * The use of a_proc is preferred to handle the case where curproc is 16896430daecSBrandon Baker * a door_call server and is allocating memory in the client's (a_proc) 16906430daecSBrandon Baker * address space. 16916430daecSBrandon Baker * When creating a shared memory segment a_proc will be NULL so we 16926430daecSBrandon Baker * fallback to curproc in that case. 16936430daecSBrandon Baker */ 16946430daecSBrandon Baker struct proc *p = (as->a_proc == NULL) ? curproc : as->a_proc; 1695232cfe63Ssusans struct segvn_crargs crargs; 16967c478bd9Sstevel@tonic-gate 16977c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 16987c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 1699843e1988Sjohnlev (size_t)raddr; 17007c478bd9Sstevel@tonic-gate 17017c478bd9Sstevel@tonic-gate /* 17027c478bd9Sstevel@tonic-gate * check for wrap around 17037c478bd9Sstevel@tonic-gate */ 17047c478bd9Sstevel@tonic-gate if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) { 1705dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 17067c478bd9Sstevel@tonic-gate return (ENOMEM); 17077c478bd9Sstevel@tonic-gate } 17087c478bd9Sstevel@tonic-gate 17097c478bd9Sstevel@tonic-gate as->a_updatedir = 1; /* inform /proc */ 17107c478bd9Sstevel@tonic-gate gethrestime(&as->a_updatetime); 17117c478bd9Sstevel@tonic-gate 1712*284ce987SPatrick Mooney if (as != &kas) { 1713*284ce987SPatrick Mooney /* 1714*284ce987SPatrick Mooney * Ensure that the virtual size of the process will not exceed 1715*284ce987SPatrick Mooney * the configured limit. Since seg_hole segments will later 1716*284ce987SPatrick Mooney * set the S_HOLE flag indicating their status as a hole in the 1717*284ce987SPatrick Mooney * AS, they are excluded from this check. 1718*284ce987SPatrick Mooney */ 1719*284ce987SPatrick Mooney if (as->a_size + rsize > (size_t)p->p_vmem_ctl && 1720*284ce987SPatrick Mooney !AS_MAP_CHECK_SEGHOLE(crfp)) { 1721*284ce987SPatrick Mooney AS_LOCK_EXIT(as); 17227c478bd9Sstevel@tonic-gate 1723*284ce987SPatrick Mooney (void) rctl_action(rctlproc_legacy[RLIMIT_VMEM], 1724*284ce987SPatrick Mooney p->p_rctls, p, RCA_UNSAFE_ALL); 1725*284ce987SPatrick Mooney return (ENOMEM); 1726*284ce987SPatrick Mooney } 17277c478bd9Sstevel@tonic-gate } 17287c478bd9Sstevel@tonic-gate 1729ec25b48fSsusans if (AS_MAP_CHECK_VNODE_LPOOB(crfp, argsp)) { 1730*284ce987SPatrick Mooney boolean_t do_unmap = B_FALSE; 1731*284ce987SPatrick Mooney 1732232cfe63Ssusans crargs = *(struct segvn_crargs *)argsp; 1733*284ce987SPatrick Mooney error = as_map_vnsegs(as, raddr, rsize, crfp, &crargs, 1734*284ce987SPatrick Mooney &do_unmap); 1735ec25b48fSsusans if (error != 0) { 1736dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 1737*284ce987SPatrick Mooney if (do_unmap) { 1738ec25b48fSsusans (void) as_unmap(as, addr, size); 1739ec25b48fSsusans } 1740ec25b48fSsusans return (error); 174107b65a64Saguzovsk } 1742ec25b48fSsusans } else if (AS_MAP_CHECK_ANON_LPOOB(crfp, argsp)) { 1743*284ce987SPatrick Mooney boolean_t do_unmap = B_FALSE; 1744*284ce987SPatrick Mooney 1745232cfe63Ssusans crargs = *(struct segvn_crargs *)argsp; 1746*284ce987SPatrick Mooney error = as_map_ansegs(as, raddr, rsize, crfp, &crargs, 1747*284ce987SPatrick Mooney &do_unmap); 17487c478bd9Sstevel@tonic-gate if (error != 0) { 1749dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 1750*284ce987SPatrick Mooney if (do_unmap) { 17517c478bd9Sstevel@tonic-gate (void) as_unmap(as, addr, size); 17527c478bd9Sstevel@tonic-gate } 17537c478bd9Sstevel@tonic-gate return (error); 17547c478bd9Sstevel@tonic-gate } 17557c478bd9Sstevel@tonic-gate } else { 1756*284ce987SPatrick Mooney struct seg *seg, *segref; 1757*284ce987SPatrick Mooney 1758*284ce987SPatrick Mooney seg = segref = seg_alloc(as, addr, size); 17597c478bd9Sstevel@tonic-gate if (seg == NULL) { 1760dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 17617c478bd9Sstevel@tonic-gate return (ENOMEM); 17627c478bd9Sstevel@tonic-gate } 17637c478bd9Sstevel@tonic-gate 1764*284ce987SPatrick Mooney /* 1765*284ce987SPatrick Mooney * It is possible that the segment creation routine will free 1766*284ce987SPatrick Mooney * 'seg' as part of a more advanced operation, such as when 1767*284ce987SPatrick Mooney * segvn concatenates adjacent segments together. When this 1768*284ce987SPatrick Mooney * occurs, the seg*_create routine must communicate the 1769*284ce987SPatrick Mooney * resulting segment out via the 'struct seg **' parameter. 1770*284ce987SPatrick Mooney * 1771*284ce987SPatrick Mooney * If segment creation fails, it must not free the passed-in 1772*284ce987SPatrick Mooney * segment, nor alter the argument pointer. 1773*284ce987SPatrick Mooney */ 1774*284ce987SPatrick Mooney error = (*crfp)(&seg, argsp); 17757c478bd9Sstevel@tonic-gate if (error != 0) { 1776*284ce987SPatrick Mooney VERIFY3P(seg, ==, segref); 17777c478bd9Sstevel@tonic-gate seg_free(seg); 1778dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 17797c478bd9Sstevel@tonic-gate return (error); 17807c478bd9Sstevel@tonic-gate } 1781*284ce987SPatrick Mooney 17823a30c6acSsusans /* 1783*284ce987SPatrick Mooney * Check if the resulting segment represents a hole in the 1784*284ce987SPatrick Mooney * address space, rather than contributing to the AS size. 17853a30c6acSsusans */ 1786*284ce987SPatrick Mooney is_hole = ((seg->s_flags & S_HOLE) != 0); 1787*284ce987SPatrick Mooney 1788*284ce987SPatrick Mooney /* Add size now so as_unmap will work if as_ctl fails. */ 1789*284ce987SPatrick Mooney if (!is_hole) { 1790*284ce987SPatrick Mooney as->a_size += rsize; 1791*284ce987SPatrick Mooney as->a_resvsize += rsize; 1792*284ce987SPatrick Mooney } 17937c478bd9Sstevel@tonic-gate } 17947c478bd9Sstevel@tonic-gate 17957c478bd9Sstevel@tonic-gate as_setwatch(as); 17967c478bd9Sstevel@tonic-gate 17977c478bd9Sstevel@tonic-gate /* 1798*284ce987SPatrick Mooney * Establish memory locks for the segment if the address space is 1799*284ce987SPatrick Mooney * locked, provided it's not an explicit hole in the AS. 18007c478bd9Sstevel@tonic-gate */ 18017c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 1802*284ce987SPatrick Mooney if (AS_ISPGLCK(as) && !is_hole) { 18037c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 1804dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 18057c478bd9Sstevel@tonic-gate error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0); 18067c478bd9Sstevel@tonic-gate if (error != 0) 18077c478bd9Sstevel@tonic-gate (void) as_unmap(as, addr, size); 18087c478bd9Sstevel@tonic-gate } else { 18097c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 1810dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 18117c478bd9Sstevel@tonic-gate } 18127c478bd9Sstevel@tonic-gate return (error); 18137c478bd9Sstevel@tonic-gate } 18147c478bd9Sstevel@tonic-gate 18157c478bd9Sstevel@tonic-gate 18167c478bd9Sstevel@tonic-gate /* 18177c478bd9Sstevel@tonic-gate * Delete all segments in the address space marked with S_PURGE. 18187c478bd9Sstevel@tonic-gate * This is currently used for Sparc V9 nofault ASI segments (seg_nf.c). 18197c478bd9Sstevel@tonic-gate * These segments are deleted as a first step before calls to as_gap(), so 18207c478bd9Sstevel@tonic-gate * that they don't affect mmap() or shmat(). 18217c478bd9Sstevel@tonic-gate */ 18227c478bd9Sstevel@tonic-gate void 18237c478bd9Sstevel@tonic-gate as_purge(struct as *as) 18247c478bd9Sstevel@tonic-gate { 18257c478bd9Sstevel@tonic-gate struct seg *seg; 18267c478bd9Sstevel@tonic-gate struct seg *next_seg; 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate /* 18297c478bd9Sstevel@tonic-gate * the setting of NEEDSPURGE is protect by as_rangelock(), so 18307c478bd9Sstevel@tonic-gate * no need to grab a_contents mutex for this check 18317c478bd9Sstevel@tonic-gate */ 18327c478bd9Sstevel@tonic-gate if ((as->a_flags & AS_NEEDSPURGE) == 0) 18337c478bd9Sstevel@tonic-gate return; 18347c478bd9Sstevel@tonic-gate 1835dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 18367c478bd9Sstevel@tonic-gate next_seg = NULL; 18377c478bd9Sstevel@tonic-gate seg = AS_SEGFIRST(as); 18387c478bd9Sstevel@tonic-gate while (seg != NULL) { 18397c478bd9Sstevel@tonic-gate next_seg = AS_SEGNEXT(as, seg); 18407c478bd9Sstevel@tonic-gate if (seg->s_flags & S_PURGE) 18417c478bd9Sstevel@tonic-gate SEGOP_UNMAP(seg, seg->s_base, seg->s_size); 18427c478bd9Sstevel@tonic-gate seg = next_seg; 18437c478bd9Sstevel@tonic-gate } 1844dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 18477c478bd9Sstevel@tonic-gate as->a_flags &= ~AS_NEEDSPURGE; 18487c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 18497c478bd9Sstevel@tonic-gate } 18507c478bd9Sstevel@tonic-gate 18517c478bd9Sstevel@tonic-gate /* 185246ab9534Smec * Find a hole within [*basep, *basep + *lenp), which contains a mappable 185346ab9534Smec * range of addresses at least "minlen" long, where the base of the range is 185446ab9534Smec * at "off" phase from an "align" boundary and there is space for a 185546ab9534Smec * "redzone"-sized redzone on eithe rside of the range. Thus, 185646ab9534Smec * if align was 4M and off was 16k, the user wants a hole which will start 185746ab9534Smec * 16k into a 4M page. 18587c478bd9Sstevel@tonic-gate * 18597c478bd9Sstevel@tonic-gate * If flags specifies AH_HI, the hole will have the highest possible address 18607c478bd9Sstevel@tonic-gate * in the range. We use the as->a_lastgap field to figure out where to 18617c478bd9Sstevel@tonic-gate * start looking for a gap. 18627c478bd9Sstevel@tonic-gate * 18637c478bd9Sstevel@tonic-gate * Otherwise, the gap will have the lowest possible address. 18647c478bd9Sstevel@tonic-gate * 18657c478bd9Sstevel@tonic-gate * If flags specifies AH_CONTAIN, the hole will contain the address addr. 18667c478bd9Sstevel@tonic-gate * 186746ab9534Smec * If an adequate hole is found, *basep and *lenp are set to reflect the part of 186846ab9534Smec * the hole that is within range, and 0 is returned. On failure, -1 is returned. 18697c478bd9Sstevel@tonic-gate * 18707c478bd9Sstevel@tonic-gate * NOTE: This routine is not correct when base+len overflows caddr_t. 18717c478bd9Sstevel@tonic-gate */ 18727c478bd9Sstevel@tonic-gate int 187346ab9534Smec as_gap_aligned(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, 187446ab9534Smec uint_t flags, caddr_t addr, size_t align, size_t redzone, size_t off) 18757c478bd9Sstevel@tonic-gate { 18767c478bd9Sstevel@tonic-gate caddr_t lobound = *basep; 18777c478bd9Sstevel@tonic-gate caddr_t hibound = lobound + *lenp; 18787c478bd9Sstevel@tonic-gate struct seg *lseg, *hseg; 18797c478bd9Sstevel@tonic-gate caddr_t lo, hi; 18807c478bd9Sstevel@tonic-gate int forward; 18817c478bd9Sstevel@tonic-gate caddr_t save_base; 18827c478bd9Sstevel@tonic-gate size_t save_len; 18837d87bd0eSmec size_t save_minlen; 18847d87bd0eSmec size_t save_redzone; 18857d87bd0eSmec int fast_path = 1; 18867c478bd9Sstevel@tonic-gate 18877c478bd9Sstevel@tonic-gate save_base = *basep; 18887c478bd9Sstevel@tonic-gate save_len = *lenp; 18897d87bd0eSmec save_minlen = minlen; 18907d87bd0eSmec save_redzone = redzone; 18917d87bd0eSmec 18927d87bd0eSmec /* 18937d87bd0eSmec * For the first pass/fast_path, just add align and redzone into 18947d87bd0eSmec * minlen since if we get an allocation, we can guarantee that it 18957d87bd0eSmec * will fit the alignment and redzone requested. 18967d87bd0eSmec * This increases the chance that hibound will be adjusted to 18977d87bd0eSmec * a_lastgap->s_base which will likely allow us to find an 18987d87bd0eSmec * acceptable hole in the address space quicker. 18997d87bd0eSmec * If we can't find a hole with this fast_path, then we look for 19007d87bd0eSmec * smaller holes in which the alignment and offset may allow 19017d87bd0eSmec * the allocation to fit. 19027d87bd0eSmec */ 19037d87bd0eSmec minlen += align; 19047d87bd0eSmec minlen += 2 * redzone; 19057d87bd0eSmec redzone = 0; 19067d87bd0eSmec 1907dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 19087c478bd9Sstevel@tonic-gate if (AS_SEGFIRST(as) == NULL) { 190946ab9534Smec if (valid_va_range_aligned(basep, lenp, minlen, flags & AH_DIR, 191046ab9534Smec align, redzone, off)) { 1911dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 19127c478bd9Sstevel@tonic-gate return (0); 19137c478bd9Sstevel@tonic-gate } else { 1914dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 19157c478bd9Sstevel@tonic-gate *basep = save_base; 19167c478bd9Sstevel@tonic-gate *lenp = save_len; 19177c478bd9Sstevel@tonic-gate return (-1); 19187c478bd9Sstevel@tonic-gate } 19197c478bd9Sstevel@tonic-gate } 19207c478bd9Sstevel@tonic-gate 19217d87bd0eSmec retry: 19227c478bd9Sstevel@tonic-gate /* 19237c478bd9Sstevel@tonic-gate * Set up to iterate over all the inter-segment holes in the given 19247c478bd9Sstevel@tonic-gate * direction. lseg is NULL for the lowest-addressed hole and hseg is 19257c478bd9Sstevel@tonic-gate * NULL for the highest-addressed hole. If moving backwards, we reset 19267c478bd9Sstevel@tonic-gate * sseg to denote the highest-addressed segment. 19277c478bd9Sstevel@tonic-gate */ 19287c478bd9Sstevel@tonic-gate forward = (flags & AH_DIR) == AH_LO; 19297c478bd9Sstevel@tonic-gate if (forward) { 19307c478bd9Sstevel@tonic-gate hseg = as_findseg(as, lobound, 1); 19317c478bd9Sstevel@tonic-gate lseg = AS_SEGPREV(as, hseg); 19327c478bd9Sstevel@tonic-gate } else { 19337c478bd9Sstevel@tonic-gate 19347c478bd9Sstevel@tonic-gate /* 19357c478bd9Sstevel@tonic-gate * If allocating at least as much as the last allocation, 19367c478bd9Sstevel@tonic-gate * use a_lastgap's base as a better estimate of hibound. 19377c478bd9Sstevel@tonic-gate */ 19387c478bd9Sstevel@tonic-gate if (as->a_lastgap && 19397c478bd9Sstevel@tonic-gate minlen >= as->a_lastgap->s_size && 19407c478bd9Sstevel@tonic-gate hibound >= as->a_lastgap->s_base) 19417c478bd9Sstevel@tonic-gate hibound = as->a_lastgap->s_base; 19427c478bd9Sstevel@tonic-gate 19437c478bd9Sstevel@tonic-gate hseg = as_findseg(as, hibound, 1); 19447c478bd9Sstevel@tonic-gate if (hseg->s_base + hseg->s_size < hibound) { 19457c478bd9Sstevel@tonic-gate lseg = hseg; 19467c478bd9Sstevel@tonic-gate hseg = NULL; 19477c478bd9Sstevel@tonic-gate } else { 19487c478bd9Sstevel@tonic-gate lseg = AS_SEGPREV(as, hseg); 19497c478bd9Sstevel@tonic-gate } 19507c478bd9Sstevel@tonic-gate } 19517c478bd9Sstevel@tonic-gate 19527c478bd9Sstevel@tonic-gate for (;;) { 19537c478bd9Sstevel@tonic-gate /* 19547c478bd9Sstevel@tonic-gate * Set lo and hi to the hole's boundaries. (We should really 19557c478bd9Sstevel@tonic-gate * use MAXADDR in place of hibound in the expression below, 19567c478bd9Sstevel@tonic-gate * but can't express it easily; using hibound in its place is 19577c478bd9Sstevel@tonic-gate * harmless.) 19587c478bd9Sstevel@tonic-gate */ 19597c478bd9Sstevel@tonic-gate lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size; 19607c478bd9Sstevel@tonic-gate hi = (hseg == NULL) ? hibound : hseg->s_base; 19617c478bd9Sstevel@tonic-gate /* 19627c478bd9Sstevel@tonic-gate * If the iteration has moved past the interval from lobound 19637c478bd9Sstevel@tonic-gate * to hibound it's pointless to continue. 19647c478bd9Sstevel@tonic-gate */ 19657c478bd9Sstevel@tonic-gate if ((forward && lo > hibound) || (!forward && hi < lobound)) 19667c478bd9Sstevel@tonic-gate break; 19677c478bd9Sstevel@tonic-gate else if (lo > hibound || hi < lobound) 19687c478bd9Sstevel@tonic-gate goto cont; 19697c478bd9Sstevel@tonic-gate /* 19707c478bd9Sstevel@tonic-gate * Candidate hole lies at least partially within the allowable 19717c478bd9Sstevel@tonic-gate * range. Restrict it to fall completely within that range, 19727c478bd9Sstevel@tonic-gate * i.e., to [max(lo, lobound), min(hi, hibound)]. 19737c478bd9Sstevel@tonic-gate */ 19747c478bd9Sstevel@tonic-gate if (lo < lobound) 19757c478bd9Sstevel@tonic-gate lo = lobound; 19767c478bd9Sstevel@tonic-gate if (hi > hibound) 19777c478bd9Sstevel@tonic-gate hi = hibound; 19787c478bd9Sstevel@tonic-gate /* 19797c478bd9Sstevel@tonic-gate * Verify that the candidate hole is big enough and meets 19807d87bd0eSmec * hardware constraints. If the hole is too small, no need 19817d87bd0eSmec * to do the further checks since they will fail. 19827c478bd9Sstevel@tonic-gate */ 19837c478bd9Sstevel@tonic-gate *basep = lo; 19847c478bd9Sstevel@tonic-gate *lenp = hi - lo; 19857d87bd0eSmec if (*lenp >= minlen && valid_va_range_aligned(basep, lenp, 19867d87bd0eSmec minlen, forward ? AH_LO : AH_HI, align, redzone, off) && 19877c478bd9Sstevel@tonic-gate ((flags & AH_CONTAIN) == 0 || 19887c478bd9Sstevel@tonic-gate (*basep <= addr && *basep + *lenp > addr))) { 19897c478bd9Sstevel@tonic-gate if (!forward) 19907c478bd9Sstevel@tonic-gate as->a_lastgap = hseg; 19917c478bd9Sstevel@tonic-gate if (hseg != NULL) 19927c478bd9Sstevel@tonic-gate as->a_lastgaphl = hseg; 19937c478bd9Sstevel@tonic-gate else 19947c478bd9Sstevel@tonic-gate as->a_lastgaphl = lseg; 1995dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 19967c478bd9Sstevel@tonic-gate return (0); 19977c478bd9Sstevel@tonic-gate } 19987c478bd9Sstevel@tonic-gate cont: 19997c478bd9Sstevel@tonic-gate /* 20007c478bd9Sstevel@tonic-gate * Move to the next hole. 20017c478bd9Sstevel@tonic-gate */ 20027c478bd9Sstevel@tonic-gate if (forward) { 20037c478bd9Sstevel@tonic-gate lseg = hseg; 20047c478bd9Sstevel@tonic-gate if (lseg == NULL) 20057c478bd9Sstevel@tonic-gate break; 20067c478bd9Sstevel@tonic-gate hseg = AS_SEGNEXT(as, hseg); 20077c478bd9Sstevel@tonic-gate } else { 20087c478bd9Sstevel@tonic-gate hseg = lseg; 20097c478bd9Sstevel@tonic-gate if (hseg == NULL) 20107c478bd9Sstevel@tonic-gate break; 20117c478bd9Sstevel@tonic-gate lseg = AS_SEGPREV(as, lseg); 20127c478bd9Sstevel@tonic-gate } 20137c478bd9Sstevel@tonic-gate } 20147d87bd0eSmec if (fast_path && (align != 0 || save_redzone != 0)) { 20157d87bd0eSmec fast_path = 0; 20167d87bd0eSmec minlen = save_minlen; 20177d87bd0eSmec redzone = save_redzone; 20187d87bd0eSmec goto retry; 20197d87bd0eSmec } 20207c478bd9Sstevel@tonic-gate *basep = save_base; 20217c478bd9Sstevel@tonic-gate *lenp = save_len; 2022dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 20237c478bd9Sstevel@tonic-gate return (-1); 20247c478bd9Sstevel@tonic-gate } 20257c478bd9Sstevel@tonic-gate 202646ab9534Smec /* 202746ab9534Smec * Find a hole of at least size minlen within [*basep, *basep + *lenp). 202846ab9534Smec * 202946ab9534Smec * If flags specifies AH_HI, the hole will have the highest possible address 203046ab9534Smec * in the range. We use the as->a_lastgap field to figure out where to 203146ab9534Smec * start looking for a gap. 203246ab9534Smec * 203346ab9534Smec * Otherwise, the gap will have the lowest possible address. 203446ab9534Smec * 203546ab9534Smec * If flags specifies AH_CONTAIN, the hole will contain the address addr. 203646ab9534Smec * 203746ab9534Smec * If an adequate hole is found, base and len are set to reflect the part of 203846ab9534Smec * the hole that is within range, and 0 is returned, otherwise, 203946ab9534Smec * -1 is returned. 204046ab9534Smec * 204146ab9534Smec * NOTE: This routine is not correct when base+len overflows caddr_t. 204246ab9534Smec */ 204346ab9534Smec int 204446ab9534Smec as_gap(struct as *as, size_t minlen, caddr_t *basep, size_t *lenp, uint_t flags, 204546ab9534Smec caddr_t addr) 204646ab9534Smec { 204746ab9534Smec 204846ab9534Smec return (as_gap_aligned(as, minlen, basep, lenp, flags, addr, 0, 0, 0)); 204946ab9534Smec } 205046ab9534Smec 20517c478bd9Sstevel@tonic-gate /* 20527c478bd9Sstevel@tonic-gate * Return the next range within [base, base + len) that is backed 20537c478bd9Sstevel@tonic-gate * with "real memory". Skip holes and non-seg_vn segments. 20547c478bd9Sstevel@tonic-gate * We're lazy and only return one segment at a time. 20557c478bd9Sstevel@tonic-gate */ 20567c478bd9Sstevel@tonic-gate int 20577c478bd9Sstevel@tonic-gate as_memory(struct as *as, caddr_t *basep, size_t *lenp) 20587c478bd9Sstevel@tonic-gate { 20597c478bd9Sstevel@tonic-gate extern struct seg_ops segspt_shmops; /* needs a header file */ 20607c478bd9Sstevel@tonic-gate struct seg *seg; 20617c478bd9Sstevel@tonic-gate caddr_t addr, eaddr; 20627c478bd9Sstevel@tonic-gate caddr_t segend; 20637c478bd9Sstevel@tonic-gate 2064dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 20657c478bd9Sstevel@tonic-gate 20667c478bd9Sstevel@tonic-gate addr = *basep; 20677c478bd9Sstevel@tonic-gate eaddr = addr + *lenp; 20687c478bd9Sstevel@tonic-gate 20697c478bd9Sstevel@tonic-gate seg = as_findseg(as, addr, 0); 20707c478bd9Sstevel@tonic-gate if (seg != NULL) 20717c478bd9Sstevel@tonic-gate addr = MAX(seg->s_base, addr); 20727c478bd9Sstevel@tonic-gate 20737c478bd9Sstevel@tonic-gate for (;;) { 20747c478bd9Sstevel@tonic-gate if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) { 2075dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 20767c478bd9Sstevel@tonic-gate return (EINVAL); 20777c478bd9Sstevel@tonic-gate } 20787c478bd9Sstevel@tonic-gate 20797c478bd9Sstevel@tonic-gate if (seg->s_ops == &segvn_ops) { 20807c478bd9Sstevel@tonic-gate segend = seg->s_base + seg->s_size; 20817c478bd9Sstevel@tonic-gate break; 20827c478bd9Sstevel@tonic-gate } 20837c478bd9Sstevel@tonic-gate 20847c478bd9Sstevel@tonic-gate /* 20857c478bd9Sstevel@tonic-gate * We do ISM by looking into the private data 20867c478bd9Sstevel@tonic-gate * to determine the real size of the segment. 20877c478bd9Sstevel@tonic-gate */ 20887c478bd9Sstevel@tonic-gate if (seg->s_ops == &segspt_shmops) { 20897c478bd9Sstevel@tonic-gate segend = seg->s_base + spt_realsize(seg); 20907c478bd9Sstevel@tonic-gate if (addr < segend) 20917c478bd9Sstevel@tonic-gate break; 20927c478bd9Sstevel@tonic-gate } 20937c478bd9Sstevel@tonic-gate 20947c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 20957c478bd9Sstevel@tonic-gate 20967c478bd9Sstevel@tonic-gate if (seg != NULL) 20977c478bd9Sstevel@tonic-gate addr = seg->s_base; 20987c478bd9Sstevel@tonic-gate } 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate *basep = addr; 21017c478bd9Sstevel@tonic-gate 21027c478bd9Sstevel@tonic-gate if (segend > eaddr) 21037c478bd9Sstevel@tonic-gate *lenp = eaddr - addr; 21047c478bd9Sstevel@tonic-gate else 21057c478bd9Sstevel@tonic-gate *lenp = segend - addr; 21067c478bd9Sstevel@tonic-gate 2107dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 21087c478bd9Sstevel@tonic-gate return (0); 21097c478bd9Sstevel@tonic-gate } 21107c478bd9Sstevel@tonic-gate 21117c478bd9Sstevel@tonic-gate /* 21127c478bd9Sstevel@tonic-gate * Swap the pages associated with the address space as out to 21137c478bd9Sstevel@tonic-gate * secondary storage, returning the number of bytes actually 21147c478bd9Sstevel@tonic-gate * swapped. 21157c478bd9Sstevel@tonic-gate * 21167c478bd9Sstevel@tonic-gate * The value returned is intended to correlate well with the process's 21177c478bd9Sstevel@tonic-gate * memory requirements. Its usefulness for this purpose depends on 21187c478bd9Sstevel@tonic-gate * how well the segment-level routines do at returning accurate 21197c478bd9Sstevel@tonic-gate * information. 21207c478bd9Sstevel@tonic-gate */ 21217c478bd9Sstevel@tonic-gate size_t 21227c478bd9Sstevel@tonic-gate as_swapout(struct as *as) 21237c478bd9Sstevel@tonic-gate { 21247c478bd9Sstevel@tonic-gate struct seg *seg; 21257c478bd9Sstevel@tonic-gate size_t swpcnt = 0; 21267c478bd9Sstevel@tonic-gate 21277c478bd9Sstevel@tonic-gate /* 21287c478bd9Sstevel@tonic-gate * Kernel-only processes have given up their address 21297c478bd9Sstevel@tonic-gate * spaces. Of course, we shouldn't be attempting to 21307c478bd9Sstevel@tonic-gate * swap out such processes in the first place... 21317c478bd9Sstevel@tonic-gate */ 21327c478bd9Sstevel@tonic-gate if (as == NULL) 21337c478bd9Sstevel@tonic-gate return (0); 21347c478bd9Sstevel@tonic-gate 2135dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 21367c478bd9Sstevel@tonic-gate 21377c478bd9Sstevel@tonic-gate /* 21387c478bd9Sstevel@tonic-gate * Free all mapping resources associated with the address 21397c478bd9Sstevel@tonic-gate * space. The segment-level swapout routines capitalize 21407c478bd9Sstevel@tonic-gate * on this unmapping by scavanging pages that have become 21417c478bd9Sstevel@tonic-gate * unmapped here. 21427c478bd9Sstevel@tonic-gate */ 21437c478bd9Sstevel@tonic-gate hat_swapout(as->a_hat); 21447c478bd9Sstevel@tonic-gate 21457c478bd9Sstevel@tonic-gate /* 21467c478bd9Sstevel@tonic-gate * Call the swapout routines of all segments in the address 21477c478bd9Sstevel@tonic-gate * space to do the actual work, accumulating the amount of 21487c478bd9Sstevel@tonic-gate * space reclaimed. 21497c478bd9Sstevel@tonic-gate */ 21507c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; seg = AS_SEGNEXT(as, seg)) { 21517c478bd9Sstevel@tonic-gate struct seg_ops *ov = seg->s_ops; 21527c478bd9Sstevel@tonic-gate 21537c478bd9Sstevel@tonic-gate /* 21547c478bd9Sstevel@tonic-gate * We have to check to see if the seg has 21557c478bd9Sstevel@tonic-gate * an ops vector because the seg may have 21567c478bd9Sstevel@tonic-gate * been in the middle of being set up when 21577c478bd9Sstevel@tonic-gate * the process was picked for swapout. 21587c478bd9Sstevel@tonic-gate */ 21597c478bd9Sstevel@tonic-gate if ((ov != NULL) && (ov->swapout != NULL)) 21607c478bd9Sstevel@tonic-gate swpcnt += SEGOP_SWAPOUT(seg); 21617c478bd9Sstevel@tonic-gate } 2162dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 21637c478bd9Sstevel@tonic-gate return (swpcnt); 21647c478bd9Sstevel@tonic-gate } 21657c478bd9Sstevel@tonic-gate 21667c478bd9Sstevel@tonic-gate /* 21677c478bd9Sstevel@tonic-gate * Determine whether data from the mappings in interval [addr, addr + size) 21687c478bd9Sstevel@tonic-gate * are in the primary memory (core) cache. 21697c478bd9Sstevel@tonic-gate */ 21707c478bd9Sstevel@tonic-gate int 21717c478bd9Sstevel@tonic-gate as_incore(struct as *as, caddr_t addr, 21727c478bd9Sstevel@tonic-gate size_t size, char *vec, size_t *sizep) 21737c478bd9Sstevel@tonic-gate { 21747c478bd9Sstevel@tonic-gate struct seg *seg; 21757c478bd9Sstevel@tonic-gate size_t ssize; 21767c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 21777c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 21787c478bd9Sstevel@tonic-gate size_t isize; /* iteration size */ 21797c478bd9Sstevel@tonic-gate int error = 0; /* result, assume success */ 21807c478bd9Sstevel@tonic-gate 21817c478bd9Sstevel@tonic-gate *sizep = 0; 21827c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 21837c478bd9Sstevel@tonic-gate rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) - 2184843e1988Sjohnlev (size_t)raddr; 21857c478bd9Sstevel@tonic-gate 21867c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 21877c478bd9Sstevel@tonic-gate return (ENOMEM); 21887c478bd9Sstevel@tonic-gate 2189dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 21907c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 21917c478bd9Sstevel@tonic-gate if (seg == NULL) { 2192dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 21937c478bd9Sstevel@tonic-gate return (-1); 21947c478bd9Sstevel@tonic-gate } 21957c478bd9Sstevel@tonic-gate 21967c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 21977c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 21987c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 21997c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 22007c478bd9Sstevel@tonic-gate error = -1; 22017c478bd9Sstevel@tonic-gate break; 22027c478bd9Sstevel@tonic-gate } 22037c478bd9Sstevel@tonic-gate } 22047c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 22057c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 22067c478bd9Sstevel@tonic-gate else 22077c478bd9Sstevel@tonic-gate ssize = rsize; 22087c478bd9Sstevel@tonic-gate *sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec); 22097c478bd9Sstevel@tonic-gate if (isize != ssize) { 22107c478bd9Sstevel@tonic-gate error = -1; 22117c478bd9Sstevel@tonic-gate break; 22127c478bd9Sstevel@tonic-gate } 22137c478bd9Sstevel@tonic-gate vec += btopr(ssize); 22147c478bd9Sstevel@tonic-gate } 2215dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 22167c478bd9Sstevel@tonic-gate return (error); 22177c478bd9Sstevel@tonic-gate } 22187c478bd9Sstevel@tonic-gate 22197c478bd9Sstevel@tonic-gate static void 22207c478bd9Sstevel@tonic-gate as_segunlock(struct seg *seg, caddr_t addr, int attr, 22216430daecSBrandon Baker ulong_t *bitmap, size_t position, size_t npages) 22227c478bd9Sstevel@tonic-gate { 22237c478bd9Sstevel@tonic-gate caddr_t range_start; 22247c478bd9Sstevel@tonic-gate size_t pos1 = position; 22257c478bd9Sstevel@tonic-gate size_t pos2; 22267c478bd9Sstevel@tonic-gate size_t size; 22277c478bd9Sstevel@tonic-gate size_t end_pos = npages + position; 22287c478bd9Sstevel@tonic-gate 22297c478bd9Sstevel@tonic-gate while (bt_range(bitmap, &pos1, &pos2, end_pos)) { 22307c478bd9Sstevel@tonic-gate size = ptob((pos2 - pos1)); 22317c478bd9Sstevel@tonic-gate range_start = (caddr_t)((uintptr_t)addr + 2232843e1988Sjohnlev ptob(pos1 - position)); 22337c478bd9Sstevel@tonic-gate 22347c478bd9Sstevel@tonic-gate (void) SEGOP_LOCKOP(seg, range_start, size, attr, MC_UNLOCK, 2235843e1988Sjohnlev (ulong_t *)NULL, (size_t)NULL); 22367c478bd9Sstevel@tonic-gate pos1 = pos2; 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate } 22397c478bd9Sstevel@tonic-gate 22407c478bd9Sstevel@tonic-gate static void 22417c478bd9Sstevel@tonic-gate as_unlockerr(struct as *as, int attr, ulong_t *mlock_map, 22426430daecSBrandon Baker caddr_t raddr, size_t rsize) 22437c478bd9Sstevel@tonic-gate { 22447c478bd9Sstevel@tonic-gate struct seg *seg = as_segat(as, raddr); 22457c478bd9Sstevel@tonic-gate size_t ssize; 22467c478bd9Sstevel@tonic-gate 22477c478bd9Sstevel@tonic-gate while (rsize != 0) { 22487c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) 22497c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 22507c478bd9Sstevel@tonic-gate 22517c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 22527c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 22537c478bd9Sstevel@tonic-gate else 22547c478bd9Sstevel@tonic-gate ssize = rsize; 22557c478bd9Sstevel@tonic-gate 22567c478bd9Sstevel@tonic-gate as_segunlock(seg, raddr, attr, mlock_map, 0, btopr(ssize)); 22577c478bd9Sstevel@tonic-gate 22587c478bd9Sstevel@tonic-gate rsize -= ssize; 22597c478bd9Sstevel@tonic-gate raddr += ssize; 22607c478bd9Sstevel@tonic-gate } 22617c478bd9Sstevel@tonic-gate } 22627c478bd9Sstevel@tonic-gate 22637c478bd9Sstevel@tonic-gate /* 22647c478bd9Sstevel@tonic-gate * Cache control operations over the interval [addr, addr + size) in 22657c478bd9Sstevel@tonic-gate * address space "as". 22667c478bd9Sstevel@tonic-gate */ 22677c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 22687c478bd9Sstevel@tonic-gate int 22697c478bd9Sstevel@tonic-gate as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr, 22707c478bd9Sstevel@tonic-gate uintptr_t arg, ulong_t *lock_map, size_t pos) 22717c478bd9Sstevel@tonic-gate { 22727c478bd9Sstevel@tonic-gate struct seg *seg; /* working segment */ 22737c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 22747c478bd9Sstevel@tonic-gate caddr_t initraddr; /* saved initial rounded down addr */ 22757c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 22767c478bd9Sstevel@tonic-gate size_t initrsize; /* saved initial rounded up size */ 22777c478bd9Sstevel@tonic-gate size_t ssize; /* size of seg */ 22787c478bd9Sstevel@tonic-gate int error = 0; /* result */ 22797c478bd9Sstevel@tonic-gate size_t mlock_size; /* size of bitmap */ 22807c478bd9Sstevel@tonic-gate ulong_t *mlock_map; /* pointer to bitmap used */ 22817c478bd9Sstevel@tonic-gate /* to represent the locked */ 22827c478bd9Sstevel@tonic-gate /* pages. */ 22837c478bd9Sstevel@tonic-gate retry: 22847c478bd9Sstevel@tonic-gate if (error == IE_RETRY) 2285dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 22867c478bd9Sstevel@tonic-gate else 2287dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 22887c478bd9Sstevel@tonic-gate 22897c478bd9Sstevel@tonic-gate /* 22907c478bd9Sstevel@tonic-gate * If these are address space lock/unlock operations, loop over 22917c478bd9Sstevel@tonic-gate * all segments in the address space, as appropriate. 22927c478bd9Sstevel@tonic-gate */ 22937c478bd9Sstevel@tonic-gate if (func == MC_LOCKAS) { 22947c478bd9Sstevel@tonic-gate size_t npages, idx; 22957c478bd9Sstevel@tonic-gate size_t rlen = 0; /* rounded as length */ 22967c478bd9Sstevel@tonic-gate 22977c478bd9Sstevel@tonic-gate idx = pos; 22987c478bd9Sstevel@tonic-gate 22997c478bd9Sstevel@tonic-gate if (arg & MCL_FUTURE) { 23007c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 23017c478bd9Sstevel@tonic-gate AS_SETPGLCK(as); 23027c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 23037c478bd9Sstevel@tonic-gate } 23047c478bd9Sstevel@tonic-gate if ((arg & MCL_CURRENT) == 0) { 2305dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23067c478bd9Sstevel@tonic-gate return (0); 23077c478bd9Sstevel@tonic-gate } 23087c478bd9Sstevel@tonic-gate 23097c478bd9Sstevel@tonic-gate seg = AS_SEGFIRST(as); 23107c478bd9Sstevel@tonic-gate if (seg == NULL) { 2311dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23127c478bd9Sstevel@tonic-gate return (0); 23137c478bd9Sstevel@tonic-gate } 23147c478bd9Sstevel@tonic-gate 23157c478bd9Sstevel@tonic-gate do { 23167c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)seg->s_base & 23177c478bd9Sstevel@tonic-gate (uintptr_t)PAGEMASK); 23187c478bd9Sstevel@tonic-gate rlen += (((uintptr_t)(seg->s_base + seg->s_size) + 2319843e1988Sjohnlev PAGEOFFSET) & PAGEMASK) - (uintptr_t)raddr; 23207c478bd9Sstevel@tonic-gate } while ((seg = AS_SEGNEXT(as, seg)) != NULL); 23217c478bd9Sstevel@tonic-gate 23227c478bd9Sstevel@tonic-gate mlock_size = BT_BITOUL(btopr(rlen)); 23237c478bd9Sstevel@tonic-gate if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 2324843e1988Sjohnlev sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 2325dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23267c478bd9Sstevel@tonic-gate return (EAGAIN); 23277c478bd9Sstevel@tonic-gate } 23287c478bd9Sstevel@tonic-gate 23297c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 2330*284ce987SPatrick Mooney if ((seg->s_flags & S_HOLE) != 0) { 2331*284ce987SPatrick Mooney continue; 2332*284ce987SPatrick Mooney } 23337c478bd9Sstevel@tonic-gate error = SEGOP_LOCKOP(seg, seg->s_base, 23347c478bd9Sstevel@tonic-gate seg->s_size, attr, MC_LOCK, mlock_map, pos); 23357c478bd9Sstevel@tonic-gate if (error != 0) 23367c478bd9Sstevel@tonic-gate break; 23377c478bd9Sstevel@tonic-gate pos += seg_pages(seg); 23387c478bd9Sstevel@tonic-gate } 23397c478bd9Sstevel@tonic-gate 23407c478bd9Sstevel@tonic-gate if (error) { 23417c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg != NULL; 2342843e1988Sjohnlev seg = AS_SEGNEXT(as, seg)) { 23437c478bd9Sstevel@tonic-gate 23447c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)seg->s_base & 2345843e1988Sjohnlev (uintptr_t)PAGEMASK); 23467c478bd9Sstevel@tonic-gate npages = seg_pages(seg); 23477c478bd9Sstevel@tonic-gate as_segunlock(seg, raddr, attr, mlock_map, 2348843e1988Sjohnlev idx, npages); 23497c478bd9Sstevel@tonic-gate idx += npages; 23507c478bd9Sstevel@tonic-gate } 23517c478bd9Sstevel@tonic-gate } 23527c478bd9Sstevel@tonic-gate 23537c478bd9Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 2354dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23557c478bd9Sstevel@tonic-gate goto lockerr; 23567c478bd9Sstevel@tonic-gate } else if (func == MC_UNLOCKAS) { 23577c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 23587c478bd9Sstevel@tonic-gate AS_CLRPGLCK(as); 23597c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 23607c478bd9Sstevel@tonic-gate 23617c478bd9Sstevel@tonic-gate for (seg = AS_SEGFIRST(as); seg; seg = AS_SEGNEXT(as, seg)) { 2362*284ce987SPatrick Mooney if ((seg->s_flags & S_HOLE) != 0) { 2363*284ce987SPatrick Mooney continue; 2364*284ce987SPatrick Mooney } 23657c478bd9Sstevel@tonic-gate error = SEGOP_LOCKOP(seg, seg->s_base, 23667c478bd9Sstevel@tonic-gate seg->s_size, attr, MC_UNLOCK, NULL, 0); 23677c478bd9Sstevel@tonic-gate if (error != 0) 23687c478bd9Sstevel@tonic-gate break; 23697c478bd9Sstevel@tonic-gate } 23707c478bd9Sstevel@tonic-gate 2371dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23727c478bd9Sstevel@tonic-gate goto lockerr; 23737c478bd9Sstevel@tonic-gate } 23747c478bd9Sstevel@tonic-gate 23757c478bd9Sstevel@tonic-gate /* 23767c478bd9Sstevel@tonic-gate * Normalize addresses and sizes. 23777c478bd9Sstevel@tonic-gate */ 23787c478bd9Sstevel@tonic-gate initraddr = raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 23797c478bd9Sstevel@tonic-gate initrsize = rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2380843e1988Sjohnlev (size_t)raddr; 23817c478bd9Sstevel@tonic-gate 23827c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) { /* check for wraparound */ 2383dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23847c478bd9Sstevel@tonic-gate return (ENOMEM); 23857c478bd9Sstevel@tonic-gate } 23867c478bd9Sstevel@tonic-gate 23877c478bd9Sstevel@tonic-gate /* 23887c478bd9Sstevel@tonic-gate * Get initial segment. 23897c478bd9Sstevel@tonic-gate */ 23907c478bd9Sstevel@tonic-gate if ((seg = as_segat(as, raddr)) == NULL) { 2391dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 23927c478bd9Sstevel@tonic-gate return (ENOMEM); 23937c478bd9Sstevel@tonic-gate } 23947c478bd9Sstevel@tonic-gate 23957c478bd9Sstevel@tonic-gate if (func == MC_LOCK) { 23967c478bd9Sstevel@tonic-gate mlock_size = BT_BITOUL(btopr(rsize)); 23977c478bd9Sstevel@tonic-gate if ((mlock_map = (ulong_t *)kmem_zalloc(mlock_size * 2398843e1988Sjohnlev sizeof (ulong_t), KM_NOSLEEP)) == NULL) { 2399dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 24007c478bd9Sstevel@tonic-gate return (EAGAIN); 24017c478bd9Sstevel@tonic-gate } 24027c478bd9Sstevel@tonic-gate } 24037c478bd9Sstevel@tonic-gate 24047c478bd9Sstevel@tonic-gate /* 24057c478bd9Sstevel@tonic-gate * Loop over all segments. If a hole in the address range is 24067c478bd9Sstevel@tonic-gate * discovered, then fail. For each segment, perform the appropriate 24077c478bd9Sstevel@tonic-gate * control operation. 24087c478bd9Sstevel@tonic-gate */ 24097c478bd9Sstevel@tonic-gate while (rsize != 0) { 24107c478bd9Sstevel@tonic-gate 24117c478bd9Sstevel@tonic-gate /* 24127c478bd9Sstevel@tonic-gate * Make sure there's no hole, calculate the portion 24137c478bd9Sstevel@tonic-gate * of the next segment to be operated over. 24147c478bd9Sstevel@tonic-gate */ 24157c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 24167c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 24177c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 24187c478bd9Sstevel@tonic-gate if (func == MC_LOCK) { 24197c478bd9Sstevel@tonic-gate as_unlockerr(as, attr, mlock_map, 2420843e1988Sjohnlev initraddr, initrsize - rsize); 24217c478bd9Sstevel@tonic-gate kmem_free(mlock_map, 2422843e1988Sjohnlev mlock_size * sizeof (ulong_t)); 24237c478bd9Sstevel@tonic-gate } 2424dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 24257c478bd9Sstevel@tonic-gate return (ENOMEM); 24267c478bd9Sstevel@tonic-gate } 24277c478bd9Sstevel@tonic-gate } 24287c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) 24297c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 24307c478bd9Sstevel@tonic-gate else 24317c478bd9Sstevel@tonic-gate ssize = rsize; 24327c478bd9Sstevel@tonic-gate 24337c478bd9Sstevel@tonic-gate /* 24347c478bd9Sstevel@tonic-gate * Dispatch on specific function. 24357c478bd9Sstevel@tonic-gate */ 24367c478bd9Sstevel@tonic-gate switch (func) { 24377c478bd9Sstevel@tonic-gate 24387c478bd9Sstevel@tonic-gate /* 24397c478bd9Sstevel@tonic-gate * Synchronize cached data from mappings with backing 24407c478bd9Sstevel@tonic-gate * objects. 24417c478bd9Sstevel@tonic-gate */ 24427c478bd9Sstevel@tonic-gate case MC_SYNC: 24437c478bd9Sstevel@tonic-gate if (error = SEGOP_SYNC(seg, raddr, ssize, 24447c478bd9Sstevel@tonic-gate attr, (uint_t)arg)) { 2445dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 24467c478bd9Sstevel@tonic-gate return (error); 24477c478bd9Sstevel@tonic-gate } 24487c478bd9Sstevel@tonic-gate break; 24497c478bd9Sstevel@tonic-gate 24507c478bd9Sstevel@tonic-gate /* 24517c478bd9Sstevel@tonic-gate * Lock pages in memory. 24527c478bd9Sstevel@tonic-gate */ 24537c478bd9Sstevel@tonic-gate case MC_LOCK: 24547c478bd9Sstevel@tonic-gate if (error = SEGOP_LOCKOP(seg, raddr, ssize, 2455843e1988Sjohnlev attr, func, mlock_map, pos)) { 24567c478bd9Sstevel@tonic-gate as_unlockerr(as, attr, mlock_map, initraddr, 2457843e1988Sjohnlev initrsize - rsize + ssize); 24587c478bd9Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * 2459843e1988Sjohnlev sizeof (ulong_t)); 2460dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 24617c478bd9Sstevel@tonic-gate goto lockerr; 24627c478bd9Sstevel@tonic-gate } 24637c478bd9Sstevel@tonic-gate break; 24647c478bd9Sstevel@tonic-gate 24657c478bd9Sstevel@tonic-gate /* 24667c478bd9Sstevel@tonic-gate * Unlock mapped pages. 24677c478bd9Sstevel@tonic-gate */ 24687c478bd9Sstevel@tonic-gate case MC_UNLOCK: 24697c478bd9Sstevel@tonic-gate (void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func, 2470843e1988Sjohnlev (ulong_t *)NULL, (size_t)NULL); 24717c478bd9Sstevel@tonic-gate break; 24727c478bd9Sstevel@tonic-gate 24737c478bd9Sstevel@tonic-gate /* 24747c478bd9Sstevel@tonic-gate * Store VM advise for mapped pages in segment layer. 24757c478bd9Sstevel@tonic-gate */ 24767c478bd9Sstevel@tonic-gate case MC_ADVISE: 24777c478bd9Sstevel@tonic-gate error = SEGOP_ADVISE(seg, raddr, ssize, (uint_t)arg); 24787c478bd9Sstevel@tonic-gate 24797c478bd9Sstevel@tonic-gate /* 24807c478bd9Sstevel@tonic-gate * Check for regular errors and special retry error 24817c478bd9Sstevel@tonic-gate */ 24827c478bd9Sstevel@tonic-gate if (error) { 24837c478bd9Sstevel@tonic-gate if (error == IE_RETRY) { 24847c478bd9Sstevel@tonic-gate /* 24857c478bd9Sstevel@tonic-gate * Need to acquire writers lock, so 24867c478bd9Sstevel@tonic-gate * have to drop readers lock and start 24877c478bd9Sstevel@tonic-gate * all over again 24887c478bd9Sstevel@tonic-gate */ 2489dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 24907c478bd9Sstevel@tonic-gate goto retry; 24917c478bd9Sstevel@tonic-gate } else if (error == IE_REATTACH) { 24927c478bd9Sstevel@tonic-gate /* 24937c478bd9Sstevel@tonic-gate * Find segment for current address 24947c478bd9Sstevel@tonic-gate * because current segment just got 24957c478bd9Sstevel@tonic-gate * split or concatenated 24967c478bd9Sstevel@tonic-gate */ 24977c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 24987c478bd9Sstevel@tonic-gate if (seg == NULL) { 2499dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 25007c478bd9Sstevel@tonic-gate return (ENOMEM); 25017c478bd9Sstevel@tonic-gate } 25027c478bd9Sstevel@tonic-gate } else { 25037c478bd9Sstevel@tonic-gate /* 25047c478bd9Sstevel@tonic-gate * Regular error 25057c478bd9Sstevel@tonic-gate */ 2506dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 25077c478bd9Sstevel@tonic-gate return (error); 25087c478bd9Sstevel@tonic-gate } 25097c478bd9Sstevel@tonic-gate } 25107c478bd9Sstevel@tonic-gate break; 25117c478bd9Sstevel@tonic-gate 25129d12795fSRobert Mustacchi case MC_INHERIT_ZERO: 25139d12795fSRobert Mustacchi if (seg->s_ops->inherit == NULL) { 25149d12795fSRobert Mustacchi error = ENOTSUP; 25159d12795fSRobert Mustacchi } else { 25169d12795fSRobert Mustacchi error = SEGOP_INHERIT(seg, raddr, ssize, 25179d12795fSRobert Mustacchi SEGP_INH_ZERO); 25189d12795fSRobert Mustacchi } 25199d12795fSRobert Mustacchi if (error != 0) { 2520dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 25219d12795fSRobert Mustacchi return (error); 25229d12795fSRobert Mustacchi } 25239d12795fSRobert Mustacchi break; 25249d12795fSRobert Mustacchi 25257c478bd9Sstevel@tonic-gate /* 25267c478bd9Sstevel@tonic-gate * Can't happen. 25277c478bd9Sstevel@tonic-gate */ 25287c478bd9Sstevel@tonic-gate default: 25297c478bd9Sstevel@tonic-gate panic("as_ctl: bad operation %d", func); 25307c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 25317c478bd9Sstevel@tonic-gate } 25327c478bd9Sstevel@tonic-gate 25337c478bd9Sstevel@tonic-gate rsize -= ssize; 25347c478bd9Sstevel@tonic-gate raddr += ssize; 25357c478bd9Sstevel@tonic-gate } 25367c478bd9Sstevel@tonic-gate 25377c478bd9Sstevel@tonic-gate if (func == MC_LOCK) 25387c478bd9Sstevel@tonic-gate kmem_free(mlock_map, mlock_size * sizeof (ulong_t)); 2539dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 25407c478bd9Sstevel@tonic-gate return (0); 25417c478bd9Sstevel@tonic-gate lockerr: 25427c478bd9Sstevel@tonic-gate 25437c478bd9Sstevel@tonic-gate /* 25447c478bd9Sstevel@tonic-gate * If the lower levels returned EDEADLK for a segment lockop, 25457c478bd9Sstevel@tonic-gate * it means that we should retry the operation. Let's wait 25467c478bd9Sstevel@tonic-gate * a bit also to let the deadlock causing condition clear. 25477c478bd9Sstevel@tonic-gate * This is part of a gross hack to work around a design flaw 25487c478bd9Sstevel@tonic-gate * in the ufs/sds logging code and should go away when the 25497c478bd9Sstevel@tonic-gate * logging code is re-designed to fix the problem. See bug 25507c478bd9Sstevel@tonic-gate * 4125102 for details of the problem. 25517c478bd9Sstevel@tonic-gate */ 25527c478bd9Sstevel@tonic-gate if (error == EDEADLK) { 25537c478bd9Sstevel@tonic-gate delay(deadlk_wait); 25547c478bd9Sstevel@tonic-gate error = 0; 25557c478bd9Sstevel@tonic-gate goto retry; 25567c478bd9Sstevel@tonic-gate } 25577c478bd9Sstevel@tonic-gate return (error); 25587c478bd9Sstevel@tonic-gate } 25597c478bd9Sstevel@tonic-gate 2560843e1988Sjohnlev int 2561843e1988Sjohnlev fc_decode(faultcode_t fault_err) 25627c478bd9Sstevel@tonic-gate { 25637c478bd9Sstevel@tonic-gate int error = 0; 25647c478bd9Sstevel@tonic-gate 25657c478bd9Sstevel@tonic-gate switch (FC_CODE(fault_err)) { 25667c478bd9Sstevel@tonic-gate case FC_OBJERR: 25677c478bd9Sstevel@tonic-gate error = FC_ERRNO(fault_err); 25687c478bd9Sstevel@tonic-gate break; 25697c478bd9Sstevel@tonic-gate case FC_PROT: 25707c478bd9Sstevel@tonic-gate error = EACCES; 25717c478bd9Sstevel@tonic-gate break; 25727c478bd9Sstevel@tonic-gate default: 25737c478bd9Sstevel@tonic-gate error = EFAULT; 25747c478bd9Sstevel@tonic-gate break; 25757c478bd9Sstevel@tonic-gate } 25767c478bd9Sstevel@tonic-gate return (error); 25777c478bd9Sstevel@tonic-gate } 25787c478bd9Sstevel@tonic-gate 2579a98e9dbfSaguzovsk /* 2580a98e9dbfSaguzovsk * Pagelock pages from a range that spans more than 1 segment. Obtain shadow 2581a98e9dbfSaguzovsk * lists from each segment and copy them to one contiguous shadow list (plist) 2582a98e9dbfSaguzovsk * as expected by the caller. Save pointers to per segment shadow lists at 2583a98e9dbfSaguzovsk * the tail of plist so that they can be used during as_pageunlock(). 2584a98e9dbfSaguzovsk */ 2585a98e9dbfSaguzovsk static int 2586a98e9dbfSaguzovsk as_pagelock_segs(struct as *as, struct seg *seg, struct page ***ppp, 2587a98e9dbfSaguzovsk caddr_t addr, size_t size, enum seg_rw rw) 2588a98e9dbfSaguzovsk { 2589a98e9dbfSaguzovsk caddr_t sv_addr = addr; 2590a98e9dbfSaguzovsk size_t sv_size = size; 2591a98e9dbfSaguzovsk struct seg *sv_seg = seg; 2592a98e9dbfSaguzovsk ulong_t segcnt = 1; 2593a98e9dbfSaguzovsk ulong_t cnt; 2594a98e9dbfSaguzovsk size_t ssize; 2595a98e9dbfSaguzovsk pgcnt_t npages = btop(size); 2596a98e9dbfSaguzovsk page_t **plist; 2597a98e9dbfSaguzovsk page_t **pl; 2598a98e9dbfSaguzovsk int error; 2599a98e9dbfSaguzovsk caddr_t eaddr; 2600a98e9dbfSaguzovsk faultcode_t fault_err = 0; 2601a98e9dbfSaguzovsk pgcnt_t pl_off; 2602a98e9dbfSaguzovsk extern struct seg_ops segspt_shmops; 2603a98e9dbfSaguzovsk 2604dc32d872SJosef 'Jeff' Sipek ASSERT(AS_LOCK_HELD(as)); 2605a98e9dbfSaguzovsk ASSERT(seg != NULL); 2606a98e9dbfSaguzovsk ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); 2607a98e9dbfSaguzovsk ASSERT(addr + size > seg->s_base + seg->s_size); 2608a98e9dbfSaguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 2609a98e9dbfSaguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 2610a98e9dbfSaguzovsk 2611a98e9dbfSaguzovsk /* 2612a98e9dbfSaguzovsk * Count the number of segments covered by the range we are about to 2613a98e9dbfSaguzovsk * lock. The segment count is used to size the shadow list we return 2614a98e9dbfSaguzovsk * back to the caller. 2615a98e9dbfSaguzovsk */ 2616a98e9dbfSaguzovsk for (; size != 0; size -= ssize, addr += ssize) { 2617a98e9dbfSaguzovsk if (addr >= seg->s_base + seg->s_size) { 2618a98e9dbfSaguzovsk 2619a98e9dbfSaguzovsk seg = AS_SEGNEXT(as, seg); 2620a98e9dbfSaguzovsk if (seg == NULL || addr != seg->s_base) { 2621dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2622a98e9dbfSaguzovsk return (EFAULT); 2623a98e9dbfSaguzovsk } 2624a98e9dbfSaguzovsk /* 2625a98e9dbfSaguzovsk * Do a quick check if subsequent segments 2626a98e9dbfSaguzovsk * will most likely support pagelock. 2627a98e9dbfSaguzovsk */ 2628a98e9dbfSaguzovsk if (seg->s_ops == &segvn_ops) { 2629a98e9dbfSaguzovsk vnode_t *vp; 2630a98e9dbfSaguzovsk 2631a98e9dbfSaguzovsk if (SEGOP_GETVP(seg, addr, &vp) != 0 || 2632a98e9dbfSaguzovsk vp != NULL) { 2633dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2634a98e9dbfSaguzovsk goto slow; 2635a98e9dbfSaguzovsk } 2636a98e9dbfSaguzovsk } else if (seg->s_ops != &segspt_shmops) { 2637dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2638a98e9dbfSaguzovsk goto slow; 2639a98e9dbfSaguzovsk } 2640a98e9dbfSaguzovsk segcnt++; 2641a98e9dbfSaguzovsk } 2642a98e9dbfSaguzovsk if (addr + size > seg->s_base + seg->s_size) { 2643a98e9dbfSaguzovsk ssize = seg->s_base + seg->s_size - addr; 2644a98e9dbfSaguzovsk } else { 2645a98e9dbfSaguzovsk ssize = size; 2646a98e9dbfSaguzovsk } 2647a98e9dbfSaguzovsk } 2648a98e9dbfSaguzovsk ASSERT(segcnt > 1); 2649a98e9dbfSaguzovsk 2650a98e9dbfSaguzovsk plist = kmem_zalloc((npages + segcnt) * sizeof (page_t *), KM_SLEEP); 2651a98e9dbfSaguzovsk 2652a98e9dbfSaguzovsk addr = sv_addr; 2653a98e9dbfSaguzovsk size = sv_size; 2654a98e9dbfSaguzovsk seg = sv_seg; 2655a98e9dbfSaguzovsk 2656a98e9dbfSaguzovsk for (cnt = 0, pl_off = 0; size != 0; size -= ssize, addr += ssize) { 2657a98e9dbfSaguzovsk if (addr >= seg->s_base + seg->s_size) { 2658a98e9dbfSaguzovsk seg = AS_SEGNEXT(as, seg); 2659a98e9dbfSaguzovsk ASSERT(seg != NULL && addr == seg->s_base); 2660a98e9dbfSaguzovsk cnt++; 2661a98e9dbfSaguzovsk ASSERT(cnt < segcnt); 2662a98e9dbfSaguzovsk } 2663a98e9dbfSaguzovsk if (addr + size > seg->s_base + seg->s_size) { 2664a98e9dbfSaguzovsk ssize = seg->s_base + seg->s_size - addr; 2665a98e9dbfSaguzovsk } else { 2666a98e9dbfSaguzovsk ssize = size; 2667a98e9dbfSaguzovsk } 2668a98e9dbfSaguzovsk pl = &plist[npages + cnt]; 2669a98e9dbfSaguzovsk error = SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, 2670a98e9dbfSaguzovsk L_PAGELOCK, rw); 2671a98e9dbfSaguzovsk if (error) { 2672a98e9dbfSaguzovsk break; 2673a98e9dbfSaguzovsk } 2674a98e9dbfSaguzovsk ASSERT(plist[npages + cnt] != NULL); 2675a98e9dbfSaguzovsk ASSERT(pl_off + btop(ssize) <= npages); 2676a98e9dbfSaguzovsk bcopy(plist[npages + cnt], &plist[pl_off], 2677a98e9dbfSaguzovsk btop(ssize) * sizeof (page_t *)); 2678a98e9dbfSaguzovsk pl_off += btop(ssize); 2679a98e9dbfSaguzovsk } 2680a98e9dbfSaguzovsk 2681a98e9dbfSaguzovsk if (size == 0) { 2682dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2683a98e9dbfSaguzovsk ASSERT(cnt == segcnt - 1); 2684a98e9dbfSaguzovsk *ppp = plist; 2685a98e9dbfSaguzovsk return (0); 2686a98e9dbfSaguzovsk } 2687a98e9dbfSaguzovsk 2688a98e9dbfSaguzovsk /* 2689a98e9dbfSaguzovsk * one of pagelock calls failed. The error type is in error variable. 2690a98e9dbfSaguzovsk * Unlock what we've locked so far and retry with F_SOFTLOCK if error 2691a98e9dbfSaguzovsk * type is either EFAULT or ENOTSUP. Otherwise just return the error 2692a98e9dbfSaguzovsk * back to the caller. 2693a98e9dbfSaguzovsk */ 2694a98e9dbfSaguzovsk 2695a98e9dbfSaguzovsk eaddr = addr; 2696a98e9dbfSaguzovsk seg = sv_seg; 2697a98e9dbfSaguzovsk 2698a98e9dbfSaguzovsk for (cnt = 0, addr = sv_addr; addr < eaddr; addr += ssize) { 2699a98e9dbfSaguzovsk if (addr >= seg->s_base + seg->s_size) { 2700a98e9dbfSaguzovsk seg = AS_SEGNEXT(as, seg); 2701a98e9dbfSaguzovsk ASSERT(seg != NULL && addr == seg->s_base); 2702a98e9dbfSaguzovsk cnt++; 2703a98e9dbfSaguzovsk ASSERT(cnt < segcnt); 2704a98e9dbfSaguzovsk } 2705a98e9dbfSaguzovsk if (eaddr > seg->s_base + seg->s_size) { 2706a98e9dbfSaguzovsk ssize = seg->s_base + seg->s_size - addr; 2707a98e9dbfSaguzovsk } else { 2708a98e9dbfSaguzovsk ssize = eaddr - addr; 2709a98e9dbfSaguzovsk } 2710a98e9dbfSaguzovsk pl = &plist[npages + cnt]; 2711a98e9dbfSaguzovsk ASSERT(*pl != NULL); 2712a98e9dbfSaguzovsk (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, 2713a98e9dbfSaguzovsk L_PAGEUNLOCK, rw); 2714a98e9dbfSaguzovsk } 2715a98e9dbfSaguzovsk 2716dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2717a98e9dbfSaguzovsk 2718a98e9dbfSaguzovsk kmem_free(plist, (npages + segcnt) * sizeof (page_t *)); 2719a98e9dbfSaguzovsk 2720a98e9dbfSaguzovsk if (error != ENOTSUP && error != EFAULT) { 2721a98e9dbfSaguzovsk return (error); 2722a98e9dbfSaguzovsk } 2723a98e9dbfSaguzovsk 2724a98e9dbfSaguzovsk slow: 2725a98e9dbfSaguzovsk /* 2726a98e9dbfSaguzovsk * If we are here because pagelock failed due to the need to cow fault 2727a98e9dbfSaguzovsk * in the pages we want to lock F_SOFTLOCK will do this job and in 2728a98e9dbfSaguzovsk * next as_pagelock() call for this address range pagelock will 2729a98e9dbfSaguzovsk * hopefully succeed. 2730a98e9dbfSaguzovsk */ 2731a98e9dbfSaguzovsk fault_err = as_fault(as->a_hat, as, sv_addr, sv_size, F_SOFTLOCK, rw); 2732a98e9dbfSaguzovsk if (fault_err != 0) { 2733a98e9dbfSaguzovsk return (fc_decode(fault_err)); 2734a98e9dbfSaguzovsk } 2735a98e9dbfSaguzovsk *ppp = NULL; 2736a98e9dbfSaguzovsk 2737a98e9dbfSaguzovsk return (0); 2738a98e9dbfSaguzovsk } 2739a98e9dbfSaguzovsk 27407c478bd9Sstevel@tonic-gate /* 27417c478bd9Sstevel@tonic-gate * lock pages in a given address space. Return shadow list. If 27427c478bd9Sstevel@tonic-gate * the list is NULL, the MMU mapping is also locked. 27437c478bd9Sstevel@tonic-gate */ 27447c478bd9Sstevel@tonic-gate int 27457c478bd9Sstevel@tonic-gate as_pagelock(struct as *as, struct page ***ppp, caddr_t addr, 27467c478bd9Sstevel@tonic-gate size_t size, enum seg_rw rw) 27477c478bd9Sstevel@tonic-gate { 27487c478bd9Sstevel@tonic-gate size_t rsize; 27497c478bd9Sstevel@tonic-gate caddr_t raddr; 27507c478bd9Sstevel@tonic-gate faultcode_t fault_err; 27517c478bd9Sstevel@tonic-gate struct seg *seg; 2752a98e9dbfSaguzovsk int err; 27537c478bd9Sstevel@tonic-gate 27547c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START, 27557c478bd9Sstevel@tonic-gate "as_pagelock_start: addr %p size %ld", addr, size); 27567c478bd9Sstevel@tonic-gate 27577c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 27587c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2759843e1988Sjohnlev (size_t)raddr; 2760a98e9dbfSaguzovsk 27617c478bd9Sstevel@tonic-gate /* 27627c478bd9Sstevel@tonic-gate * if the request crosses two segments let 27637c478bd9Sstevel@tonic-gate * as_fault handle it. 27647c478bd9Sstevel@tonic-gate */ 2765dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 2766a98e9dbfSaguzovsk 2767a98e9dbfSaguzovsk seg = as_segat(as, raddr); 2768a98e9dbfSaguzovsk if (seg == NULL) { 2769dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2770a98e9dbfSaguzovsk return (EFAULT); 2771a98e9dbfSaguzovsk } 2772a98e9dbfSaguzovsk ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size); 2773a98e9dbfSaguzovsk if (raddr + rsize > seg->s_base + seg->s_size) { 2774a98e9dbfSaguzovsk return (as_pagelock_segs(as, seg, ppp, raddr, rsize, rw)); 2775a98e9dbfSaguzovsk } 2776a98e9dbfSaguzovsk if (raddr + rsize <= raddr) { 2777dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2778a98e9dbfSaguzovsk return (EFAULT); 27797c478bd9Sstevel@tonic-gate } 27807c478bd9Sstevel@tonic-gate 27817c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START, 27827c478bd9Sstevel@tonic-gate "seg_lock_1_start: raddr %p rsize %ld", raddr, rsize); 27837c478bd9Sstevel@tonic-gate 27847c478bd9Sstevel@tonic-gate /* 27857c478bd9Sstevel@tonic-gate * try to lock pages and pass back shadow list 27867c478bd9Sstevel@tonic-gate */ 2787a98e9dbfSaguzovsk err = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw); 27887c478bd9Sstevel@tonic-gate 27897c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end"); 27907c478bd9Sstevel@tonic-gate 2791dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 27927c478bd9Sstevel@tonic-gate 2793a98e9dbfSaguzovsk if (err == 0 || (err != ENOTSUP && err != EFAULT)) { 2794a98e9dbfSaguzovsk return (err); 27957c478bd9Sstevel@tonic-gate } 27967c478bd9Sstevel@tonic-gate 27977c478bd9Sstevel@tonic-gate /* 2798a98e9dbfSaguzovsk * Use F_SOFTLOCK to lock the pages because pagelock failed either due 2799a98e9dbfSaguzovsk * to no pagelock support for this segment or pages need to be cow 2800a98e9dbfSaguzovsk * faulted in. If fault is needed F_SOFTLOCK will do this job for 2801a98e9dbfSaguzovsk * this as_pagelock() call and in the next as_pagelock() call for the 2802a98e9dbfSaguzovsk * same address range pagelock call will hopefull succeed. 28037c478bd9Sstevel@tonic-gate */ 28047c478bd9Sstevel@tonic-gate fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw); 28057c478bd9Sstevel@tonic-gate if (fault_err != 0) { 2806843e1988Sjohnlev return (fc_decode(fault_err)); 28077c478bd9Sstevel@tonic-gate } 28087c478bd9Sstevel@tonic-gate *ppp = NULL; 28097c478bd9Sstevel@tonic-gate 28107c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end"); 28117c478bd9Sstevel@tonic-gate return (0); 28127c478bd9Sstevel@tonic-gate } 28137c478bd9Sstevel@tonic-gate 2814a98e9dbfSaguzovsk /* 2815a98e9dbfSaguzovsk * unlock pages locked by as_pagelock_segs(). Retrieve per segment shadow 2816a98e9dbfSaguzovsk * lists from the end of plist and call pageunlock interface for each segment. 2817a98e9dbfSaguzovsk * Drop as lock and free plist. 2818a98e9dbfSaguzovsk */ 2819a98e9dbfSaguzovsk static void 2820a98e9dbfSaguzovsk as_pageunlock_segs(struct as *as, struct seg *seg, caddr_t addr, size_t size, 2821a98e9dbfSaguzovsk struct page **plist, enum seg_rw rw) 2822a98e9dbfSaguzovsk { 2823a98e9dbfSaguzovsk ulong_t cnt; 2824a98e9dbfSaguzovsk caddr_t eaddr = addr + size; 2825a98e9dbfSaguzovsk pgcnt_t npages = btop(size); 2826a98e9dbfSaguzovsk size_t ssize; 2827a98e9dbfSaguzovsk page_t **pl; 2828a98e9dbfSaguzovsk 2829dc32d872SJosef 'Jeff' Sipek ASSERT(AS_LOCK_HELD(as)); 2830a98e9dbfSaguzovsk ASSERT(seg != NULL); 2831a98e9dbfSaguzovsk ASSERT(addr >= seg->s_base && addr < seg->s_base + seg->s_size); 2832a98e9dbfSaguzovsk ASSERT(addr + size > seg->s_base + seg->s_size); 2833a98e9dbfSaguzovsk ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 2834a98e9dbfSaguzovsk ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 2835a98e9dbfSaguzovsk ASSERT(plist != NULL); 2836a98e9dbfSaguzovsk 2837a98e9dbfSaguzovsk for (cnt = 0; addr < eaddr; addr += ssize) { 2838a98e9dbfSaguzovsk if (addr >= seg->s_base + seg->s_size) { 2839a98e9dbfSaguzovsk seg = AS_SEGNEXT(as, seg); 2840a98e9dbfSaguzovsk ASSERT(seg != NULL && addr == seg->s_base); 2841a98e9dbfSaguzovsk cnt++; 2842a98e9dbfSaguzovsk } 2843a98e9dbfSaguzovsk if (eaddr > seg->s_base + seg->s_size) { 2844a98e9dbfSaguzovsk ssize = seg->s_base + seg->s_size - addr; 2845a98e9dbfSaguzovsk } else { 2846a98e9dbfSaguzovsk ssize = eaddr - addr; 2847a98e9dbfSaguzovsk } 2848a98e9dbfSaguzovsk pl = &plist[npages + cnt]; 2849a98e9dbfSaguzovsk ASSERT(*pl != NULL); 2850a98e9dbfSaguzovsk (void) SEGOP_PAGELOCK(seg, addr, ssize, (page_t ***)pl, 2851a98e9dbfSaguzovsk L_PAGEUNLOCK, rw); 2852a98e9dbfSaguzovsk } 2853a98e9dbfSaguzovsk ASSERT(cnt > 0); 2854dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 2855a98e9dbfSaguzovsk 2856a98e9dbfSaguzovsk cnt++; 2857a98e9dbfSaguzovsk kmem_free(plist, (npages + cnt) * sizeof (page_t *)); 2858a98e9dbfSaguzovsk } 2859a98e9dbfSaguzovsk 28607c478bd9Sstevel@tonic-gate /* 28617c478bd9Sstevel@tonic-gate * unlock pages in a given address range 28627c478bd9Sstevel@tonic-gate */ 28637c478bd9Sstevel@tonic-gate void 28647c478bd9Sstevel@tonic-gate as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size, 28657c478bd9Sstevel@tonic-gate enum seg_rw rw) 28667c478bd9Sstevel@tonic-gate { 28677c478bd9Sstevel@tonic-gate struct seg *seg; 28687c478bd9Sstevel@tonic-gate size_t rsize; 28697c478bd9Sstevel@tonic-gate caddr_t raddr; 28707c478bd9Sstevel@tonic-gate 28717c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START, 28727c478bd9Sstevel@tonic-gate "as_pageunlock_start: addr %p size %ld", addr, size); 28737c478bd9Sstevel@tonic-gate 28747c478bd9Sstevel@tonic-gate /* 28757c478bd9Sstevel@tonic-gate * if the shadow list is NULL, as_pagelock was 28767c478bd9Sstevel@tonic-gate * falling back to as_fault 28777c478bd9Sstevel@tonic-gate */ 28787c478bd9Sstevel@tonic-gate if (pp == NULL) { 28797c478bd9Sstevel@tonic-gate (void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw); 28807c478bd9Sstevel@tonic-gate return; 28817c478bd9Sstevel@tonic-gate } 2882a98e9dbfSaguzovsk 28837c478bd9Sstevel@tonic-gate raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 28847c478bd9Sstevel@tonic-gate rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 2885843e1988Sjohnlev (size_t)raddr; 2886a98e9dbfSaguzovsk 2887dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 2888a98e9dbfSaguzovsk seg = as_segat(as, raddr); 2889a98e9dbfSaguzovsk ASSERT(seg != NULL); 2890a98e9dbfSaguzovsk 28917c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START, 28927c478bd9Sstevel@tonic-gate "seg_unlock_start: raddr %p rsize %ld", raddr, rsize); 2893a98e9dbfSaguzovsk 2894a98e9dbfSaguzovsk ASSERT(raddr >= seg->s_base && raddr < seg->s_base + seg->s_size); 2895a98e9dbfSaguzovsk if (raddr + rsize <= seg->s_base + seg->s_size) { 2896a98e9dbfSaguzovsk SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw); 2897a98e9dbfSaguzovsk } else { 2898a98e9dbfSaguzovsk as_pageunlock_segs(as, seg, raddr, rsize, pp, rw); 2899a98e9dbfSaguzovsk return; 2900a98e9dbfSaguzovsk } 2901dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 29027c478bd9Sstevel@tonic-gate TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end"); 29037c478bd9Sstevel@tonic-gate } 29047c478bd9Sstevel@tonic-gate 29057c478bd9Sstevel@tonic-gate int 29067c478bd9Sstevel@tonic-gate as_setpagesize(struct as *as, caddr_t addr, size_t size, uint_t szc, 29077c478bd9Sstevel@tonic-gate boolean_t wait) 29087c478bd9Sstevel@tonic-gate { 29097c478bd9Sstevel@tonic-gate struct seg *seg; 29107c478bd9Sstevel@tonic-gate size_t ssize; 29117c478bd9Sstevel@tonic-gate caddr_t raddr; /* rounded down addr */ 29127c478bd9Sstevel@tonic-gate size_t rsize; /* rounded up size */ 29137c478bd9Sstevel@tonic-gate int error = 0; 29147c478bd9Sstevel@tonic-gate size_t pgsz = page_get_pagesize(szc); 29157c478bd9Sstevel@tonic-gate 29167c478bd9Sstevel@tonic-gate setpgsz_top: 29177c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(addr, pgsz) || !IS_P2ALIGNED(size, pgsz)) { 29187c478bd9Sstevel@tonic-gate return (EINVAL); 29197c478bd9Sstevel@tonic-gate } 29207c478bd9Sstevel@tonic-gate 29217c478bd9Sstevel@tonic-gate raddr = addr; 29227c478bd9Sstevel@tonic-gate rsize = size; 29237c478bd9Sstevel@tonic-gate 29247c478bd9Sstevel@tonic-gate if (raddr + rsize < raddr) /* check for wraparound */ 29257c478bd9Sstevel@tonic-gate return (ENOMEM); 29267c478bd9Sstevel@tonic-gate 2927dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 29287c478bd9Sstevel@tonic-gate as_clearwatchprot(as, raddr, rsize); 29297c478bd9Sstevel@tonic-gate seg = as_segat(as, raddr); 29307c478bd9Sstevel@tonic-gate if (seg == NULL) { 29317c478bd9Sstevel@tonic-gate as_setwatch(as); 2932dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 29337c478bd9Sstevel@tonic-gate return (ENOMEM); 29347c478bd9Sstevel@tonic-gate } 29357c478bd9Sstevel@tonic-gate 29367c478bd9Sstevel@tonic-gate for (; rsize != 0; rsize -= ssize, raddr += ssize) { 29377c478bd9Sstevel@tonic-gate if (raddr >= seg->s_base + seg->s_size) { 29387c478bd9Sstevel@tonic-gate seg = AS_SEGNEXT(as, seg); 29397c478bd9Sstevel@tonic-gate if (seg == NULL || raddr != seg->s_base) { 29407c478bd9Sstevel@tonic-gate error = ENOMEM; 29417c478bd9Sstevel@tonic-gate break; 29427c478bd9Sstevel@tonic-gate } 29437c478bd9Sstevel@tonic-gate } 29447c478bd9Sstevel@tonic-gate if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 29457c478bd9Sstevel@tonic-gate ssize = seg->s_base + seg->s_size - raddr; 29467c478bd9Sstevel@tonic-gate } else { 29477c478bd9Sstevel@tonic-gate ssize = rsize; 29487c478bd9Sstevel@tonic-gate } 29497c478bd9Sstevel@tonic-gate 2950a98e9dbfSaguzovsk retry: 29517c478bd9Sstevel@tonic-gate error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 29527c478bd9Sstevel@tonic-gate 29537c478bd9Sstevel@tonic-gate if (error == IE_NOMEM) { 29547c478bd9Sstevel@tonic-gate error = EAGAIN; 29557c478bd9Sstevel@tonic-gate break; 29567c478bd9Sstevel@tonic-gate } 29577c478bd9Sstevel@tonic-gate 29587c478bd9Sstevel@tonic-gate if (error == IE_RETRY) { 2959dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 29607c478bd9Sstevel@tonic-gate goto setpgsz_top; 29617c478bd9Sstevel@tonic-gate } 29627c478bd9Sstevel@tonic-gate 29637c478bd9Sstevel@tonic-gate if (error == ENOTSUP) { 29647c478bd9Sstevel@tonic-gate error = EINVAL; 29657c478bd9Sstevel@tonic-gate break; 29667c478bd9Sstevel@tonic-gate } 29677c478bd9Sstevel@tonic-gate 29687c478bd9Sstevel@tonic-gate if (wait && (error == EAGAIN)) { 29697c478bd9Sstevel@tonic-gate /* 29707c478bd9Sstevel@tonic-gate * Memory is currently locked. It must be unlocked 29717c478bd9Sstevel@tonic-gate * before this operation can succeed through a retry. 29727c478bd9Sstevel@tonic-gate * The possible reasons for locked memory and 29737c478bd9Sstevel@tonic-gate * corresponding strategies for unlocking are: 29747c478bd9Sstevel@tonic-gate * (1) Normal I/O 29757c478bd9Sstevel@tonic-gate * wait for a signal that the I/O operation 29767c478bd9Sstevel@tonic-gate * has completed and the memory is unlocked. 29777c478bd9Sstevel@tonic-gate * (2) Asynchronous I/O 29787c478bd9Sstevel@tonic-gate * The aio subsystem does not unlock pages when 29797c478bd9Sstevel@tonic-gate * the I/O is completed. Those pages are unlocked 29807c478bd9Sstevel@tonic-gate * when the application calls aiowait/aioerror. 29817c478bd9Sstevel@tonic-gate * So, to prevent blocking forever, cv_broadcast() 29827c478bd9Sstevel@tonic-gate * is done to wake up aio_cleanup_thread. 29837c478bd9Sstevel@tonic-gate * Subsequently, segvn_reclaim will be called, and 29847c478bd9Sstevel@tonic-gate * that will do AS_CLRUNMAPWAIT() and wake us up. 29857c478bd9Sstevel@tonic-gate * (3) Long term page locking: 29867c478bd9Sstevel@tonic-gate * This is not relevant for as_setpagesize() 29877c478bd9Sstevel@tonic-gate * because we cannot change the page size for 29887c478bd9Sstevel@tonic-gate * driver memory. The attempt to do so will 29897c478bd9Sstevel@tonic-gate * fail with a different error than EAGAIN so 29907c478bd9Sstevel@tonic-gate * there's no need to trigger as callbacks like 29917c478bd9Sstevel@tonic-gate * as_unmap, as_setprot or as_free would do. 29927c478bd9Sstevel@tonic-gate */ 29937c478bd9Sstevel@tonic-gate mutex_enter(&as->a_contents); 2994a98e9dbfSaguzovsk if (!AS_ISNOUNMAPWAIT(as)) { 2995a98e9dbfSaguzovsk if (AS_ISUNMAPWAIT(as) == 0) { 2996a98e9dbfSaguzovsk cv_broadcast(&as->a_cv); 2997a98e9dbfSaguzovsk } 2998a98e9dbfSaguzovsk AS_SETUNMAPWAIT(as); 2999dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 3000a98e9dbfSaguzovsk while (AS_ISUNMAPWAIT(as)) { 3001a98e9dbfSaguzovsk cv_wait(&as->a_cv, &as->a_contents); 3002a98e9dbfSaguzovsk } 3003a98e9dbfSaguzovsk } else { 3004a98e9dbfSaguzovsk /* 3005a98e9dbfSaguzovsk * We may have raced with 3006a98e9dbfSaguzovsk * segvn_reclaim()/segspt_reclaim(). In this 3007a98e9dbfSaguzovsk * case clean nounmapwait flag and retry since 3008a98e9dbfSaguzovsk * softlockcnt in this segment may be already 3009a98e9dbfSaguzovsk * 0. We don't drop as writer lock so our 3010a98e9dbfSaguzovsk * number of retries without sleeping should 3011a98e9dbfSaguzovsk * be very small. See segvn_reclaim() for 3012a98e9dbfSaguzovsk * more comments. 3013a98e9dbfSaguzovsk */ 3014a98e9dbfSaguzovsk AS_CLRNOUNMAPWAIT(as); 3015a98e9dbfSaguzovsk mutex_exit(&as->a_contents); 3016a98e9dbfSaguzovsk goto retry; 30177c478bd9Sstevel@tonic-gate } 30187c478bd9Sstevel@tonic-gate mutex_exit(&as->a_contents); 30197c478bd9Sstevel@tonic-gate goto setpgsz_top; 30207c478bd9Sstevel@tonic-gate } else if (error != 0) { 30217c478bd9Sstevel@tonic-gate break; 30227c478bd9Sstevel@tonic-gate } 30237c478bd9Sstevel@tonic-gate } 30247c478bd9Sstevel@tonic-gate as_setwatch(as); 3025dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 30267c478bd9Sstevel@tonic-gate return (error); 30277c478bd9Sstevel@tonic-gate } 30287c478bd9Sstevel@tonic-gate 3029ec25b48fSsusans /* 3030ec25b48fSsusans * as_iset3_default_lpsize() just calls SEGOP_SETPAGESIZE() on all segments 3031ec25b48fSsusans * in its chunk where s_szc is less than the szc we want to set. 3032ec25b48fSsusans */ 3033ec25b48fSsusans static int 3034ec25b48fSsusans as_iset3_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc, 3035ec25b48fSsusans int *retry) 3036ec25b48fSsusans { 3037ec25b48fSsusans struct seg *seg; 3038ec25b48fSsusans size_t ssize; 3039ec25b48fSsusans int error; 3040ec25b48fSsusans 3041dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 3042a98e9dbfSaguzovsk 3043ec25b48fSsusans seg = as_segat(as, raddr); 3044ec25b48fSsusans if (seg == NULL) { 3045ec25b48fSsusans panic("as_iset3_default_lpsize: no seg"); 3046ec25b48fSsusans } 3047ec25b48fSsusans 3048ec25b48fSsusans for (; rsize != 0; rsize -= ssize, raddr += ssize) { 3049ec25b48fSsusans if (raddr >= seg->s_base + seg->s_size) { 3050ec25b48fSsusans seg = AS_SEGNEXT(as, seg); 3051ec25b48fSsusans if (seg == NULL || raddr != seg->s_base) { 3052ec25b48fSsusans panic("as_iset3_default_lpsize: as changed"); 3053ec25b48fSsusans } 3054ec25b48fSsusans } 3055ec25b48fSsusans if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 3056ec25b48fSsusans ssize = seg->s_base + seg->s_size - raddr; 3057ec25b48fSsusans } else { 3058ec25b48fSsusans ssize = rsize; 3059ec25b48fSsusans } 3060ec25b48fSsusans 3061ec25b48fSsusans if (szc > seg->s_szc) { 3062ec25b48fSsusans error = SEGOP_SETPAGESIZE(seg, raddr, ssize, szc); 3063ec25b48fSsusans /* Only retry on EINVAL segments that have no vnode. */ 3064ec25b48fSsusans if (error == EINVAL) { 3065ec25b48fSsusans vnode_t *vp = NULL; 3066ec25b48fSsusans if ((SEGOP_GETTYPE(seg, raddr) & MAP_SHARED) && 3067ec25b48fSsusans (SEGOP_GETVP(seg, raddr, &vp) != 0 || 3068ec25b48fSsusans vp == NULL)) { 3069ec25b48fSsusans *retry = 1; 3070ec25b48fSsusans } else { 3071ec25b48fSsusans *retry = 0; 3072ec25b48fSsusans } 3073ec25b48fSsusans } 3074ec25b48fSsusans if (error) { 3075ec25b48fSsusans return (error); 3076ec25b48fSsusans } 3077ec25b48fSsusans } 3078ec25b48fSsusans } 3079ec25b48fSsusans return (0); 3080ec25b48fSsusans } 3081ec25b48fSsusans 3082ec25b48fSsusans /* 3083ec25b48fSsusans * as_iset2_default_lpsize() calls as_iset3_default_lpsize() to set the 3084ec25b48fSsusans * pagesize on each segment in its range, but if any fails with EINVAL, 3085ec25b48fSsusans * then it reduces the pagesizes to the next size in the bitmap and 3086ec25b48fSsusans * retries as_iset3_default_lpsize(). The reason why the code retries 3087ec25b48fSsusans * smaller allowed sizes on EINVAL is because (a) the anon offset may not 3088ec25b48fSsusans * match the bigger sizes, and (b) it's hard to get this offset (to begin 3089ec25b48fSsusans * with) to pass to map_pgszcvec(). 3090ec25b48fSsusans */ 3091ec25b48fSsusans static int 3092ec25b48fSsusans as_iset2_default_lpsize(struct as *as, caddr_t addr, size_t size, uint_t szc, 3093ec25b48fSsusans uint_t szcvec) 3094ec25b48fSsusans { 3095ec25b48fSsusans int error; 3096ec25b48fSsusans int retry; 3097ec25b48fSsusans 3098dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 3099a98e9dbfSaguzovsk 3100ec25b48fSsusans for (;;) { 3101ec25b48fSsusans error = as_iset3_default_lpsize(as, addr, size, szc, &retry); 3102ec25b48fSsusans if (error == EINVAL && retry) { 3103ec25b48fSsusans szcvec &= ~(1 << szc); 3104ec25b48fSsusans if (szcvec <= 1) { 3105ec25b48fSsusans return (EINVAL); 3106ec25b48fSsusans } 3107ec25b48fSsusans szc = highbit(szcvec) - 1; 3108ec25b48fSsusans } else { 3109ec25b48fSsusans return (error); 3110ec25b48fSsusans } 3111ec25b48fSsusans } 3112ec25b48fSsusans } 3113ec25b48fSsusans 3114ec25b48fSsusans /* 3115ec25b48fSsusans * as_iset1_default_lpsize() breaks its chunk into areas where existing 3116ec25b48fSsusans * segments have a smaller szc than we want to set. For each such area, 3117ec25b48fSsusans * it calls as_iset2_default_lpsize() 3118ec25b48fSsusans */ 3119ec25b48fSsusans static int 3120ec25b48fSsusans as_iset1_default_lpsize(struct as *as, caddr_t raddr, size_t rsize, uint_t szc, 3121ec25b48fSsusans uint_t szcvec) 3122ec25b48fSsusans { 3123ec25b48fSsusans struct seg *seg; 3124ec25b48fSsusans size_t ssize; 3125ec25b48fSsusans caddr_t setaddr = raddr; 3126ec25b48fSsusans size_t setsize = 0; 3127ec25b48fSsusans int set; 3128ec25b48fSsusans int error; 3129ec25b48fSsusans 3130dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 3131ec25b48fSsusans 3132ec25b48fSsusans seg = as_segat(as, raddr); 3133ec25b48fSsusans if (seg == NULL) { 3134ec25b48fSsusans panic("as_iset1_default_lpsize: no seg"); 3135ec25b48fSsusans } 3136ec25b48fSsusans if (seg->s_szc < szc) { 3137ec25b48fSsusans set = 1; 3138ec25b48fSsusans } else { 3139ec25b48fSsusans set = 0; 3140ec25b48fSsusans } 3141ec25b48fSsusans 3142ec25b48fSsusans for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) { 3143ec25b48fSsusans if (raddr >= seg->s_base + seg->s_size) { 3144ec25b48fSsusans seg = AS_SEGNEXT(as, seg); 3145ec25b48fSsusans if (seg == NULL || raddr != seg->s_base) { 3146ec25b48fSsusans panic("as_iset1_default_lpsize: as changed"); 3147ec25b48fSsusans } 3148ec25b48fSsusans if (seg->s_szc >= szc && set) { 3149ec25b48fSsusans ASSERT(setsize != 0); 3150ec25b48fSsusans error = as_iset2_default_lpsize(as, 3151ec25b48fSsusans setaddr, setsize, szc, szcvec); 3152ec25b48fSsusans if (error) { 3153ec25b48fSsusans return (error); 3154ec25b48fSsusans } 3155ec25b48fSsusans set = 0; 3156ec25b48fSsusans } else if (seg->s_szc < szc && !set) { 3157ec25b48fSsusans setaddr = raddr; 3158ec25b48fSsusans setsize = 0; 3159ec25b48fSsusans set = 1; 3160ec25b48fSsusans } 3161ec25b48fSsusans } 3162ec25b48fSsusans if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 3163ec25b48fSsusans ssize = seg->s_base + seg->s_size - raddr; 3164ec25b48fSsusans } else { 3165ec25b48fSsusans ssize = rsize; 3166ec25b48fSsusans } 3167ec25b48fSsusans } 3168ec25b48fSsusans error = 0; 3169ec25b48fSsusans if (set) { 3170ec25b48fSsusans ASSERT(setsize != 0); 3171ec25b48fSsusans error = as_iset2_default_lpsize(as, setaddr, setsize, 3172ec25b48fSsusans szc, szcvec); 3173ec25b48fSsusans } 3174ec25b48fSsusans return (error); 3175ec25b48fSsusans } 3176ec25b48fSsusans 3177ec25b48fSsusans /* 3178ec25b48fSsusans * as_iset_default_lpsize() breaks its chunk according to the size code bitmap 3179ec25b48fSsusans * returned by map_pgszcvec() (similar to as_map_segvn_segs()), and passes each 3180ec25b48fSsusans * chunk to as_iset1_default_lpsize(). 3181ec25b48fSsusans */ 3182ec25b48fSsusans static int 3183ec25b48fSsusans as_iset_default_lpsize(struct as *as, caddr_t addr, size_t size, int flags, 3184ec25b48fSsusans int type) 3185ec25b48fSsusans { 3186ec25b48fSsusans int rtype = (type & MAP_SHARED) ? MAPPGSZC_SHM : MAPPGSZC_PRIVM; 3187ec25b48fSsusans uint_t szcvec = map_pgszcvec(addr, size, (uintptr_t)addr, 3188843e1988Sjohnlev flags, rtype, 1); 3189ec25b48fSsusans uint_t szc; 3190ec25b48fSsusans uint_t nszc; 3191ec25b48fSsusans int error; 3192ec25b48fSsusans caddr_t a; 3193ec25b48fSsusans caddr_t eaddr; 3194ec25b48fSsusans size_t segsize; 3195ec25b48fSsusans size_t pgsz; 3196ec25b48fSsusans uint_t save_szcvec; 3197ec25b48fSsusans 3198dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 3199ec25b48fSsusans ASSERT(IS_P2ALIGNED(addr, PAGESIZE)); 3200ec25b48fSsusans ASSERT(IS_P2ALIGNED(size, PAGESIZE)); 3201ec25b48fSsusans 3202ec25b48fSsusans szcvec &= ~1; 3203ec25b48fSsusans if (szcvec <= 1) { /* skip if base page size */ 3204ec25b48fSsusans return (0); 3205ec25b48fSsusans } 3206ec25b48fSsusans 3207ec25b48fSsusans /* Get the pagesize of the first larger page size. */ 3208ec25b48fSsusans szc = lowbit(szcvec) - 1; 3209ec25b48fSsusans pgsz = page_get_pagesize(szc); 3210ec25b48fSsusans eaddr = addr + size; 3211ec25b48fSsusans addr = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 3212ec25b48fSsusans eaddr = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 3213ec25b48fSsusans 3214ec25b48fSsusans save_szcvec = szcvec; 3215ec25b48fSsusans szcvec >>= (szc + 1); 3216ec25b48fSsusans nszc = szc; 3217ec25b48fSsusans while (szcvec) { 3218ec25b48fSsusans if ((szcvec & 0x1) == 0) { 3219ec25b48fSsusans nszc++; 3220ec25b48fSsusans szcvec >>= 1; 3221ec25b48fSsusans continue; 3222ec25b48fSsusans } 3223ec25b48fSsusans nszc++; 3224ec25b48fSsusans pgsz = page_get_pagesize(nszc); 3225ec25b48fSsusans a = (caddr_t)P2ROUNDUP((uintptr_t)addr, pgsz); 3226ec25b48fSsusans if (a != addr) { 3227ec25b48fSsusans ASSERT(szc > 0); 3228ec25b48fSsusans ASSERT(a < eaddr); 3229ec25b48fSsusans segsize = a - addr; 3230ec25b48fSsusans error = as_iset1_default_lpsize(as, addr, segsize, szc, 3231ec25b48fSsusans save_szcvec); 3232ec25b48fSsusans if (error) { 3233ec25b48fSsusans return (error); 3234ec25b48fSsusans } 3235ec25b48fSsusans addr = a; 3236ec25b48fSsusans } 3237ec25b48fSsusans szc = nszc; 3238ec25b48fSsusans szcvec >>= 1; 3239ec25b48fSsusans } 3240ec25b48fSsusans 3241ec25b48fSsusans ASSERT(addr < eaddr); 3242ec25b48fSsusans szcvec = save_szcvec; 3243ec25b48fSsusans while (szcvec) { 3244ec25b48fSsusans a = (caddr_t)P2ALIGN((uintptr_t)eaddr, pgsz); 3245ec25b48fSsusans ASSERT(a >= addr); 3246ec25b48fSsusans if (a != addr) { 3247ec25b48fSsusans ASSERT(szc > 0); 3248ec25b48fSsusans segsize = a - addr; 3249ec25b48fSsusans error = as_iset1_default_lpsize(as, addr, segsize, szc, 3250ec25b48fSsusans save_szcvec); 3251ec25b48fSsusans if (error) { 3252ec25b48fSsusans return (error); 3253ec25b48fSsusans } 3254ec25b48fSsusans addr = a; 3255ec25b48fSsusans } 3256ec25b48fSsusans szcvec &= ~(1 << szc); 3257ec25b48fSsusans if (szcvec) { 3258ec25b48fSsusans szc = highbit(szcvec) - 1; 3259ec25b48fSsusans pgsz = page_get_pagesize(szc); 3260ec25b48fSsusans } 3261ec25b48fSsusans } 3262ec25b48fSsusans ASSERT(addr == eaddr); 3263ec25b48fSsusans 3264ec25b48fSsusans return (0); 3265ec25b48fSsusans } 3266ec25b48fSsusans 3267ec25b48fSsusans /* 3268ec25b48fSsusans * Set the default large page size for the range. Called via memcntl with 3269ec25b48fSsusans * page size set to 0. as_set_default_lpsize breaks the range down into 3270ec25b48fSsusans * chunks with the same type/flags, ignores-non segvn segments, and passes 3271ec25b48fSsusans * each chunk to as_iset_default_lpsize(). 3272ec25b48fSsusans */ 3273ec25b48fSsusans int 3274ec25b48fSsusans as_set_default_lpsize(struct as *as, caddr_t addr, size_t size) 3275ec25b48fSsusans { 3276ec25b48fSsusans struct seg *seg; 3277ec25b48fSsusans caddr_t raddr; 3278ec25b48fSsusans size_t rsize; 3279ec25b48fSsusans size_t ssize; 3280ec25b48fSsusans int rtype, rflags; 3281ec25b48fSsusans int stype, sflags; 3282ec25b48fSsusans int error; 3283ec25b48fSsusans caddr_t setaddr; 3284ec25b48fSsusans size_t setsize; 3285ec25b48fSsusans int segvn; 3286ec25b48fSsusans 3287ec25b48fSsusans if (size == 0) 3288ec25b48fSsusans return (0); 3289ec25b48fSsusans 3290dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 3291ec25b48fSsusans again: 3292ec25b48fSsusans error = 0; 3293ec25b48fSsusans 3294ec25b48fSsusans raddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 3295ec25b48fSsusans rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) - 3296ec25b48fSsusans (size_t)raddr; 3297ec25b48fSsusans 3298ec25b48fSsusans if (raddr + rsize < raddr) { /* check for wraparound */ 3299dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 3300ec25b48fSsusans return (ENOMEM); 3301ec25b48fSsusans } 3302ec25b48fSsusans as_clearwatchprot(as, raddr, rsize); 3303ec25b48fSsusans seg = as_segat(as, raddr); 3304ec25b48fSsusans if (seg == NULL) { 3305ec25b48fSsusans as_setwatch(as); 3306dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 3307ec25b48fSsusans return (ENOMEM); 3308ec25b48fSsusans } 3309ec25b48fSsusans if (seg->s_ops == &segvn_ops) { 3310ec25b48fSsusans rtype = SEGOP_GETTYPE(seg, addr); 3311ec25b48fSsusans rflags = rtype & (MAP_TEXT | MAP_INITDATA); 3312ec25b48fSsusans rtype = rtype & (MAP_SHARED | MAP_PRIVATE); 3313ec25b48fSsusans segvn = 1; 3314ec25b48fSsusans } else { 3315ec25b48fSsusans segvn = 0; 3316ec25b48fSsusans } 3317ec25b48fSsusans setaddr = raddr; 3318ec25b48fSsusans setsize = 0; 3319ec25b48fSsusans 3320ec25b48fSsusans for (; rsize != 0; rsize -= ssize, raddr += ssize, setsize += ssize) { 3321ec25b48fSsusans if (raddr >= (seg->s_base + seg->s_size)) { 3322ec25b48fSsusans seg = AS_SEGNEXT(as, seg); 3323ec25b48fSsusans if (seg == NULL || raddr != seg->s_base) { 3324ec25b48fSsusans error = ENOMEM; 3325ec25b48fSsusans break; 3326ec25b48fSsusans } 3327ec25b48fSsusans if (seg->s_ops == &segvn_ops) { 3328ec25b48fSsusans stype = SEGOP_GETTYPE(seg, raddr); 3329ec25b48fSsusans sflags = stype & (MAP_TEXT | MAP_INITDATA); 3330ec25b48fSsusans stype &= (MAP_SHARED | MAP_PRIVATE); 3331ec25b48fSsusans if (segvn && (rflags != sflags || 3332ec25b48fSsusans rtype != stype)) { 3333ec25b48fSsusans /* 3334ec25b48fSsusans * The next segment is also segvn but 3335ec25b48fSsusans * has different flags and/or type. 3336ec25b48fSsusans */ 3337ec25b48fSsusans ASSERT(setsize != 0); 3338ec25b48fSsusans error = as_iset_default_lpsize(as, 3339ec25b48fSsusans setaddr, setsize, rflags, rtype); 3340ec25b48fSsusans if (error) { 3341ec25b48fSsusans break; 3342ec25b48fSsusans } 3343ec25b48fSsusans rflags = sflags; 3344ec25b48fSsusans rtype = stype; 3345ec25b48fSsusans setaddr = raddr; 3346ec25b48fSsusans setsize = 0; 3347ec25b48fSsusans } else if (!segvn) { 3348ec25b48fSsusans rflags = sflags; 3349ec25b48fSsusans rtype = stype; 3350ec25b48fSsusans setaddr = raddr; 3351ec25b48fSsusans setsize = 0; 3352ec25b48fSsusans segvn = 1; 3353ec25b48fSsusans } 3354ec25b48fSsusans } else if (segvn) { 3355ec25b48fSsusans /* The next segment is not segvn. */ 3356ec25b48fSsusans ASSERT(setsize != 0); 3357ec25b48fSsusans error = as_iset_default_lpsize(as, 3358ec25b48fSsusans setaddr, setsize, rflags, rtype); 3359ec25b48fSsusans if (error) { 3360ec25b48fSsusans break; 3361ec25b48fSsusans } 3362ec25b48fSsusans segvn = 0; 3363ec25b48fSsusans } 3364ec25b48fSsusans } 3365ec25b48fSsusans if ((raddr + rsize) > (seg->s_base + seg->s_size)) { 3366ec25b48fSsusans ssize = seg->s_base + seg->s_size - raddr; 3367ec25b48fSsusans } else { 3368ec25b48fSsusans ssize = rsize; 3369ec25b48fSsusans } 3370ec25b48fSsusans } 3371ec25b48fSsusans if (error == 0 && segvn) { 3372ec25b48fSsusans /* The last chunk when rsize == 0. */ 3373ec25b48fSsusans ASSERT(setsize != 0); 3374ec25b48fSsusans error = as_iset_default_lpsize(as, setaddr, setsize, 3375ec25b48fSsusans rflags, rtype); 3376ec25b48fSsusans } 3377ec25b48fSsusans 3378ec25b48fSsusans if (error == IE_RETRY) { 3379ec25b48fSsusans goto again; 3380ec25b48fSsusans } else if (error == IE_NOMEM) { 3381ec25b48fSsusans error = EAGAIN; 3382ec25b48fSsusans } else if (error == ENOTSUP) { 3383ec25b48fSsusans error = EINVAL; 3384ec25b48fSsusans } else if (error == EAGAIN) { 3385ec25b48fSsusans mutex_enter(&as->a_contents); 3386a98e9dbfSaguzovsk if (!AS_ISNOUNMAPWAIT(as)) { 3387a98e9dbfSaguzovsk if (AS_ISUNMAPWAIT(as) == 0) { 3388a98e9dbfSaguzovsk cv_broadcast(&as->a_cv); 3389a98e9dbfSaguzovsk } 3390a98e9dbfSaguzovsk AS_SETUNMAPWAIT(as); 3391dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 3392a98e9dbfSaguzovsk while (AS_ISUNMAPWAIT(as)) { 3393a98e9dbfSaguzovsk cv_wait(&as->a_cv, &as->a_contents); 3394a98e9dbfSaguzovsk } 3395a98e9dbfSaguzovsk mutex_exit(&as->a_contents); 3396dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_WRITER); 3397a98e9dbfSaguzovsk } else { 3398a98e9dbfSaguzovsk /* 3399a98e9dbfSaguzovsk * We may have raced with 3400a98e9dbfSaguzovsk * segvn_reclaim()/segspt_reclaim(). In this case 3401a98e9dbfSaguzovsk * clean nounmapwait flag and retry since softlockcnt 3402a98e9dbfSaguzovsk * in this segment may be already 0. We don't drop as 3403a98e9dbfSaguzovsk * writer lock so our number of retries without 3404a98e9dbfSaguzovsk * sleeping should be very small. See segvn_reclaim() 3405a98e9dbfSaguzovsk * for more comments. 3406a98e9dbfSaguzovsk */ 3407a98e9dbfSaguzovsk AS_CLRNOUNMAPWAIT(as); 3408a98e9dbfSaguzovsk mutex_exit(&as->a_contents); 3409ec25b48fSsusans } 3410ec25b48fSsusans goto again; 3411ec25b48fSsusans } 3412ec25b48fSsusans 3413ec25b48fSsusans as_setwatch(as); 3414dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 3415ec25b48fSsusans return (error); 3416ec25b48fSsusans } 3417ec25b48fSsusans 34187c478bd9Sstevel@tonic-gate /* 34197c478bd9Sstevel@tonic-gate * Setup all of the uninitialized watched pages that we can. 34207c478bd9Sstevel@tonic-gate */ 34217c478bd9Sstevel@tonic-gate void 34227c478bd9Sstevel@tonic-gate as_setwatch(struct as *as) 34237c478bd9Sstevel@tonic-gate { 34247c478bd9Sstevel@tonic-gate struct watched_page *pwp; 34257c478bd9Sstevel@tonic-gate struct seg *seg; 34267c478bd9Sstevel@tonic-gate caddr_t vaddr; 34277c478bd9Sstevel@tonic-gate uint_t prot; 34287c478bd9Sstevel@tonic-gate int err, retrycnt; 34297c478bd9Sstevel@tonic-gate 34307c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 34317c478bd9Sstevel@tonic-gate return; 34327c478bd9Sstevel@tonic-gate 3433dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 34347c478bd9Sstevel@tonic-gate 34357c478bd9Sstevel@tonic-gate for (pwp = avl_first(&as->a_wpage); pwp != NULL; 34367c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp)) { 34377c478bd9Sstevel@tonic-gate retrycnt = 0; 34387c478bd9Sstevel@tonic-gate retry: 34397c478bd9Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 34407c478bd9Sstevel@tonic-gate if (pwp->wp_oprot != 0 || /* already set up */ 34417c478bd9Sstevel@tonic-gate (seg = as_segat(as, vaddr)) == NULL || 34427c478bd9Sstevel@tonic-gate SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0) 34437c478bd9Sstevel@tonic-gate continue; 34447c478bd9Sstevel@tonic-gate 34457c478bd9Sstevel@tonic-gate pwp->wp_oprot = prot; 34467c478bd9Sstevel@tonic-gate if (pwp->wp_read) 34477c478bd9Sstevel@tonic-gate prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 34487c478bd9Sstevel@tonic-gate if (pwp->wp_write) 34497c478bd9Sstevel@tonic-gate prot &= ~PROT_WRITE; 34507c478bd9Sstevel@tonic-gate if (pwp->wp_exec) 34517c478bd9Sstevel@tonic-gate prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 34527c478bd9Sstevel@tonic-gate if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot) { 34537c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 34547c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 34557c478bd9Sstevel@tonic-gate pwp->wp_oprot = 0; 34567c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 34577c478bd9Sstevel@tonic-gate retrycnt++; 34587c478bd9Sstevel@tonic-gate goto retry; 34597c478bd9Sstevel@tonic-gate } 34607c478bd9Sstevel@tonic-gate } 34617c478bd9Sstevel@tonic-gate pwp->wp_prot = prot; 34627c478bd9Sstevel@tonic-gate } 34637c478bd9Sstevel@tonic-gate } 34647c478bd9Sstevel@tonic-gate 34657c478bd9Sstevel@tonic-gate /* 34667c478bd9Sstevel@tonic-gate * Clear all of the watched pages in the address space. 34677c478bd9Sstevel@tonic-gate */ 34687c478bd9Sstevel@tonic-gate void 34697c478bd9Sstevel@tonic-gate as_clearwatch(struct as *as) 34707c478bd9Sstevel@tonic-gate { 34717c478bd9Sstevel@tonic-gate struct watched_page *pwp; 34727c478bd9Sstevel@tonic-gate struct seg *seg; 34737c478bd9Sstevel@tonic-gate caddr_t vaddr; 34747c478bd9Sstevel@tonic-gate uint_t prot; 34757c478bd9Sstevel@tonic-gate int err, retrycnt; 34767c478bd9Sstevel@tonic-gate 34777c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 34787c478bd9Sstevel@tonic-gate return; 34797c478bd9Sstevel@tonic-gate 3480dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 34817c478bd9Sstevel@tonic-gate 34827c478bd9Sstevel@tonic-gate for (pwp = avl_first(&as->a_wpage); pwp != NULL; 34837c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp)) { 34847c478bd9Sstevel@tonic-gate retrycnt = 0; 34857c478bd9Sstevel@tonic-gate retry: 34867c478bd9Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 34877c478bd9Sstevel@tonic-gate if (pwp->wp_oprot == 0 || /* not set up */ 34887c478bd9Sstevel@tonic-gate (seg = as_segat(as, vaddr)) == NULL) 34897c478bd9Sstevel@tonic-gate continue; 34907c478bd9Sstevel@tonic-gate 34917c478bd9Sstevel@tonic-gate if ((prot = pwp->wp_oprot) != pwp->wp_prot) { 34927c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot); 34937c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 34947c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 34957c478bd9Sstevel@tonic-gate retrycnt++; 34967c478bd9Sstevel@tonic-gate goto retry; 34977c478bd9Sstevel@tonic-gate } 34987c478bd9Sstevel@tonic-gate } 34997c478bd9Sstevel@tonic-gate pwp->wp_oprot = 0; 35007c478bd9Sstevel@tonic-gate pwp->wp_prot = 0; 35017c478bd9Sstevel@tonic-gate } 35027c478bd9Sstevel@tonic-gate } 35037c478bd9Sstevel@tonic-gate 35047c478bd9Sstevel@tonic-gate /* 35057c478bd9Sstevel@tonic-gate * Force a new setup for all the watched pages in the range. 35067c478bd9Sstevel@tonic-gate */ 35077c478bd9Sstevel@tonic-gate static void 35087c478bd9Sstevel@tonic-gate as_setwatchprot(struct as *as, caddr_t addr, size_t size, uint_t prot) 35097c478bd9Sstevel@tonic-gate { 35107c478bd9Sstevel@tonic-gate struct watched_page *pwp; 35117c478bd9Sstevel@tonic-gate struct watched_page tpw; 35127c478bd9Sstevel@tonic-gate caddr_t eaddr = addr + size; 35137c478bd9Sstevel@tonic-gate caddr_t vaddr; 35147c478bd9Sstevel@tonic-gate struct seg *seg; 35157c478bd9Sstevel@tonic-gate int err, retrycnt; 35167c478bd9Sstevel@tonic-gate uint_t wprot; 35177c478bd9Sstevel@tonic-gate avl_index_t where; 35187c478bd9Sstevel@tonic-gate 35197c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 35207c478bd9Sstevel@tonic-gate return; 35217c478bd9Sstevel@tonic-gate 3522dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 35237c478bd9Sstevel@tonic-gate 35247c478bd9Sstevel@tonic-gate tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 35257c478bd9Sstevel@tonic-gate if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 35267c478bd9Sstevel@tonic-gate pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 35277c478bd9Sstevel@tonic-gate 35287c478bd9Sstevel@tonic-gate while (pwp != NULL && pwp->wp_vaddr < eaddr) { 35297c478bd9Sstevel@tonic-gate retrycnt = 0; 35307c478bd9Sstevel@tonic-gate vaddr = pwp->wp_vaddr; 35317c478bd9Sstevel@tonic-gate 35327c478bd9Sstevel@tonic-gate wprot = prot; 35337c478bd9Sstevel@tonic-gate if (pwp->wp_read) 35347c478bd9Sstevel@tonic-gate wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 35357c478bd9Sstevel@tonic-gate if (pwp->wp_write) 35367c478bd9Sstevel@tonic-gate wprot &= ~PROT_WRITE; 35377c478bd9Sstevel@tonic-gate if (pwp->wp_exec) 35387c478bd9Sstevel@tonic-gate wprot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC); 35397c478bd9Sstevel@tonic-gate if (!(pwp->wp_flags & WP_NOWATCH) && wprot != pwp->wp_oprot) { 35407c478bd9Sstevel@tonic-gate retry: 35417c478bd9Sstevel@tonic-gate seg = as_segat(as, vaddr); 35427c478bd9Sstevel@tonic-gate if (seg == NULL) { 35437c478bd9Sstevel@tonic-gate panic("as_setwatchprot: no seg"); 35447c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 35457c478bd9Sstevel@tonic-gate } 35467c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, vaddr, PAGESIZE, wprot); 35477c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 35487c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 35497c478bd9Sstevel@tonic-gate retrycnt++; 35507c478bd9Sstevel@tonic-gate goto retry; 35517c478bd9Sstevel@tonic-gate } 35527c478bd9Sstevel@tonic-gate } 35537c478bd9Sstevel@tonic-gate pwp->wp_oprot = prot; 35547c478bd9Sstevel@tonic-gate pwp->wp_prot = wprot; 35557c478bd9Sstevel@tonic-gate 35567c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp); 35577c478bd9Sstevel@tonic-gate } 35587c478bd9Sstevel@tonic-gate } 35597c478bd9Sstevel@tonic-gate 35607c478bd9Sstevel@tonic-gate /* 35617c478bd9Sstevel@tonic-gate * Clear all of the watched pages in the range. 35627c478bd9Sstevel@tonic-gate */ 35637c478bd9Sstevel@tonic-gate static void 35647c478bd9Sstevel@tonic-gate as_clearwatchprot(struct as *as, caddr_t addr, size_t size) 35657c478bd9Sstevel@tonic-gate { 35667c478bd9Sstevel@tonic-gate caddr_t eaddr = addr + size; 35677c478bd9Sstevel@tonic-gate struct watched_page *pwp; 35687c478bd9Sstevel@tonic-gate struct watched_page tpw; 35697c478bd9Sstevel@tonic-gate uint_t prot; 35707c478bd9Sstevel@tonic-gate struct seg *seg; 35717c478bd9Sstevel@tonic-gate int err, retrycnt; 35727c478bd9Sstevel@tonic-gate avl_index_t where; 35737c478bd9Sstevel@tonic-gate 35747c478bd9Sstevel@tonic-gate if (avl_numnodes(&as->a_wpage) == 0) 35757c478bd9Sstevel@tonic-gate return; 35767c478bd9Sstevel@tonic-gate 35777c478bd9Sstevel@tonic-gate tpw.wp_vaddr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 35787c478bd9Sstevel@tonic-gate if ((pwp = avl_find(&as->a_wpage, &tpw, &where)) == NULL) 35797c478bd9Sstevel@tonic-gate pwp = avl_nearest(&as->a_wpage, where, AVL_AFTER); 35807c478bd9Sstevel@tonic-gate 3581dc32d872SJosef 'Jeff' Sipek ASSERT(AS_WRITE_HELD(as)); 35827c478bd9Sstevel@tonic-gate 35837c478bd9Sstevel@tonic-gate while (pwp != NULL && pwp->wp_vaddr < eaddr) { 35847c478bd9Sstevel@tonic-gate 35857c478bd9Sstevel@tonic-gate if ((prot = pwp->wp_oprot) != 0) { 35867c478bd9Sstevel@tonic-gate retrycnt = 0; 35877c478bd9Sstevel@tonic-gate 35887c478bd9Sstevel@tonic-gate if (prot != pwp->wp_prot) { 35897c478bd9Sstevel@tonic-gate retry: 35907c478bd9Sstevel@tonic-gate seg = as_segat(as, pwp->wp_vaddr); 35917c478bd9Sstevel@tonic-gate if (seg == NULL) 35927c478bd9Sstevel@tonic-gate continue; 35937c478bd9Sstevel@tonic-gate err = SEGOP_SETPROT(seg, pwp->wp_vaddr, 35947c478bd9Sstevel@tonic-gate PAGESIZE, prot); 35957c478bd9Sstevel@tonic-gate if (err == IE_RETRY) { 35967c478bd9Sstevel@tonic-gate ASSERT(retrycnt == 0); 35977c478bd9Sstevel@tonic-gate retrycnt++; 35987c478bd9Sstevel@tonic-gate goto retry; 35997c478bd9Sstevel@tonic-gate 36007c478bd9Sstevel@tonic-gate } 36017c478bd9Sstevel@tonic-gate } 36027c478bd9Sstevel@tonic-gate pwp->wp_oprot = 0; 36037c478bd9Sstevel@tonic-gate pwp->wp_prot = 0; 36047c478bd9Sstevel@tonic-gate } 36057c478bd9Sstevel@tonic-gate 36067c478bd9Sstevel@tonic-gate pwp = AVL_NEXT(&as->a_wpage, pwp); 36077c478bd9Sstevel@tonic-gate } 36087c478bd9Sstevel@tonic-gate } 36097c478bd9Sstevel@tonic-gate 36107c478bd9Sstevel@tonic-gate void 36117c478bd9Sstevel@tonic-gate as_signal_proc(struct as *as, k_siginfo_t *siginfo) 36127c478bd9Sstevel@tonic-gate { 36137c478bd9Sstevel@tonic-gate struct proc *p; 36147c478bd9Sstevel@tonic-gate 36157c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 36167c478bd9Sstevel@tonic-gate for (p = practive; p; p = p->p_next) { 36177c478bd9Sstevel@tonic-gate if (p->p_as == as) { 36187c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 36197c478bd9Sstevel@tonic-gate if (p->p_as == as) 36207c478bd9Sstevel@tonic-gate sigaddq(p, NULL, siginfo, KM_NOSLEEP); 36217c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 36227c478bd9Sstevel@tonic-gate } 36237c478bd9Sstevel@tonic-gate } 36247c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 36257c478bd9Sstevel@tonic-gate } 36267c478bd9Sstevel@tonic-gate 36277c478bd9Sstevel@tonic-gate /* 36287c478bd9Sstevel@tonic-gate * return memory object ID 36297c478bd9Sstevel@tonic-gate */ 36307c478bd9Sstevel@tonic-gate int 36317c478bd9Sstevel@tonic-gate as_getmemid(struct as *as, caddr_t addr, memid_t *memidp) 36327c478bd9Sstevel@tonic-gate { 36337c478bd9Sstevel@tonic-gate struct seg *seg; 36347c478bd9Sstevel@tonic-gate int sts; 36357c478bd9Sstevel@tonic-gate 3636dc32d872SJosef 'Jeff' Sipek AS_LOCK_ENTER(as, RW_READER); 36377c478bd9Sstevel@tonic-gate seg = as_segat(as, addr); 36387c478bd9Sstevel@tonic-gate if (seg == NULL) { 3639dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 36407c478bd9Sstevel@tonic-gate return (EFAULT); 36417c478bd9Sstevel@tonic-gate } 36427c478bd9Sstevel@tonic-gate /* 36437c478bd9Sstevel@tonic-gate * catch old drivers which may not support getmemid 36447c478bd9Sstevel@tonic-gate */ 36457c478bd9Sstevel@tonic-gate if (seg->s_ops->getmemid == NULL) { 3646dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 36477c478bd9Sstevel@tonic-gate return (ENODEV); 36487c478bd9Sstevel@tonic-gate } 36497c478bd9Sstevel@tonic-gate 36507c478bd9Sstevel@tonic-gate sts = SEGOP_GETMEMID(seg, addr, memidp); 36517c478bd9Sstevel@tonic-gate 3652dc32d872SJosef 'Jeff' Sipek AS_LOCK_EXIT(as); 36537c478bd9Sstevel@tonic-gate return (sts); 36547c478bd9Sstevel@tonic-gate } 3655