17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 53cff2f43Sstans * Common Development and Distribution License (the "License"). 63cff2f43Sstans * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*cee1d74bSjfrank * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * University Copyright- Copyright (c) 1982, 1986, 1988 317c478bd9Sstevel@tonic-gate * The Regents of the University of California 327c478bd9Sstevel@tonic-gate * All Rights Reserved 337c478bd9Sstevel@tonic-gate * 347c478bd9Sstevel@tonic-gate * University Acknowledgment- Portions of this document are derived from 357c478bd9Sstevel@tonic-gate * software developed by the University of California, Berkeley, and its 367c478bd9Sstevel@tonic-gate * contributors. 377c478bd9Sstevel@tonic-gate */ 387c478bd9Sstevel@tonic-gate 397c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate /* 427c478bd9Sstevel@tonic-gate * VM - physical page management. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate 457c478bd9Sstevel@tonic-gate #include <sys/types.h> 467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 477c478bd9Sstevel@tonic-gate #include <sys/param.h> 487c478bd9Sstevel@tonic-gate #include <sys/systm.h> 497c478bd9Sstevel@tonic-gate #include <sys/errno.h> 507c478bd9Sstevel@tonic-gate #include <sys/time.h> 517c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 527c478bd9Sstevel@tonic-gate #include <sys/vm.h> 537c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 547c478bd9Sstevel@tonic-gate #include <sys/swap.h> 557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 567c478bd9Sstevel@tonic-gate #include <sys/tuneable.h> 577c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 587c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 597c478bd9Sstevel@tonic-gate #include <sys/callb.h> 607c478bd9Sstevel@tonic-gate #include <sys/debug.h> 617c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h> 627c478bd9Sstevel@tonic-gate #include <sys/condvar_impl.h> 637c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 647c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 657c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 667c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 677c478bd9Sstevel@tonic-gate #include <sys/strlog.h> 687c478bd9Sstevel@tonic-gate #include <sys/mman.h> 697c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 707c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 717c478bd9Sstevel@tonic-gate #include <sys/vfs.h> 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate #include <vm/hat.h> 747c478bd9Sstevel@tonic-gate #include <vm/anon.h> 757c478bd9Sstevel@tonic-gate #include <vm/page.h> 767c478bd9Sstevel@tonic-gate #include <vm/seg.h> 777c478bd9Sstevel@tonic-gate #include <vm/pvn.h> 787c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 797c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 800209230bSgjelinek #include <sys/vm_usage.h> 817c478bd9Sstevel@tonic-gate #include <fs/fs_subr.h> 82*cee1d74bSjfrank #include <sys/ddi.h> 83*cee1d74bSjfrank #include <sys/modctl.h> 847c478bd9Sstevel@tonic-gate 857c478bd9Sstevel@tonic-gate static int nopageage = 0; 867c478bd9Sstevel@tonic-gate 877c478bd9Sstevel@tonic-gate static pgcnt_t max_page_get; /* max page_get request size in pages */ 887c478bd9Sstevel@tonic-gate pgcnt_t total_pages = 0; /* total number of pages (used by /proc) */ 897c478bd9Sstevel@tonic-gate 907c478bd9Sstevel@tonic-gate /* 917c478bd9Sstevel@tonic-gate * freemem_lock protects all freemem variables: 927c478bd9Sstevel@tonic-gate * availrmem. Also this lock protects the globals which track the 937c478bd9Sstevel@tonic-gate * availrmem changes for accurate kernel footprint calculation. 947c478bd9Sstevel@tonic-gate * See below for an explanation of these 957c478bd9Sstevel@tonic-gate * globals. 967c478bd9Sstevel@tonic-gate */ 977c478bd9Sstevel@tonic-gate kmutex_t freemem_lock; 987c478bd9Sstevel@tonic-gate pgcnt_t availrmem; 997c478bd9Sstevel@tonic-gate pgcnt_t availrmem_initial; 1007c478bd9Sstevel@tonic-gate 1017c478bd9Sstevel@tonic-gate /* 1027c478bd9Sstevel@tonic-gate * These globals track availrmem changes to get a more accurate 1037c478bd9Sstevel@tonic-gate * estimate of tke kernel size. Historically pp_kernel is used for 1047c478bd9Sstevel@tonic-gate * kernel size and is based on availrmem. But availrmem is adjusted for 1057c478bd9Sstevel@tonic-gate * locked pages in the system not just for kernel locked pages. 1067c478bd9Sstevel@tonic-gate * These new counters will track the pages locked through segvn and 1077c478bd9Sstevel@tonic-gate * by explicit user locking. 1087c478bd9Sstevel@tonic-gate * 1097c478bd9Sstevel@tonic-gate * segvn_pages_locked : This keeps track on a global basis how many pages 1107c478bd9Sstevel@tonic-gate * are currently locked because of I/O. 1117c478bd9Sstevel@tonic-gate * 1127c478bd9Sstevel@tonic-gate * pages_locked : How many pages are locked becuase of user specified 1137c478bd9Sstevel@tonic-gate * locking through mlock or plock. 1147c478bd9Sstevel@tonic-gate * 1157c478bd9Sstevel@tonic-gate * pages_useclaim,pages_claimed : These two variables track the 1167c478bd9Sstevel@tonic-gate * cliam adjustments because of the protection changes on a segvn segment. 1177c478bd9Sstevel@tonic-gate * 1187c478bd9Sstevel@tonic-gate * All these globals are protected by the same lock which protects availrmem. 1197c478bd9Sstevel@tonic-gate */ 1207c478bd9Sstevel@tonic-gate pgcnt_t segvn_pages_locked; 1217c478bd9Sstevel@tonic-gate pgcnt_t pages_locked; 1227c478bd9Sstevel@tonic-gate pgcnt_t pages_useclaim; 1237c478bd9Sstevel@tonic-gate pgcnt_t pages_claimed; 1247c478bd9Sstevel@tonic-gate 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate /* 1277c478bd9Sstevel@tonic-gate * new_freemem_lock protects freemem, freemem_wait & freemem_cv. 1287c478bd9Sstevel@tonic-gate */ 1297c478bd9Sstevel@tonic-gate static kmutex_t new_freemem_lock; 1307c478bd9Sstevel@tonic-gate static uint_t freemem_wait; /* someone waiting for freemem */ 1317c478bd9Sstevel@tonic-gate static kcondvar_t freemem_cv; 1327c478bd9Sstevel@tonic-gate 1337c478bd9Sstevel@tonic-gate /* 1347c478bd9Sstevel@tonic-gate * The logical page free list is maintained as two lists, the 'free' 1357c478bd9Sstevel@tonic-gate * and the 'cache' lists. 1367c478bd9Sstevel@tonic-gate * The free list contains those pages that should be reused first. 1377c478bd9Sstevel@tonic-gate * 1387c478bd9Sstevel@tonic-gate * The implementation of the lists is machine dependent. 1397c478bd9Sstevel@tonic-gate * page_get_freelist(), page_get_cachelist(), 1407c478bd9Sstevel@tonic-gate * page_list_sub(), and page_list_add() 1417c478bd9Sstevel@tonic-gate * form the interface to the machine dependent implementation. 1427c478bd9Sstevel@tonic-gate * 1437c478bd9Sstevel@tonic-gate * Pages with p_free set are on the cache list. 1447c478bd9Sstevel@tonic-gate * Pages with p_free and p_age set are on the free list, 1457c478bd9Sstevel@tonic-gate * 1467c478bd9Sstevel@tonic-gate * A page may be locked while on either list. 1477c478bd9Sstevel@tonic-gate */ 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate /* 1507c478bd9Sstevel@tonic-gate * free list accounting stuff. 1517c478bd9Sstevel@tonic-gate * 1527c478bd9Sstevel@tonic-gate * 1537c478bd9Sstevel@tonic-gate * Spread out the value for the number of pages on the 1547c478bd9Sstevel@tonic-gate * page free and page cache lists. If there is just one 1557c478bd9Sstevel@tonic-gate * value, then it must be under just one lock. 1567c478bd9Sstevel@tonic-gate * The lock contention and cache traffic are a real bother. 1577c478bd9Sstevel@tonic-gate * 1587c478bd9Sstevel@tonic-gate * When we acquire and then drop a single pcf lock 1597c478bd9Sstevel@tonic-gate * we can start in the middle of the array of pcf structures. 1607c478bd9Sstevel@tonic-gate * If we acquire more than one pcf lock at a time, we need to 1617c478bd9Sstevel@tonic-gate * start at the front to avoid deadlocking. 1627c478bd9Sstevel@tonic-gate * 1637c478bd9Sstevel@tonic-gate * pcf_count holds the number of pages in each pool. 1647c478bd9Sstevel@tonic-gate * 1657c478bd9Sstevel@tonic-gate * pcf_block is set when page_create_get_something() has asked the 1667c478bd9Sstevel@tonic-gate * PSM page freelist and page cachelist routines without specifying 1677c478bd9Sstevel@tonic-gate * a color and nothing came back. This is used to block anything 1687c478bd9Sstevel@tonic-gate * else from moving pages from one list to the other while the 1697c478bd9Sstevel@tonic-gate * lists are searched again. If a page is freeed while pcf_block is 1707c478bd9Sstevel@tonic-gate * set, then pcf_reserve is incremented. pcgs_unblock() takes care 1717c478bd9Sstevel@tonic-gate * of clearning pcf_block, doing the wakeups, etc. 1727c478bd9Sstevel@tonic-gate */ 1737c478bd9Sstevel@tonic-gate 1747c478bd9Sstevel@tonic-gate #if NCPU <= 4 175f2b37d75Sfr #define PAD 2 1767c478bd9Sstevel@tonic-gate #define PCF_FANOUT 4 1777c478bd9Sstevel@tonic-gate static uint_t pcf_mask = PCF_FANOUT - 1; 1787c478bd9Sstevel@tonic-gate #else 179f2b37d75Sfr #define PAD 10 1807c478bd9Sstevel@tonic-gate #ifdef sun4v 1817c478bd9Sstevel@tonic-gate #define PCF_FANOUT 32 1827c478bd9Sstevel@tonic-gate #else 1837c478bd9Sstevel@tonic-gate #define PCF_FANOUT 128 1847c478bd9Sstevel@tonic-gate #endif 1857c478bd9Sstevel@tonic-gate static uint_t pcf_mask = PCF_FANOUT - 1; 1867c478bd9Sstevel@tonic-gate #endif 1877c478bd9Sstevel@tonic-gate 1887c478bd9Sstevel@tonic-gate struct pcf { 1897c478bd9Sstevel@tonic-gate kmutex_t pcf_lock; /* protects the structure */ 190f2b37d75Sfr uint_t pcf_count; /* page count */ 1917c478bd9Sstevel@tonic-gate uint_t pcf_wait; /* number of waiters */ 1927c478bd9Sstevel@tonic-gate uint_t pcf_block; /* pcgs flag to page_free() */ 1937c478bd9Sstevel@tonic-gate uint_t pcf_reserve; /* pages freed after pcf_block set */ 1947c478bd9Sstevel@tonic-gate uint_t pcf_fill[PAD]; /* to line up on the caches */ 1957c478bd9Sstevel@tonic-gate }; 1967c478bd9Sstevel@tonic-gate 1977c478bd9Sstevel@tonic-gate static struct pcf pcf[PCF_FANOUT]; 1987c478bd9Sstevel@tonic-gate #define PCF_INDEX() ((CPU->cpu_id) & (pcf_mask)) 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate kmutex_t pcgs_lock; /* serializes page_create_get_ */ 2017c478bd9Sstevel@tonic-gate kmutex_t pcgs_cagelock; /* serializes NOSLEEP cage allocs */ 2027c478bd9Sstevel@tonic-gate kmutex_t pcgs_wait_lock; /* used for delay in pcgs */ 2037c478bd9Sstevel@tonic-gate static kcondvar_t pcgs_cv; /* cv for delay in pcgs */ 2047c478bd9Sstevel@tonic-gate 2057c478bd9Sstevel@tonic-gate #ifdef VM_STATS 2067c478bd9Sstevel@tonic-gate 2077c478bd9Sstevel@tonic-gate /* 2087c478bd9Sstevel@tonic-gate * No locks, but so what, they are only statistics. 2097c478bd9Sstevel@tonic-gate */ 2107c478bd9Sstevel@tonic-gate 2117c478bd9Sstevel@tonic-gate static struct page_tcnt { 2127c478bd9Sstevel@tonic-gate int pc_free_cache; /* free's into cache list */ 2137c478bd9Sstevel@tonic-gate int pc_free_dontneed; /* free's with dontneed */ 2147c478bd9Sstevel@tonic-gate int pc_free_pageout; /* free's from pageout */ 2157c478bd9Sstevel@tonic-gate int pc_free_free; /* free's into free list */ 2167c478bd9Sstevel@tonic-gate int pc_free_pages; /* free's into large page free list */ 2177c478bd9Sstevel@tonic-gate int pc_destroy_pages; /* large page destroy's */ 2187c478bd9Sstevel@tonic-gate int pc_get_cache; /* get's from cache list */ 2197c478bd9Sstevel@tonic-gate int pc_get_free; /* get's from free list */ 2207c478bd9Sstevel@tonic-gate int pc_reclaim; /* reclaim's */ 2217c478bd9Sstevel@tonic-gate int pc_abortfree; /* abort's of free pages */ 2227c478bd9Sstevel@tonic-gate int pc_find_hit; /* find's that find page */ 2237c478bd9Sstevel@tonic-gate int pc_find_miss; /* find's that don't find page */ 2247c478bd9Sstevel@tonic-gate int pc_destroy_free; /* # of free pages destroyed */ 2257c478bd9Sstevel@tonic-gate #define PC_HASH_CNT (4*PAGE_HASHAVELEN) 2267c478bd9Sstevel@tonic-gate int pc_find_hashlen[PC_HASH_CNT+1]; 2277c478bd9Sstevel@tonic-gate int pc_addclaim_pages; 2287c478bd9Sstevel@tonic-gate int pc_subclaim_pages; 2297c478bd9Sstevel@tonic-gate int pc_free_replacement_page[2]; 2307c478bd9Sstevel@tonic-gate int pc_try_demote_pages[6]; 2317c478bd9Sstevel@tonic-gate int pc_demote_pages[2]; 2327c478bd9Sstevel@tonic-gate } pagecnt; 2337c478bd9Sstevel@tonic-gate 2347c478bd9Sstevel@tonic-gate uint_t hashin_count; 2357c478bd9Sstevel@tonic-gate uint_t hashin_not_held; 2367c478bd9Sstevel@tonic-gate uint_t hashin_already; 2377c478bd9Sstevel@tonic-gate 2387c478bd9Sstevel@tonic-gate uint_t hashout_count; 2397c478bd9Sstevel@tonic-gate uint_t hashout_not_held; 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate uint_t page_create_count; 2427c478bd9Sstevel@tonic-gate uint_t page_create_not_enough; 2437c478bd9Sstevel@tonic-gate uint_t page_create_not_enough_again; 2447c478bd9Sstevel@tonic-gate uint_t page_create_zero; 2457c478bd9Sstevel@tonic-gate uint_t page_create_hashout; 2467c478bd9Sstevel@tonic-gate uint_t page_create_page_lock_failed; 2477c478bd9Sstevel@tonic-gate uint_t page_create_trylock_failed; 2487c478bd9Sstevel@tonic-gate uint_t page_create_found_one; 2497c478bd9Sstevel@tonic-gate uint_t page_create_hashin_failed; 2507c478bd9Sstevel@tonic-gate uint_t page_create_dropped_phm; 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate uint_t page_create_new; 2537c478bd9Sstevel@tonic-gate uint_t page_create_exists; 2547c478bd9Sstevel@tonic-gate uint_t page_create_putbacks; 2557c478bd9Sstevel@tonic-gate uint_t page_create_overshoot; 2567c478bd9Sstevel@tonic-gate 2577c478bd9Sstevel@tonic-gate uint_t page_reclaim_zero; 2587c478bd9Sstevel@tonic-gate uint_t page_reclaim_zero_locked; 2597c478bd9Sstevel@tonic-gate 2607c478bd9Sstevel@tonic-gate uint_t page_rename_exists; 2617c478bd9Sstevel@tonic-gate uint_t page_rename_count; 2627c478bd9Sstevel@tonic-gate 2637c478bd9Sstevel@tonic-gate uint_t page_lookup_cnt[20]; 2647c478bd9Sstevel@tonic-gate uint_t page_lookup_nowait_cnt[10]; 2657c478bd9Sstevel@tonic-gate uint_t page_find_cnt; 2667c478bd9Sstevel@tonic-gate uint_t page_exists_cnt; 2677c478bd9Sstevel@tonic-gate uint_t page_exists_forreal_cnt; 2687c478bd9Sstevel@tonic-gate uint_t page_lookup_dev_cnt; 2697c478bd9Sstevel@tonic-gate uint_t get_cachelist_cnt; 2707c478bd9Sstevel@tonic-gate uint_t page_create_cnt[10]; 2717c478bd9Sstevel@tonic-gate uint_t alloc_pages[8]; 2727c478bd9Sstevel@tonic-gate uint_t page_exphcontg[19]; 2737c478bd9Sstevel@tonic-gate uint_t page_create_large_cnt[10]; 2747c478bd9Sstevel@tonic-gate 2757c478bd9Sstevel@tonic-gate /* 2767c478bd9Sstevel@tonic-gate * Collects statistics. 2777c478bd9Sstevel@tonic-gate */ 2787c478bd9Sstevel@tonic-gate #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 2797c478bd9Sstevel@tonic-gate uint_t mylen = 0; \ 2807c478bd9Sstevel@tonic-gate \ 2817c478bd9Sstevel@tonic-gate for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \ 2827c478bd9Sstevel@tonic-gate if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 2837c478bd9Sstevel@tonic-gate break; \ 2847c478bd9Sstevel@tonic-gate } \ 2857c478bd9Sstevel@tonic-gate if ((pp) != NULL) \ 2867c478bd9Sstevel@tonic-gate pagecnt.pc_find_hit++; \ 2877c478bd9Sstevel@tonic-gate else \ 2887c478bd9Sstevel@tonic-gate pagecnt.pc_find_miss++; \ 2897c478bd9Sstevel@tonic-gate if (mylen > PC_HASH_CNT) \ 2907c478bd9Sstevel@tonic-gate mylen = PC_HASH_CNT; \ 2917c478bd9Sstevel@tonic-gate pagecnt.pc_find_hashlen[mylen]++; \ 2927c478bd9Sstevel@tonic-gate } 2937c478bd9Sstevel@tonic-gate 2947c478bd9Sstevel@tonic-gate #else /* VM_STATS */ 2957c478bd9Sstevel@tonic-gate 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * Don't collect statistics 2987c478bd9Sstevel@tonic-gate */ 2997c478bd9Sstevel@tonic-gate #define PAGE_HASH_SEARCH(index, pp, vp, off) { \ 3007c478bd9Sstevel@tonic-gate for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \ 3017c478bd9Sstevel@tonic-gate if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \ 3027c478bd9Sstevel@tonic-gate break; \ 3037c478bd9Sstevel@tonic-gate } \ 3047c478bd9Sstevel@tonic-gate } 3057c478bd9Sstevel@tonic-gate 3067c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 3077c478bd9Sstevel@tonic-gate 3087c478bd9Sstevel@tonic-gate 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate #ifdef DEBUG 3117c478bd9Sstevel@tonic-gate #define MEMSEG_SEARCH_STATS 3127c478bd9Sstevel@tonic-gate #endif 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate #ifdef MEMSEG_SEARCH_STATS 3157c478bd9Sstevel@tonic-gate struct memseg_stats { 3167c478bd9Sstevel@tonic-gate uint_t nsearch; 3177c478bd9Sstevel@tonic-gate uint_t nlastwon; 3187c478bd9Sstevel@tonic-gate uint_t nhashwon; 3197c478bd9Sstevel@tonic-gate uint_t nnotfound; 3207c478bd9Sstevel@tonic-gate } memseg_stats; 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate #define MEMSEG_STAT_INCR(v) \ 3237c478bd9Sstevel@tonic-gate atomic_add_32(&memseg_stats.v, 1) 3247c478bd9Sstevel@tonic-gate #else 3257c478bd9Sstevel@tonic-gate #define MEMSEG_STAT_INCR(x) 3267c478bd9Sstevel@tonic-gate #endif 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate struct memseg *memsegs; /* list of memory segments */ 3297c478bd9Sstevel@tonic-gate 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate static void page_init_mem_config(void); 3327c478bd9Sstevel@tonic-gate static int page_do_hashin(page_t *, vnode_t *, u_offset_t); 3337c478bd9Sstevel@tonic-gate static void page_do_hashout(page_t *); 3348b464eb8Smec static void page_capture_init(); 3358b464eb8Smec int page_capture_take_action(page_t *, uint_t, void *); 3367c478bd9Sstevel@tonic-gate 3377c478bd9Sstevel@tonic-gate static void page_demote_vp_pages(page_t *); 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate /* 3407c478bd9Sstevel@tonic-gate * vm subsystem related initialization 3417c478bd9Sstevel@tonic-gate */ 3427c478bd9Sstevel@tonic-gate void 3437c478bd9Sstevel@tonic-gate vm_init(void) 3447c478bd9Sstevel@tonic-gate { 3457c478bd9Sstevel@tonic-gate boolean_t callb_vm_cpr(void *, int); 3467c478bd9Sstevel@tonic-gate 3477c478bd9Sstevel@tonic-gate (void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm"); 3487c478bd9Sstevel@tonic-gate page_init_mem_config(); 349db874c57Selowe page_retire_init(); 3500209230bSgjelinek vm_usage_init(); 3518b464eb8Smec page_capture_init(); 3527c478bd9Sstevel@tonic-gate } 3537c478bd9Sstevel@tonic-gate 3547c478bd9Sstevel@tonic-gate /* 3557c478bd9Sstevel@tonic-gate * This function is called at startup and when memory is added or deleted. 3567c478bd9Sstevel@tonic-gate */ 3577c478bd9Sstevel@tonic-gate void 3587c478bd9Sstevel@tonic-gate init_pages_pp_maximum() 3597c478bd9Sstevel@tonic-gate { 3607c478bd9Sstevel@tonic-gate static pgcnt_t p_min; 3617c478bd9Sstevel@tonic-gate static pgcnt_t pages_pp_maximum_startup; 3627c478bd9Sstevel@tonic-gate static pgcnt_t avrmem_delta; 3637c478bd9Sstevel@tonic-gate static int init_done; 3647c478bd9Sstevel@tonic-gate static int user_set; /* true if set in /etc/system */ 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate if (init_done == 0) { 3677c478bd9Sstevel@tonic-gate 3687c478bd9Sstevel@tonic-gate /* If the user specified a value, save it */ 3697c478bd9Sstevel@tonic-gate if (pages_pp_maximum != 0) { 3707c478bd9Sstevel@tonic-gate user_set = 1; 3717c478bd9Sstevel@tonic-gate pages_pp_maximum_startup = pages_pp_maximum; 3727c478bd9Sstevel@tonic-gate } 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate /* 3757c478bd9Sstevel@tonic-gate * Setting of pages_pp_maximum is based first time 3767c478bd9Sstevel@tonic-gate * on the value of availrmem just after the start-up 3777c478bd9Sstevel@tonic-gate * allocations. To preserve this relationship at run 3787c478bd9Sstevel@tonic-gate * time, use a delta from availrmem_initial. 3797c478bd9Sstevel@tonic-gate */ 3807c478bd9Sstevel@tonic-gate ASSERT(availrmem_initial >= availrmem); 3817c478bd9Sstevel@tonic-gate avrmem_delta = availrmem_initial - availrmem; 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate /* The allowable floor of pages_pp_maximum */ 3847c478bd9Sstevel@tonic-gate p_min = tune.t_minarmem + 100; 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate /* Make sure we don't come through here again. */ 3877c478bd9Sstevel@tonic-gate init_done = 1; 3887c478bd9Sstevel@tonic-gate } 3897c478bd9Sstevel@tonic-gate /* 3907c478bd9Sstevel@tonic-gate * Determine pages_pp_maximum, the number of currently available 3917c478bd9Sstevel@tonic-gate * pages (availrmem) that can't be `locked'. If not set by 3927c478bd9Sstevel@tonic-gate * the user, we set it to 4% of the currently available memory 3937c478bd9Sstevel@tonic-gate * plus 4MB. 3947c478bd9Sstevel@tonic-gate * But we also insist that it be greater than tune.t_minarmem; 3957c478bd9Sstevel@tonic-gate * otherwise a process could lock down a lot of memory, get swapped 3967c478bd9Sstevel@tonic-gate * out, and never have enough to get swapped back in. 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate if (user_set) 3997c478bd9Sstevel@tonic-gate pages_pp_maximum = pages_pp_maximum_startup; 4007c478bd9Sstevel@tonic-gate else 4017c478bd9Sstevel@tonic-gate pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25) 4027c478bd9Sstevel@tonic-gate + btop(4 * 1024 * 1024); 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate if (pages_pp_maximum <= p_min) { 4057c478bd9Sstevel@tonic-gate pages_pp_maximum = p_min; 4067c478bd9Sstevel@tonic-gate } 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate void 4107c478bd9Sstevel@tonic-gate set_max_page_get(pgcnt_t target_total_pages) 4117c478bd9Sstevel@tonic-gate { 4127c478bd9Sstevel@tonic-gate max_page_get = target_total_pages / 2; 4137c478bd9Sstevel@tonic-gate } 4147c478bd9Sstevel@tonic-gate 4157c478bd9Sstevel@tonic-gate static pgcnt_t pending_delete; 4167c478bd9Sstevel@tonic-gate 4177c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4187c478bd9Sstevel@tonic-gate static void 4197c478bd9Sstevel@tonic-gate page_mem_config_post_add( 4207c478bd9Sstevel@tonic-gate void *arg, 4217c478bd9Sstevel@tonic-gate pgcnt_t delta_pages) 4227c478bd9Sstevel@tonic-gate { 4237c478bd9Sstevel@tonic-gate set_max_page_get(total_pages - pending_delete); 4247c478bd9Sstevel@tonic-gate init_pages_pp_maximum(); 4257c478bd9Sstevel@tonic-gate } 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4287c478bd9Sstevel@tonic-gate static int 4297c478bd9Sstevel@tonic-gate page_mem_config_pre_del( 4307c478bd9Sstevel@tonic-gate void *arg, 4317c478bd9Sstevel@tonic-gate pgcnt_t delta_pages) 4327c478bd9Sstevel@tonic-gate { 4337c478bd9Sstevel@tonic-gate pgcnt_t nv; 4347c478bd9Sstevel@tonic-gate 4357c478bd9Sstevel@tonic-gate nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages); 4367c478bd9Sstevel@tonic-gate set_max_page_get(total_pages - nv); 4377c478bd9Sstevel@tonic-gate return (0); 4387c478bd9Sstevel@tonic-gate } 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4417c478bd9Sstevel@tonic-gate static void 4427c478bd9Sstevel@tonic-gate page_mem_config_post_del( 4437c478bd9Sstevel@tonic-gate void *arg, 4447c478bd9Sstevel@tonic-gate pgcnt_t delta_pages, 4457c478bd9Sstevel@tonic-gate int cancelled) 4467c478bd9Sstevel@tonic-gate { 4477c478bd9Sstevel@tonic-gate pgcnt_t nv; 4487c478bd9Sstevel@tonic-gate 4497c478bd9Sstevel@tonic-gate nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages); 4507c478bd9Sstevel@tonic-gate set_max_page_get(total_pages - nv); 4517c478bd9Sstevel@tonic-gate if (!cancelled) 4527c478bd9Sstevel@tonic-gate init_pages_pp_maximum(); 4537c478bd9Sstevel@tonic-gate } 4547c478bd9Sstevel@tonic-gate 4557c478bd9Sstevel@tonic-gate static kphysm_setup_vector_t page_mem_config_vec = { 4567c478bd9Sstevel@tonic-gate KPHYSM_SETUP_VECTOR_VERSION, 4577c478bd9Sstevel@tonic-gate page_mem_config_post_add, 4587c478bd9Sstevel@tonic-gate page_mem_config_pre_del, 4597c478bd9Sstevel@tonic-gate page_mem_config_post_del, 4607c478bd9Sstevel@tonic-gate }; 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate static void 4637c478bd9Sstevel@tonic-gate page_init_mem_config(void) 4647c478bd9Sstevel@tonic-gate { 4657c478bd9Sstevel@tonic-gate int ret; 4667c478bd9Sstevel@tonic-gate 4677c478bd9Sstevel@tonic-gate ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL); 4687c478bd9Sstevel@tonic-gate ASSERT(ret == 0); 4697c478bd9Sstevel@tonic-gate } 4707c478bd9Sstevel@tonic-gate 4717c478bd9Sstevel@tonic-gate /* 4727c478bd9Sstevel@tonic-gate * Evenly spread out the PCF counters for large free pages 4737c478bd9Sstevel@tonic-gate */ 4747c478bd9Sstevel@tonic-gate static void 4757c478bd9Sstevel@tonic-gate page_free_large_ctr(pgcnt_t npages) 4767c478bd9Sstevel@tonic-gate { 4777c478bd9Sstevel@tonic-gate static struct pcf *p = pcf; 4787c478bd9Sstevel@tonic-gate pgcnt_t lump; 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate freemem += npages; 4817c478bd9Sstevel@tonic-gate 4827c478bd9Sstevel@tonic-gate lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 4837c478bd9Sstevel@tonic-gate 4847c478bd9Sstevel@tonic-gate while (npages > 0) { 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate ASSERT(!p->pcf_block); 4877c478bd9Sstevel@tonic-gate 4887c478bd9Sstevel@tonic-gate if (lump < npages) { 4897c478bd9Sstevel@tonic-gate p->pcf_count += (uint_t)lump; 4907c478bd9Sstevel@tonic-gate npages -= lump; 4917c478bd9Sstevel@tonic-gate } else { 4927c478bd9Sstevel@tonic-gate p->pcf_count += (uint_t)npages; 4937c478bd9Sstevel@tonic-gate npages = 0; 4947c478bd9Sstevel@tonic-gate } 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate ASSERT(!p->pcf_wait); 4977c478bd9Sstevel@tonic-gate 4987c478bd9Sstevel@tonic-gate if (++p > &pcf[PCF_FANOUT - 1]) 4997c478bd9Sstevel@tonic-gate p = pcf; 5007c478bd9Sstevel@tonic-gate } 5017c478bd9Sstevel@tonic-gate 5027c478bd9Sstevel@tonic-gate ASSERT(npages == 0); 5037c478bd9Sstevel@tonic-gate } 5047c478bd9Sstevel@tonic-gate 5057c478bd9Sstevel@tonic-gate /* 5067c478bd9Sstevel@tonic-gate * Add a physical chunk of memory to the system freee lists during startup. 5077c478bd9Sstevel@tonic-gate * Platform specific startup() allocates the memory for the page structs. 5087c478bd9Sstevel@tonic-gate * 5097c478bd9Sstevel@tonic-gate * num - number of page structures 5107c478bd9Sstevel@tonic-gate * base - page number (pfn) to be associated with the first page. 5117c478bd9Sstevel@tonic-gate * 5127c478bd9Sstevel@tonic-gate * Since we are doing this during startup (ie. single threaded), we will 5137c478bd9Sstevel@tonic-gate * use shortcut routines to avoid any locking overhead while putting all 5147c478bd9Sstevel@tonic-gate * these pages on the freelists. 5157c478bd9Sstevel@tonic-gate * 5167c478bd9Sstevel@tonic-gate * NOTE: Any changes performed to page_free(), must also be performed to 5177c478bd9Sstevel@tonic-gate * add_physmem() since this is how we initialize all page_t's at 5187c478bd9Sstevel@tonic-gate * boot time. 5197c478bd9Sstevel@tonic-gate */ 5207c478bd9Sstevel@tonic-gate void 5217c478bd9Sstevel@tonic-gate add_physmem( 5227c478bd9Sstevel@tonic-gate page_t *pp, 5237c478bd9Sstevel@tonic-gate pgcnt_t num, 5247c478bd9Sstevel@tonic-gate pfn_t pnum) 5257c478bd9Sstevel@tonic-gate { 5267c478bd9Sstevel@tonic-gate page_t *root = NULL; 5277c478bd9Sstevel@tonic-gate uint_t szc = page_num_pagesizes() - 1; 5287c478bd9Sstevel@tonic-gate pgcnt_t large = page_get_pagecnt(szc); 5297c478bd9Sstevel@tonic-gate pgcnt_t cnt = 0; 5307c478bd9Sstevel@tonic-gate 5317c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_INIT, 5327c478bd9Sstevel@tonic-gate "add_physmem:pp %p num %lu", pp, num); 5337c478bd9Sstevel@tonic-gate 5347c478bd9Sstevel@tonic-gate /* 5357c478bd9Sstevel@tonic-gate * Arbitrarily limit the max page_get request 5367c478bd9Sstevel@tonic-gate * to 1/2 of the page structs we have. 5377c478bd9Sstevel@tonic-gate */ 5387c478bd9Sstevel@tonic-gate total_pages += num; 5397c478bd9Sstevel@tonic-gate set_max_page_get(total_pages); 5407c478bd9Sstevel@tonic-gate 541e21bae1bSkchow PLCNT_MODIFY_MAX(pnum, (long)num); 542e21bae1bSkchow 5437c478bd9Sstevel@tonic-gate /* 5447c478bd9Sstevel@tonic-gate * The physical space for the pages array 5457c478bd9Sstevel@tonic-gate * representing ram pages has already been 5467c478bd9Sstevel@tonic-gate * allocated. Here we initialize each lock 5477c478bd9Sstevel@tonic-gate * in the page structure, and put each on 5487c478bd9Sstevel@tonic-gate * the free list 5497c478bd9Sstevel@tonic-gate */ 550affbd3ccSkchow for (; num; pp++, pnum++, num--) { 5517c478bd9Sstevel@tonic-gate 5527c478bd9Sstevel@tonic-gate /* 5537c478bd9Sstevel@tonic-gate * this needs to fill in the page number 5547c478bd9Sstevel@tonic-gate * and do any other arch specific initialization 5557c478bd9Sstevel@tonic-gate */ 5567c478bd9Sstevel@tonic-gate add_physmem_cb(pp, pnum); 5577c478bd9Sstevel@tonic-gate 55807b65a64Saguzovsk pp->p_lckcnt = 0; 55907b65a64Saguzovsk pp->p_cowcnt = 0; 56007b65a64Saguzovsk pp->p_slckcnt = 0; 56107b65a64Saguzovsk 5627c478bd9Sstevel@tonic-gate /* 5637c478bd9Sstevel@tonic-gate * Initialize the page lock as unlocked, since nobody 5647c478bd9Sstevel@tonic-gate * can see or access this page yet. 5657c478bd9Sstevel@tonic-gate */ 5667c478bd9Sstevel@tonic-gate pp->p_selock = 0; 5677c478bd9Sstevel@tonic-gate 5687c478bd9Sstevel@tonic-gate /* 5697c478bd9Sstevel@tonic-gate * Initialize IO lock 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate page_iolock_init(pp); 5727c478bd9Sstevel@tonic-gate 5737c478bd9Sstevel@tonic-gate /* 5747c478bd9Sstevel@tonic-gate * initialize other fields in the page_t 5757c478bd9Sstevel@tonic-gate */ 5767c478bd9Sstevel@tonic-gate PP_SETFREE(pp); 5777c478bd9Sstevel@tonic-gate page_clr_all_props(pp); 5787c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 5797c478bd9Sstevel@tonic-gate pp->p_offset = (u_offset_t)-1; 5807c478bd9Sstevel@tonic-gate pp->p_next = pp; 5817c478bd9Sstevel@tonic-gate pp->p_prev = pp; 5827c478bd9Sstevel@tonic-gate 5837c478bd9Sstevel@tonic-gate /* 5847c478bd9Sstevel@tonic-gate * Simple case: System doesn't support large pages. 5857c478bd9Sstevel@tonic-gate */ 5867c478bd9Sstevel@tonic-gate if (szc == 0) { 5877c478bd9Sstevel@tonic-gate pp->p_szc = 0; 5887c478bd9Sstevel@tonic-gate page_free_at_startup(pp); 5897c478bd9Sstevel@tonic-gate continue; 5907c478bd9Sstevel@tonic-gate } 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate /* 5937c478bd9Sstevel@tonic-gate * Handle unaligned pages, we collect them up onto 5947c478bd9Sstevel@tonic-gate * the root page until we have a full large page. 5957c478bd9Sstevel@tonic-gate */ 5967c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(pnum, large)) { 5977c478bd9Sstevel@tonic-gate 5987c478bd9Sstevel@tonic-gate /* 5997c478bd9Sstevel@tonic-gate * If not in a large page, 6007c478bd9Sstevel@tonic-gate * just free as small page. 6017c478bd9Sstevel@tonic-gate */ 6027c478bd9Sstevel@tonic-gate if (root == NULL) { 6037c478bd9Sstevel@tonic-gate pp->p_szc = 0; 6047c478bd9Sstevel@tonic-gate page_free_at_startup(pp); 6057c478bd9Sstevel@tonic-gate continue; 6067c478bd9Sstevel@tonic-gate } 6077c478bd9Sstevel@tonic-gate 6087c478bd9Sstevel@tonic-gate /* 6097c478bd9Sstevel@tonic-gate * Link a constituent page into the large page. 6107c478bd9Sstevel@tonic-gate */ 6117c478bd9Sstevel@tonic-gate pp->p_szc = szc; 6127c478bd9Sstevel@tonic-gate page_list_concat(&root, &pp); 6137c478bd9Sstevel@tonic-gate 6147c478bd9Sstevel@tonic-gate /* 6157c478bd9Sstevel@tonic-gate * When large page is fully formed, free it. 6167c478bd9Sstevel@tonic-gate */ 6177c478bd9Sstevel@tonic-gate if (++cnt == large) { 6187c478bd9Sstevel@tonic-gate page_free_large_ctr(cnt); 6197c478bd9Sstevel@tonic-gate page_list_add_pages(root, PG_LIST_ISINIT); 6207c478bd9Sstevel@tonic-gate root = NULL; 6217c478bd9Sstevel@tonic-gate cnt = 0; 6227c478bd9Sstevel@tonic-gate } 6237c478bd9Sstevel@tonic-gate continue; 6247c478bd9Sstevel@tonic-gate } 6257c478bd9Sstevel@tonic-gate 6267c478bd9Sstevel@tonic-gate /* 6277c478bd9Sstevel@tonic-gate * At this point we have a page number which 6287c478bd9Sstevel@tonic-gate * is aligned. We assert that we aren't already 6297c478bd9Sstevel@tonic-gate * in a different large page. 6307c478bd9Sstevel@tonic-gate */ 6317c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pnum, large)); 6327c478bd9Sstevel@tonic-gate ASSERT(root == NULL && cnt == 0); 6337c478bd9Sstevel@tonic-gate 6347c478bd9Sstevel@tonic-gate /* 6357c478bd9Sstevel@tonic-gate * If insufficient number of pages left to form 6367c478bd9Sstevel@tonic-gate * a large page, just free the small page. 6377c478bd9Sstevel@tonic-gate */ 6387c478bd9Sstevel@tonic-gate if (num < large) { 6397c478bd9Sstevel@tonic-gate pp->p_szc = 0; 6407c478bd9Sstevel@tonic-gate page_free_at_startup(pp); 6417c478bd9Sstevel@tonic-gate continue; 6427c478bd9Sstevel@tonic-gate } 6437c478bd9Sstevel@tonic-gate 6447c478bd9Sstevel@tonic-gate /* 6457c478bd9Sstevel@tonic-gate * Otherwise start a new large page. 6467c478bd9Sstevel@tonic-gate */ 6477c478bd9Sstevel@tonic-gate pp->p_szc = szc; 6487c478bd9Sstevel@tonic-gate cnt++; 6497c478bd9Sstevel@tonic-gate root = pp; 6507c478bd9Sstevel@tonic-gate } 6517c478bd9Sstevel@tonic-gate ASSERT(root == NULL && cnt == 0); 6527c478bd9Sstevel@tonic-gate } 6537c478bd9Sstevel@tonic-gate 6547c478bd9Sstevel@tonic-gate /* 6557c478bd9Sstevel@tonic-gate * Find a page representing the specified [vp, offset]. 6567c478bd9Sstevel@tonic-gate * If we find the page but it is intransit coming in, 6577c478bd9Sstevel@tonic-gate * it will have an "exclusive" lock and we wait for 6587c478bd9Sstevel@tonic-gate * the i/o to complete. A page found on the free list 6597c478bd9Sstevel@tonic-gate * is always reclaimed and then locked. On success, the page 6607c478bd9Sstevel@tonic-gate * is locked, its data is valid and it isn't on the free 6617c478bd9Sstevel@tonic-gate * list, while a NULL is returned if the page doesn't exist. 6627c478bd9Sstevel@tonic-gate */ 6637c478bd9Sstevel@tonic-gate page_t * 6647c478bd9Sstevel@tonic-gate page_lookup(vnode_t *vp, u_offset_t off, se_t se) 6657c478bd9Sstevel@tonic-gate { 6667c478bd9Sstevel@tonic-gate return (page_lookup_create(vp, off, se, NULL, NULL, 0)); 6677c478bd9Sstevel@tonic-gate } 6687c478bd9Sstevel@tonic-gate 6697c478bd9Sstevel@tonic-gate /* 6707c478bd9Sstevel@tonic-gate * Find a page representing the specified [vp, offset]. 6717c478bd9Sstevel@tonic-gate * We either return the one we found or, if passed in, 6727c478bd9Sstevel@tonic-gate * create one with identity of [vp, offset] of the 6737c478bd9Sstevel@tonic-gate * pre-allocated page. If we find exsisting page but it is 6747c478bd9Sstevel@tonic-gate * intransit coming in, it will have an "exclusive" lock 6757c478bd9Sstevel@tonic-gate * and we wait for the i/o to complete. A page found on 6767c478bd9Sstevel@tonic-gate * the free list is always reclaimed and then locked. 6777c478bd9Sstevel@tonic-gate * On success, the page is locked, its data is valid and 6787c478bd9Sstevel@tonic-gate * it isn't on the free list, while a NULL is returned 6797c478bd9Sstevel@tonic-gate * if the page doesn't exist and newpp is NULL; 6807c478bd9Sstevel@tonic-gate */ 6817c478bd9Sstevel@tonic-gate page_t * 6827c478bd9Sstevel@tonic-gate page_lookup_create( 6837c478bd9Sstevel@tonic-gate vnode_t *vp, 6847c478bd9Sstevel@tonic-gate u_offset_t off, 6857c478bd9Sstevel@tonic-gate se_t se, 6867c478bd9Sstevel@tonic-gate page_t *newpp, 6877c478bd9Sstevel@tonic-gate spgcnt_t *nrelocp, 6887c478bd9Sstevel@tonic-gate int flags) 6897c478bd9Sstevel@tonic-gate { 6907c478bd9Sstevel@tonic-gate page_t *pp; 6917c478bd9Sstevel@tonic-gate kmutex_t *phm; 6927c478bd9Sstevel@tonic-gate ulong_t index; 6937c478bd9Sstevel@tonic-gate uint_t hash_locked; 6947c478bd9Sstevel@tonic-gate uint_t es; 6957c478bd9Sstevel@tonic-gate 6967c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 6977c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[0]); 6987c478bd9Sstevel@tonic-gate ASSERT(newpp ? PAGE_EXCL(newpp) : 1); 6997c478bd9Sstevel@tonic-gate 7007c478bd9Sstevel@tonic-gate /* 7017c478bd9Sstevel@tonic-gate * Acquire the appropriate page hash lock since 7027c478bd9Sstevel@tonic-gate * we have to search the hash list. Pages that 7037c478bd9Sstevel@tonic-gate * hash to this list can't change identity while 7047c478bd9Sstevel@tonic-gate * this lock is held. 7057c478bd9Sstevel@tonic-gate */ 7067c478bd9Sstevel@tonic-gate hash_locked = 0; 7077c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 7087c478bd9Sstevel@tonic-gate phm = NULL; 7097c478bd9Sstevel@tonic-gate top: 7107c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 7117c478bd9Sstevel@tonic-gate if (pp != NULL) { 7127c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[1]); 7137c478bd9Sstevel@tonic-gate es = (newpp != NULL) ? 1 : 0; 7147c478bd9Sstevel@tonic-gate es |= flags; 7157c478bd9Sstevel@tonic-gate if (!hash_locked) { 7167c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[2]); 7177c478bd9Sstevel@tonic-gate if (!page_try_reclaim_lock(pp, se, es)) { 7187c478bd9Sstevel@tonic-gate /* 7197c478bd9Sstevel@tonic-gate * On a miss, acquire the phm. Then 7207c478bd9Sstevel@tonic-gate * next time, page_lock() will be called, 7217c478bd9Sstevel@tonic-gate * causing a wait if the page is busy. 7227c478bd9Sstevel@tonic-gate * just looping with page_trylock() would 7237c478bd9Sstevel@tonic-gate * get pretty boring. 7247c478bd9Sstevel@tonic-gate */ 7257c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[3]); 7267c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 7277c478bd9Sstevel@tonic-gate mutex_enter(phm); 7287c478bd9Sstevel@tonic-gate hash_locked = 1; 7297c478bd9Sstevel@tonic-gate goto top; 7307c478bd9Sstevel@tonic-gate } 7317c478bd9Sstevel@tonic-gate } else { 7327c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[4]); 7337c478bd9Sstevel@tonic-gate if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) { 7347c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[5]); 7357c478bd9Sstevel@tonic-gate goto top; 7367c478bd9Sstevel@tonic-gate } 7377c478bd9Sstevel@tonic-gate } 7387c478bd9Sstevel@tonic-gate 7397c478bd9Sstevel@tonic-gate /* 7407c478bd9Sstevel@tonic-gate * Since `pp' is locked it can not change identity now. 7417c478bd9Sstevel@tonic-gate * Reconfirm we locked the correct page. 7427c478bd9Sstevel@tonic-gate * 7437c478bd9Sstevel@tonic-gate * Both the p_vnode and p_offset *must* be cast volatile 7447c478bd9Sstevel@tonic-gate * to force a reload of their values: The PAGE_HASH_SEARCH 7457c478bd9Sstevel@tonic-gate * macro will have stuffed p_vnode and p_offset into 7467c478bd9Sstevel@tonic-gate * registers before calling page_trylock(); another thread, 7477c478bd9Sstevel@tonic-gate * actually holding the hash lock, could have changed the 7487c478bd9Sstevel@tonic-gate * page's identity in memory, but our registers would not 7497c478bd9Sstevel@tonic-gate * be changed, fooling the reconfirmation. If the hash 7507c478bd9Sstevel@tonic-gate * lock was held during the search, the casting would 7517c478bd9Sstevel@tonic-gate * not be needed. 7527c478bd9Sstevel@tonic-gate */ 7537c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[6]); 7547c478bd9Sstevel@tonic-gate if (((volatile struct vnode *)(pp->p_vnode) != vp) || 7557c478bd9Sstevel@tonic-gate ((volatile u_offset_t)(pp->p_offset) != off)) { 7567c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[7]); 7577c478bd9Sstevel@tonic-gate if (hash_locked) { 7587c478bd9Sstevel@tonic-gate panic("page_lookup_create: lost page %p", 7597c478bd9Sstevel@tonic-gate (void *)pp); 7607c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 7617c478bd9Sstevel@tonic-gate } 7627c478bd9Sstevel@tonic-gate page_unlock(pp); 7637c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 7647c478bd9Sstevel@tonic-gate mutex_enter(phm); 7657c478bd9Sstevel@tonic-gate hash_locked = 1; 7667c478bd9Sstevel@tonic-gate goto top; 7677c478bd9Sstevel@tonic-gate } 7687c478bd9Sstevel@tonic-gate 7697c478bd9Sstevel@tonic-gate /* 7707c478bd9Sstevel@tonic-gate * If page_trylock() was called, then pp may still be on 7717c478bd9Sstevel@tonic-gate * the cachelist (can't be on the free list, it would not 7727c478bd9Sstevel@tonic-gate * have been found in the search). If it is on the 7737c478bd9Sstevel@tonic-gate * cachelist it must be pulled now. To pull the page from 7747c478bd9Sstevel@tonic-gate * the cachelist, it must be exclusively locked. 7757c478bd9Sstevel@tonic-gate * 7767c478bd9Sstevel@tonic-gate * The other big difference between page_trylock() and 7777c478bd9Sstevel@tonic-gate * page_lock(), is that page_lock() will pull the 7787c478bd9Sstevel@tonic-gate * page from whatever free list (the cache list in this 7797c478bd9Sstevel@tonic-gate * case) the page is on. If page_trylock() was used 7807c478bd9Sstevel@tonic-gate * above, then we have to do the reclaim ourselves. 7817c478bd9Sstevel@tonic-gate */ 7827c478bd9Sstevel@tonic-gate if ((!hash_locked) && (PP_ISFREE(pp))) { 7837c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0); 7847c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[8]); 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate /* 7877c478bd9Sstevel@tonic-gate * page_relcaim will insure that we 7887c478bd9Sstevel@tonic-gate * have this page exclusively 7897c478bd9Sstevel@tonic-gate */ 7907c478bd9Sstevel@tonic-gate 7917c478bd9Sstevel@tonic-gate if (!page_reclaim(pp, NULL)) { 7927c478bd9Sstevel@tonic-gate /* 7937c478bd9Sstevel@tonic-gate * Page_reclaim dropped whatever lock 7947c478bd9Sstevel@tonic-gate * we held. 7957c478bd9Sstevel@tonic-gate */ 7967c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[9]); 7977c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 7987c478bd9Sstevel@tonic-gate mutex_enter(phm); 7997c478bd9Sstevel@tonic-gate hash_locked = 1; 8007c478bd9Sstevel@tonic-gate goto top; 8017c478bd9Sstevel@tonic-gate } else if (se == SE_SHARED && newpp == NULL) { 8027c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[10]); 8037c478bd9Sstevel@tonic-gate page_downgrade(pp); 8047c478bd9Sstevel@tonic-gate } 8057c478bd9Sstevel@tonic-gate } 8067c478bd9Sstevel@tonic-gate 8077c478bd9Sstevel@tonic-gate if (hash_locked) { 8087c478bd9Sstevel@tonic-gate mutex_exit(phm); 8097c478bd9Sstevel@tonic-gate } 8107c478bd9Sstevel@tonic-gate 8117c478bd9Sstevel@tonic-gate if (newpp != NULL && pp->p_szc < newpp->p_szc && 8127c478bd9Sstevel@tonic-gate PAGE_EXCL(pp) && nrelocp != NULL) { 8137c478bd9Sstevel@tonic-gate ASSERT(nrelocp != NULL); 8147c478bd9Sstevel@tonic-gate (void) page_relocate(&pp, &newpp, 1, 1, nrelocp, 8157c478bd9Sstevel@tonic-gate NULL); 8167c478bd9Sstevel@tonic-gate if (*nrelocp > 0) { 8177c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(*nrelocp == 1, 8187c478bd9Sstevel@tonic-gate page_lookup_cnt[11]); 8197c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(*nrelocp > 1, 8207c478bd9Sstevel@tonic-gate page_lookup_cnt[12]); 8217c478bd9Sstevel@tonic-gate pp = newpp; 8227c478bd9Sstevel@tonic-gate se = SE_EXCL; 8237c478bd9Sstevel@tonic-gate } else { 8247c478bd9Sstevel@tonic-gate if (se == SE_SHARED) { 8257c478bd9Sstevel@tonic-gate page_downgrade(pp); 8267c478bd9Sstevel@tonic-gate } 8277c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[13]); 8287c478bd9Sstevel@tonic-gate } 8297c478bd9Sstevel@tonic-gate } else if (newpp != NULL && nrelocp != NULL) { 8307c478bd9Sstevel@tonic-gate if (PAGE_EXCL(pp) && se == SE_SHARED) { 8317c478bd9Sstevel@tonic-gate page_downgrade(pp); 8327c478bd9Sstevel@tonic-gate } 8337c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc, 8347c478bd9Sstevel@tonic-gate page_lookup_cnt[14]); 8357c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc, 8367c478bd9Sstevel@tonic-gate page_lookup_cnt[15]); 8377c478bd9Sstevel@tonic-gate VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc, 8387c478bd9Sstevel@tonic-gate page_lookup_cnt[16]); 8397c478bd9Sstevel@tonic-gate } else if (newpp != NULL && PAGE_EXCL(pp)) { 8407c478bd9Sstevel@tonic-gate se = SE_EXCL; 8417c478bd9Sstevel@tonic-gate } 8427c478bd9Sstevel@tonic-gate } else if (!hash_locked) { 8437c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[17]); 8447c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 8457c478bd9Sstevel@tonic-gate mutex_enter(phm); 8467c478bd9Sstevel@tonic-gate hash_locked = 1; 8477c478bd9Sstevel@tonic-gate goto top; 8487c478bd9Sstevel@tonic-gate } else if (newpp != NULL) { 8497c478bd9Sstevel@tonic-gate /* 8507c478bd9Sstevel@tonic-gate * If we have a preallocated page then 8517c478bd9Sstevel@tonic-gate * insert it now and basically behave like 8527c478bd9Sstevel@tonic-gate * page_create. 8537c478bd9Sstevel@tonic-gate */ 8547c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[18]); 8557c478bd9Sstevel@tonic-gate /* 8567c478bd9Sstevel@tonic-gate * Since we hold the page hash mutex and 8577c478bd9Sstevel@tonic-gate * just searched for this page, page_hashin 8587c478bd9Sstevel@tonic-gate * had better not fail. If it does, that 8597c478bd9Sstevel@tonic-gate * means some thread did not follow the 8607c478bd9Sstevel@tonic-gate * page hash mutex rules. Panic now and 8617c478bd9Sstevel@tonic-gate * get it over with. As usual, go down 8627c478bd9Sstevel@tonic-gate * holding all the locks. 8637c478bd9Sstevel@tonic-gate */ 8647c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 8657c478bd9Sstevel@tonic-gate if (!page_hashin(newpp, vp, off, phm)) { 8667c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 8677c478bd9Sstevel@tonic-gate panic("page_lookup_create: hashin failed %p %p %llx %p", 8687c478bd9Sstevel@tonic-gate (void *)newpp, (void *)vp, off, (void *)phm); 8697c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 8707c478bd9Sstevel@tonic-gate } 8717c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 8727c478bd9Sstevel@tonic-gate mutex_exit(phm); 8737c478bd9Sstevel@tonic-gate phm = NULL; 8747c478bd9Sstevel@tonic-gate page_set_props(newpp, P_REF); 8757c478bd9Sstevel@tonic-gate page_io_lock(newpp); 8767c478bd9Sstevel@tonic-gate pp = newpp; 8777c478bd9Sstevel@tonic-gate se = SE_EXCL; 8787c478bd9Sstevel@tonic-gate } else { 8797c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_cnt[19]); 8807c478bd9Sstevel@tonic-gate mutex_exit(phm); 8817c478bd9Sstevel@tonic-gate } 8827c478bd9Sstevel@tonic-gate 8837c478bd9Sstevel@tonic-gate ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1); 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate return (pp); 8887c478bd9Sstevel@tonic-gate } 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate /* 8917c478bd9Sstevel@tonic-gate * Search the hash list for the page representing the 8927c478bd9Sstevel@tonic-gate * specified [vp, offset] and return it locked. Skip 8937c478bd9Sstevel@tonic-gate * free pages and pages that cannot be locked as requested. 8947c478bd9Sstevel@tonic-gate * Used while attempting to kluster pages. 8957c478bd9Sstevel@tonic-gate */ 8967c478bd9Sstevel@tonic-gate page_t * 8977c478bd9Sstevel@tonic-gate page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se) 8987c478bd9Sstevel@tonic-gate { 8997c478bd9Sstevel@tonic-gate page_t *pp; 9007c478bd9Sstevel@tonic-gate kmutex_t *phm; 9017c478bd9Sstevel@tonic-gate ulong_t index; 9027c478bd9Sstevel@tonic-gate uint_t locked; 9037c478bd9Sstevel@tonic-gate 9047c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 9057c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[0]); 9067c478bd9Sstevel@tonic-gate 9077c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 9087c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 9097c478bd9Sstevel@tonic-gate locked = 0; 9107c478bd9Sstevel@tonic-gate if (pp == NULL) { 9117c478bd9Sstevel@tonic-gate top: 9127c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[1]); 9137c478bd9Sstevel@tonic-gate locked = 1; 9147c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 9157c478bd9Sstevel@tonic-gate mutex_enter(phm); 9167c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 9177c478bd9Sstevel@tonic-gate } 9187c478bd9Sstevel@tonic-gate 9197c478bd9Sstevel@tonic-gate if (pp == NULL || PP_ISFREE(pp)) { 9207c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[2]); 9217c478bd9Sstevel@tonic-gate pp = NULL; 9227c478bd9Sstevel@tonic-gate } else { 9237c478bd9Sstevel@tonic-gate if (!page_trylock(pp, se)) { 9247c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[3]); 9257c478bd9Sstevel@tonic-gate pp = NULL; 9267c478bd9Sstevel@tonic-gate } else { 9277c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[4]); 9287c478bd9Sstevel@tonic-gate /* 9297c478bd9Sstevel@tonic-gate * See the comment in page_lookup() 9307c478bd9Sstevel@tonic-gate */ 9317c478bd9Sstevel@tonic-gate if (((volatile struct vnode *)(pp->p_vnode) != vp) || 9327c478bd9Sstevel@tonic-gate ((u_offset_t)(pp->p_offset) != off)) { 9337c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[5]); 9347c478bd9Sstevel@tonic-gate if (locked) { 9357c478bd9Sstevel@tonic-gate panic("page_lookup_nowait %p", 9367c478bd9Sstevel@tonic-gate (void *)pp); 9377c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 9387c478bd9Sstevel@tonic-gate } 9397c478bd9Sstevel@tonic-gate page_unlock(pp); 9407c478bd9Sstevel@tonic-gate goto top; 9417c478bd9Sstevel@tonic-gate } 9427c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 9437c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[6]); 9447c478bd9Sstevel@tonic-gate page_unlock(pp); 9457c478bd9Sstevel@tonic-gate pp = NULL; 9467c478bd9Sstevel@tonic-gate } 9477c478bd9Sstevel@tonic-gate } 9487c478bd9Sstevel@tonic-gate } 9497c478bd9Sstevel@tonic-gate if (locked) { 9507c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_lookup_nowait_cnt[7]); 9517c478bd9Sstevel@tonic-gate mutex_exit(phm); 9527c478bd9Sstevel@tonic-gate } 9537c478bd9Sstevel@tonic-gate 9547c478bd9Sstevel@tonic-gate ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1); 9557c478bd9Sstevel@tonic-gate 9567c478bd9Sstevel@tonic-gate return (pp); 9577c478bd9Sstevel@tonic-gate } 9587c478bd9Sstevel@tonic-gate 9597c478bd9Sstevel@tonic-gate /* 9607c478bd9Sstevel@tonic-gate * Search the hash list for a page with the specified [vp, off] 9617c478bd9Sstevel@tonic-gate * that is known to exist and is already locked. This routine 9627c478bd9Sstevel@tonic-gate * is typically used by segment SOFTUNLOCK routines. 9637c478bd9Sstevel@tonic-gate */ 9647c478bd9Sstevel@tonic-gate page_t * 9657c478bd9Sstevel@tonic-gate page_find(vnode_t *vp, u_offset_t off) 9667c478bd9Sstevel@tonic-gate { 9677c478bd9Sstevel@tonic-gate page_t *pp; 9687c478bd9Sstevel@tonic-gate kmutex_t *phm; 9697c478bd9Sstevel@tonic-gate ulong_t index; 9707c478bd9Sstevel@tonic-gate 9717c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 9727c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_find_cnt); 9737c478bd9Sstevel@tonic-gate 9747c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 9757c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate mutex_enter(phm); 9787c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 9797c478bd9Sstevel@tonic-gate mutex_exit(phm); 9807c478bd9Sstevel@tonic-gate 9814fc2445aSelowe ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr); 9827c478bd9Sstevel@tonic-gate return (pp); 9837c478bd9Sstevel@tonic-gate } 9847c478bd9Sstevel@tonic-gate 9857c478bd9Sstevel@tonic-gate /* 9867c478bd9Sstevel@tonic-gate * Determine whether a page with the specified [vp, off] 9877c478bd9Sstevel@tonic-gate * currently exists in the system. Obviously this should 9887c478bd9Sstevel@tonic-gate * only be considered as a hint since nothing prevents the 9897c478bd9Sstevel@tonic-gate * page from disappearing or appearing immediately after 9907c478bd9Sstevel@tonic-gate * the return from this routine. Subsequently, we don't 9917c478bd9Sstevel@tonic-gate * even bother to lock the list. 9927c478bd9Sstevel@tonic-gate */ 9937c478bd9Sstevel@tonic-gate page_t * 9947c478bd9Sstevel@tonic-gate page_exists(vnode_t *vp, u_offset_t off) 9957c478bd9Sstevel@tonic-gate { 9967c478bd9Sstevel@tonic-gate page_t *pp; 9977c478bd9Sstevel@tonic-gate ulong_t index; 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 10007c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exists_cnt); 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 10037c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate return (pp); 10067c478bd9Sstevel@tonic-gate } 10077c478bd9Sstevel@tonic-gate 10087c478bd9Sstevel@tonic-gate /* 10097c478bd9Sstevel@tonic-gate * Determine if physically contiguous pages exist for [vp, off] - [vp, off + 10107c478bd9Sstevel@tonic-gate * page_size(szc)) range. if they exist and ppa is not NULL fill ppa array 10117c478bd9Sstevel@tonic-gate * with these pages locked SHARED. If necessary reclaim pages from 10127c478bd9Sstevel@tonic-gate * freelist. Return 1 if contiguous pages exist and 0 otherwise. 10137c478bd9Sstevel@tonic-gate * 10147c478bd9Sstevel@tonic-gate * If we fail to lock pages still return 1 if pages exist and contiguous. 10157c478bd9Sstevel@tonic-gate * But in this case return value is just a hint. ppa array won't be filled. 10167c478bd9Sstevel@tonic-gate * Caller should initialize ppa[0] as NULL to distinguish return value. 10177c478bd9Sstevel@tonic-gate * 10187c478bd9Sstevel@tonic-gate * Returns 0 if pages don't exist or not physically contiguous. 10197c478bd9Sstevel@tonic-gate * 10207c478bd9Sstevel@tonic-gate * This routine doesn't work for anonymous(swapfs) pages. 10217c478bd9Sstevel@tonic-gate */ 10227c478bd9Sstevel@tonic-gate int 10237c478bd9Sstevel@tonic-gate page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[]) 10247c478bd9Sstevel@tonic-gate { 10257c478bd9Sstevel@tonic-gate pgcnt_t pages; 10267c478bd9Sstevel@tonic-gate pfn_t pfn; 10277c478bd9Sstevel@tonic-gate page_t *rootpp; 10287c478bd9Sstevel@tonic-gate pgcnt_t i; 10297c478bd9Sstevel@tonic-gate pgcnt_t j; 10307c478bd9Sstevel@tonic-gate u_offset_t save_off = off; 10317c478bd9Sstevel@tonic-gate ulong_t index; 10327c478bd9Sstevel@tonic-gate kmutex_t *phm; 10337c478bd9Sstevel@tonic-gate page_t *pp; 10347c478bd9Sstevel@tonic-gate uint_t pszc; 10357c478bd9Sstevel@tonic-gate int loopcnt = 0; 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate ASSERT(szc != 0); 10387c478bd9Sstevel@tonic-gate ASSERT(vp != NULL); 10397c478bd9Sstevel@tonic-gate ASSERT(!IS_SWAPFSVP(vp)); 1040ad23a2dbSjohansen ASSERT(!VN_ISKAS(vp)); 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate again: 10437c478bd9Sstevel@tonic-gate if (++loopcnt > 3) { 10447c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[0]); 10457c478bd9Sstevel@tonic-gate return (0); 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 10497c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 10507c478bd9Sstevel@tonic-gate 10517c478bd9Sstevel@tonic-gate mutex_enter(phm); 10527c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 10537c478bd9Sstevel@tonic-gate mutex_exit(phm); 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[1]); 10567c478bd9Sstevel@tonic-gate 10577c478bd9Sstevel@tonic-gate if (pp == NULL) { 10587c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[2]); 10597c478bd9Sstevel@tonic-gate return (0); 10607c478bd9Sstevel@tonic-gate } 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate pages = page_get_pagecnt(szc); 10637c478bd9Sstevel@tonic-gate rootpp = pp; 10647c478bd9Sstevel@tonic-gate pfn = rootpp->p_pagenum; 10657c478bd9Sstevel@tonic-gate 10667c478bd9Sstevel@tonic-gate if ((pszc = pp->p_szc) >= szc && ppa != NULL) { 10677c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[3]); 10687c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_SHARED)) { 10697c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[4]); 10707c478bd9Sstevel@tonic-gate return (1); 10717c478bd9Sstevel@tonic-gate } 10727c478bd9Sstevel@tonic-gate if (pp->p_szc != pszc || pp->p_vnode != vp || 10737c478bd9Sstevel@tonic-gate pp->p_offset != off) { 10747c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[5]); 10757c478bd9Sstevel@tonic-gate page_unlock(pp); 10767c478bd9Sstevel@tonic-gate off = save_off; 10777c478bd9Sstevel@tonic-gate goto again; 10787c478bd9Sstevel@tonic-gate } 10797c478bd9Sstevel@tonic-gate /* 10807c478bd9Sstevel@tonic-gate * szc was non zero and vnode and offset matched after we 10817c478bd9Sstevel@tonic-gate * locked the page it means it can't become free on us. 10827c478bd9Sstevel@tonic-gate */ 10837c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(pp)); 10847c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(pfn, pages)) { 10857c478bd9Sstevel@tonic-gate page_unlock(pp); 10867c478bd9Sstevel@tonic-gate return (0); 10877c478bd9Sstevel@tonic-gate } 10887c478bd9Sstevel@tonic-gate ppa[0] = pp; 10897c478bd9Sstevel@tonic-gate pp++; 10907c478bd9Sstevel@tonic-gate off += PAGESIZE; 10917c478bd9Sstevel@tonic-gate pfn++; 10927c478bd9Sstevel@tonic-gate for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 10937c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_SHARED)) { 10947c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[6]); 10957c478bd9Sstevel@tonic-gate pp--; 10967c478bd9Sstevel@tonic-gate while (i-- > 0) { 10977c478bd9Sstevel@tonic-gate page_unlock(pp); 10987c478bd9Sstevel@tonic-gate pp--; 10997c478bd9Sstevel@tonic-gate } 11007c478bd9Sstevel@tonic-gate ppa[0] = NULL; 11017c478bd9Sstevel@tonic-gate return (1); 11027c478bd9Sstevel@tonic-gate } 11037c478bd9Sstevel@tonic-gate if (pp->p_szc != pszc) { 11047c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[7]); 11057c478bd9Sstevel@tonic-gate page_unlock(pp); 11067c478bd9Sstevel@tonic-gate pp--; 11077c478bd9Sstevel@tonic-gate while (i-- > 0) { 11087c478bd9Sstevel@tonic-gate page_unlock(pp); 11097c478bd9Sstevel@tonic-gate pp--; 11107c478bd9Sstevel@tonic-gate } 11117c478bd9Sstevel@tonic-gate ppa[0] = NULL; 11127c478bd9Sstevel@tonic-gate off = save_off; 11137c478bd9Sstevel@tonic-gate goto again; 11147c478bd9Sstevel@tonic-gate } 11157c478bd9Sstevel@tonic-gate /* 11167c478bd9Sstevel@tonic-gate * szc the same as for previous already locked pages 11177c478bd9Sstevel@tonic-gate * with right identity. Since this page had correct 11187c478bd9Sstevel@tonic-gate * szc after we locked it can't get freed or destroyed 11197c478bd9Sstevel@tonic-gate * and therefore must have the expected identity. 11207c478bd9Sstevel@tonic-gate */ 11217c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(pp)); 11227c478bd9Sstevel@tonic-gate if (pp->p_vnode != vp || 11237c478bd9Sstevel@tonic-gate pp->p_offset != off) { 11247c478bd9Sstevel@tonic-gate panic("page_exists_physcontig: " 11257c478bd9Sstevel@tonic-gate "large page identity doesn't match"); 11267c478bd9Sstevel@tonic-gate } 11277c478bd9Sstevel@tonic-gate ppa[i] = pp; 11287c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == pfn); 11297c478bd9Sstevel@tonic-gate } 11307c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[8]); 11317c478bd9Sstevel@tonic-gate ppa[pages] = NULL; 11327c478bd9Sstevel@tonic-gate return (1); 11337c478bd9Sstevel@tonic-gate } else if (pszc >= szc) { 11347c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[9]); 11357c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(pfn, pages)) { 11367c478bd9Sstevel@tonic-gate return (0); 11377c478bd9Sstevel@tonic-gate } 11387c478bd9Sstevel@tonic-gate return (1); 11397c478bd9Sstevel@tonic-gate } 11407c478bd9Sstevel@tonic-gate 11417c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(pfn, pages)) { 11427c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[10]); 11437c478bd9Sstevel@tonic-gate return (0); 11447c478bd9Sstevel@tonic-gate } 11457c478bd9Sstevel@tonic-gate 11467c478bd9Sstevel@tonic-gate if (page_numtomemseg_nolock(pfn) != 11477c478bd9Sstevel@tonic-gate page_numtomemseg_nolock(pfn + pages - 1)) { 11487c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[11]); 11497c478bd9Sstevel@tonic-gate return (0); 11507c478bd9Sstevel@tonic-gate } 11517c478bd9Sstevel@tonic-gate 11527c478bd9Sstevel@tonic-gate /* 11537c478bd9Sstevel@tonic-gate * We loop up 4 times across pages to promote page size. 11547c478bd9Sstevel@tonic-gate * We're extra cautious to promote page size atomically with respect 11557c478bd9Sstevel@tonic-gate * to everybody else. But we can probably optimize into 1 loop if 11567c478bd9Sstevel@tonic-gate * this becomes an issue. 11577c478bd9Sstevel@tonic-gate */ 11587c478bd9Sstevel@tonic-gate 11597c478bd9Sstevel@tonic-gate for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) { 11607c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == pfn); 11617c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 11627c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[12]); 11637c478bd9Sstevel@tonic-gate break; 11647c478bd9Sstevel@tonic-gate } 11657c478bd9Sstevel@tonic-gate if (pp->p_vnode != vp || 11667c478bd9Sstevel@tonic-gate pp->p_offset != off) { 11677c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[13]); 11687c478bd9Sstevel@tonic-gate page_unlock(pp); 11697c478bd9Sstevel@tonic-gate break; 11707c478bd9Sstevel@tonic-gate } 11717c478bd9Sstevel@tonic-gate if (pp->p_szc >= szc) { 11727c478bd9Sstevel@tonic-gate ASSERT(i == 0); 11737c478bd9Sstevel@tonic-gate page_unlock(pp); 11747c478bd9Sstevel@tonic-gate off = save_off; 11757c478bd9Sstevel@tonic-gate goto again; 11767c478bd9Sstevel@tonic-gate } 11777c478bd9Sstevel@tonic-gate } 11787c478bd9Sstevel@tonic-gate 11797c478bd9Sstevel@tonic-gate if (i != pages) { 11807c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[14]); 11817c478bd9Sstevel@tonic-gate --pp; 11827c478bd9Sstevel@tonic-gate while (i-- > 0) { 11837c478bd9Sstevel@tonic-gate page_unlock(pp); 11847c478bd9Sstevel@tonic-gate --pp; 11857c478bd9Sstevel@tonic-gate } 11867c478bd9Sstevel@tonic-gate return (0); 11877c478bd9Sstevel@tonic-gate } 11887c478bd9Sstevel@tonic-gate 11897c478bd9Sstevel@tonic-gate pp = rootpp; 11907c478bd9Sstevel@tonic-gate for (i = 0; i < pages; i++, pp++) { 11917c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 11927c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[15]); 11937c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 11947c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 11957c478bd9Sstevel@tonic-gate if (!page_reclaim(pp, NULL)) { 11967c478bd9Sstevel@tonic-gate break; 11977c478bd9Sstevel@tonic-gate } 11987c478bd9Sstevel@tonic-gate } else { 11997c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < szc); 12007c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[16]); 12017c478bd9Sstevel@tonic-gate (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 12027c478bd9Sstevel@tonic-gate } 12037c478bd9Sstevel@tonic-gate } 12047c478bd9Sstevel@tonic-gate if (i < pages) { 12057c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[17]); 12067c478bd9Sstevel@tonic-gate /* 12077c478bd9Sstevel@tonic-gate * page_reclaim failed because we were out of memory. 12087c478bd9Sstevel@tonic-gate * drop the rest of the locks and return because this page 12097c478bd9Sstevel@tonic-gate * must be already reallocated anyway. 12107c478bd9Sstevel@tonic-gate */ 12117c478bd9Sstevel@tonic-gate pp = rootpp; 12127c478bd9Sstevel@tonic-gate for (j = 0; j < pages; j++, pp++) { 12137c478bd9Sstevel@tonic-gate if (j != i) { 12147c478bd9Sstevel@tonic-gate page_unlock(pp); 12157c478bd9Sstevel@tonic-gate } 12167c478bd9Sstevel@tonic-gate } 12177c478bd9Sstevel@tonic-gate return (0); 12187c478bd9Sstevel@tonic-gate } 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate off = save_off; 12217c478bd9Sstevel@tonic-gate pp = rootpp; 12227c478bd9Sstevel@tonic-gate for (i = 0; i < pages; i++, pp++, off += PAGESIZE) { 12237c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 12247c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(pp)); 12257c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 12267c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == vp); 12277c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == off); 12287c478bd9Sstevel@tonic-gate pp->p_szc = szc; 12297c478bd9Sstevel@tonic-gate } 12307c478bd9Sstevel@tonic-gate pp = rootpp; 12317c478bd9Sstevel@tonic-gate for (i = 0; i < pages; i++, pp++) { 12327c478bd9Sstevel@tonic-gate if (ppa == NULL) { 12337c478bd9Sstevel@tonic-gate page_unlock(pp); 12347c478bd9Sstevel@tonic-gate } else { 12357c478bd9Sstevel@tonic-gate ppa[i] = pp; 12367c478bd9Sstevel@tonic-gate page_downgrade(ppa[i]); 12377c478bd9Sstevel@tonic-gate } 12387c478bd9Sstevel@tonic-gate } 12397c478bd9Sstevel@tonic-gate if (ppa != NULL) { 12407c478bd9Sstevel@tonic-gate ppa[pages] = NULL; 12417c478bd9Sstevel@tonic-gate } 12427c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exphcontg[18]); 12437c478bd9Sstevel@tonic-gate ASSERT(vp->v_pages != NULL); 12447c478bd9Sstevel@tonic-gate return (1); 12457c478bd9Sstevel@tonic-gate } 12467c478bd9Sstevel@tonic-gate 12477c478bd9Sstevel@tonic-gate /* 12487c478bd9Sstevel@tonic-gate * Determine whether a page with the specified [vp, off] 12497c478bd9Sstevel@tonic-gate * currently exists in the system and if so return its 12507c478bd9Sstevel@tonic-gate * size code. Obviously this should only be considered as 12517c478bd9Sstevel@tonic-gate * a hint since nothing prevents the page from disappearing 12527c478bd9Sstevel@tonic-gate * or appearing immediately after the return from this routine. 12537c478bd9Sstevel@tonic-gate */ 12547c478bd9Sstevel@tonic-gate int 12557c478bd9Sstevel@tonic-gate page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc) 12567c478bd9Sstevel@tonic-gate { 12577c478bd9Sstevel@tonic-gate page_t *pp; 12587c478bd9Sstevel@tonic-gate kmutex_t *phm; 12597c478bd9Sstevel@tonic-gate ulong_t index; 12607c478bd9Sstevel@tonic-gate int rc = 0; 12617c478bd9Sstevel@tonic-gate 12627c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 12637c478bd9Sstevel@tonic-gate ASSERT(szc != NULL); 12647c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_exists_forreal_cnt); 12657c478bd9Sstevel@tonic-gate 12667c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 12677c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate mutex_enter(phm); 12707c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 12717c478bd9Sstevel@tonic-gate if (pp != NULL) { 12727c478bd9Sstevel@tonic-gate *szc = pp->p_szc; 12737c478bd9Sstevel@tonic-gate rc = 1; 12747c478bd9Sstevel@tonic-gate } 12757c478bd9Sstevel@tonic-gate mutex_exit(phm); 12767c478bd9Sstevel@tonic-gate return (rc); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate 12797c478bd9Sstevel@tonic-gate /* wakeup threads waiting for pages in page_create_get_something() */ 12807c478bd9Sstevel@tonic-gate void 12817c478bd9Sstevel@tonic-gate wakeup_pcgs(void) 12827c478bd9Sstevel@tonic-gate { 12837c478bd9Sstevel@tonic-gate if (!CV_HAS_WAITERS(&pcgs_cv)) 12847c478bd9Sstevel@tonic-gate return; 12857c478bd9Sstevel@tonic-gate cv_broadcast(&pcgs_cv); 12867c478bd9Sstevel@tonic-gate } 12877c478bd9Sstevel@tonic-gate 12887c478bd9Sstevel@tonic-gate /* 12897c478bd9Sstevel@tonic-gate * 'freemem' is used all over the kernel as an indication of how many 12907c478bd9Sstevel@tonic-gate * pages are free (either on the cache list or on the free page list) 12917c478bd9Sstevel@tonic-gate * in the system. In very few places is a really accurate 'freemem' 12927c478bd9Sstevel@tonic-gate * needed. To avoid contention of the lock protecting a the 12937c478bd9Sstevel@tonic-gate * single freemem, it was spread out into NCPU buckets. Set_freemem 12947c478bd9Sstevel@tonic-gate * sets freemem to the total of all NCPU buckets. It is called from 12957c478bd9Sstevel@tonic-gate * clock() on each TICK. 12967c478bd9Sstevel@tonic-gate */ 12977c478bd9Sstevel@tonic-gate void 12987c478bd9Sstevel@tonic-gate set_freemem() 12997c478bd9Sstevel@tonic-gate { 13007c478bd9Sstevel@tonic-gate struct pcf *p; 13017c478bd9Sstevel@tonic-gate ulong_t t; 13027c478bd9Sstevel@tonic-gate uint_t i; 13037c478bd9Sstevel@tonic-gate 13047c478bd9Sstevel@tonic-gate t = 0; 13057c478bd9Sstevel@tonic-gate p = pcf; 13067c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 13077c478bd9Sstevel@tonic-gate t += p->pcf_count; 13087c478bd9Sstevel@tonic-gate p++; 13097c478bd9Sstevel@tonic-gate } 13107c478bd9Sstevel@tonic-gate freemem = t; 13117c478bd9Sstevel@tonic-gate 13127c478bd9Sstevel@tonic-gate /* 13137c478bd9Sstevel@tonic-gate * Don't worry about grabbing mutex. It's not that 13147c478bd9Sstevel@tonic-gate * critical if we miss a tick or two. This is 13157c478bd9Sstevel@tonic-gate * where we wakeup possible delayers in 13167c478bd9Sstevel@tonic-gate * page_create_get_something(). 13177c478bd9Sstevel@tonic-gate */ 13187c478bd9Sstevel@tonic-gate wakeup_pcgs(); 13197c478bd9Sstevel@tonic-gate } 13207c478bd9Sstevel@tonic-gate 13217c478bd9Sstevel@tonic-gate ulong_t 13227c478bd9Sstevel@tonic-gate get_freemem() 13237c478bd9Sstevel@tonic-gate { 13247c478bd9Sstevel@tonic-gate struct pcf *p; 13257c478bd9Sstevel@tonic-gate ulong_t t; 13267c478bd9Sstevel@tonic-gate uint_t i; 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate t = 0; 13297c478bd9Sstevel@tonic-gate p = pcf; 13307c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 13317c478bd9Sstevel@tonic-gate t += p->pcf_count; 13327c478bd9Sstevel@tonic-gate p++; 13337c478bd9Sstevel@tonic-gate } 13347c478bd9Sstevel@tonic-gate /* 13357c478bd9Sstevel@tonic-gate * We just calculated it, might as well set it. 13367c478bd9Sstevel@tonic-gate */ 13377c478bd9Sstevel@tonic-gate freemem = t; 13387c478bd9Sstevel@tonic-gate return (t); 13397c478bd9Sstevel@tonic-gate } 13407c478bd9Sstevel@tonic-gate 13417c478bd9Sstevel@tonic-gate /* 13427c478bd9Sstevel@tonic-gate * Acquire all of the page cache & free (pcf) locks. 13437c478bd9Sstevel@tonic-gate */ 13447c478bd9Sstevel@tonic-gate void 13457c478bd9Sstevel@tonic-gate pcf_acquire_all() 13467c478bd9Sstevel@tonic-gate { 13477c478bd9Sstevel@tonic-gate struct pcf *p; 13487c478bd9Sstevel@tonic-gate uint_t i; 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate p = pcf; 13517c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 13527c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 13537c478bd9Sstevel@tonic-gate p++; 13547c478bd9Sstevel@tonic-gate } 13557c478bd9Sstevel@tonic-gate } 13567c478bd9Sstevel@tonic-gate 13577c478bd9Sstevel@tonic-gate /* 13587c478bd9Sstevel@tonic-gate * Release all the pcf_locks. 13597c478bd9Sstevel@tonic-gate */ 13607c478bd9Sstevel@tonic-gate void 13617c478bd9Sstevel@tonic-gate pcf_release_all() 13627c478bd9Sstevel@tonic-gate { 13637c478bd9Sstevel@tonic-gate struct pcf *p; 13647c478bd9Sstevel@tonic-gate uint_t i; 13657c478bd9Sstevel@tonic-gate 13667c478bd9Sstevel@tonic-gate p = pcf; 13677c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 13687c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 13697c478bd9Sstevel@tonic-gate p++; 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate } 13727c478bd9Sstevel@tonic-gate 13737c478bd9Sstevel@tonic-gate /* 13747c478bd9Sstevel@tonic-gate * Inform the VM system that we need some pages freed up. 13757c478bd9Sstevel@tonic-gate * Calls must be symmetric, e.g.: 13767c478bd9Sstevel@tonic-gate * 13777c478bd9Sstevel@tonic-gate * page_needfree(100); 13787c478bd9Sstevel@tonic-gate * wait a bit; 13797c478bd9Sstevel@tonic-gate * page_needfree(-100); 13807c478bd9Sstevel@tonic-gate */ 13817c478bd9Sstevel@tonic-gate void 13827c478bd9Sstevel@tonic-gate page_needfree(spgcnt_t npages) 13837c478bd9Sstevel@tonic-gate { 13847c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 13857c478bd9Sstevel@tonic-gate needfree += npages; 13867c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 13877c478bd9Sstevel@tonic-gate } 13887c478bd9Sstevel@tonic-gate 13897c478bd9Sstevel@tonic-gate /* 13907c478bd9Sstevel@tonic-gate * Throttle for page_create(): try to prevent freemem from dropping 13917c478bd9Sstevel@tonic-gate * below throttlefree. We can't provide a 100% guarantee because 13927c478bd9Sstevel@tonic-gate * KM_NOSLEEP allocations, page_reclaim(), and various other things 13937c478bd9Sstevel@tonic-gate * nibble away at the freelist. However, we can block all PG_WAIT 13947c478bd9Sstevel@tonic-gate * allocations until memory becomes available. The motivation is 13957c478bd9Sstevel@tonic-gate * that several things can fall apart when there's no free memory: 13967c478bd9Sstevel@tonic-gate * 13977c478bd9Sstevel@tonic-gate * (1) If pageout() needs memory to push a page, the system deadlocks. 13987c478bd9Sstevel@tonic-gate * 13997c478bd9Sstevel@tonic-gate * (2) By (broken) specification, timeout(9F) can neither fail nor 14007c478bd9Sstevel@tonic-gate * block, so it has no choice but to panic the system if it 14017c478bd9Sstevel@tonic-gate * cannot allocate a callout structure. 14027c478bd9Sstevel@tonic-gate * 14037c478bd9Sstevel@tonic-gate * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block; 14047c478bd9Sstevel@tonic-gate * it panics if it cannot allocate a callback structure. 14057c478bd9Sstevel@tonic-gate * 14067c478bd9Sstevel@tonic-gate * (4) Untold numbers of third-party drivers have not yet been hardened 14077c478bd9Sstevel@tonic-gate * against KM_NOSLEEP and/or allocb() failures; they simply assume 14087c478bd9Sstevel@tonic-gate * success and panic the system with a data fault on failure. 14097c478bd9Sstevel@tonic-gate * (The long-term solution to this particular problem is to ship 14107c478bd9Sstevel@tonic-gate * hostile fault-injecting DEBUG kernels with the DDK.) 14117c478bd9Sstevel@tonic-gate * 14127c478bd9Sstevel@tonic-gate * It is theoretically impossible to guarantee success of non-blocking 14137c478bd9Sstevel@tonic-gate * allocations, but in practice, this throttle is very hard to break. 14147c478bd9Sstevel@tonic-gate */ 14157c478bd9Sstevel@tonic-gate static int 14167c478bd9Sstevel@tonic-gate page_create_throttle(pgcnt_t npages, int flags) 14177c478bd9Sstevel@tonic-gate { 14187c478bd9Sstevel@tonic-gate ulong_t fm; 14197c478bd9Sstevel@tonic-gate uint_t i; 14207c478bd9Sstevel@tonic-gate pgcnt_t tf; /* effective value of throttlefree */ 14217c478bd9Sstevel@tonic-gate 14227c478bd9Sstevel@tonic-gate /* 14237c478bd9Sstevel@tonic-gate * Never deny pages when: 14247c478bd9Sstevel@tonic-gate * - it's a thread that cannot block [NOMEMWAIT()] 14257c478bd9Sstevel@tonic-gate * - the allocation cannot block and must not fail 14267c478bd9Sstevel@tonic-gate * - the allocation cannot block and is pageout dispensated 14277c478bd9Sstevel@tonic-gate */ 14287c478bd9Sstevel@tonic-gate if (NOMEMWAIT() || 14297c478bd9Sstevel@tonic-gate ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) || 14307c478bd9Sstevel@tonic-gate ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE)) 14317c478bd9Sstevel@tonic-gate return (1); 14327c478bd9Sstevel@tonic-gate 14337c478bd9Sstevel@tonic-gate /* 14347c478bd9Sstevel@tonic-gate * If the allocation can't block, we look favorably upon it 14357c478bd9Sstevel@tonic-gate * unless we're below pageout_reserve. In that case we fail 14367c478bd9Sstevel@tonic-gate * the allocation because we want to make sure there are a few 14377c478bd9Sstevel@tonic-gate * pages available for pageout. 14387c478bd9Sstevel@tonic-gate */ 14397c478bd9Sstevel@tonic-gate if ((flags & PG_WAIT) == 0) 14407c478bd9Sstevel@tonic-gate return (freemem >= npages + pageout_reserve); 14417c478bd9Sstevel@tonic-gate 14427c478bd9Sstevel@tonic-gate /* Calculate the effective throttlefree value */ 14437c478bd9Sstevel@tonic-gate tf = throttlefree - 14447c478bd9Sstevel@tonic-gate ((flags & PG_PUSHPAGE) ? pageout_reserve : 0); 14457c478bd9Sstevel@tonic-gate 14467c478bd9Sstevel@tonic-gate cv_signal(&proc_pageout->p_cv); 14477c478bd9Sstevel@tonic-gate 14487c478bd9Sstevel@tonic-gate while (freemem < npages + tf) { 14497c478bd9Sstevel@tonic-gate pcf_acquire_all(); 14507c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 14517c478bd9Sstevel@tonic-gate fm = 0; 14527c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 14537c478bd9Sstevel@tonic-gate fm += pcf[i].pcf_count; 14547c478bd9Sstevel@tonic-gate pcf[i].pcf_wait++; 14557c478bd9Sstevel@tonic-gate mutex_exit(&pcf[i].pcf_lock); 14567c478bd9Sstevel@tonic-gate } 14577c478bd9Sstevel@tonic-gate freemem = fm; 14587c478bd9Sstevel@tonic-gate needfree += npages; 14597c478bd9Sstevel@tonic-gate freemem_wait++; 14607c478bd9Sstevel@tonic-gate cv_wait(&freemem_cv, &new_freemem_lock); 14617c478bd9Sstevel@tonic-gate freemem_wait--; 14627c478bd9Sstevel@tonic-gate needfree -= npages; 14637c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 14647c478bd9Sstevel@tonic-gate } 14657c478bd9Sstevel@tonic-gate return (1); 14667c478bd9Sstevel@tonic-gate } 14677c478bd9Sstevel@tonic-gate 14687c478bd9Sstevel@tonic-gate /* 14697c478bd9Sstevel@tonic-gate * page_create_wait() is called to either coalecse pages from the 14707c478bd9Sstevel@tonic-gate * different pcf buckets or to wait because there simply are not 14717c478bd9Sstevel@tonic-gate * enough pages to satisfy the caller's request. 14727c478bd9Sstevel@tonic-gate * 14737c478bd9Sstevel@tonic-gate * Sadly, this is called from platform/vm/vm_machdep.c 14747c478bd9Sstevel@tonic-gate */ 14757c478bd9Sstevel@tonic-gate int 14767c478bd9Sstevel@tonic-gate page_create_wait(size_t npages, uint_t flags) 14777c478bd9Sstevel@tonic-gate { 14787c478bd9Sstevel@tonic-gate pgcnt_t total; 14797c478bd9Sstevel@tonic-gate uint_t i; 14807c478bd9Sstevel@tonic-gate struct pcf *p; 14817c478bd9Sstevel@tonic-gate 14827c478bd9Sstevel@tonic-gate /* 14837c478bd9Sstevel@tonic-gate * Wait until there are enough free pages to satisfy our 14847c478bd9Sstevel@tonic-gate * entire request. 14857c478bd9Sstevel@tonic-gate * We set needfree += npages before prodding pageout, to make sure 14867c478bd9Sstevel@tonic-gate * it does real work when npages > lotsfree > freemem. 14877c478bd9Sstevel@tonic-gate */ 14887c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_not_enough); 14897c478bd9Sstevel@tonic-gate 14907c478bd9Sstevel@tonic-gate ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1); 14917c478bd9Sstevel@tonic-gate checkagain: 14927c478bd9Sstevel@tonic-gate if ((flags & PG_NORELOC) && 14937c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + npages) 14947c478bd9Sstevel@tonic-gate (void) kcage_create_throttle(npages, flags); 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate if (freemem < npages + throttlefree) 14977c478bd9Sstevel@tonic-gate if (!page_create_throttle(npages, flags)) 14987c478bd9Sstevel@tonic-gate return (0); 14997c478bd9Sstevel@tonic-gate 15007c478bd9Sstevel@tonic-gate /* 15017c478bd9Sstevel@tonic-gate * Since page_create_va() looked at every 15027c478bd9Sstevel@tonic-gate * bucket, assume we are going to have to wait. 15037c478bd9Sstevel@tonic-gate * Get all of the pcf locks. 15047c478bd9Sstevel@tonic-gate */ 15057c478bd9Sstevel@tonic-gate total = 0; 15067c478bd9Sstevel@tonic-gate p = pcf; 15077c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 15087c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 15097c478bd9Sstevel@tonic-gate total += p->pcf_count; 15107c478bd9Sstevel@tonic-gate if (total >= npages) { 15117c478bd9Sstevel@tonic-gate /* 15127c478bd9Sstevel@tonic-gate * Wow! There are enough pages laying around 15137c478bd9Sstevel@tonic-gate * to satisfy the request. Do the accounting, 15147c478bd9Sstevel@tonic-gate * drop the locks we acquired, and go back. 15157c478bd9Sstevel@tonic-gate * 15167c478bd9Sstevel@tonic-gate * freemem is not protected by any lock. So, 15177c478bd9Sstevel@tonic-gate * we cannot have any assertion containing 15187c478bd9Sstevel@tonic-gate * freemem. 15197c478bd9Sstevel@tonic-gate */ 15207c478bd9Sstevel@tonic-gate freemem -= npages; 15217c478bd9Sstevel@tonic-gate 15227c478bd9Sstevel@tonic-gate while (p >= pcf) { 15237c478bd9Sstevel@tonic-gate if (p->pcf_count <= npages) { 15247c478bd9Sstevel@tonic-gate npages -= p->pcf_count; 15257c478bd9Sstevel@tonic-gate p->pcf_count = 0; 15267c478bd9Sstevel@tonic-gate } else { 15277c478bd9Sstevel@tonic-gate p->pcf_count -= (uint_t)npages; 15287c478bd9Sstevel@tonic-gate npages = 0; 15297c478bd9Sstevel@tonic-gate } 15307c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 15317c478bd9Sstevel@tonic-gate p--; 15327c478bd9Sstevel@tonic-gate } 15337c478bd9Sstevel@tonic-gate ASSERT(npages == 0); 15347c478bd9Sstevel@tonic-gate return (1); 15357c478bd9Sstevel@tonic-gate } 15367c478bd9Sstevel@tonic-gate p++; 15377c478bd9Sstevel@tonic-gate } 15387c478bd9Sstevel@tonic-gate 15397c478bd9Sstevel@tonic-gate /* 15407c478bd9Sstevel@tonic-gate * All of the pcf locks are held, there are not enough pages 15417c478bd9Sstevel@tonic-gate * to satisfy the request (npages < total). 15427c478bd9Sstevel@tonic-gate * Be sure to acquire the new_freemem_lock before dropping 15437c478bd9Sstevel@tonic-gate * the pcf locks. This prevents dropping wakeups in page_free(). 15447c478bd9Sstevel@tonic-gate * The order is always pcf_lock then new_freemem_lock. 15457c478bd9Sstevel@tonic-gate * 15467c478bd9Sstevel@tonic-gate * Since we hold all the pcf locks, it is a good time to set freemem. 15477c478bd9Sstevel@tonic-gate * 15487c478bd9Sstevel@tonic-gate * If the caller does not want to wait, return now. 15497c478bd9Sstevel@tonic-gate * Else turn the pageout daemon loose to find something 15507c478bd9Sstevel@tonic-gate * and wait till it does. 15517c478bd9Sstevel@tonic-gate * 15527c478bd9Sstevel@tonic-gate */ 15537c478bd9Sstevel@tonic-gate freemem = total; 15547c478bd9Sstevel@tonic-gate 15557c478bd9Sstevel@tonic-gate if ((flags & PG_WAIT) == 0) { 15567c478bd9Sstevel@tonic-gate pcf_release_all(); 15577c478bd9Sstevel@tonic-gate 15587c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM, 15597c478bd9Sstevel@tonic-gate "page_create_nomem:npages %ld freemem %ld", npages, freemem); 15607c478bd9Sstevel@tonic-gate return (0); 15617c478bd9Sstevel@tonic-gate } 15627c478bd9Sstevel@tonic-gate 15637c478bd9Sstevel@tonic-gate ASSERT(proc_pageout != NULL); 15647c478bd9Sstevel@tonic-gate cv_signal(&proc_pageout->p_cv); 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START, 15677c478bd9Sstevel@tonic-gate "page_create_sleep_start: freemem %ld needfree %ld", 15687c478bd9Sstevel@tonic-gate freemem, needfree); 15697c478bd9Sstevel@tonic-gate 15707c478bd9Sstevel@tonic-gate /* 15717c478bd9Sstevel@tonic-gate * We are going to wait. 15727c478bd9Sstevel@tonic-gate * We currently hold all of the pcf_locks, 15737c478bd9Sstevel@tonic-gate * get the new_freemem_lock (it protects freemem_wait), 15747c478bd9Sstevel@tonic-gate * before dropping the pcf_locks. 15757c478bd9Sstevel@tonic-gate */ 15767c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 15777c478bd9Sstevel@tonic-gate 15787c478bd9Sstevel@tonic-gate p = pcf; 15797c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 15807c478bd9Sstevel@tonic-gate p->pcf_wait++; 15817c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 15827c478bd9Sstevel@tonic-gate p++; 15837c478bd9Sstevel@tonic-gate } 15847c478bd9Sstevel@tonic-gate 15857c478bd9Sstevel@tonic-gate needfree += npages; 15867c478bd9Sstevel@tonic-gate freemem_wait++; 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate cv_wait(&freemem_cv, &new_freemem_lock); 15897c478bd9Sstevel@tonic-gate 15907c478bd9Sstevel@tonic-gate freemem_wait--; 15917c478bd9Sstevel@tonic-gate needfree -= npages; 15927c478bd9Sstevel@tonic-gate 15937c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 15947c478bd9Sstevel@tonic-gate 15957c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END, 15967c478bd9Sstevel@tonic-gate "page_create_sleep_end: freemem %ld needfree %ld", 15977c478bd9Sstevel@tonic-gate freemem, needfree); 15987c478bd9Sstevel@tonic-gate 15997c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_not_enough_again); 16007c478bd9Sstevel@tonic-gate goto checkagain; 16017c478bd9Sstevel@tonic-gate } 16027c478bd9Sstevel@tonic-gate 16037c478bd9Sstevel@tonic-gate /* 16047c478bd9Sstevel@tonic-gate * A routine to do the opposite of page_create_wait(). 16057c478bd9Sstevel@tonic-gate */ 16067c478bd9Sstevel@tonic-gate void 16077c478bd9Sstevel@tonic-gate page_create_putback(spgcnt_t npages) 16087c478bd9Sstevel@tonic-gate { 16097c478bd9Sstevel@tonic-gate struct pcf *p; 16107c478bd9Sstevel@tonic-gate pgcnt_t lump; 16117c478bd9Sstevel@tonic-gate uint_t *which; 16127c478bd9Sstevel@tonic-gate 16137c478bd9Sstevel@tonic-gate /* 16147c478bd9Sstevel@tonic-gate * When a contiguous lump is broken up, we have to 16157c478bd9Sstevel@tonic-gate * deal with lots of pages (min 64) so lets spread 16167c478bd9Sstevel@tonic-gate * the wealth around. 16177c478bd9Sstevel@tonic-gate */ 16187c478bd9Sstevel@tonic-gate lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT; 16197c478bd9Sstevel@tonic-gate freemem += npages; 16207c478bd9Sstevel@tonic-gate 16217c478bd9Sstevel@tonic-gate for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) { 16227c478bd9Sstevel@tonic-gate which = &p->pcf_count; 16237c478bd9Sstevel@tonic-gate 16247c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 16257c478bd9Sstevel@tonic-gate 16267c478bd9Sstevel@tonic-gate if (p->pcf_block) { 16277c478bd9Sstevel@tonic-gate which = &p->pcf_reserve; 16287c478bd9Sstevel@tonic-gate } 16297c478bd9Sstevel@tonic-gate 16307c478bd9Sstevel@tonic-gate if (lump < npages) { 16317c478bd9Sstevel@tonic-gate *which += (uint_t)lump; 16327c478bd9Sstevel@tonic-gate npages -= lump; 16337c478bd9Sstevel@tonic-gate } else { 16347c478bd9Sstevel@tonic-gate *which += (uint_t)npages; 16357c478bd9Sstevel@tonic-gate npages = 0; 16367c478bd9Sstevel@tonic-gate } 16377c478bd9Sstevel@tonic-gate 16387c478bd9Sstevel@tonic-gate if (p->pcf_wait) { 16397c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 16407c478bd9Sstevel@tonic-gate /* 16417c478bd9Sstevel@tonic-gate * Check to see if some other thread 16427c478bd9Sstevel@tonic-gate * is actually waiting. Another bucket 16437c478bd9Sstevel@tonic-gate * may have woken it up by now. If there 16447c478bd9Sstevel@tonic-gate * are no waiters, then set our pcf_wait 16457c478bd9Sstevel@tonic-gate * count to zero to avoid coming in here 16467c478bd9Sstevel@tonic-gate * next time. 16477c478bd9Sstevel@tonic-gate */ 16487c478bd9Sstevel@tonic-gate if (freemem_wait) { 16497c478bd9Sstevel@tonic-gate if (npages > 1) { 16507c478bd9Sstevel@tonic-gate cv_broadcast(&freemem_cv); 16517c478bd9Sstevel@tonic-gate } else { 16527c478bd9Sstevel@tonic-gate cv_signal(&freemem_cv); 16537c478bd9Sstevel@tonic-gate } 16547c478bd9Sstevel@tonic-gate p->pcf_wait--; 16557c478bd9Sstevel@tonic-gate } else { 16567c478bd9Sstevel@tonic-gate p->pcf_wait = 0; 16577c478bd9Sstevel@tonic-gate } 16587c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 16617c478bd9Sstevel@tonic-gate } 16627c478bd9Sstevel@tonic-gate ASSERT(npages == 0); 16637c478bd9Sstevel@tonic-gate } 16647c478bd9Sstevel@tonic-gate 16657c478bd9Sstevel@tonic-gate /* 16667c478bd9Sstevel@tonic-gate * A helper routine for page_create_get_something. 16677c478bd9Sstevel@tonic-gate * The indenting got to deep down there. 16687c478bd9Sstevel@tonic-gate * Unblock the pcf counters. Any pages freed after 16697c478bd9Sstevel@tonic-gate * pcf_block got set are moved to pcf_count and 16707c478bd9Sstevel@tonic-gate * wakeups (cv_broadcast() or cv_signal()) are done as needed. 16717c478bd9Sstevel@tonic-gate */ 16727c478bd9Sstevel@tonic-gate static void 16737c478bd9Sstevel@tonic-gate pcgs_unblock(void) 16747c478bd9Sstevel@tonic-gate { 16757c478bd9Sstevel@tonic-gate int i; 16767c478bd9Sstevel@tonic-gate struct pcf *p; 16777c478bd9Sstevel@tonic-gate 16787c478bd9Sstevel@tonic-gate /* Update freemem while we're here. */ 16797c478bd9Sstevel@tonic-gate freemem = 0; 16807c478bd9Sstevel@tonic-gate p = pcf; 16817c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 16827c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 16837c478bd9Sstevel@tonic-gate ASSERT(p->pcf_count == 0); 16847c478bd9Sstevel@tonic-gate p->pcf_count = p->pcf_reserve; 16857c478bd9Sstevel@tonic-gate p->pcf_block = 0; 16867c478bd9Sstevel@tonic-gate freemem += p->pcf_count; 16877c478bd9Sstevel@tonic-gate if (p->pcf_wait) { 16887c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 16897c478bd9Sstevel@tonic-gate if (freemem_wait) { 16907c478bd9Sstevel@tonic-gate if (p->pcf_reserve > 1) { 16917c478bd9Sstevel@tonic-gate cv_broadcast(&freemem_cv); 16927c478bd9Sstevel@tonic-gate p->pcf_wait = 0; 16937c478bd9Sstevel@tonic-gate } else { 16947c478bd9Sstevel@tonic-gate cv_signal(&freemem_cv); 16957c478bd9Sstevel@tonic-gate p->pcf_wait--; 16967c478bd9Sstevel@tonic-gate } 16977c478bd9Sstevel@tonic-gate } else { 16987c478bd9Sstevel@tonic-gate p->pcf_wait = 0; 16997c478bd9Sstevel@tonic-gate } 17007c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 17017c478bd9Sstevel@tonic-gate } 17027c478bd9Sstevel@tonic-gate p->pcf_reserve = 0; 17037c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 17047c478bd9Sstevel@tonic-gate p++; 17057c478bd9Sstevel@tonic-gate } 17067c478bd9Sstevel@tonic-gate } 17077c478bd9Sstevel@tonic-gate 17087c478bd9Sstevel@tonic-gate /* 17097c478bd9Sstevel@tonic-gate * Called from page_create_va() when both the cache and free lists 17107c478bd9Sstevel@tonic-gate * have been checked once. 17117c478bd9Sstevel@tonic-gate * 17127c478bd9Sstevel@tonic-gate * Either returns a page or panics since the accounting was done 17137c478bd9Sstevel@tonic-gate * way before we got here. 17147c478bd9Sstevel@tonic-gate * 17157c478bd9Sstevel@tonic-gate * We don't come here often, so leave the accounting on permanently. 17167c478bd9Sstevel@tonic-gate */ 17177c478bd9Sstevel@tonic-gate 17187c478bd9Sstevel@tonic-gate #define MAX_PCGS 100 17197c478bd9Sstevel@tonic-gate 17207c478bd9Sstevel@tonic-gate #ifdef DEBUG 17217c478bd9Sstevel@tonic-gate #define PCGS_TRIES 100 17227c478bd9Sstevel@tonic-gate #else /* DEBUG */ 17237c478bd9Sstevel@tonic-gate #define PCGS_TRIES 10 17247c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 17257c478bd9Sstevel@tonic-gate 17267c478bd9Sstevel@tonic-gate #ifdef VM_STATS 17277c478bd9Sstevel@tonic-gate uint_t pcgs_counts[PCGS_TRIES]; 17287c478bd9Sstevel@tonic-gate uint_t pcgs_too_many; 17297c478bd9Sstevel@tonic-gate uint_t pcgs_entered; 17307c478bd9Sstevel@tonic-gate uint_t pcgs_entered_noreloc; 17317c478bd9Sstevel@tonic-gate uint_t pcgs_locked; 17327c478bd9Sstevel@tonic-gate uint_t pcgs_cagelocked; 17337c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 17347c478bd9Sstevel@tonic-gate 17357c478bd9Sstevel@tonic-gate static page_t * 17367c478bd9Sstevel@tonic-gate page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg, 17377c478bd9Sstevel@tonic-gate caddr_t vaddr, uint_t flags) 17387c478bd9Sstevel@tonic-gate { 17397c478bd9Sstevel@tonic-gate uint_t count; 17407c478bd9Sstevel@tonic-gate page_t *pp; 17417c478bd9Sstevel@tonic-gate uint_t locked, i; 17427c478bd9Sstevel@tonic-gate struct pcf *p; 17437c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 17447c478bd9Sstevel@tonic-gate int cagelocked = 0; 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate VM_STAT_ADD(pcgs_entered); 17477c478bd9Sstevel@tonic-gate 17487c478bd9Sstevel@tonic-gate /* 17497c478bd9Sstevel@tonic-gate * Tap any reserve freelists: if we fail now, we'll die 17507c478bd9Sstevel@tonic-gate * since the page(s) we're looking for have already been 17517c478bd9Sstevel@tonic-gate * accounted for. 17527c478bd9Sstevel@tonic-gate */ 17537c478bd9Sstevel@tonic-gate flags |= PG_PANIC; 17547c478bd9Sstevel@tonic-gate 17557c478bd9Sstevel@tonic-gate if ((flags & PG_NORELOC) != 0) { 17567c478bd9Sstevel@tonic-gate VM_STAT_ADD(pcgs_entered_noreloc); 17577c478bd9Sstevel@tonic-gate /* 17587c478bd9Sstevel@tonic-gate * Requests for free pages from critical threads 17597c478bd9Sstevel@tonic-gate * such as pageout still won't throttle here, but 17607c478bd9Sstevel@tonic-gate * we must try again, to give the cageout thread 17617c478bd9Sstevel@tonic-gate * another chance to catch up. Since we already 17627c478bd9Sstevel@tonic-gate * accounted for the pages, we had better get them 17637c478bd9Sstevel@tonic-gate * this time. 17647c478bd9Sstevel@tonic-gate * 17657c478bd9Sstevel@tonic-gate * N.B. All non-critical threads acquire the pcgs_cagelock 17667c478bd9Sstevel@tonic-gate * to serialize access to the freelists. This implements a 17677c478bd9Sstevel@tonic-gate * turnstile-type synchornization to avoid starvation of 17687c478bd9Sstevel@tonic-gate * critical requests for PG_NORELOC memory by non-critical 17697c478bd9Sstevel@tonic-gate * threads: all non-critical threads must acquire a 'ticket' 17707c478bd9Sstevel@tonic-gate * before passing through, which entails making sure 17717c478bd9Sstevel@tonic-gate * kcage_freemem won't fall below minfree prior to grabbing 17727c478bd9Sstevel@tonic-gate * pages from the freelists. 17737c478bd9Sstevel@tonic-gate */ 17747c478bd9Sstevel@tonic-gate if (kcage_create_throttle(1, flags) == KCT_NONCRIT) { 17757c478bd9Sstevel@tonic-gate mutex_enter(&pcgs_cagelock); 17767c478bd9Sstevel@tonic-gate cagelocked = 1; 17777c478bd9Sstevel@tonic-gate VM_STAT_ADD(pcgs_cagelocked); 17787c478bd9Sstevel@tonic-gate } 17797c478bd9Sstevel@tonic-gate } 17807c478bd9Sstevel@tonic-gate 17817c478bd9Sstevel@tonic-gate /* 17827c478bd9Sstevel@tonic-gate * Time to get serious. 17837c478bd9Sstevel@tonic-gate * We failed to get a `correctly colored' page from both the 17847c478bd9Sstevel@tonic-gate * free and cache lists. 17857c478bd9Sstevel@tonic-gate * We escalate in stage. 17867c478bd9Sstevel@tonic-gate * 17877c478bd9Sstevel@tonic-gate * First try both lists without worring about color. 17887c478bd9Sstevel@tonic-gate * 17897c478bd9Sstevel@tonic-gate * Then, grab all page accounting locks (ie. pcf[]) and 17907c478bd9Sstevel@tonic-gate * steal any pages that they have and set the pcf_block flag to 17917c478bd9Sstevel@tonic-gate * stop deletions from the lists. This will help because 17927c478bd9Sstevel@tonic-gate * a page can get added to the free list while we are looking 17937c478bd9Sstevel@tonic-gate * at the cache list, then another page could be added to the cache 17947c478bd9Sstevel@tonic-gate * list allowing the page on the free list to be removed as we 17957c478bd9Sstevel@tonic-gate * move from looking at the cache list to the free list. This 17967c478bd9Sstevel@tonic-gate * could happen over and over. We would never find the page 17977c478bd9Sstevel@tonic-gate * we have accounted for. 17987c478bd9Sstevel@tonic-gate * 17997c478bd9Sstevel@tonic-gate * Noreloc pages are a subset of the global (relocatable) page pool. 18007c478bd9Sstevel@tonic-gate * They are not tracked separately in the pcf bins, so it is 18017c478bd9Sstevel@tonic-gate * impossible to know when doing pcf accounting if the available 18027c478bd9Sstevel@tonic-gate * page(s) are noreloc pages or not. When looking for a noreloc page 18037c478bd9Sstevel@tonic-gate * it is quite easy to end up here even if the global (relocatable) 18047c478bd9Sstevel@tonic-gate * page pool has plenty of free pages but the noreloc pool is empty. 18057c478bd9Sstevel@tonic-gate * 18067c478bd9Sstevel@tonic-gate * When the noreloc pool is empty (or low), additional noreloc pages 18077c478bd9Sstevel@tonic-gate * are created by converting pages from the global page pool. This 18087c478bd9Sstevel@tonic-gate * process will stall during pcf accounting if the pcf bins are 18097c478bd9Sstevel@tonic-gate * already locked. Such is the case when a noreloc allocation is 18107c478bd9Sstevel@tonic-gate * looping here in page_create_get_something waiting for more noreloc 18117c478bd9Sstevel@tonic-gate * pages to appear. 18127c478bd9Sstevel@tonic-gate * 18137c478bd9Sstevel@tonic-gate * Short of adding a new field to the pcf bins to accurately track 18147c478bd9Sstevel@tonic-gate * the number of free noreloc pages, we instead do not grab the 18157c478bd9Sstevel@tonic-gate * pcgs_lock, do not set the pcf blocks and do not timeout when 18167c478bd9Sstevel@tonic-gate * allocating a noreloc page. This allows noreloc allocations to 18177c478bd9Sstevel@tonic-gate * loop without blocking global page pool allocations. 18187c478bd9Sstevel@tonic-gate * 18197c478bd9Sstevel@tonic-gate * NOTE: the behaviour of page_create_get_something has not changed 18207c478bd9Sstevel@tonic-gate * for the case of global page pool allocations. 18217c478bd9Sstevel@tonic-gate */ 18227c478bd9Sstevel@tonic-gate 18237c478bd9Sstevel@tonic-gate flags &= ~PG_MATCH_COLOR; 18247c478bd9Sstevel@tonic-gate locked = 0; 182507ad560dSkchow #if defined(__i386) || defined(__amd64) 18267c478bd9Sstevel@tonic-gate /* 18277c478bd9Sstevel@tonic-gate * page_create_get_something may be called because 4g memory may be 18287c478bd9Sstevel@tonic-gate * depleted. Set flags to allow for relocation of base page below 18297c478bd9Sstevel@tonic-gate * 4g if necessary. 18307c478bd9Sstevel@tonic-gate */ 18317c478bd9Sstevel@tonic-gate if (physmax4g) 18327c478bd9Sstevel@tonic-gate flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI); 18337c478bd9Sstevel@tonic-gate #endif 18347c478bd9Sstevel@tonic-gate 18357c478bd9Sstevel@tonic-gate lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 18367c478bd9Sstevel@tonic-gate 18377c478bd9Sstevel@tonic-gate for (count = 0; kcage_on || count < MAX_PCGS; count++) { 18387c478bd9Sstevel@tonic-gate pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 18397c478bd9Sstevel@tonic-gate flags, lgrp); 18407c478bd9Sstevel@tonic-gate if (pp == NULL) { 18417c478bd9Sstevel@tonic-gate pp = page_get_cachelist(vp, off, seg, vaddr, 18427c478bd9Sstevel@tonic-gate flags, lgrp); 18437c478bd9Sstevel@tonic-gate } 18447c478bd9Sstevel@tonic-gate if (pp == NULL) { 18457c478bd9Sstevel@tonic-gate /* 18467c478bd9Sstevel@tonic-gate * Serialize. Don't fight with other pcgs(). 18477c478bd9Sstevel@tonic-gate */ 18487c478bd9Sstevel@tonic-gate if (!locked && (!kcage_on || !(flags & PG_NORELOC))) { 18497c478bd9Sstevel@tonic-gate mutex_enter(&pcgs_lock); 18507c478bd9Sstevel@tonic-gate VM_STAT_ADD(pcgs_locked); 18517c478bd9Sstevel@tonic-gate locked = 1; 18527c478bd9Sstevel@tonic-gate p = pcf; 18537c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 18547c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 18557c478bd9Sstevel@tonic-gate ASSERT(p->pcf_block == 0); 18567c478bd9Sstevel@tonic-gate p->pcf_block = 1; 18577c478bd9Sstevel@tonic-gate p->pcf_reserve = p->pcf_count; 18587c478bd9Sstevel@tonic-gate p->pcf_count = 0; 18597c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 18607c478bd9Sstevel@tonic-gate p++; 18617c478bd9Sstevel@tonic-gate } 18627c478bd9Sstevel@tonic-gate freemem = 0; 18637c478bd9Sstevel@tonic-gate } 18647c478bd9Sstevel@tonic-gate 18657c478bd9Sstevel@tonic-gate if (count) { 18667c478bd9Sstevel@tonic-gate /* 18677c478bd9Sstevel@tonic-gate * Since page_free() puts pages on 18687c478bd9Sstevel@tonic-gate * a list then accounts for it, we 18697c478bd9Sstevel@tonic-gate * just have to wait for page_free() 18707c478bd9Sstevel@tonic-gate * to unlock any page it was working 18717c478bd9Sstevel@tonic-gate * with. The page_lock()-page_reclaim() 18727c478bd9Sstevel@tonic-gate * path falls in the same boat. 18737c478bd9Sstevel@tonic-gate * 18747c478bd9Sstevel@tonic-gate * We don't need to check on the 18757c478bd9Sstevel@tonic-gate * PG_WAIT flag, we have already 18767c478bd9Sstevel@tonic-gate * accounted for the page we are 18777c478bd9Sstevel@tonic-gate * looking for in page_create_va(). 18787c478bd9Sstevel@tonic-gate * 18797c478bd9Sstevel@tonic-gate * We just wait a moment to let any 18807c478bd9Sstevel@tonic-gate * locked pages on the lists free up, 18817c478bd9Sstevel@tonic-gate * then continue around and try again. 18827c478bd9Sstevel@tonic-gate * 18837c478bd9Sstevel@tonic-gate * Will be awakened by set_freemem(). 18847c478bd9Sstevel@tonic-gate */ 18857c478bd9Sstevel@tonic-gate mutex_enter(&pcgs_wait_lock); 18867c478bd9Sstevel@tonic-gate cv_wait(&pcgs_cv, &pcgs_wait_lock); 18877c478bd9Sstevel@tonic-gate mutex_exit(&pcgs_wait_lock); 18887c478bd9Sstevel@tonic-gate } 18897c478bd9Sstevel@tonic-gate } else { 18907c478bd9Sstevel@tonic-gate #ifdef VM_STATS 18917c478bd9Sstevel@tonic-gate if (count >= PCGS_TRIES) { 18927c478bd9Sstevel@tonic-gate VM_STAT_ADD(pcgs_too_many); 18937c478bd9Sstevel@tonic-gate } else { 18947c478bd9Sstevel@tonic-gate VM_STAT_ADD(pcgs_counts[count]); 18957c478bd9Sstevel@tonic-gate } 18967c478bd9Sstevel@tonic-gate #endif 18977c478bd9Sstevel@tonic-gate if (locked) { 18987c478bd9Sstevel@tonic-gate pcgs_unblock(); 18997c478bd9Sstevel@tonic-gate mutex_exit(&pcgs_lock); 19007c478bd9Sstevel@tonic-gate } 19017c478bd9Sstevel@tonic-gate if (cagelocked) 19027c478bd9Sstevel@tonic-gate mutex_exit(&pcgs_cagelock); 19037c478bd9Sstevel@tonic-gate return (pp); 19047c478bd9Sstevel@tonic-gate } 19057c478bd9Sstevel@tonic-gate } 19067c478bd9Sstevel@tonic-gate /* 19077c478bd9Sstevel@tonic-gate * we go down holding the pcf locks. 19087c478bd9Sstevel@tonic-gate */ 19097c478bd9Sstevel@tonic-gate panic("no %spage found %d", 19107c478bd9Sstevel@tonic-gate ((flags & PG_NORELOC) ? "non-reloc " : ""), count); 19117c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 19127c478bd9Sstevel@tonic-gate } 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate /* 19157c478bd9Sstevel@tonic-gate * Create enough pages for "bytes" worth of data starting at 19167c478bd9Sstevel@tonic-gate * "off" in "vp". 19177c478bd9Sstevel@tonic-gate * 19187c478bd9Sstevel@tonic-gate * Where flag must be one of: 19197c478bd9Sstevel@tonic-gate * 19207c478bd9Sstevel@tonic-gate * PG_EXCL: Exclusive create (fail if any page already 19217c478bd9Sstevel@tonic-gate * exists in the page cache) which does not 19227c478bd9Sstevel@tonic-gate * wait for memory to become available. 19237c478bd9Sstevel@tonic-gate * 19247c478bd9Sstevel@tonic-gate * PG_WAIT: Non-exclusive create which can wait for 19257c478bd9Sstevel@tonic-gate * memory to become available. 19267c478bd9Sstevel@tonic-gate * 19277c478bd9Sstevel@tonic-gate * PG_PHYSCONTIG: Allocate physically contiguous pages. 19287c478bd9Sstevel@tonic-gate * (Not Supported) 19297c478bd9Sstevel@tonic-gate * 19307c478bd9Sstevel@tonic-gate * A doubly linked list of pages is returned to the caller. Each page 19317c478bd9Sstevel@tonic-gate * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock) 19327c478bd9Sstevel@tonic-gate * lock. 19337c478bd9Sstevel@tonic-gate * 19347c478bd9Sstevel@tonic-gate * Unable to change the parameters to page_create() in a minor release, 19357c478bd9Sstevel@tonic-gate * we renamed page_create() to page_create_va(), changed all known calls 19367c478bd9Sstevel@tonic-gate * from page_create() to page_create_va(), and created this wrapper. 19377c478bd9Sstevel@tonic-gate * 19387c478bd9Sstevel@tonic-gate * Upon a major release, we should break compatibility by deleting this 19397c478bd9Sstevel@tonic-gate * wrapper, and replacing all the strings "page_create_va", with "page_create". 19407c478bd9Sstevel@tonic-gate * 19417c478bd9Sstevel@tonic-gate * NOTE: There is a copy of this interface as page_create_io() in 19427c478bd9Sstevel@tonic-gate * i86/vm/vm_machdep.c. Any bugs fixed here should be applied 19437c478bd9Sstevel@tonic-gate * there. 19447c478bd9Sstevel@tonic-gate */ 19457c478bd9Sstevel@tonic-gate page_t * 19467c478bd9Sstevel@tonic-gate page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags) 19477c478bd9Sstevel@tonic-gate { 19487c478bd9Sstevel@tonic-gate caddr_t random_vaddr; 19497c478bd9Sstevel@tonic-gate struct seg kseg; 19507c478bd9Sstevel@tonic-gate 19517c478bd9Sstevel@tonic-gate #ifdef DEBUG 19527c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p", 19537c478bd9Sstevel@tonic-gate (void *)caller()); 19547c478bd9Sstevel@tonic-gate #endif 19557c478bd9Sstevel@tonic-gate 19567c478bd9Sstevel@tonic-gate random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^ 19577c478bd9Sstevel@tonic-gate (uintptr_t)(off >> PAGESHIFT)); 19587c478bd9Sstevel@tonic-gate kseg.s_as = &kas; 19597c478bd9Sstevel@tonic-gate 19607c478bd9Sstevel@tonic-gate return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr)); 19617c478bd9Sstevel@tonic-gate } 19627c478bd9Sstevel@tonic-gate 19637c478bd9Sstevel@tonic-gate #ifdef DEBUG 19647c478bd9Sstevel@tonic-gate uint32_t pg_alloc_pgs_mtbf = 0; 19657c478bd9Sstevel@tonic-gate #endif 19667c478bd9Sstevel@tonic-gate 19677c478bd9Sstevel@tonic-gate /* 19687c478bd9Sstevel@tonic-gate * Used for large page support. It will attempt to allocate 19697c478bd9Sstevel@tonic-gate * a large page(s) off the freelist. 19707c478bd9Sstevel@tonic-gate * 19717c478bd9Sstevel@tonic-gate * Returns non zero on failure. 19727c478bd9Sstevel@tonic-gate */ 19737c478bd9Sstevel@tonic-gate int 1974e44bd21cSsusans page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr, 1975e44bd21cSsusans page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz) 19767c478bd9Sstevel@tonic-gate { 19777c478bd9Sstevel@tonic-gate pgcnt_t npgs, curnpgs, totpgs; 19787c478bd9Sstevel@tonic-gate size_t pgsz; 19797c478bd9Sstevel@tonic-gate page_t *pplist = NULL, *pp; 19807c478bd9Sstevel@tonic-gate int err = 0; 19817c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 19827c478bd9Sstevel@tonic-gate 19837c478bd9Sstevel@tonic-gate ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1)); 19847c478bd9Sstevel@tonic-gate 19857c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[0]); 19867c478bd9Sstevel@tonic-gate 19877c478bd9Sstevel@tonic-gate #ifdef DEBUG 19887c478bd9Sstevel@tonic-gate if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) { 19897c478bd9Sstevel@tonic-gate return (ENOMEM); 19907c478bd9Sstevel@tonic-gate } 19917c478bd9Sstevel@tonic-gate #endif 19927c478bd9Sstevel@tonic-gate 19937c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(szc); 19947c478bd9Sstevel@tonic-gate totpgs = curnpgs = npgs = pgsz >> PAGESHIFT; 19957c478bd9Sstevel@tonic-gate 19967c478bd9Sstevel@tonic-gate ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0); 19977c478bd9Sstevel@tonic-gate /* 19987c478bd9Sstevel@tonic-gate * One must be NULL but not both. 19997c478bd9Sstevel@tonic-gate * And one must be non NULL but not both. 20007c478bd9Sstevel@tonic-gate */ 20017c478bd9Sstevel@tonic-gate ASSERT(basepp != NULL || ppa != NULL); 20027c478bd9Sstevel@tonic-gate ASSERT(basepp == NULL || ppa == NULL); 20037c478bd9Sstevel@tonic-gate 20047c478bd9Sstevel@tonic-gate (void) page_create_wait(npgs, PG_WAIT); 20057c478bd9Sstevel@tonic-gate 20067c478bd9Sstevel@tonic-gate while (npgs && szc) { 20077c478bd9Sstevel@tonic-gate lgrp = lgrp_mem_choose(seg, addr, pgsz); 2008e44bd21cSsusans pp = page_get_freelist(vp, 0, seg, addr, pgsz, 0, lgrp); 20097c478bd9Sstevel@tonic-gate if (pp != NULL) { 20107c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[1]); 20117c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 20127c478bd9Sstevel@tonic-gate ASSERT(npgs >= curnpgs); 20137c478bd9Sstevel@tonic-gate npgs -= curnpgs; 20147c478bd9Sstevel@tonic-gate } else if (anypgsz) { 20157c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[2]); 20167c478bd9Sstevel@tonic-gate szc--; 20177c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(szc); 20187c478bd9Sstevel@tonic-gate curnpgs = pgsz >> PAGESHIFT; 20197c478bd9Sstevel@tonic-gate } else { 20207c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[3]); 20217c478bd9Sstevel@tonic-gate ASSERT(npgs == totpgs); 20227c478bd9Sstevel@tonic-gate page_create_putback(npgs); 20237c478bd9Sstevel@tonic-gate return (ENOMEM); 20247c478bd9Sstevel@tonic-gate } 20257c478bd9Sstevel@tonic-gate } 20267c478bd9Sstevel@tonic-gate if (szc == 0) { 20277c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[4]); 20287c478bd9Sstevel@tonic-gate ASSERT(npgs != 0); 20297c478bd9Sstevel@tonic-gate page_create_putback(npgs); 20307c478bd9Sstevel@tonic-gate err = ENOMEM; 20317c478bd9Sstevel@tonic-gate } else if (basepp != NULL) { 20327c478bd9Sstevel@tonic-gate ASSERT(npgs == 0); 20337c478bd9Sstevel@tonic-gate ASSERT(ppa == NULL); 20347c478bd9Sstevel@tonic-gate *basepp = pplist; 20357c478bd9Sstevel@tonic-gate } 20367c478bd9Sstevel@tonic-gate 20377c478bd9Sstevel@tonic-gate npgs = totpgs - npgs; 20387c478bd9Sstevel@tonic-gate pp = pplist; 20397c478bd9Sstevel@tonic-gate 20407c478bd9Sstevel@tonic-gate /* 20417c478bd9Sstevel@tonic-gate * Clear the free and age bits. Also if we were passed in a ppa then 20427c478bd9Sstevel@tonic-gate * fill it in with all the constituent pages from the large page. But 20437c478bd9Sstevel@tonic-gate * if we failed to allocate all the pages just free what we got. 20447c478bd9Sstevel@tonic-gate */ 20457c478bd9Sstevel@tonic-gate while (npgs != 0) { 20467c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 20477c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 20487c478bd9Sstevel@tonic-gate if (ppa != NULL || err != 0) { 20497c478bd9Sstevel@tonic-gate if (err == 0) { 20507c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[5]); 20517c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 20527c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 20537c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 20547c478bd9Sstevel@tonic-gate *ppa++ = pp; 20557c478bd9Sstevel@tonic-gate npgs--; 20567c478bd9Sstevel@tonic-gate } else { 20577c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[6]); 20587c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0); 20597c478bd9Sstevel@tonic-gate curnpgs = page_get_pagecnt(pp->p_szc); 20607c478bd9Sstevel@tonic-gate page_list_break(&pp, &pplist, curnpgs); 20617c478bd9Sstevel@tonic-gate page_list_add_pages(pp, 0); 20627c478bd9Sstevel@tonic-gate page_create_putback(curnpgs); 20637c478bd9Sstevel@tonic-gate ASSERT(npgs >= curnpgs); 20647c478bd9Sstevel@tonic-gate npgs -= curnpgs; 20657c478bd9Sstevel@tonic-gate } 20667c478bd9Sstevel@tonic-gate pp = pplist; 20677c478bd9Sstevel@tonic-gate } else { 20687c478bd9Sstevel@tonic-gate VM_STAT_ADD(alloc_pages[7]); 20697c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 20707c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 20717c478bd9Sstevel@tonic-gate pp = pp->p_next; 20727c478bd9Sstevel@tonic-gate npgs--; 20737c478bd9Sstevel@tonic-gate } 20747c478bd9Sstevel@tonic-gate } 20757c478bd9Sstevel@tonic-gate return (err); 20767c478bd9Sstevel@tonic-gate } 20777c478bd9Sstevel@tonic-gate 20787c478bd9Sstevel@tonic-gate /* 20797c478bd9Sstevel@tonic-gate * Get a single large page off of the freelists, and set it up for use. 20807c478bd9Sstevel@tonic-gate * Number of bytes requested must be a supported page size. 20817c478bd9Sstevel@tonic-gate * 20827c478bd9Sstevel@tonic-gate * Note that this call may fail even if there is sufficient 20837c478bd9Sstevel@tonic-gate * memory available or PG_WAIT is set, so the caller must 20847c478bd9Sstevel@tonic-gate * be willing to fallback on page_create_va(), block and retry, 20857c478bd9Sstevel@tonic-gate * or fail the requester. 20867c478bd9Sstevel@tonic-gate */ 20877c478bd9Sstevel@tonic-gate page_t * 20887c478bd9Sstevel@tonic-gate page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 20897c478bd9Sstevel@tonic-gate struct seg *seg, caddr_t vaddr, void *arg) 20907c478bd9Sstevel@tonic-gate { 20917c478bd9Sstevel@tonic-gate pgcnt_t npages, pcftotal; 20927c478bd9Sstevel@tonic-gate page_t *pp; 20937c478bd9Sstevel@tonic-gate page_t *rootpp; 20947c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 20957c478bd9Sstevel@tonic-gate uint_t enough; 20967c478bd9Sstevel@tonic-gate uint_t pcf_index; 20977c478bd9Sstevel@tonic-gate uint_t i; 20987c478bd9Sstevel@tonic-gate struct pcf *p; 20997c478bd9Sstevel@tonic-gate struct pcf *q; 21007c478bd9Sstevel@tonic-gate lgrp_id_t *lgrpid = (lgrp_id_t *)arg; 21017c478bd9Sstevel@tonic-gate 21027c478bd9Sstevel@tonic-gate ASSERT(vp != NULL); 21037c478bd9Sstevel@tonic-gate 21047c478bd9Sstevel@tonic-gate ASSERT((flags & ~(PG_EXCL | PG_WAIT | 21057c478bd9Sstevel@tonic-gate PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 21067c478bd9Sstevel@tonic-gate /* but no others */ 21077c478bd9Sstevel@tonic-gate 21087c478bd9Sstevel@tonic-gate ASSERT((flags & PG_EXCL) == PG_EXCL); 21097c478bd9Sstevel@tonic-gate 21107c478bd9Sstevel@tonic-gate npages = btop(bytes); 21117c478bd9Sstevel@tonic-gate 21127c478bd9Sstevel@tonic-gate if (!kcage_on || panicstr) { 21137c478bd9Sstevel@tonic-gate /* 21147c478bd9Sstevel@tonic-gate * Cage is OFF, or we are single threaded in 21157c478bd9Sstevel@tonic-gate * panic, so make everything a RELOC request. 21167c478bd9Sstevel@tonic-gate */ 21177c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 21187c478bd9Sstevel@tonic-gate } 21197c478bd9Sstevel@tonic-gate 21207c478bd9Sstevel@tonic-gate /* 21217c478bd9Sstevel@tonic-gate * Make sure there's adequate physical memory available. 21227c478bd9Sstevel@tonic-gate * Note: PG_WAIT is ignored here. 21237c478bd9Sstevel@tonic-gate */ 21247c478bd9Sstevel@tonic-gate if (freemem <= throttlefree + npages) { 21257c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[1]); 21267c478bd9Sstevel@tonic-gate return (NULL); 21277c478bd9Sstevel@tonic-gate } 21287c478bd9Sstevel@tonic-gate 21297c478bd9Sstevel@tonic-gate /* 21307c478bd9Sstevel@tonic-gate * If cage is on, dampen draw from cage when available 21317c478bd9Sstevel@tonic-gate * cage space is low. 21327c478bd9Sstevel@tonic-gate */ 21337c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_WAIT)) == (PG_NORELOC | PG_WAIT) && 21347c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + npages) { 21357c478bd9Sstevel@tonic-gate 21367c478bd9Sstevel@tonic-gate /* 21377c478bd9Sstevel@tonic-gate * The cage is on, the caller wants PG_NORELOC 21387c478bd9Sstevel@tonic-gate * pages and available cage memory is very low. 21397c478bd9Sstevel@tonic-gate * Call kcage_create_throttle() to attempt to 21407c478bd9Sstevel@tonic-gate * control demand on the cage. 21417c478bd9Sstevel@tonic-gate */ 21427c478bd9Sstevel@tonic-gate if (kcage_create_throttle(npages, flags) == KCT_FAILURE) { 21437c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[2]); 21447c478bd9Sstevel@tonic-gate return (NULL); 21457c478bd9Sstevel@tonic-gate } 21467c478bd9Sstevel@tonic-gate } 21477c478bd9Sstevel@tonic-gate 21487c478bd9Sstevel@tonic-gate enough = 0; 21497c478bd9Sstevel@tonic-gate pcf_index = PCF_INDEX(); 21507c478bd9Sstevel@tonic-gate p = &pcf[pcf_index]; 21517c478bd9Sstevel@tonic-gate q = &pcf[PCF_FANOUT]; 21527c478bd9Sstevel@tonic-gate for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 21537c478bd9Sstevel@tonic-gate if (p->pcf_count > npages) { 21547c478bd9Sstevel@tonic-gate /* 21557c478bd9Sstevel@tonic-gate * a good one to try. 21567c478bd9Sstevel@tonic-gate */ 21577c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 21587c478bd9Sstevel@tonic-gate if (p->pcf_count > npages) { 21597c478bd9Sstevel@tonic-gate p->pcf_count -= (uint_t)npages; 21607c478bd9Sstevel@tonic-gate /* 21617c478bd9Sstevel@tonic-gate * freemem is not protected by any lock. 21627c478bd9Sstevel@tonic-gate * Thus, we cannot have any assertion 21637c478bd9Sstevel@tonic-gate * containing freemem here. 21647c478bd9Sstevel@tonic-gate */ 21657c478bd9Sstevel@tonic-gate freemem -= npages; 21667c478bd9Sstevel@tonic-gate enough = 1; 21677c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 21687c478bd9Sstevel@tonic-gate break; 21697c478bd9Sstevel@tonic-gate } 21707c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 21717c478bd9Sstevel@tonic-gate } 21727c478bd9Sstevel@tonic-gate pcftotal += p->pcf_count; 21737c478bd9Sstevel@tonic-gate p++; 21747c478bd9Sstevel@tonic-gate if (p >= q) { 21757c478bd9Sstevel@tonic-gate p = pcf; 21767c478bd9Sstevel@tonic-gate } 21777c478bd9Sstevel@tonic-gate } 21787c478bd9Sstevel@tonic-gate 21797c478bd9Sstevel@tonic-gate if (!enough) { 21807c478bd9Sstevel@tonic-gate /* If there isn't enough memory available, give up. */ 21817c478bd9Sstevel@tonic-gate if (pcftotal < npages) { 21827c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[3]); 21837c478bd9Sstevel@tonic-gate return (NULL); 21847c478bd9Sstevel@tonic-gate } 21857c478bd9Sstevel@tonic-gate 21867c478bd9Sstevel@tonic-gate /* try to collect pages from several pcf bins */ 21877c478bd9Sstevel@tonic-gate for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) { 21887c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 21897c478bd9Sstevel@tonic-gate pcftotal += p->pcf_count; 21907c478bd9Sstevel@tonic-gate if (pcftotal >= npages) { 21917c478bd9Sstevel@tonic-gate /* 21927c478bd9Sstevel@tonic-gate * Wow! There are enough pages laying around 21937c478bd9Sstevel@tonic-gate * to satisfy the request. Do the accounting, 21947c478bd9Sstevel@tonic-gate * drop the locks we acquired, and go back. 21957c478bd9Sstevel@tonic-gate * 21967c478bd9Sstevel@tonic-gate * freemem is not protected by any lock. So, 21977c478bd9Sstevel@tonic-gate * we cannot have any assertion containing 21987c478bd9Sstevel@tonic-gate * freemem. 21997c478bd9Sstevel@tonic-gate */ 22007c478bd9Sstevel@tonic-gate pgcnt_t tpages = npages; 22017c478bd9Sstevel@tonic-gate freemem -= npages; 22027c478bd9Sstevel@tonic-gate while (p >= pcf) { 22037c478bd9Sstevel@tonic-gate if (p->pcf_count <= tpages) { 22047c478bd9Sstevel@tonic-gate tpages -= p->pcf_count; 22057c478bd9Sstevel@tonic-gate p->pcf_count = 0; 22067c478bd9Sstevel@tonic-gate } else { 22077c478bd9Sstevel@tonic-gate p->pcf_count -= (uint_t)tpages; 22087c478bd9Sstevel@tonic-gate tpages = 0; 22097c478bd9Sstevel@tonic-gate } 22107c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 22117c478bd9Sstevel@tonic-gate p--; 22127c478bd9Sstevel@tonic-gate } 22137c478bd9Sstevel@tonic-gate ASSERT(tpages == 0); 22147c478bd9Sstevel@tonic-gate break; 22157c478bd9Sstevel@tonic-gate } 22167c478bd9Sstevel@tonic-gate p++; 22177c478bd9Sstevel@tonic-gate } 22187c478bd9Sstevel@tonic-gate if (i == PCF_FANOUT) { 22197c478bd9Sstevel@tonic-gate /* failed to collect pages - release the locks */ 22207c478bd9Sstevel@tonic-gate while (--p >= pcf) { 22217c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 22227c478bd9Sstevel@tonic-gate } 22237c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[4]); 22247c478bd9Sstevel@tonic-gate return (NULL); 22257c478bd9Sstevel@tonic-gate } 22267c478bd9Sstevel@tonic-gate } 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate /* 22297c478bd9Sstevel@tonic-gate * This is where this function behaves fundamentally differently 22307c478bd9Sstevel@tonic-gate * than page_create_va(); since we're intending to map the page 22317c478bd9Sstevel@tonic-gate * with a single TTE, we have to get it as a physically contiguous 22327c478bd9Sstevel@tonic-gate * hardware pagesize chunk. If we can't, we fail. 22337c478bd9Sstevel@tonic-gate */ 22347c478bd9Sstevel@tonic-gate if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max && 22357c478bd9Sstevel@tonic-gate LGRP_EXISTS(lgrp_table[*lgrpid])) 22367c478bd9Sstevel@tonic-gate lgrp = lgrp_table[*lgrpid]; 22377c478bd9Sstevel@tonic-gate else 22387c478bd9Sstevel@tonic-gate lgrp = lgrp_mem_choose(seg, vaddr, bytes); 22397c478bd9Sstevel@tonic-gate 22407c478bd9Sstevel@tonic-gate if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr, 22417c478bd9Sstevel@tonic-gate bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) { 22427c478bd9Sstevel@tonic-gate page_create_putback(npages); 22437c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[5]); 22447c478bd9Sstevel@tonic-gate return (NULL); 22457c478bd9Sstevel@tonic-gate } 22467c478bd9Sstevel@tonic-gate 22477c478bd9Sstevel@tonic-gate /* 22487c478bd9Sstevel@tonic-gate * if we got the page with the wrong mtype give it back this is a 22497c478bd9Sstevel@tonic-gate * workaround for CR 6249718. When CR 6249718 is fixed we never get 22507c478bd9Sstevel@tonic-gate * inside "if" and the workaround becomes just a nop 22517c478bd9Sstevel@tonic-gate */ 22527c478bd9Sstevel@tonic-gate if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) { 22537c478bd9Sstevel@tonic-gate page_list_add_pages(rootpp, 0); 22547c478bd9Sstevel@tonic-gate page_create_putback(npages); 22557c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[6]); 22567c478bd9Sstevel@tonic-gate return (NULL); 22577c478bd9Sstevel@tonic-gate } 22587c478bd9Sstevel@tonic-gate 22597c478bd9Sstevel@tonic-gate /* 22607c478bd9Sstevel@tonic-gate * If satisfying this request has left us with too little 22617c478bd9Sstevel@tonic-gate * memory, start the wheels turning to get some back. The 22627c478bd9Sstevel@tonic-gate * first clause of the test prevents waking up the pageout 22637c478bd9Sstevel@tonic-gate * daemon in situations where it would decide that there's 22647c478bd9Sstevel@tonic-gate * nothing to do. 22657c478bd9Sstevel@tonic-gate */ 22667c478bd9Sstevel@tonic-gate if (nscan < desscan && freemem < minfree) { 22677c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 22687c478bd9Sstevel@tonic-gate "pageout_cv_signal:freemem %ld", freemem); 22697c478bd9Sstevel@tonic-gate cv_signal(&proc_pageout->p_cv); 22707c478bd9Sstevel@tonic-gate } 22717c478bd9Sstevel@tonic-gate 22727c478bd9Sstevel@tonic-gate pp = rootpp; 22737c478bd9Sstevel@tonic-gate while (npages--) { 22747c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 22757c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 22767c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 22777c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 22787c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 22797c478bd9Sstevel@tonic-gate if (!page_hashin(pp, vp, off, NULL)) 22807c478bd9Sstevel@tonic-gate panic("page_create_large: hashin failed: page %p", 22817c478bd9Sstevel@tonic-gate (void *)pp); 22827c478bd9Sstevel@tonic-gate page_io_lock(pp); 22837c478bd9Sstevel@tonic-gate off += PAGESIZE; 22847c478bd9Sstevel@tonic-gate pp = pp->p_next; 22857c478bd9Sstevel@tonic-gate } 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_large_cnt[0]); 22887c478bd9Sstevel@tonic-gate return (rootpp); 22897c478bd9Sstevel@tonic-gate } 22907c478bd9Sstevel@tonic-gate 22917c478bd9Sstevel@tonic-gate page_t * 22927c478bd9Sstevel@tonic-gate page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags, 22937c478bd9Sstevel@tonic-gate struct seg *seg, caddr_t vaddr) 22947c478bd9Sstevel@tonic-gate { 22957c478bd9Sstevel@tonic-gate page_t *plist = NULL; 22967c478bd9Sstevel@tonic-gate pgcnt_t npages; 22977c478bd9Sstevel@tonic-gate pgcnt_t found_on_free = 0; 22987c478bd9Sstevel@tonic-gate pgcnt_t pages_req; 22997c478bd9Sstevel@tonic-gate page_t *npp = NULL; 23007c478bd9Sstevel@tonic-gate uint_t enough; 23017c478bd9Sstevel@tonic-gate uint_t i; 23027c478bd9Sstevel@tonic-gate uint_t pcf_index; 23037c478bd9Sstevel@tonic-gate struct pcf *p; 23047c478bd9Sstevel@tonic-gate struct pcf *q; 23057c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 23067c478bd9Sstevel@tonic-gate 23077c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START, 23087c478bd9Sstevel@tonic-gate "page_create_start:vp %p off %llx bytes %lu flags %x", 23097c478bd9Sstevel@tonic-gate vp, off, bytes, flags); 23107c478bd9Sstevel@tonic-gate 23117c478bd9Sstevel@tonic-gate ASSERT(bytes != 0 && vp != NULL); 23127c478bd9Sstevel@tonic-gate 23137c478bd9Sstevel@tonic-gate if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) { 23147c478bd9Sstevel@tonic-gate panic("page_create: invalid flags"); 23157c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 23167c478bd9Sstevel@tonic-gate } 23177c478bd9Sstevel@tonic-gate ASSERT((flags & ~(PG_EXCL | PG_WAIT | 23187c478bd9Sstevel@tonic-gate PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0); 23197c478bd9Sstevel@tonic-gate /* but no others */ 23207c478bd9Sstevel@tonic-gate 23217c478bd9Sstevel@tonic-gate pages_req = npages = btopr(bytes); 23227c478bd9Sstevel@tonic-gate /* 23237c478bd9Sstevel@tonic-gate * Try to see whether request is too large to *ever* be 23247c478bd9Sstevel@tonic-gate * satisfied, in order to prevent deadlock. We arbitrarily 23257c478bd9Sstevel@tonic-gate * decide to limit maximum size requests to max_page_get. 23267c478bd9Sstevel@tonic-gate */ 23277c478bd9Sstevel@tonic-gate if (npages >= max_page_get) { 23287c478bd9Sstevel@tonic-gate if ((flags & PG_WAIT) == 0) { 23297c478bd9Sstevel@tonic-gate TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG, 23307c478bd9Sstevel@tonic-gate "page_create_toobig:vp %p off %llx npages " 23317c478bd9Sstevel@tonic-gate "%lu max_page_get %lu", 23327c478bd9Sstevel@tonic-gate vp, off, npages, max_page_get); 23337c478bd9Sstevel@tonic-gate return (NULL); 23347c478bd9Sstevel@tonic-gate } else { 23357c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 23367c478bd9Sstevel@tonic-gate "Request for too much kernel memory " 23377c478bd9Sstevel@tonic-gate "(%lu bytes), will hang forever", bytes); 23387c478bd9Sstevel@tonic-gate for (;;) 23397c478bd9Sstevel@tonic-gate delay(1000000000); 23407c478bd9Sstevel@tonic-gate } 23417c478bd9Sstevel@tonic-gate } 23427c478bd9Sstevel@tonic-gate 23437c478bd9Sstevel@tonic-gate if (!kcage_on || panicstr) { 23447c478bd9Sstevel@tonic-gate /* 23457c478bd9Sstevel@tonic-gate * Cage is OFF, or we are single threaded in 23467c478bd9Sstevel@tonic-gate * panic, so make everything a RELOC request. 23477c478bd9Sstevel@tonic-gate */ 23487c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 23497c478bd9Sstevel@tonic-gate } 23507c478bd9Sstevel@tonic-gate 23517c478bd9Sstevel@tonic-gate if (freemem <= throttlefree + npages) 23527c478bd9Sstevel@tonic-gate if (!page_create_throttle(npages, flags)) 23537c478bd9Sstevel@tonic-gate return (NULL); 23547c478bd9Sstevel@tonic-gate 23557c478bd9Sstevel@tonic-gate /* 23567c478bd9Sstevel@tonic-gate * If cage is on, dampen draw from cage when available 23577c478bd9Sstevel@tonic-gate * cage space is low. 23587c478bd9Sstevel@tonic-gate */ 23597c478bd9Sstevel@tonic-gate if ((flags & PG_NORELOC) && 23607c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + npages) { 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate /* 23637c478bd9Sstevel@tonic-gate * The cage is on, the caller wants PG_NORELOC 23647c478bd9Sstevel@tonic-gate * pages and available cage memory is very low. 23657c478bd9Sstevel@tonic-gate * Call kcage_create_throttle() to attempt to 23667c478bd9Sstevel@tonic-gate * control demand on the cage. 23677c478bd9Sstevel@tonic-gate */ 23687c478bd9Sstevel@tonic-gate if (kcage_create_throttle(npages, flags) == KCT_FAILURE) 23697c478bd9Sstevel@tonic-gate return (NULL); 23707c478bd9Sstevel@tonic-gate } 23717c478bd9Sstevel@tonic-gate 23727c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_cnt[0]); 23737c478bd9Sstevel@tonic-gate 23747c478bd9Sstevel@tonic-gate enough = 0; 23757c478bd9Sstevel@tonic-gate pcf_index = PCF_INDEX(); 23767c478bd9Sstevel@tonic-gate 23777c478bd9Sstevel@tonic-gate p = &pcf[pcf_index]; 23787c478bd9Sstevel@tonic-gate q = &pcf[PCF_FANOUT]; 23797c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 23807c478bd9Sstevel@tonic-gate if (p->pcf_count > npages) { 23817c478bd9Sstevel@tonic-gate /* 23827c478bd9Sstevel@tonic-gate * a good one to try. 23837c478bd9Sstevel@tonic-gate */ 23847c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 23857c478bd9Sstevel@tonic-gate if (p->pcf_count > npages) { 23867c478bd9Sstevel@tonic-gate p->pcf_count -= (uint_t)npages; 23877c478bd9Sstevel@tonic-gate /* 23887c478bd9Sstevel@tonic-gate * freemem is not protected by any lock. 23897c478bd9Sstevel@tonic-gate * Thus, we cannot have any assertion 23907c478bd9Sstevel@tonic-gate * containing freemem here. 23917c478bd9Sstevel@tonic-gate */ 23927c478bd9Sstevel@tonic-gate freemem -= npages; 23937c478bd9Sstevel@tonic-gate enough = 1; 23947c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 23957c478bd9Sstevel@tonic-gate break; 23967c478bd9Sstevel@tonic-gate } 23977c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 23987c478bd9Sstevel@tonic-gate } 23997c478bd9Sstevel@tonic-gate p++; 24007c478bd9Sstevel@tonic-gate if (p >= q) { 24017c478bd9Sstevel@tonic-gate p = pcf; 24027c478bd9Sstevel@tonic-gate } 24037c478bd9Sstevel@tonic-gate } 24047c478bd9Sstevel@tonic-gate 24057c478bd9Sstevel@tonic-gate if (!enough) { 24067c478bd9Sstevel@tonic-gate /* 24077c478bd9Sstevel@tonic-gate * Have to look harder. If npages is greater than 24087c478bd9Sstevel@tonic-gate * one, then we might have to coalecse the counters. 24097c478bd9Sstevel@tonic-gate * 24107c478bd9Sstevel@tonic-gate * Go wait. We come back having accounted 24117c478bd9Sstevel@tonic-gate * for the memory. 24127c478bd9Sstevel@tonic-gate */ 24137c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_cnt[1]); 24147c478bd9Sstevel@tonic-gate if (!page_create_wait(npages, flags)) { 24157c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_cnt[2]); 24167c478bd9Sstevel@tonic-gate return (NULL); 24177c478bd9Sstevel@tonic-gate } 24187c478bd9Sstevel@tonic-gate } 24197c478bd9Sstevel@tonic-gate 24207c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS, 24217c478bd9Sstevel@tonic-gate "page_create_success:vp %p off %llx", vp, off); 24227c478bd9Sstevel@tonic-gate 24237c478bd9Sstevel@tonic-gate /* 24247c478bd9Sstevel@tonic-gate * If satisfying this request has left us with too little 24257c478bd9Sstevel@tonic-gate * memory, start the wheels turning to get some back. The 24267c478bd9Sstevel@tonic-gate * first clause of the test prevents waking up the pageout 24277c478bd9Sstevel@tonic-gate * daemon in situations where it would decide that there's 24287c478bd9Sstevel@tonic-gate * nothing to do. 24297c478bd9Sstevel@tonic-gate */ 24307c478bd9Sstevel@tonic-gate if (nscan < desscan && freemem < minfree) { 24317c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL, 24327c478bd9Sstevel@tonic-gate "pageout_cv_signal:freemem %ld", freemem); 24337c478bd9Sstevel@tonic-gate cv_signal(&proc_pageout->p_cv); 24347c478bd9Sstevel@tonic-gate } 24357c478bd9Sstevel@tonic-gate 24367c478bd9Sstevel@tonic-gate /* 24377c478bd9Sstevel@tonic-gate * Loop around collecting the requested number of pages. 24387c478bd9Sstevel@tonic-gate * Most of the time, we have to `create' a new page. With 24397c478bd9Sstevel@tonic-gate * this in mind, pull the page off the free list before 24407c478bd9Sstevel@tonic-gate * getting the hash lock. This will minimize the hash 24417c478bd9Sstevel@tonic-gate * lock hold time, nesting, and the like. If it turns 24427c478bd9Sstevel@tonic-gate * out we don't need the page, we put it back at the end. 24437c478bd9Sstevel@tonic-gate */ 24447c478bd9Sstevel@tonic-gate while (npages--) { 24457c478bd9Sstevel@tonic-gate page_t *pp; 24467c478bd9Sstevel@tonic-gate kmutex_t *phm = NULL; 24477c478bd9Sstevel@tonic-gate ulong_t index; 24487c478bd9Sstevel@tonic-gate 24497c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 24507c478bd9Sstevel@tonic-gate top: 24517c478bd9Sstevel@tonic-gate ASSERT(phm == NULL); 24527c478bd9Sstevel@tonic-gate ASSERT(index == PAGE_HASH_FUNC(vp, off)); 24537c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate if (npp == NULL) { 24567c478bd9Sstevel@tonic-gate /* 24577c478bd9Sstevel@tonic-gate * Try to get a page from the freelist (ie, 24587c478bd9Sstevel@tonic-gate * a page with no [vp, off] tag). If that 24597c478bd9Sstevel@tonic-gate * fails, use the cachelist. 24607c478bd9Sstevel@tonic-gate * 24617c478bd9Sstevel@tonic-gate * During the first attempt at both the free 24627c478bd9Sstevel@tonic-gate * and cache lists we try for the correct color. 24637c478bd9Sstevel@tonic-gate */ 24647c478bd9Sstevel@tonic-gate /* 24657c478bd9Sstevel@tonic-gate * XXXX-how do we deal with virtual indexed 24667c478bd9Sstevel@tonic-gate * caches and and colors? 24677c478bd9Sstevel@tonic-gate */ 24687c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_cnt[4]); 24697c478bd9Sstevel@tonic-gate /* 24707c478bd9Sstevel@tonic-gate * Get lgroup to allocate next page of shared memory 24717c478bd9Sstevel@tonic-gate * from and use it to specify where to allocate 24727c478bd9Sstevel@tonic-gate * the physical memory 24737c478bd9Sstevel@tonic-gate */ 24747c478bd9Sstevel@tonic-gate lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE); 24757c478bd9Sstevel@tonic-gate npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE, 24767c478bd9Sstevel@tonic-gate flags | PG_MATCH_COLOR, lgrp); 24777c478bd9Sstevel@tonic-gate if (npp == NULL) { 24787c478bd9Sstevel@tonic-gate npp = page_get_cachelist(vp, off, seg, 24797c478bd9Sstevel@tonic-gate vaddr, flags | PG_MATCH_COLOR, lgrp); 24807c478bd9Sstevel@tonic-gate if (npp == NULL) { 24817c478bd9Sstevel@tonic-gate npp = page_create_get_something(vp, 24827c478bd9Sstevel@tonic-gate off, seg, vaddr, 24837c478bd9Sstevel@tonic-gate flags & ~PG_MATCH_COLOR); 24847c478bd9Sstevel@tonic-gate } 24857c478bd9Sstevel@tonic-gate 24867c478bd9Sstevel@tonic-gate if (PP_ISAGED(npp) == 0) { 24877c478bd9Sstevel@tonic-gate /* 24887c478bd9Sstevel@tonic-gate * Since this page came from the 24897c478bd9Sstevel@tonic-gate * cachelist, we must destroy the 24907c478bd9Sstevel@tonic-gate * old vnode association. 24917c478bd9Sstevel@tonic-gate */ 24927c478bd9Sstevel@tonic-gate page_hashout(npp, NULL); 24937c478bd9Sstevel@tonic-gate } 24947c478bd9Sstevel@tonic-gate } 24957c478bd9Sstevel@tonic-gate } 24967c478bd9Sstevel@tonic-gate 24977c478bd9Sstevel@tonic-gate /* 24987c478bd9Sstevel@tonic-gate * We own this page! 24997c478bd9Sstevel@tonic-gate */ 25007c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(npp)); 25017c478bd9Sstevel@tonic-gate ASSERT(npp->p_vnode == NULL); 25027c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(npp)); 25037c478bd9Sstevel@tonic-gate PP_CLRFREE(npp); 25047c478bd9Sstevel@tonic-gate PP_CLRAGED(npp); 25057c478bd9Sstevel@tonic-gate 25067c478bd9Sstevel@tonic-gate /* 25077c478bd9Sstevel@tonic-gate * Here we have a page in our hot little mits and are 25087c478bd9Sstevel@tonic-gate * just waiting to stuff it on the appropriate lists. 25097c478bd9Sstevel@tonic-gate * Get the mutex and check to see if it really does 25107c478bd9Sstevel@tonic-gate * not exist. 25117c478bd9Sstevel@tonic-gate */ 25127c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 25137c478bd9Sstevel@tonic-gate mutex_enter(phm); 25147c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 25157c478bd9Sstevel@tonic-gate if (pp == NULL) { 25167c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_new); 25177c478bd9Sstevel@tonic-gate pp = npp; 25187c478bd9Sstevel@tonic-gate npp = NULL; 25197c478bd9Sstevel@tonic-gate if (!page_hashin(pp, vp, off, phm)) { 25207c478bd9Sstevel@tonic-gate /* 25217c478bd9Sstevel@tonic-gate * Since we hold the page hash mutex and 25227c478bd9Sstevel@tonic-gate * just searched for this page, page_hashin 25237c478bd9Sstevel@tonic-gate * had better not fail. If it does, that 25247c478bd9Sstevel@tonic-gate * means somethread did not follow the 25257c478bd9Sstevel@tonic-gate * page hash mutex rules. Panic now and 25267c478bd9Sstevel@tonic-gate * get it over with. As usual, go down 25277c478bd9Sstevel@tonic-gate * holding all the locks. 25287c478bd9Sstevel@tonic-gate */ 25297c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 25307c478bd9Sstevel@tonic-gate panic("page_create: " 25317c478bd9Sstevel@tonic-gate "hashin failed %p %p %llx %p", 25327c478bd9Sstevel@tonic-gate (void *)pp, (void *)vp, off, (void *)phm); 25337c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 25347c478bd9Sstevel@tonic-gate } 25357c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 25367c478bd9Sstevel@tonic-gate mutex_exit(phm); 25377c478bd9Sstevel@tonic-gate phm = NULL; 25387c478bd9Sstevel@tonic-gate 25397c478bd9Sstevel@tonic-gate /* 25407c478bd9Sstevel@tonic-gate * Hat layer locking need not be done to set 25417c478bd9Sstevel@tonic-gate * the following bits since the page is not hashed 25427c478bd9Sstevel@tonic-gate * and was on the free list (i.e., had no mappings). 25437c478bd9Sstevel@tonic-gate * 25447c478bd9Sstevel@tonic-gate * Set the reference bit to protect 25457c478bd9Sstevel@tonic-gate * against immediate pageout 25467c478bd9Sstevel@tonic-gate * 25477c478bd9Sstevel@tonic-gate * XXXmh modify freelist code to set reference 25487c478bd9Sstevel@tonic-gate * bit so we don't have to do it here. 25497c478bd9Sstevel@tonic-gate */ 25507c478bd9Sstevel@tonic-gate page_set_props(pp, P_REF); 25517c478bd9Sstevel@tonic-gate found_on_free++; 25527c478bd9Sstevel@tonic-gate } else { 25537c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_exists); 25547c478bd9Sstevel@tonic-gate if (flags & PG_EXCL) { 25557c478bd9Sstevel@tonic-gate /* 25567c478bd9Sstevel@tonic-gate * Found an existing page, and the caller 25577c478bd9Sstevel@tonic-gate * wanted all new pages. Undo all of the work 25587c478bd9Sstevel@tonic-gate * we have done. 25597c478bd9Sstevel@tonic-gate */ 25607c478bd9Sstevel@tonic-gate mutex_exit(phm); 25617c478bd9Sstevel@tonic-gate phm = NULL; 25627c478bd9Sstevel@tonic-gate while (plist != NULL) { 25637c478bd9Sstevel@tonic-gate pp = plist; 25647c478bd9Sstevel@tonic-gate page_sub(&plist, pp); 25657c478bd9Sstevel@tonic-gate page_io_unlock(pp); 25667c478bd9Sstevel@tonic-gate /* large pages should not end up here */ 25677c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 25687c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional ctx*/ 25697c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_INVAL, 0, kcred); 25707c478bd9Sstevel@tonic-gate } 25717c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_found_one); 25727c478bd9Sstevel@tonic-gate goto fail; 25737c478bd9Sstevel@tonic-gate } 25747c478bd9Sstevel@tonic-gate ASSERT(flags & PG_WAIT); 25757c478bd9Sstevel@tonic-gate if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) { 25767c478bd9Sstevel@tonic-gate /* 25777c478bd9Sstevel@tonic-gate * Start all over again if we blocked trying 25787c478bd9Sstevel@tonic-gate * to lock the page. 25797c478bd9Sstevel@tonic-gate */ 25807c478bd9Sstevel@tonic-gate mutex_exit(phm); 25817c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_page_lock_failed); 25827c478bd9Sstevel@tonic-gate phm = NULL; 25837c478bd9Sstevel@tonic-gate goto top; 25847c478bd9Sstevel@tonic-gate } 25857c478bd9Sstevel@tonic-gate mutex_exit(phm); 25867c478bd9Sstevel@tonic-gate phm = NULL; 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 25897c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0); 25907c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_get_cache); 25917c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 25927c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 25937c478bd9Sstevel@tonic-gate found_on_free++; 25947c478bd9Sstevel@tonic-gate } 25957c478bd9Sstevel@tonic-gate } 25967c478bd9Sstevel@tonic-gate 25977c478bd9Sstevel@tonic-gate /* 25987c478bd9Sstevel@tonic-gate * Got a page! It is locked. Acquire the i/o 25997c478bd9Sstevel@tonic-gate * lock since we are going to use the p_next and 26007c478bd9Sstevel@tonic-gate * p_prev fields to link the requested pages together. 26017c478bd9Sstevel@tonic-gate */ 26027c478bd9Sstevel@tonic-gate page_io_lock(pp); 26037c478bd9Sstevel@tonic-gate page_add(&plist, pp); 26047c478bd9Sstevel@tonic-gate plist = plist->p_next; 26057c478bd9Sstevel@tonic-gate off += PAGESIZE; 26067c478bd9Sstevel@tonic-gate vaddr += PAGESIZE; 26077c478bd9Sstevel@tonic-gate } 26087c478bd9Sstevel@tonic-gate 26097c478bd9Sstevel@tonic-gate ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1); 26107c478bd9Sstevel@tonic-gate fail: 26117c478bd9Sstevel@tonic-gate if (npp != NULL) { 26127c478bd9Sstevel@tonic-gate /* 26137c478bd9Sstevel@tonic-gate * Did not need this page after all. 26147c478bd9Sstevel@tonic-gate * Put it back on the free list. 26157c478bd9Sstevel@tonic-gate */ 26167c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_putbacks); 26177c478bd9Sstevel@tonic-gate PP_SETFREE(npp); 26187c478bd9Sstevel@tonic-gate PP_SETAGED(npp); 26197c478bd9Sstevel@tonic-gate npp->p_offset = (u_offset_t)-1; 26207c478bd9Sstevel@tonic-gate page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL); 26217c478bd9Sstevel@tonic-gate page_unlock(npp); 26227c478bd9Sstevel@tonic-gate 26237c478bd9Sstevel@tonic-gate } 26247c478bd9Sstevel@tonic-gate 26257c478bd9Sstevel@tonic-gate ASSERT(pages_req >= found_on_free); 26267c478bd9Sstevel@tonic-gate 26277c478bd9Sstevel@tonic-gate { 26287c478bd9Sstevel@tonic-gate uint_t overshoot = (uint_t)(pages_req - found_on_free); 26297c478bd9Sstevel@tonic-gate 26307c478bd9Sstevel@tonic-gate if (overshoot) { 26317c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_create_overshoot); 26327c478bd9Sstevel@tonic-gate p = &pcf[pcf_index]; 26337c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 26347c478bd9Sstevel@tonic-gate if (p->pcf_block) { 26357c478bd9Sstevel@tonic-gate p->pcf_reserve += overshoot; 26367c478bd9Sstevel@tonic-gate } else { 26377c478bd9Sstevel@tonic-gate p->pcf_count += overshoot; 26387c478bd9Sstevel@tonic-gate if (p->pcf_wait) { 26397c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 26407c478bd9Sstevel@tonic-gate if (freemem_wait) { 26417c478bd9Sstevel@tonic-gate cv_signal(&freemem_cv); 26427c478bd9Sstevel@tonic-gate p->pcf_wait--; 26437c478bd9Sstevel@tonic-gate } else { 26447c478bd9Sstevel@tonic-gate p->pcf_wait = 0; 26457c478bd9Sstevel@tonic-gate } 26467c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 26477c478bd9Sstevel@tonic-gate } 26487c478bd9Sstevel@tonic-gate } 26497c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 26507c478bd9Sstevel@tonic-gate /* freemem is approximate, so this test OK */ 26517c478bd9Sstevel@tonic-gate if (!p->pcf_block) 26527c478bd9Sstevel@tonic-gate freemem += overshoot; 26537c478bd9Sstevel@tonic-gate } 26547c478bd9Sstevel@tonic-gate } 26557c478bd9Sstevel@tonic-gate 26567c478bd9Sstevel@tonic-gate return (plist); 26577c478bd9Sstevel@tonic-gate } 26587c478bd9Sstevel@tonic-gate 26597c478bd9Sstevel@tonic-gate /* 26607c478bd9Sstevel@tonic-gate * One or more constituent pages of this large page has been marked 26617c478bd9Sstevel@tonic-gate * toxic. Simply demote the large page to PAGESIZE pages and let 26627c478bd9Sstevel@tonic-gate * page_free() handle it. This routine should only be called by 26637c478bd9Sstevel@tonic-gate * large page free routines (page_free_pages() and page_destroy_pages(). 26647c478bd9Sstevel@tonic-gate * All pages are locked SE_EXCL and have already been marked free. 26657c478bd9Sstevel@tonic-gate */ 26667c478bd9Sstevel@tonic-gate static void 26677c478bd9Sstevel@tonic-gate page_free_toxic_pages(page_t *rootpp) 26687c478bd9Sstevel@tonic-gate { 26697c478bd9Sstevel@tonic-gate page_t *tpp; 26707c478bd9Sstevel@tonic-gate pgcnt_t i, pgcnt = page_get_pagecnt(rootpp->p_szc); 26717c478bd9Sstevel@tonic-gate uint_t szc = rootpp->p_szc; 26727c478bd9Sstevel@tonic-gate 26737c478bd9Sstevel@tonic-gate for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) { 26747c478bd9Sstevel@tonic-gate ASSERT(tpp->p_szc == szc); 26757c478bd9Sstevel@tonic-gate ASSERT((PAGE_EXCL(tpp) && 26767c478bd9Sstevel@tonic-gate !page_iolock_assert(tpp)) || panicstr); 26777c478bd9Sstevel@tonic-gate tpp->p_szc = 0; 26787c478bd9Sstevel@tonic-gate } 26797c478bd9Sstevel@tonic-gate 26807c478bd9Sstevel@tonic-gate while (rootpp != NULL) { 26817c478bd9Sstevel@tonic-gate tpp = rootpp; 26827c478bd9Sstevel@tonic-gate page_sub(&rootpp, tpp); 26837c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(tpp)); 26847c478bd9Sstevel@tonic-gate PP_CLRFREE(tpp); 26857c478bd9Sstevel@tonic-gate page_free(tpp, 1); 26867c478bd9Sstevel@tonic-gate } 26877c478bd9Sstevel@tonic-gate } 26887c478bd9Sstevel@tonic-gate 26897c478bd9Sstevel@tonic-gate /* 26907c478bd9Sstevel@tonic-gate * Put page on the "free" list. 26917c478bd9Sstevel@tonic-gate * The free list is really two lists maintained by 26927c478bd9Sstevel@tonic-gate * the PSM of whatever machine we happen to be on. 26937c478bd9Sstevel@tonic-gate */ 26947c478bd9Sstevel@tonic-gate void 26957c478bd9Sstevel@tonic-gate page_free(page_t *pp, int dontneed) 26967c478bd9Sstevel@tonic-gate { 26977c478bd9Sstevel@tonic-gate struct pcf *p; 26987c478bd9Sstevel@tonic-gate uint_t pcf_index; 26997c478bd9Sstevel@tonic-gate 27007c478bd9Sstevel@tonic-gate ASSERT((PAGE_EXCL(pp) && 27017c478bd9Sstevel@tonic-gate !page_iolock_assert(pp)) || panicstr); 27027c478bd9Sstevel@tonic-gate 27037c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 27047c478bd9Sstevel@tonic-gate panic("page_free: page %p is free", (void *)pp); 27057c478bd9Sstevel@tonic-gate } 27067c478bd9Sstevel@tonic-gate 27077c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 27087c478bd9Sstevel@tonic-gate if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 2709ad23a2dbSjohansen PP_ISKAS(pp)) { 27107c478bd9Sstevel@tonic-gate panic("page_free: anon or kernel " 27117c478bd9Sstevel@tonic-gate "or no vnode large page %p", (void *)pp); 27127c478bd9Sstevel@tonic-gate } 27137c478bd9Sstevel@tonic-gate page_demote_vp_pages(pp); 27147c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 27157c478bd9Sstevel@tonic-gate } 27167c478bd9Sstevel@tonic-gate 27177c478bd9Sstevel@tonic-gate /* 27187c478bd9Sstevel@tonic-gate * The page_struct_lock need not be acquired to examine these 27197c478bd9Sstevel@tonic-gate * fields since the page has an "exclusive" lock. 27207c478bd9Sstevel@tonic-gate */ 272107b65a64Saguzovsk if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 272207b65a64Saguzovsk pp->p_slckcnt != 0) { 272307b65a64Saguzovsk panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d " 272407b65a64Saguzovsk "slckcnt = %d", pp, page_pptonum(pp), pp->p_lckcnt, 272507b65a64Saguzovsk pp->p_cowcnt, pp->p_slckcnt); 27267c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 27277c478bd9Sstevel@tonic-gate } 27287c478bd9Sstevel@tonic-gate 27297c478bd9Sstevel@tonic-gate ASSERT(!hat_page_getshare(pp)); 27307c478bd9Sstevel@tonic-gate 27317c478bd9Sstevel@tonic-gate PP_SETFREE(pp); 27327c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) || 27337c478bd9Sstevel@tonic-gate !hat_ismod(pp)); 27347c478bd9Sstevel@tonic-gate page_clr_all_props(pp); 27357c478bd9Sstevel@tonic-gate ASSERT(!hat_page_getshare(pp)); 27367c478bd9Sstevel@tonic-gate 27377c478bd9Sstevel@tonic-gate /* 27387c478bd9Sstevel@tonic-gate * Now we add the page to the head of the free list. 27397c478bd9Sstevel@tonic-gate * But if this page is associated with a paged vnode 27407c478bd9Sstevel@tonic-gate * then we adjust the head forward so that the page is 27417c478bd9Sstevel@tonic-gate * effectively at the end of the list. 27427c478bd9Sstevel@tonic-gate */ 27437c478bd9Sstevel@tonic-gate if (pp->p_vnode == NULL) { 27447c478bd9Sstevel@tonic-gate /* 27457c478bd9Sstevel@tonic-gate * Page has no identity, put it on the free list. 27467c478bd9Sstevel@tonic-gate */ 27477c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 27487c478bd9Sstevel@tonic-gate pp->p_offset = (u_offset_t)-1; 27497c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 27507c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_free); 27517c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 27527c478bd9Sstevel@tonic-gate "page_free_free:pp %p", pp); 27537c478bd9Sstevel@tonic-gate } else { 27547c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 27557c478bd9Sstevel@tonic-gate 27567c478bd9Sstevel@tonic-gate if (!dontneed || nopageage) { 27577c478bd9Sstevel@tonic-gate /* move it to the tail of the list */ 27587c478bd9Sstevel@tonic-gate page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL); 27597c478bd9Sstevel@tonic-gate 27607c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_cache); 27617c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL, 27627c478bd9Sstevel@tonic-gate "page_free_cache_tail:pp %p", pp); 27637c478bd9Sstevel@tonic-gate } else { 27647c478bd9Sstevel@tonic-gate page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD); 27657c478bd9Sstevel@tonic-gate 27667c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_dontneed); 27677c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD, 27687c478bd9Sstevel@tonic-gate "page_free_cache_head:pp %p", pp); 27697c478bd9Sstevel@tonic-gate } 27707c478bd9Sstevel@tonic-gate } 27717c478bd9Sstevel@tonic-gate page_unlock(pp); 27727c478bd9Sstevel@tonic-gate 27737c478bd9Sstevel@tonic-gate /* 27747c478bd9Sstevel@tonic-gate * Now do the `freemem' accounting. 27757c478bd9Sstevel@tonic-gate */ 27767c478bd9Sstevel@tonic-gate pcf_index = PCF_INDEX(); 27777c478bd9Sstevel@tonic-gate p = &pcf[pcf_index]; 27787c478bd9Sstevel@tonic-gate 27797c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 27807c478bd9Sstevel@tonic-gate if (p->pcf_block) { 27817c478bd9Sstevel@tonic-gate p->pcf_reserve += 1; 27827c478bd9Sstevel@tonic-gate } else { 27837c478bd9Sstevel@tonic-gate p->pcf_count += 1; 27847c478bd9Sstevel@tonic-gate if (p->pcf_wait) { 27857c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 27867c478bd9Sstevel@tonic-gate /* 27877c478bd9Sstevel@tonic-gate * Check to see if some other thread 27887c478bd9Sstevel@tonic-gate * is actually waiting. Another bucket 27897c478bd9Sstevel@tonic-gate * may have woken it up by now. If there 27907c478bd9Sstevel@tonic-gate * are no waiters, then set our pcf_wait 27917c478bd9Sstevel@tonic-gate * count to zero to avoid coming in here 27927c478bd9Sstevel@tonic-gate * next time. Also, since only one page 27937c478bd9Sstevel@tonic-gate * was put on the free list, just wake 27947c478bd9Sstevel@tonic-gate * up one waiter. 27957c478bd9Sstevel@tonic-gate */ 27967c478bd9Sstevel@tonic-gate if (freemem_wait) { 27977c478bd9Sstevel@tonic-gate cv_signal(&freemem_cv); 27987c478bd9Sstevel@tonic-gate p->pcf_wait--; 27997c478bd9Sstevel@tonic-gate } else { 28007c478bd9Sstevel@tonic-gate p->pcf_wait = 0; 28017c478bd9Sstevel@tonic-gate } 28027c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 28037c478bd9Sstevel@tonic-gate } 28047c478bd9Sstevel@tonic-gate } 28057c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 28067c478bd9Sstevel@tonic-gate 28077c478bd9Sstevel@tonic-gate /* freemem is approximate, so this test OK */ 28087c478bd9Sstevel@tonic-gate if (!p->pcf_block) 28097c478bd9Sstevel@tonic-gate freemem += 1; 28107c478bd9Sstevel@tonic-gate } 28117c478bd9Sstevel@tonic-gate 28127c478bd9Sstevel@tonic-gate /* 28137c478bd9Sstevel@tonic-gate * Put page on the "free" list during intial startup. 28147c478bd9Sstevel@tonic-gate * This happens during initial single threaded execution. 28157c478bd9Sstevel@tonic-gate */ 28167c478bd9Sstevel@tonic-gate void 28177c478bd9Sstevel@tonic-gate page_free_at_startup(page_t *pp) 28187c478bd9Sstevel@tonic-gate { 28197c478bd9Sstevel@tonic-gate struct pcf *p; 28207c478bd9Sstevel@tonic-gate uint_t pcf_index; 28217c478bd9Sstevel@tonic-gate 28227c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT); 28237c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_free); 28247c478bd9Sstevel@tonic-gate 28257c478bd9Sstevel@tonic-gate /* 28267c478bd9Sstevel@tonic-gate * Now do the `freemem' accounting. 28277c478bd9Sstevel@tonic-gate */ 28287c478bd9Sstevel@tonic-gate pcf_index = PCF_INDEX(); 28297c478bd9Sstevel@tonic-gate p = &pcf[pcf_index]; 28307c478bd9Sstevel@tonic-gate 28317c478bd9Sstevel@tonic-gate ASSERT(p->pcf_block == 0); 28327c478bd9Sstevel@tonic-gate ASSERT(p->pcf_wait == 0); 28337c478bd9Sstevel@tonic-gate p->pcf_count += 1; 28347c478bd9Sstevel@tonic-gate 28357c478bd9Sstevel@tonic-gate /* freemem is approximate, so this is OK */ 28367c478bd9Sstevel@tonic-gate freemem += 1; 28377c478bd9Sstevel@tonic-gate } 28387c478bd9Sstevel@tonic-gate 28397c478bd9Sstevel@tonic-gate void 28407c478bd9Sstevel@tonic-gate page_free_pages(page_t *pp) 28417c478bd9Sstevel@tonic-gate { 28427c478bd9Sstevel@tonic-gate page_t *tpp, *rootpp = NULL; 28437c478bd9Sstevel@tonic-gate pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 28447c478bd9Sstevel@tonic-gate pgcnt_t i; 28457c478bd9Sstevel@tonic-gate uint_t szc = pp->p_szc; 28467c478bd9Sstevel@tonic-gate 28477c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_pages); 28487c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE, 28497c478bd9Sstevel@tonic-gate "page_free_free:pp %p", pp); 28507c478bd9Sstevel@tonic-gate 28517c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 28527c478bd9Sstevel@tonic-gate if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 28537c478bd9Sstevel@tonic-gate panic("page_free_pages: not root page %p", (void *)pp); 28547c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 28557c478bd9Sstevel@tonic-gate } 28567c478bd9Sstevel@tonic-gate 2857affbd3ccSkchow for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 28587c478bd9Sstevel@tonic-gate ASSERT((PAGE_EXCL(tpp) && 28597c478bd9Sstevel@tonic-gate !page_iolock_assert(tpp)) || panicstr); 28607c478bd9Sstevel@tonic-gate if (PP_ISFREE(tpp)) { 28617c478bd9Sstevel@tonic-gate panic("page_free_pages: page %p is free", (void *)tpp); 28627c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 28637c478bd9Sstevel@tonic-gate } 28647c478bd9Sstevel@tonic-gate if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 || 286507b65a64Saguzovsk tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) { 28667c478bd9Sstevel@tonic-gate panic("page_free_pages %p", (void *)tpp); 28677c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 28687c478bd9Sstevel@tonic-gate } 28697c478bd9Sstevel@tonic-gate 28707c478bd9Sstevel@tonic-gate ASSERT(!hat_page_getshare(tpp)); 28717c478bd9Sstevel@tonic-gate ASSERT(tpp->p_vnode == NULL); 28727c478bd9Sstevel@tonic-gate ASSERT(tpp->p_szc == szc); 28737c478bd9Sstevel@tonic-gate 28747c478bd9Sstevel@tonic-gate PP_SETFREE(tpp); 28757c478bd9Sstevel@tonic-gate page_clr_all_props(tpp); 28767c478bd9Sstevel@tonic-gate PP_SETAGED(tpp); 28777c478bd9Sstevel@tonic-gate tpp->p_offset = (u_offset_t)-1; 28787c478bd9Sstevel@tonic-gate ASSERT(tpp->p_next == tpp); 28797c478bd9Sstevel@tonic-gate ASSERT(tpp->p_prev == tpp); 28807c478bd9Sstevel@tonic-gate page_list_concat(&rootpp, &tpp); 28817c478bd9Sstevel@tonic-gate } 28827c478bd9Sstevel@tonic-gate ASSERT(rootpp == pp); 28837c478bd9Sstevel@tonic-gate 28847c478bd9Sstevel@tonic-gate page_list_add_pages(rootpp, 0); 28857c478bd9Sstevel@tonic-gate page_create_putback(pgcnt); 28867c478bd9Sstevel@tonic-gate } 28877c478bd9Sstevel@tonic-gate 28887c478bd9Sstevel@tonic-gate int free_pages = 1; 28897c478bd9Sstevel@tonic-gate 28907c478bd9Sstevel@tonic-gate /* 28917c478bd9Sstevel@tonic-gate * This routine attempts to return pages to the cachelist via page_release(). 28927c478bd9Sstevel@tonic-gate * It does not *have* to be successful in all cases, since the pageout scanner 28937c478bd9Sstevel@tonic-gate * will catch any pages it misses. It does need to be fast and not introduce 28947c478bd9Sstevel@tonic-gate * too much overhead. 28957c478bd9Sstevel@tonic-gate * 28967c478bd9Sstevel@tonic-gate * If a page isn't found on the unlocked sweep of the page_hash bucket, we 28977c478bd9Sstevel@tonic-gate * don't lock and retry. This is ok, since the page scanner will eventually 28987c478bd9Sstevel@tonic-gate * find any page we miss in free_vp_pages(). 28997c478bd9Sstevel@tonic-gate */ 29007c478bd9Sstevel@tonic-gate void 29017c478bd9Sstevel@tonic-gate free_vp_pages(vnode_t *vp, u_offset_t off, size_t len) 29027c478bd9Sstevel@tonic-gate { 29037c478bd9Sstevel@tonic-gate page_t *pp; 29047c478bd9Sstevel@tonic-gate u_offset_t eoff; 29057c478bd9Sstevel@tonic-gate extern int swap_in_range(vnode_t *, u_offset_t, size_t); 29067c478bd9Sstevel@tonic-gate 29077c478bd9Sstevel@tonic-gate eoff = off + len; 29087c478bd9Sstevel@tonic-gate 29097c478bd9Sstevel@tonic-gate if (free_pages == 0) 29107c478bd9Sstevel@tonic-gate return; 29117c478bd9Sstevel@tonic-gate if (swap_in_range(vp, off, len)) 29127c478bd9Sstevel@tonic-gate return; 29137c478bd9Sstevel@tonic-gate 29147c478bd9Sstevel@tonic-gate for (; off < eoff; off += PAGESIZE) { 29157c478bd9Sstevel@tonic-gate 29167c478bd9Sstevel@tonic-gate /* 29177c478bd9Sstevel@tonic-gate * find the page using a fast, but inexact search. It'll be OK 29187c478bd9Sstevel@tonic-gate * if a few pages slip through the cracks here. 29197c478bd9Sstevel@tonic-gate */ 29207c478bd9Sstevel@tonic-gate pp = page_exists(vp, off); 29217c478bd9Sstevel@tonic-gate 29227c478bd9Sstevel@tonic-gate /* 29237c478bd9Sstevel@tonic-gate * If we didn't find the page (it may not exist), the page 29247c478bd9Sstevel@tonic-gate * is free, looks still in use (shared), or we can't lock it, 29257c478bd9Sstevel@tonic-gate * just give up. 29267c478bd9Sstevel@tonic-gate */ 29277c478bd9Sstevel@tonic-gate if (pp == NULL || 29287c478bd9Sstevel@tonic-gate PP_ISFREE(pp) || 29297c478bd9Sstevel@tonic-gate page_share_cnt(pp) > 0 || 29307c478bd9Sstevel@tonic-gate !page_trylock(pp, SE_EXCL)) 29317c478bd9Sstevel@tonic-gate continue; 29327c478bd9Sstevel@tonic-gate 29337c478bd9Sstevel@tonic-gate /* 29347c478bd9Sstevel@tonic-gate * Once we have locked pp, verify that it's still the 29357c478bd9Sstevel@tonic-gate * correct page and not already free 29367c478bd9Sstevel@tonic-gate */ 29377c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL)); 29387c478bd9Sstevel@tonic-gate if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) { 29397c478bd9Sstevel@tonic-gate page_unlock(pp); 29407c478bd9Sstevel@tonic-gate continue; 29417c478bd9Sstevel@tonic-gate } 29427c478bd9Sstevel@tonic-gate 29437c478bd9Sstevel@tonic-gate /* 29447c478bd9Sstevel@tonic-gate * try to release the page... 29457c478bd9Sstevel@tonic-gate */ 29467c478bd9Sstevel@tonic-gate (void) page_release(pp, 1); 29477c478bd9Sstevel@tonic-gate } 29487c478bd9Sstevel@tonic-gate } 29497c478bd9Sstevel@tonic-gate 29507c478bd9Sstevel@tonic-gate /* 29517c478bd9Sstevel@tonic-gate * Reclaim the given page from the free list. 29527c478bd9Sstevel@tonic-gate * Returns 1 on success or 0 on failure. 29537c478bd9Sstevel@tonic-gate * 29547c478bd9Sstevel@tonic-gate * The page is unlocked if it can't be reclaimed (when freemem == 0). 29557c478bd9Sstevel@tonic-gate * If `lock' is non-null, it will be dropped and re-acquired if 29567c478bd9Sstevel@tonic-gate * the routine must wait while freemem is 0. 29577c478bd9Sstevel@tonic-gate * 29587c478bd9Sstevel@tonic-gate * As it turns out, boot_getpages() does this. It picks a page, 29597c478bd9Sstevel@tonic-gate * based on where OBP mapped in some address, gets its pfn, searches 29607c478bd9Sstevel@tonic-gate * the memsegs, locks the page, then pulls it off the free list! 29617c478bd9Sstevel@tonic-gate */ 29627c478bd9Sstevel@tonic-gate int 29637c478bd9Sstevel@tonic-gate page_reclaim(page_t *pp, kmutex_t *lock) 29647c478bd9Sstevel@tonic-gate { 29657c478bd9Sstevel@tonic-gate struct pcf *p; 29667c478bd9Sstevel@tonic-gate uint_t pcf_index; 29677c478bd9Sstevel@tonic-gate struct cpu *cpup; 29687c478bd9Sstevel@tonic-gate uint_t i; 2969de62167aSmec pgcnt_t npgs, need; 2970de62167aSmec pgcnt_t collected = 0; 29717c478bd9Sstevel@tonic-gate 29727c478bd9Sstevel@tonic-gate ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 29737c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 2974db874c57Selowe 2975db874c57Selowe npgs = page_get_pagecnt(pp->p_szc); 29767c478bd9Sstevel@tonic-gate 29777c478bd9Sstevel@tonic-gate /* 29787c478bd9Sstevel@tonic-gate * If `freemem' is 0, we cannot reclaim this page from the 29797c478bd9Sstevel@tonic-gate * freelist, so release every lock we might hold: the page, 29807c478bd9Sstevel@tonic-gate * and the `lock' before blocking. 29817c478bd9Sstevel@tonic-gate * 29827c478bd9Sstevel@tonic-gate * The only way `freemem' can become 0 while there are pages 29837c478bd9Sstevel@tonic-gate * marked free (have their p->p_free bit set) is when the 29847c478bd9Sstevel@tonic-gate * system is low on memory and doing a page_create(). In 29857c478bd9Sstevel@tonic-gate * order to guarantee that once page_create() starts acquiring 29867c478bd9Sstevel@tonic-gate * pages it will be able to get all that it needs since `freemem' 29877c478bd9Sstevel@tonic-gate * was decreased by the requested amount. So, we need to release 29887c478bd9Sstevel@tonic-gate * this page, and let page_create() have it. 29897c478bd9Sstevel@tonic-gate * 29907c478bd9Sstevel@tonic-gate * Since `freemem' being zero is not supposed to happen, just 29917c478bd9Sstevel@tonic-gate * use the usual hash stuff as a starting point. If that bucket 29927c478bd9Sstevel@tonic-gate * is empty, then assume the worst, and start at the beginning 29937c478bd9Sstevel@tonic-gate * of the pcf array. If we always start at the beginning 29947c478bd9Sstevel@tonic-gate * when acquiring more than one pcf lock, there won't be any 29957c478bd9Sstevel@tonic-gate * deadlock problems. 29967c478bd9Sstevel@tonic-gate */ 29977c478bd9Sstevel@tonic-gate 29987c478bd9Sstevel@tonic-gate /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 29997c478bd9Sstevel@tonic-gate 300024e9c58bSelowe if (freemem <= throttlefree && !page_create_throttle(npgs, 0)) { 30017c478bd9Sstevel@tonic-gate pcf_acquire_all(); 30027c478bd9Sstevel@tonic-gate goto page_reclaim_nomem; 30037c478bd9Sstevel@tonic-gate } 30047c478bd9Sstevel@tonic-gate 30057c478bd9Sstevel@tonic-gate pcf_index = PCF_INDEX(); 30067c478bd9Sstevel@tonic-gate p = &pcf[pcf_index]; 30077c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 3008db874c57Selowe if (p->pcf_count >= npgs) { 3009db874c57Selowe collected = npgs; 3010db874c57Selowe p->pcf_count -= npgs; 30117c478bd9Sstevel@tonic-gate } 30127c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 3013db874c57Selowe need = npgs - collected; 30147c478bd9Sstevel@tonic-gate 3015db874c57Selowe if (need > 0) { 30167c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_reclaim_zero); 30177c478bd9Sstevel@tonic-gate /* 30187c478bd9Sstevel@tonic-gate * Check again. Its possible that some other thread 30197c478bd9Sstevel@tonic-gate * could have been right behind us, and added one 30207c478bd9Sstevel@tonic-gate * to a list somewhere. Acquire each of the pcf locks 30217c478bd9Sstevel@tonic-gate * until we find a page. 30227c478bd9Sstevel@tonic-gate */ 30237c478bd9Sstevel@tonic-gate p = pcf; 30247c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 30257c478bd9Sstevel@tonic-gate mutex_enter(&p->pcf_lock); 3026db874c57Selowe if (p->pcf_count) { 3027db874c57Selowe if (p->pcf_count >= need) { 3028db874c57Selowe p->pcf_count -= need; 3029db874c57Selowe collected += need; 3030db874c57Selowe need = 0; 3031db874c57Selowe break; 3032db874c57Selowe } else if (p->pcf_count) { 3033db874c57Selowe collected += p->pcf_count; 3034db874c57Selowe need -= p->pcf_count; 3035db874c57Selowe p->pcf_count = 0; 3036db874c57Selowe } 30377c478bd9Sstevel@tonic-gate } 30387c478bd9Sstevel@tonic-gate p++; 30397c478bd9Sstevel@tonic-gate } 30407c478bd9Sstevel@tonic-gate 3041db874c57Selowe if (need > 0) { 30427c478bd9Sstevel@tonic-gate page_reclaim_nomem: 30437c478bd9Sstevel@tonic-gate /* 30447c478bd9Sstevel@tonic-gate * We really can't have page `pp'. 30457c478bd9Sstevel@tonic-gate * Time for the no-memory dance with 30467c478bd9Sstevel@tonic-gate * page_free(). This is just like 30477c478bd9Sstevel@tonic-gate * page_create_wait(). Plus the added 30487c478bd9Sstevel@tonic-gate * attraction of releasing whatever mutex 30497c478bd9Sstevel@tonic-gate * we held when we were called with in `lock'. 30507c478bd9Sstevel@tonic-gate * Page_unlock() will wakeup any thread 30517c478bd9Sstevel@tonic-gate * waiting around for this page. 30527c478bd9Sstevel@tonic-gate */ 30537c478bd9Sstevel@tonic-gate if (lock) { 30547c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_reclaim_zero_locked); 30557c478bd9Sstevel@tonic-gate mutex_exit(lock); 30567c478bd9Sstevel@tonic-gate } 30577c478bd9Sstevel@tonic-gate page_unlock(pp); 30587c478bd9Sstevel@tonic-gate 30597c478bd9Sstevel@tonic-gate /* 30607c478bd9Sstevel@tonic-gate * get this before we drop all the pcf locks. 30617c478bd9Sstevel@tonic-gate */ 30627c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 30637c478bd9Sstevel@tonic-gate 30647c478bd9Sstevel@tonic-gate p = pcf; 3065db874c57Selowe p->pcf_count += collected; 30667c478bd9Sstevel@tonic-gate for (i = 0; i < PCF_FANOUT; i++) { 30677c478bd9Sstevel@tonic-gate p->pcf_wait++; 30687c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 30697c478bd9Sstevel@tonic-gate p++; 30707c478bd9Sstevel@tonic-gate } 30717c478bd9Sstevel@tonic-gate 30727c478bd9Sstevel@tonic-gate freemem_wait++; 30737c478bd9Sstevel@tonic-gate cv_wait(&freemem_cv, &new_freemem_lock); 30747c478bd9Sstevel@tonic-gate freemem_wait--; 30757c478bd9Sstevel@tonic-gate 30767c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 30777c478bd9Sstevel@tonic-gate 30787c478bd9Sstevel@tonic-gate if (lock) { 30797c478bd9Sstevel@tonic-gate mutex_enter(lock); 30807c478bd9Sstevel@tonic-gate } 30817c478bd9Sstevel@tonic-gate return (0); 30827c478bd9Sstevel@tonic-gate } 30837c478bd9Sstevel@tonic-gate 30847c478bd9Sstevel@tonic-gate /* 3085db874c57Selowe * We beat the PCF bins over the head until 3086db874c57Selowe * we got the memory that we wanted. 30877c478bd9Sstevel@tonic-gate * The pcf accounting has been done, 30887c478bd9Sstevel@tonic-gate * though none of the pcf_wait flags have been set, 30897c478bd9Sstevel@tonic-gate * drop the locks and continue on. 30907c478bd9Sstevel@tonic-gate */ 3091db874c57Selowe ASSERT(collected == npgs); 30927c478bd9Sstevel@tonic-gate while (p >= pcf) { 30937c478bd9Sstevel@tonic-gate mutex_exit(&p->pcf_lock); 30947c478bd9Sstevel@tonic-gate p--; 30957c478bd9Sstevel@tonic-gate } 30967c478bd9Sstevel@tonic-gate } 30977c478bd9Sstevel@tonic-gate 30987c478bd9Sstevel@tonic-gate /* 30997c478bd9Sstevel@tonic-gate * freemem is not protected by any lock. Thus, we cannot 31007c478bd9Sstevel@tonic-gate * have any assertion containing freemem here. 31017c478bd9Sstevel@tonic-gate */ 3102db874c57Selowe freemem -= npgs; 31037c478bd9Sstevel@tonic-gate 31047c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_reclaim); 31057c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 3106db874c57Selowe if (npgs > 1) { 3107db874c57Selowe page_list_sub_pages(pp, pp->p_szc); 3108db874c57Selowe } else { 3109db874c57Selowe page_list_sub(pp, PG_FREE_LIST); 3110db874c57Selowe } 31117c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 31127c478bd9Sstevel@tonic-gate "page_reclaim_free:pp %p", pp); 31137c478bd9Sstevel@tonic-gate } else { 3114db874c57Selowe ASSERT(npgs == 1); 31157c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 31167c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 31177c478bd9Sstevel@tonic-gate "page_reclaim_cache:pp %p", pp); 31187c478bd9Sstevel@tonic-gate } 31197c478bd9Sstevel@tonic-gate 31207c478bd9Sstevel@tonic-gate /* 31217c478bd9Sstevel@tonic-gate * clear the p_free & p_age bits since this page is no longer 31227c478bd9Sstevel@tonic-gate * on the free list. Notice that there was a brief time where 31237c478bd9Sstevel@tonic-gate * a page is marked as free, but is not on the list. 31247c478bd9Sstevel@tonic-gate * 31257c478bd9Sstevel@tonic-gate * Set the reference bit to protect against immediate pageout. 31267c478bd9Sstevel@tonic-gate */ 312724e9c58bSelowe for (i = 0; i < npgs; i++, pp++) { 3128db874c57Selowe PP_CLRFREE(pp); 3129db874c57Selowe PP_CLRAGED(pp); 3130db874c57Selowe page_set_props(pp, P_REF); 3131db874c57Selowe } 31327c478bd9Sstevel@tonic-gate 31337c478bd9Sstevel@tonic-gate CPU_STATS_ENTER_K(); 31347c478bd9Sstevel@tonic-gate cpup = CPU; /* get cpup now that CPU cannot change */ 31357c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 31367c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 31377c478bd9Sstevel@tonic-gate CPU_STATS_EXIT_K(); 31387c478bd9Sstevel@tonic-gate 31397c478bd9Sstevel@tonic-gate return (1); 31407c478bd9Sstevel@tonic-gate } 31417c478bd9Sstevel@tonic-gate 31427c478bd9Sstevel@tonic-gate 31437c478bd9Sstevel@tonic-gate 31447c478bd9Sstevel@tonic-gate /* 31457c478bd9Sstevel@tonic-gate * Destroy identity of the page and put it back on 31467c478bd9Sstevel@tonic-gate * the page free list. Assumes that the caller has 31477c478bd9Sstevel@tonic-gate * acquired the "exclusive" lock on the page. 31487c478bd9Sstevel@tonic-gate */ 31497c478bd9Sstevel@tonic-gate void 31507c478bd9Sstevel@tonic-gate page_destroy(page_t *pp, int dontfree) 31517c478bd9Sstevel@tonic-gate { 31527c478bd9Sstevel@tonic-gate ASSERT((PAGE_EXCL(pp) && 31537c478bd9Sstevel@tonic-gate !page_iolock_assert(pp)) || panicstr); 315407b65a64Saguzovsk ASSERT(pp->p_slckcnt == 0 || panicstr); 31557c478bd9Sstevel@tonic-gate 31567c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 31577c478bd9Sstevel@tonic-gate if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) || 3158ad23a2dbSjohansen PP_ISKAS(pp)) { 31597c478bd9Sstevel@tonic-gate panic("page_destroy: anon or kernel or no vnode " 31607c478bd9Sstevel@tonic-gate "large page %p", (void *)pp); 31617c478bd9Sstevel@tonic-gate } 31627c478bd9Sstevel@tonic-gate page_demote_vp_pages(pp); 31637c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 31647c478bd9Sstevel@tonic-gate } 31657c478bd9Sstevel@tonic-gate 31667c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp); 31677c478bd9Sstevel@tonic-gate 31687c478bd9Sstevel@tonic-gate /* 31697c478bd9Sstevel@tonic-gate * Unload translations, if any, then hash out the 31707c478bd9Sstevel@tonic-gate * page to erase its identity. 31717c478bd9Sstevel@tonic-gate */ 31727c478bd9Sstevel@tonic-gate (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 31737c478bd9Sstevel@tonic-gate page_hashout(pp, NULL); 31747c478bd9Sstevel@tonic-gate 31757c478bd9Sstevel@tonic-gate if (!dontfree) { 31767c478bd9Sstevel@tonic-gate /* 31777c478bd9Sstevel@tonic-gate * Acquire the "freemem_lock" for availrmem. 31787c478bd9Sstevel@tonic-gate * The page_struct_lock need not be acquired for lckcnt 31797c478bd9Sstevel@tonic-gate * and cowcnt since the page has an "exclusive" lock. 31807c478bd9Sstevel@tonic-gate */ 31817c478bd9Sstevel@tonic-gate if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) { 31827c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 31837c478bd9Sstevel@tonic-gate if (pp->p_lckcnt != 0) { 31847c478bd9Sstevel@tonic-gate availrmem++; 31857c478bd9Sstevel@tonic-gate pp->p_lckcnt = 0; 31867c478bd9Sstevel@tonic-gate } 31877c478bd9Sstevel@tonic-gate if (pp->p_cowcnt != 0) { 31887c478bd9Sstevel@tonic-gate availrmem += pp->p_cowcnt; 31897c478bd9Sstevel@tonic-gate pp->p_cowcnt = 0; 31907c478bd9Sstevel@tonic-gate } 31917c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 31927c478bd9Sstevel@tonic-gate } 31937c478bd9Sstevel@tonic-gate /* 31947c478bd9Sstevel@tonic-gate * Put the page on the "free" list. 31957c478bd9Sstevel@tonic-gate */ 31967c478bd9Sstevel@tonic-gate page_free(pp, 0); 31977c478bd9Sstevel@tonic-gate } 31987c478bd9Sstevel@tonic-gate } 31997c478bd9Sstevel@tonic-gate 32007c478bd9Sstevel@tonic-gate void 32017c478bd9Sstevel@tonic-gate page_destroy_pages(page_t *pp) 32027c478bd9Sstevel@tonic-gate { 32037c478bd9Sstevel@tonic-gate 32047c478bd9Sstevel@tonic-gate page_t *tpp, *rootpp = NULL; 32057c478bd9Sstevel@tonic-gate pgcnt_t pgcnt = page_get_pagecnt(pp->p_szc); 32067c478bd9Sstevel@tonic-gate pgcnt_t i, pglcks = 0; 32077c478bd9Sstevel@tonic-gate uint_t szc = pp->p_szc; 32087c478bd9Sstevel@tonic-gate 32097c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes()); 32107c478bd9Sstevel@tonic-gate 32117c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_destroy_pages); 32127c478bd9Sstevel@tonic-gate 32137c478bd9Sstevel@tonic-gate TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp); 32147c478bd9Sstevel@tonic-gate 32157c478bd9Sstevel@tonic-gate if ((page_pptonum(pp) & (pgcnt - 1)) != 0) { 32167c478bd9Sstevel@tonic-gate panic("page_destroy_pages: not root page %p", (void *)pp); 32177c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 32187c478bd9Sstevel@tonic-gate } 32197c478bd9Sstevel@tonic-gate 3220affbd3ccSkchow for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) { 32217c478bd9Sstevel@tonic-gate ASSERT((PAGE_EXCL(tpp) && 32227c478bd9Sstevel@tonic-gate !page_iolock_assert(tpp)) || panicstr); 322307b65a64Saguzovsk ASSERT(tpp->p_slckcnt == 0 || panicstr); 32247c478bd9Sstevel@tonic-gate (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 32257c478bd9Sstevel@tonic-gate page_hashout(tpp, NULL); 32267c478bd9Sstevel@tonic-gate ASSERT(tpp->p_offset == (u_offset_t)-1); 32277c478bd9Sstevel@tonic-gate if (tpp->p_lckcnt != 0) { 32287c478bd9Sstevel@tonic-gate pglcks++; 32297c478bd9Sstevel@tonic-gate tpp->p_lckcnt = 0; 32307c478bd9Sstevel@tonic-gate } else if (tpp->p_cowcnt != 0) { 32317c478bd9Sstevel@tonic-gate pglcks += tpp->p_cowcnt; 32327c478bd9Sstevel@tonic-gate tpp->p_cowcnt = 0; 32337c478bd9Sstevel@tonic-gate } 32347c478bd9Sstevel@tonic-gate ASSERT(!hat_page_getshare(tpp)); 32357c478bd9Sstevel@tonic-gate ASSERT(tpp->p_vnode == NULL); 32367c478bd9Sstevel@tonic-gate ASSERT(tpp->p_szc == szc); 32377c478bd9Sstevel@tonic-gate 32387c478bd9Sstevel@tonic-gate PP_SETFREE(tpp); 32397c478bd9Sstevel@tonic-gate page_clr_all_props(tpp); 32407c478bd9Sstevel@tonic-gate PP_SETAGED(tpp); 32417c478bd9Sstevel@tonic-gate ASSERT(tpp->p_next == tpp); 32427c478bd9Sstevel@tonic-gate ASSERT(tpp->p_prev == tpp); 32437c478bd9Sstevel@tonic-gate page_list_concat(&rootpp, &tpp); 32447c478bd9Sstevel@tonic-gate } 32457c478bd9Sstevel@tonic-gate 32467c478bd9Sstevel@tonic-gate ASSERT(rootpp == pp); 32477c478bd9Sstevel@tonic-gate if (pglcks != 0) { 32487c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 32497c478bd9Sstevel@tonic-gate availrmem += pglcks; 32507c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 32517c478bd9Sstevel@tonic-gate } 32527c478bd9Sstevel@tonic-gate 32537c478bd9Sstevel@tonic-gate page_list_add_pages(rootpp, 0); 32547c478bd9Sstevel@tonic-gate page_create_putback(pgcnt); 32557c478bd9Sstevel@tonic-gate } 32567c478bd9Sstevel@tonic-gate 32577c478bd9Sstevel@tonic-gate /* 32587c478bd9Sstevel@tonic-gate * Similar to page_destroy(), but destroys pages which are 32597c478bd9Sstevel@tonic-gate * locked and known to be on the page free list. Since 32607c478bd9Sstevel@tonic-gate * the page is known to be free and locked, no one can access 32617c478bd9Sstevel@tonic-gate * it. 32627c478bd9Sstevel@tonic-gate * 32637c478bd9Sstevel@tonic-gate * Also, the number of free pages does not change. 32647c478bd9Sstevel@tonic-gate */ 32657c478bd9Sstevel@tonic-gate void 32667c478bd9Sstevel@tonic-gate page_destroy_free(page_t *pp) 32677c478bd9Sstevel@tonic-gate { 32687c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32697c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 32707c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 32717c478bd9Sstevel@tonic-gate ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0); 32727c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 32737c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp) == 0); 32747c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 32757c478bd9Sstevel@tonic-gate 32767c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_destroy_free); 32777c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 32787c478bd9Sstevel@tonic-gate 32797c478bd9Sstevel@tonic-gate page_hashout(pp, NULL); 32807c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 32817c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1); 32827c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL); 32837c478bd9Sstevel@tonic-gate 32847c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 32857c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 32867c478bd9Sstevel@tonic-gate page_unlock(pp); 32877c478bd9Sstevel@tonic-gate 32887c478bd9Sstevel@tonic-gate mutex_enter(&new_freemem_lock); 32897c478bd9Sstevel@tonic-gate if (freemem_wait) { 32907c478bd9Sstevel@tonic-gate cv_signal(&freemem_cv); 32917c478bd9Sstevel@tonic-gate } 32927c478bd9Sstevel@tonic-gate mutex_exit(&new_freemem_lock); 32937c478bd9Sstevel@tonic-gate } 32947c478bd9Sstevel@tonic-gate 32957c478bd9Sstevel@tonic-gate /* 32967c478bd9Sstevel@tonic-gate * Rename the page "opp" to have an identity specified 32977c478bd9Sstevel@tonic-gate * by [vp, off]. If a page already exists with this name 32987c478bd9Sstevel@tonic-gate * it is locked and destroyed. Note that the page's 32997c478bd9Sstevel@tonic-gate * translations are not unloaded during the rename. 33007c478bd9Sstevel@tonic-gate * 33017c478bd9Sstevel@tonic-gate * This routine is used by the anon layer to "steal" the 33027c478bd9Sstevel@tonic-gate * original page and is not unlike destroying a page and 33037c478bd9Sstevel@tonic-gate * creating a new page using the same page frame. 33047c478bd9Sstevel@tonic-gate * 33057c478bd9Sstevel@tonic-gate * XXX -- Could deadlock if caller 1 tries to rename A to B while 33067c478bd9Sstevel@tonic-gate * caller 2 tries to rename B to A. 33077c478bd9Sstevel@tonic-gate */ 33087c478bd9Sstevel@tonic-gate void 33097c478bd9Sstevel@tonic-gate page_rename(page_t *opp, vnode_t *vp, u_offset_t off) 33107c478bd9Sstevel@tonic-gate { 33117c478bd9Sstevel@tonic-gate page_t *pp; 33127c478bd9Sstevel@tonic-gate int olckcnt = 0; 33137c478bd9Sstevel@tonic-gate int ocowcnt = 0; 33147c478bd9Sstevel@tonic-gate kmutex_t *phm; 33157c478bd9Sstevel@tonic-gate ulong_t index; 33167c478bd9Sstevel@tonic-gate 33177c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp)); 33187c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 33197c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(opp) == 0); 33207c478bd9Sstevel@tonic-gate 33217c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_rename_count); 33227c478bd9Sstevel@tonic-gate 33237c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_VM, TR_PAGE_RENAME, 33247c478bd9Sstevel@tonic-gate "page rename:pp %p vp %p off %llx", opp, vp, off); 33257c478bd9Sstevel@tonic-gate 332637fbc076Saguzovsk /* 332737fbc076Saguzovsk * CacheFS may call page_rename for a large NFS page 332837fbc076Saguzovsk * when both CacheFS and NFS mount points are used 332937fbc076Saguzovsk * by applications. Demote this large page before 333037fbc076Saguzovsk * renaming it, to ensure that there are no "partial" 333137fbc076Saguzovsk * large pages left lying around. 333237fbc076Saguzovsk */ 333337fbc076Saguzovsk if (opp->p_szc != 0) { 333437fbc076Saguzovsk vnode_t *ovp = opp->p_vnode; 333537fbc076Saguzovsk ASSERT(ovp != NULL); 333637fbc076Saguzovsk ASSERT(!IS_SWAPFSVP(ovp)); 3337ad23a2dbSjohansen ASSERT(!VN_ISKAS(ovp)); 333837fbc076Saguzovsk page_demote_vp_pages(opp); 333937fbc076Saguzovsk ASSERT(opp->p_szc == 0); 334037fbc076Saguzovsk } 334137fbc076Saguzovsk 33427c478bd9Sstevel@tonic-gate page_hashout(opp, NULL); 33437c478bd9Sstevel@tonic-gate PP_CLRAGED(opp); 33447c478bd9Sstevel@tonic-gate 33457c478bd9Sstevel@tonic-gate /* 33467c478bd9Sstevel@tonic-gate * Acquire the appropriate page hash lock, since 33477c478bd9Sstevel@tonic-gate * we're going to rename the page. 33487c478bd9Sstevel@tonic-gate */ 33497c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, off); 33507c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 33517c478bd9Sstevel@tonic-gate mutex_enter(phm); 33527c478bd9Sstevel@tonic-gate top: 33537c478bd9Sstevel@tonic-gate /* 33547c478bd9Sstevel@tonic-gate * Look for an existing page with this name and destroy it if found. 33557c478bd9Sstevel@tonic-gate * By holding the page hash lock all the way to the page_hashin() 33567c478bd9Sstevel@tonic-gate * call, we are assured that no page can be created with this 33577c478bd9Sstevel@tonic-gate * identity. In the case when the phm lock is dropped to undo any 33587c478bd9Sstevel@tonic-gate * hat layer mappings, the existing page is held with an "exclusive" 33597c478bd9Sstevel@tonic-gate * lock, again preventing another page from being created with 33607c478bd9Sstevel@tonic-gate * this identity. 33617c478bd9Sstevel@tonic-gate */ 33627c478bd9Sstevel@tonic-gate PAGE_HASH_SEARCH(index, pp, vp, off); 33637c478bd9Sstevel@tonic-gate if (pp != NULL) { 33647c478bd9Sstevel@tonic-gate VM_STAT_ADD(page_rename_exists); 33657c478bd9Sstevel@tonic-gate 33667c478bd9Sstevel@tonic-gate /* 33677c478bd9Sstevel@tonic-gate * As it turns out, this is one of only two places where 33687c478bd9Sstevel@tonic-gate * page_lock() needs to hold the passed in lock in the 33697c478bd9Sstevel@tonic-gate * successful case. In all of the others, the lock could 33707c478bd9Sstevel@tonic-gate * be dropped as soon as the attempt is made to lock 33717c478bd9Sstevel@tonic-gate * the page. It is tempting to add yet another arguement, 33727c478bd9Sstevel@tonic-gate * PL_KEEP or PL_DROP, to let page_lock know what to do. 33737c478bd9Sstevel@tonic-gate */ 33747c478bd9Sstevel@tonic-gate if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) { 33757c478bd9Sstevel@tonic-gate /* 33767c478bd9Sstevel@tonic-gate * Went to sleep because the page could not 33777c478bd9Sstevel@tonic-gate * be locked. We were woken up when the page 33787c478bd9Sstevel@tonic-gate * was unlocked, or when the page was destroyed. 33797c478bd9Sstevel@tonic-gate * In either case, `phm' was dropped while we 33807c478bd9Sstevel@tonic-gate * slept. Hence we should not just roar through 33817c478bd9Sstevel@tonic-gate * this loop. 33827c478bd9Sstevel@tonic-gate */ 33837c478bd9Sstevel@tonic-gate goto top; 33847c478bd9Sstevel@tonic-gate } 33857c478bd9Sstevel@tonic-gate 338637fbc076Saguzovsk /* 338737fbc076Saguzovsk * If an existing page is a large page, then demote 338837fbc076Saguzovsk * it to ensure that no "partial" large pages are 338937fbc076Saguzovsk * "created" after page_rename. An existing page 339037fbc076Saguzovsk * can be a CacheFS page, and can't belong to swapfs. 339137fbc076Saguzovsk */ 33927c478bd9Sstevel@tonic-gate if (hat_page_is_mapped(pp)) { 33937c478bd9Sstevel@tonic-gate /* 33947c478bd9Sstevel@tonic-gate * Unload translations. Since we hold the 33957c478bd9Sstevel@tonic-gate * exclusive lock on this page, the page 33967c478bd9Sstevel@tonic-gate * can not be changed while we drop phm. 33977c478bd9Sstevel@tonic-gate * This is also not a lock protocol violation, 33987c478bd9Sstevel@tonic-gate * but rather the proper way to do things. 33997c478bd9Sstevel@tonic-gate */ 34007c478bd9Sstevel@tonic-gate mutex_exit(phm); 34017c478bd9Sstevel@tonic-gate (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 340237fbc076Saguzovsk if (pp->p_szc != 0) { 340337fbc076Saguzovsk ASSERT(!IS_SWAPFSVP(vp)); 3404ad23a2dbSjohansen ASSERT(!VN_ISKAS(vp)); 340537fbc076Saguzovsk page_demote_vp_pages(pp); 340637fbc076Saguzovsk ASSERT(pp->p_szc == 0); 340737fbc076Saguzovsk } 340837fbc076Saguzovsk mutex_enter(phm); 340937fbc076Saguzovsk } else if (pp->p_szc != 0) { 341037fbc076Saguzovsk ASSERT(!IS_SWAPFSVP(vp)); 3411ad23a2dbSjohansen ASSERT(!VN_ISKAS(vp)); 341237fbc076Saguzovsk mutex_exit(phm); 341337fbc076Saguzovsk page_demote_vp_pages(pp); 341437fbc076Saguzovsk ASSERT(pp->p_szc == 0); 34157c478bd9Sstevel@tonic-gate mutex_enter(phm); 34167c478bd9Sstevel@tonic-gate } 34177c478bd9Sstevel@tonic-gate page_hashout(pp, phm); 34187c478bd9Sstevel@tonic-gate } 34197c478bd9Sstevel@tonic-gate /* 34207c478bd9Sstevel@tonic-gate * Hash in the page with the new identity. 34217c478bd9Sstevel@tonic-gate */ 34227c478bd9Sstevel@tonic-gate if (!page_hashin(opp, vp, off, phm)) { 34237c478bd9Sstevel@tonic-gate /* 34247c478bd9Sstevel@tonic-gate * We were holding phm while we searched for [vp, off] 34257c478bd9Sstevel@tonic-gate * and only dropped phm if we found and locked a page. 34267c478bd9Sstevel@tonic-gate * If we can't create this page now, then some thing 34277c478bd9Sstevel@tonic-gate * is really broken. 34287c478bd9Sstevel@tonic-gate */ 34297c478bd9Sstevel@tonic-gate panic("page_rename: Can't hash in page: %p", (void *)pp); 34307c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 34317c478bd9Sstevel@tonic-gate } 34327c478bd9Sstevel@tonic-gate 34337c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(phm)); 34347c478bd9Sstevel@tonic-gate mutex_exit(phm); 34357c478bd9Sstevel@tonic-gate 34367c478bd9Sstevel@tonic-gate /* 34377c478bd9Sstevel@tonic-gate * Now that we have dropped phm, lets get around to finishing up 34387c478bd9Sstevel@tonic-gate * with pp. 34397c478bd9Sstevel@tonic-gate */ 34407c478bd9Sstevel@tonic-gate if (pp != NULL) { 34417c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 34427c478bd9Sstevel@tonic-gate /* for now large pages should not end up here */ 34437c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 34447c478bd9Sstevel@tonic-gate /* 34457c478bd9Sstevel@tonic-gate * Save the locks for transfer to the new page and then 34467c478bd9Sstevel@tonic-gate * clear them so page_free doesn't think they're important. 34477c478bd9Sstevel@tonic-gate * The page_struct_lock need not be acquired for lckcnt and 34487c478bd9Sstevel@tonic-gate * cowcnt since the page has an "exclusive" lock. 34497c478bd9Sstevel@tonic-gate */ 34507c478bd9Sstevel@tonic-gate olckcnt = pp->p_lckcnt; 34517c478bd9Sstevel@tonic-gate ocowcnt = pp->p_cowcnt; 34527c478bd9Sstevel@tonic-gate pp->p_lckcnt = pp->p_cowcnt = 0; 34537c478bd9Sstevel@tonic-gate 34547c478bd9Sstevel@tonic-gate /* 34557c478bd9Sstevel@tonic-gate * Put the page on the "free" list after we drop 34567c478bd9Sstevel@tonic-gate * the lock. The less work under the lock the better. 34577c478bd9Sstevel@tonic-gate */ 34587c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional context*/ 34597c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_FREE, 0, kcred); 34607c478bd9Sstevel@tonic-gate } 34617c478bd9Sstevel@tonic-gate 34627c478bd9Sstevel@tonic-gate /* 34637c478bd9Sstevel@tonic-gate * Transfer the lock count from the old page (if any). 34647c478bd9Sstevel@tonic-gate * The page_struct_lock need not be acquired for lckcnt and 34657c478bd9Sstevel@tonic-gate * cowcnt since the page has an "exclusive" lock. 34667c478bd9Sstevel@tonic-gate */ 34677c478bd9Sstevel@tonic-gate opp->p_lckcnt += olckcnt; 34687c478bd9Sstevel@tonic-gate opp->p_cowcnt += ocowcnt; 34697c478bd9Sstevel@tonic-gate } 34707c478bd9Sstevel@tonic-gate 34717c478bd9Sstevel@tonic-gate /* 34727c478bd9Sstevel@tonic-gate * low level routine to add page `pp' to the hash and vp chains for [vp, offset] 34737c478bd9Sstevel@tonic-gate * 34747c478bd9Sstevel@tonic-gate * Pages are normally inserted at the start of a vnode's v_pages list. 34757c478bd9Sstevel@tonic-gate * If the vnode is VMODSORT and the page is modified, it goes at the end. 34767c478bd9Sstevel@tonic-gate * This can happen when a modified page is relocated for DR. 34777c478bd9Sstevel@tonic-gate * 34787c478bd9Sstevel@tonic-gate * Returns 1 on success and 0 on failure. 34797c478bd9Sstevel@tonic-gate */ 34807c478bd9Sstevel@tonic-gate static int 34817c478bd9Sstevel@tonic-gate page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset) 34827c478bd9Sstevel@tonic-gate { 34837c478bd9Sstevel@tonic-gate page_t **listp; 34847c478bd9Sstevel@tonic-gate page_t *tp; 34857c478bd9Sstevel@tonic-gate ulong_t index; 34867c478bd9Sstevel@tonic-gate 34877c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 34887c478bd9Sstevel@tonic-gate ASSERT(vp != NULL); 34897c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 34907c478bd9Sstevel@tonic-gate 34917c478bd9Sstevel@tonic-gate /* 34927c478bd9Sstevel@tonic-gate * Be sure to set these up before the page is inserted on the hash 34937c478bd9Sstevel@tonic-gate * list. As soon as the page is placed on the list some other 34947c478bd9Sstevel@tonic-gate * thread might get confused and wonder how this page could 34957c478bd9Sstevel@tonic-gate * possibly hash to this list. 34967c478bd9Sstevel@tonic-gate */ 34977c478bd9Sstevel@tonic-gate pp->p_vnode = vp; 34987c478bd9Sstevel@tonic-gate pp->p_offset = offset; 34997c478bd9Sstevel@tonic-gate 35007c478bd9Sstevel@tonic-gate /* 35017c478bd9Sstevel@tonic-gate * record if this page is on a swap vnode 35027c478bd9Sstevel@tonic-gate */ 35037c478bd9Sstevel@tonic-gate if ((vp->v_flag & VISSWAP) != 0) 35047c478bd9Sstevel@tonic-gate PP_SETSWAP(pp); 35057c478bd9Sstevel@tonic-gate 35067c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, offset); 35077c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index))); 35087c478bd9Sstevel@tonic-gate listp = &page_hash[index]; 35097c478bd9Sstevel@tonic-gate 35107c478bd9Sstevel@tonic-gate /* 35117c478bd9Sstevel@tonic-gate * If this page is already hashed in, fail this attempt to add it. 35127c478bd9Sstevel@tonic-gate */ 35137c478bd9Sstevel@tonic-gate for (tp = *listp; tp != NULL; tp = tp->p_hash) { 35147c478bd9Sstevel@tonic-gate if (tp->p_vnode == vp && tp->p_offset == offset) { 35157c478bd9Sstevel@tonic-gate pp->p_vnode = NULL; 35167c478bd9Sstevel@tonic-gate pp->p_offset = (u_offset_t)(-1); 35177c478bd9Sstevel@tonic-gate return (0); 35187c478bd9Sstevel@tonic-gate } 35197c478bd9Sstevel@tonic-gate } 35207c478bd9Sstevel@tonic-gate pp->p_hash = *listp; 35217c478bd9Sstevel@tonic-gate *listp = pp; 35227c478bd9Sstevel@tonic-gate 35237c478bd9Sstevel@tonic-gate /* 35247c478bd9Sstevel@tonic-gate * Add the page to the vnode's list of pages 35257c478bd9Sstevel@tonic-gate */ 35267c478bd9Sstevel@tonic-gate if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp)) 35277c478bd9Sstevel@tonic-gate listp = &vp->v_pages->p_vpprev->p_vpnext; 35287c478bd9Sstevel@tonic-gate else 35297c478bd9Sstevel@tonic-gate listp = &vp->v_pages; 35307c478bd9Sstevel@tonic-gate 35317c478bd9Sstevel@tonic-gate page_vpadd(listp, pp); 35327c478bd9Sstevel@tonic-gate 35337c478bd9Sstevel@tonic-gate return (1); 35347c478bd9Sstevel@tonic-gate } 35357c478bd9Sstevel@tonic-gate 35367c478bd9Sstevel@tonic-gate /* 35377c478bd9Sstevel@tonic-gate * Add page `pp' to both the hash and vp chains for [vp, offset]. 35387c478bd9Sstevel@tonic-gate * 35397c478bd9Sstevel@tonic-gate * Returns 1 on success and 0 on failure. 35407c478bd9Sstevel@tonic-gate * If hold is passed in, it is not dropped. 35417c478bd9Sstevel@tonic-gate */ 35427c478bd9Sstevel@tonic-gate int 35437c478bd9Sstevel@tonic-gate page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold) 35447c478bd9Sstevel@tonic-gate { 35457c478bd9Sstevel@tonic-gate kmutex_t *phm = NULL; 35467c478bd9Sstevel@tonic-gate kmutex_t *vphm; 35477c478bd9Sstevel@tonic-gate int rc; 35487c478bd9Sstevel@tonic-gate 35497c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp))); 35507c478bd9Sstevel@tonic-gate 35517c478bd9Sstevel@tonic-gate TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN, 35527c478bd9Sstevel@tonic-gate "page_hashin:pp %p vp %p offset %llx", 35537c478bd9Sstevel@tonic-gate pp, vp, offset); 35547c478bd9Sstevel@tonic-gate 35557c478bd9Sstevel@tonic-gate VM_STAT_ADD(hashin_count); 35567c478bd9Sstevel@tonic-gate 35577c478bd9Sstevel@tonic-gate if (hold != NULL) 35587c478bd9Sstevel@tonic-gate phm = hold; 35597c478bd9Sstevel@tonic-gate else { 35607c478bd9Sstevel@tonic-gate VM_STAT_ADD(hashin_not_held); 35617c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset)); 35627c478bd9Sstevel@tonic-gate mutex_enter(phm); 35637c478bd9Sstevel@tonic-gate } 35647c478bd9Sstevel@tonic-gate 35657c478bd9Sstevel@tonic-gate vphm = page_vnode_mutex(vp); 35667c478bd9Sstevel@tonic-gate mutex_enter(vphm); 35677c478bd9Sstevel@tonic-gate rc = page_do_hashin(pp, vp, offset); 35687c478bd9Sstevel@tonic-gate mutex_exit(vphm); 35697c478bd9Sstevel@tonic-gate if (hold == NULL) 35707c478bd9Sstevel@tonic-gate mutex_exit(phm); 35717c478bd9Sstevel@tonic-gate if (rc == 0) 35727c478bd9Sstevel@tonic-gate VM_STAT_ADD(hashin_already); 35737c478bd9Sstevel@tonic-gate return (rc); 35747c478bd9Sstevel@tonic-gate } 35757c478bd9Sstevel@tonic-gate 35767c478bd9Sstevel@tonic-gate /* 35777c478bd9Sstevel@tonic-gate * Remove page ``pp'' from the hash and vp chains and remove vp association. 35787c478bd9Sstevel@tonic-gate * All mutexes must be held 35797c478bd9Sstevel@tonic-gate */ 35807c478bd9Sstevel@tonic-gate static void 35817c478bd9Sstevel@tonic-gate page_do_hashout(page_t *pp) 35827c478bd9Sstevel@tonic-gate { 35837c478bd9Sstevel@tonic-gate page_t **hpp; 35847c478bd9Sstevel@tonic-gate page_t *hp; 35857c478bd9Sstevel@tonic-gate vnode_t *vp = pp->p_vnode; 35867c478bd9Sstevel@tonic-gate 35877c478bd9Sstevel@tonic-gate ASSERT(vp != NULL); 35887c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 35897c478bd9Sstevel@tonic-gate 35907c478bd9Sstevel@tonic-gate /* 35917c478bd9Sstevel@tonic-gate * First, take pp off of its hash chain. 35927c478bd9Sstevel@tonic-gate */ 35937c478bd9Sstevel@tonic-gate hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)]; 35947c478bd9Sstevel@tonic-gate 35957c478bd9Sstevel@tonic-gate for (;;) { 35967c478bd9Sstevel@tonic-gate hp = *hpp; 35977c478bd9Sstevel@tonic-gate if (hp == pp) 35987c478bd9Sstevel@tonic-gate break; 35997c478bd9Sstevel@tonic-gate if (hp == NULL) { 36007c478bd9Sstevel@tonic-gate panic("page_do_hashout"); 36017c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 36027c478bd9Sstevel@tonic-gate } 36037c478bd9Sstevel@tonic-gate hpp = &hp->p_hash; 36047c478bd9Sstevel@tonic-gate } 36057c478bd9Sstevel@tonic-gate *hpp = pp->p_hash; 36067c478bd9Sstevel@tonic-gate 36077c478bd9Sstevel@tonic-gate /* 36087c478bd9Sstevel@tonic-gate * Now remove it from its associated vnode. 36097c478bd9Sstevel@tonic-gate */ 36107c478bd9Sstevel@tonic-gate if (vp->v_pages) 36117c478bd9Sstevel@tonic-gate page_vpsub(&vp->v_pages, pp); 36127c478bd9Sstevel@tonic-gate 36137c478bd9Sstevel@tonic-gate pp->p_hash = NULL; 36147c478bd9Sstevel@tonic-gate page_clr_all_props(pp); 36157c478bd9Sstevel@tonic-gate PP_CLRSWAP(pp); 36167c478bd9Sstevel@tonic-gate pp->p_vnode = NULL; 36177c478bd9Sstevel@tonic-gate pp->p_offset = (u_offset_t)-1; 36187c478bd9Sstevel@tonic-gate } 36197c478bd9Sstevel@tonic-gate 36207c478bd9Sstevel@tonic-gate /* 36217c478bd9Sstevel@tonic-gate * Remove page ``pp'' from the hash and vp chains and remove vp association. 36227c478bd9Sstevel@tonic-gate * 36237c478bd9Sstevel@tonic-gate * When `phm' is non-NULL it contains the address of the mutex protecting the 36247c478bd9Sstevel@tonic-gate * hash list pp is on. It is not dropped. 36257c478bd9Sstevel@tonic-gate */ 36267c478bd9Sstevel@tonic-gate void 36277c478bd9Sstevel@tonic-gate page_hashout(page_t *pp, kmutex_t *phm) 36287c478bd9Sstevel@tonic-gate { 36297c478bd9Sstevel@tonic-gate vnode_t *vp; 36307c478bd9Sstevel@tonic-gate ulong_t index; 36317c478bd9Sstevel@tonic-gate kmutex_t *nphm; 36327c478bd9Sstevel@tonic-gate kmutex_t *vphm; 36337c478bd9Sstevel@tonic-gate kmutex_t *sep; 36347c478bd9Sstevel@tonic-gate 36357c478bd9Sstevel@tonic-gate ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1); 36367c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode != NULL); 36377c478bd9Sstevel@tonic-gate ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr); 36387c478bd9Sstevel@tonic-gate ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode))); 36397c478bd9Sstevel@tonic-gate 36407c478bd9Sstevel@tonic-gate vp = pp->p_vnode; 36417c478bd9Sstevel@tonic-gate 36427c478bd9Sstevel@tonic-gate TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT, 36437c478bd9Sstevel@tonic-gate "page_hashout:pp %p vp %p", pp, vp); 36447c478bd9Sstevel@tonic-gate 36457c478bd9Sstevel@tonic-gate /* Kernel probe */ 36467c478bd9Sstevel@tonic-gate TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */, 36477c478bd9Sstevel@tonic-gate tnf_opaque, vnode, vp, 36487c478bd9Sstevel@tonic-gate tnf_offset, offset, pp->p_offset); 36497c478bd9Sstevel@tonic-gate 36507c478bd9Sstevel@tonic-gate /* 36517c478bd9Sstevel@tonic-gate * 36527c478bd9Sstevel@tonic-gate */ 36537c478bd9Sstevel@tonic-gate VM_STAT_ADD(hashout_count); 36547c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(vp, pp->p_offset); 36557c478bd9Sstevel@tonic-gate if (phm == NULL) { 36567c478bd9Sstevel@tonic-gate VM_STAT_ADD(hashout_not_held); 36577c478bd9Sstevel@tonic-gate nphm = PAGE_HASH_MUTEX(index); 36587c478bd9Sstevel@tonic-gate mutex_enter(nphm); 36597c478bd9Sstevel@tonic-gate } 36607c478bd9Sstevel@tonic-gate ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1); 36617c478bd9Sstevel@tonic-gate 36627c478bd9Sstevel@tonic-gate 36637c478bd9Sstevel@tonic-gate /* 36647c478bd9Sstevel@tonic-gate * grab page vnode mutex and remove it... 36657c478bd9Sstevel@tonic-gate */ 36667c478bd9Sstevel@tonic-gate vphm = page_vnode_mutex(vp); 36677c478bd9Sstevel@tonic-gate mutex_enter(vphm); 36687c478bd9Sstevel@tonic-gate 36697c478bd9Sstevel@tonic-gate page_do_hashout(pp); 36707c478bd9Sstevel@tonic-gate 36717c478bd9Sstevel@tonic-gate mutex_exit(vphm); 36727c478bd9Sstevel@tonic-gate if (phm == NULL) 36737c478bd9Sstevel@tonic-gate mutex_exit(nphm); 36747c478bd9Sstevel@tonic-gate 36757c478bd9Sstevel@tonic-gate /* 36767c478bd9Sstevel@tonic-gate * Wake up processes waiting for this page. The page's 36777c478bd9Sstevel@tonic-gate * identity has been changed, and is probably not the 36787c478bd9Sstevel@tonic-gate * desired page any longer. 36797c478bd9Sstevel@tonic-gate */ 36807c478bd9Sstevel@tonic-gate sep = page_se_mutex(pp); 36817c478bd9Sstevel@tonic-gate mutex_enter(sep); 368242787a71Sstans pp->p_selock &= ~SE_EWANTED; 36837c478bd9Sstevel@tonic-gate if (CV_HAS_WAITERS(&pp->p_cv)) 36847c478bd9Sstevel@tonic-gate cv_broadcast(&pp->p_cv); 36857c478bd9Sstevel@tonic-gate mutex_exit(sep); 36867c478bd9Sstevel@tonic-gate } 36877c478bd9Sstevel@tonic-gate 36887c478bd9Sstevel@tonic-gate /* 36897c478bd9Sstevel@tonic-gate * Add the page to the front of a linked list of pages 36907c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 36917c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 36927c478bd9Sstevel@tonic-gate */ 36937c478bd9Sstevel@tonic-gate void 36947c478bd9Sstevel@tonic-gate page_add(page_t **ppp, page_t *pp) 36957c478bd9Sstevel@tonic-gate { 36967c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 36977c478bd9Sstevel@tonic-gate 36987c478bd9Sstevel@tonic-gate page_add_common(ppp, pp); 36997c478bd9Sstevel@tonic-gate } 37007c478bd9Sstevel@tonic-gate 37017c478bd9Sstevel@tonic-gate 37027c478bd9Sstevel@tonic-gate 37037c478bd9Sstevel@tonic-gate /* 37047c478bd9Sstevel@tonic-gate * Common code for page_add() and mach_page_add() 37057c478bd9Sstevel@tonic-gate */ 37067c478bd9Sstevel@tonic-gate void 37077c478bd9Sstevel@tonic-gate page_add_common(page_t **ppp, page_t *pp) 37087c478bd9Sstevel@tonic-gate { 37097c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 37107c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 37117c478bd9Sstevel@tonic-gate } else { 37127c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 37137c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 37147c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 37157c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 37167c478bd9Sstevel@tonic-gate } 37177c478bd9Sstevel@tonic-gate *ppp = pp; 37187c478bd9Sstevel@tonic-gate } 37197c478bd9Sstevel@tonic-gate 37207c478bd9Sstevel@tonic-gate 37217c478bd9Sstevel@tonic-gate /* 37227c478bd9Sstevel@tonic-gate * Remove this page from a linked list of pages 37237c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 37247c478bd9Sstevel@tonic-gate * 37257c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 37267c478bd9Sstevel@tonic-gate */ 37277c478bd9Sstevel@tonic-gate void 37287c478bd9Sstevel@tonic-gate page_sub(page_t **ppp, page_t *pp) 37297c478bd9Sstevel@tonic-gate { 37307c478bd9Sstevel@tonic-gate ASSERT((PP_ISFREE(pp)) ? 1 : 37317c478bd9Sstevel@tonic-gate (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp))); 37327c478bd9Sstevel@tonic-gate 37337c478bd9Sstevel@tonic-gate if (*ppp == NULL || pp == NULL) { 37347c478bd9Sstevel@tonic-gate panic("page_sub: bad arg(s): pp %p, *ppp %p", 37357c478bd9Sstevel@tonic-gate (void *)pp, (void *)(*ppp)); 37367c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 37377c478bd9Sstevel@tonic-gate } 37387c478bd9Sstevel@tonic-gate 37397c478bd9Sstevel@tonic-gate page_sub_common(ppp, pp); 37407c478bd9Sstevel@tonic-gate } 37417c478bd9Sstevel@tonic-gate 37427c478bd9Sstevel@tonic-gate 37437c478bd9Sstevel@tonic-gate /* 37447c478bd9Sstevel@tonic-gate * Common code for page_sub() and mach_page_sub() 37457c478bd9Sstevel@tonic-gate */ 37467c478bd9Sstevel@tonic-gate void 37477c478bd9Sstevel@tonic-gate page_sub_common(page_t **ppp, page_t *pp) 37487c478bd9Sstevel@tonic-gate { 37497c478bd9Sstevel@tonic-gate if (*ppp == pp) 37507c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 37517c478bd9Sstevel@tonic-gate 37527c478bd9Sstevel@tonic-gate if (*ppp == pp) 37537c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 37547c478bd9Sstevel@tonic-gate else { 37557c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 37567c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 37577c478bd9Sstevel@tonic-gate } 37587c478bd9Sstevel@tonic-gate pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 37597c478bd9Sstevel@tonic-gate } 37607c478bd9Sstevel@tonic-gate 37617c478bd9Sstevel@tonic-gate 37627c478bd9Sstevel@tonic-gate /* 37637c478bd9Sstevel@tonic-gate * Break page list cppp into two lists with npages in the first list. 37647c478bd9Sstevel@tonic-gate * The tail is returned in nppp. 37657c478bd9Sstevel@tonic-gate */ 37667c478bd9Sstevel@tonic-gate void 37677c478bd9Sstevel@tonic-gate page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages) 37687c478bd9Sstevel@tonic-gate { 37697c478bd9Sstevel@tonic-gate page_t *s1pp = *oppp; 37707c478bd9Sstevel@tonic-gate page_t *s2pp; 37717c478bd9Sstevel@tonic-gate page_t *e1pp, *e2pp; 37727c478bd9Sstevel@tonic-gate long n = 0; 37737c478bd9Sstevel@tonic-gate 37747c478bd9Sstevel@tonic-gate if (s1pp == NULL) { 37757c478bd9Sstevel@tonic-gate *nppp = NULL; 37767c478bd9Sstevel@tonic-gate return; 37777c478bd9Sstevel@tonic-gate } 37787c478bd9Sstevel@tonic-gate if (npages == 0) { 37797c478bd9Sstevel@tonic-gate *nppp = s1pp; 37807c478bd9Sstevel@tonic-gate *oppp = NULL; 37817c478bd9Sstevel@tonic-gate return; 37827c478bd9Sstevel@tonic-gate } 37837c478bd9Sstevel@tonic-gate for (n = 0, s2pp = *oppp; n < npages; n++) { 37847c478bd9Sstevel@tonic-gate s2pp = s2pp->p_next; 37857c478bd9Sstevel@tonic-gate } 37867c478bd9Sstevel@tonic-gate /* Fix head and tail of new lists */ 37877c478bd9Sstevel@tonic-gate e1pp = s2pp->p_prev; 37887c478bd9Sstevel@tonic-gate e2pp = s1pp->p_prev; 37897c478bd9Sstevel@tonic-gate s1pp->p_prev = e1pp; 37907c478bd9Sstevel@tonic-gate e1pp->p_next = s1pp; 37917c478bd9Sstevel@tonic-gate s2pp->p_prev = e2pp; 37927c478bd9Sstevel@tonic-gate e2pp->p_next = s2pp; 37937c478bd9Sstevel@tonic-gate 37947c478bd9Sstevel@tonic-gate /* second list empty */ 37957c478bd9Sstevel@tonic-gate if (s2pp == s1pp) { 37967c478bd9Sstevel@tonic-gate *oppp = s1pp; 37977c478bd9Sstevel@tonic-gate *nppp = NULL; 37987c478bd9Sstevel@tonic-gate } else { 37997c478bd9Sstevel@tonic-gate *oppp = s1pp; 38007c478bd9Sstevel@tonic-gate *nppp = s2pp; 38017c478bd9Sstevel@tonic-gate } 38027c478bd9Sstevel@tonic-gate } 38037c478bd9Sstevel@tonic-gate 38047c478bd9Sstevel@tonic-gate /* 38057c478bd9Sstevel@tonic-gate * Concatenate page list nppp onto the end of list ppp. 38067c478bd9Sstevel@tonic-gate */ 38077c478bd9Sstevel@tonic-gate void 38087c478bd9Sstevel@tonic-gate page_list_concat(page_t **ppp, page_t **nppp) 38097c478bd9Sstevel@tonic-gate { 38107c478bd9Sstevel@tonic-gate page_t *s1pp, *s2pp, *e1pp, *e2pp; 38117c478bd9Sstevel@tonic-gate 38127c478bd9Sstevel@tonic-gate if (*nppp == NULL) { 38137c478bd9Sstevel@tonic-gate return; 38147c478bd9Sstevel@tonic-gate } 38157c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 38167c478bd9Sstevel@tonic-gate *ppp = *nppp; 38177c478bd9Sstevel@tonic-gate return; 38187c478bd9Sstevel@tonic-gate } 38197c478bd9Sstevel@tonic-gate s1pp = *ppp; 38207c478bd9Sstevel@tonic-gate e1pp = s1pp->p_prev; 38217c478bd9Sstevel@tonic-gate s2pp = *nppp; 38227c478bd9Sstevel@tonic-gate e2pp = s2pp->p_prev; 38237c478bd9Sstevel@tonic-gate s1pp->p_prev = e2pp; 38247c478bd9Sstevel@tonic-gate e2pp->p_next = s1pp; 38257c478bd9Sstevel@tonic-gate e1pp->p_next = s2pp; 38267c478bd9Sstevel@tonic-gate s2pp->p_prev = e1pp; 38277c478bd9Sstevel@tonic-gate } 38287c478bd9Sstevel@tonic-gate 38297c478bd9Sstevel@tonic-gate /* 38307c478bd9Sstevel@tonic-gate * return the next page in the page list 38317c478bd9Sstevel@tonic-gate */ 38327c478bd9Sstevel@tonic-gate page_t * 38337c478bd9Sstevel@tonic-gate page_list_next(page_t *pp) 38347c478bd9Sstevel@tonic-gate { 38357c478bd9Sstevel@tonic-gate return (pp->p_next); 38367c478bd9Sstevel@tonic-gate } 38377c478bd9Sstevel@tonic-gate 38387c478bd9Sstevel@tonic-gate 38397c478bd9Sstevel@tonic-gate /* 38407c478bd9Sstevel@tonic-gate * Add the page to the front of the linked list of pages 38417c478bd9Sstevel@tonic-gate * using p_vpnext/p_vpprev pointers for the list. 38427c478bd9Sstevel@tonic-gate * 38437c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the lists. 38447c478bd9Sstevel@tonic-gate */ 38457c478bd9Sstevel@tonic-gate void 38467c478bd9Sstevel@tonic-gate page_vpadd(page_t **ppp, page_t *pp) 38477c478bd9Sstevel@tonic-gate { 38487c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 38497c478bd9Sstevel@tonic-gate pp->p_vpnext = pp->p_vpprev = pp; 38507c478bd9Sstevel@tonic-gate } else { 38517c478bd9Sstevel@tonic-gate pp->p_vpnext = *ppp; 38527c478bd9Sstevel@tonic-gate pp->p_vpprev = (*ppp)->p_vpprev; 38537c478bd9Sstevel@tonic-gate (*ppp)->p_vpprev = pp; 38547c478bd9Sstevel@tonic-gate pp->p_vpprev->p_vpnext = pp; 38557c478bd9Sstevel@tonic-gate } 38567c478bd9Sstevel@tonic-gate *ppp = pp; 38577c478bd9Sstevel@tonic-gate } 38587c478bd9Sstevel@tonic-gate 38597c478bd9Sstevel@tonic-gate /* 38607c478bd9Sstevel@tonic-gate * Remove this page from the linked list of pages 38617c478bd9Sstevel@tonic-gate * using p_vpnext/p_vpprev pointers for the list. 38627c478bd9Sstevel@tonic-gate * 38637c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the lists. 38647c478bd9Sstevel@tonic-gate */ 38657c478bd9Sstevel@tonic-gate void 38667c478bd9Sstevel@tonic-gate page_vpsub(page_t **ppp, page_t *pp) 38677c478bd9Sstevel@tonic-gate { 38687c478bd9Sstevel@tonic-gate if (*ppp == NULL || pp == NULL) { 38697c478bd9Sstevel@tonic-gate panic("page_vpsub: bad arg(s): pp %p, *ppp %p", 38707c478bd9Sstevel@tonic-gate (void *)pp, (void *)(*ppp)); 38717c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 38727c478bd9Sstevel@tonic-gate } 38737c478bd9Sstevel@tonic-gate 38747c478bd9Sstevel@tonic-gate if (*ppp == pp) 38757c478bd9Sstevel@tonic-gate *ppp = pp->p_vpnext; /* go to next page */ 38767c478bd9Sstevel@tonic-gate 38777c478bd9Sstevel@tonic-gate if (*ppp == pp) 38787c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 38797c478bd9Sstevel@tonic-gate else { 38807c478bd9Sstevel@tonic-gate pp->p_vpprev->p_vpnext = pp->p_vpnext; 38817c478bd9Sstevel@tonic-gate pp->p_vpnext->p_vpprev = pp->p_vpprev; 38827c478bd9Sstevel@tonic-gate } 38837c478bd9Sstevel@tonic-gate pp->p_vpprev = pp->p_vpnext = pp; /* make pp a list of one */ 38847c478bd9Sstevel@tonic-gate } 38857c478bd9Sstevel@tonic-gate 38867c478bd9Sstevel@tonic-gate /* 38877c478bd9Sstevel@tonic-gate * Lock a physical page into memory "long term". Used to support "lock 38887c478bd9Sstevel@tonic-gate * in memory" functions. Accepts the page to be locked, and a cow variable 38897c478bd9Sstevel@tonic-gate * to indicate whether a the lock will travel to the new page during 38907c478bd9Sstevel@tonic-gate * a potential copy-on-write. 38917c478bd9Sstevel@tonic-gate */ 38927c478bd9Sstevel@tonic-gate int 38937c478bd9Sstevel@tonic-gate page_pp_lock( 38947c478bd9Sstevel@tonic-gate page_t *pp, /* page to be locked */ 38957c478bd9Sstevel@tonic-gate int cow, /* cow lock */ 38967c478bd9Sstevel@tonic-gate int kernel) /* must succeed -- ignore checking */ 38977c478bd9Sstevel@tonic-gate { 38987c478bd9Sstevel@tonic-gate int r = 0; /* result -- assume failure */ 38997c478bd9Sstevel@tonic-gate 39007c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 39017c478bd9Sstevel@tonic-gate 39027c478bd9Sstevel@tonic-gate page_struct_lock(pp); 39037c478bd9Sstevel@tonic-gate /* 39047c478bd9Sstevel@tonic-gate * Acquire the "freemem_lock" for availrmem. 39057c478bd9Sstevel@tonic-gate */ 39067c478bd9Sstevel@tonic-gate if (cow) { 39077c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 39087c478bd9Sstevel@tonic-gate if ((availrmem > pages_pp_maximum) && 39097c478bd9Sstevel@tonic-gate (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 39107c478bd9Sstevel@tonic-gate availrmem--; 39117c478bd9Sstevel@tonic-gate pages_locked++; 39127c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 39137c478bd9Sstevel@tonic-gate r = 1; 39147c478bd9Sstevel@tonic-gate if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 39157c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 39167c478bd9Sstevel@tonic-gate "COW lock limit reached on pfn 0x%lx", 39177c478bd9Sstevel@tonic-gate page_pptonum(pp)); 39187c478bd9Sstevel@tonic-gate } 39197c478bd9Sstevel@tonic-gate } else 39207c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 39217c478bd9Sstevel@tonic-gate } else { 39227c478bd9Sstevel@tonic-gate if (pp->p_lckcnt) { 39237c478bd9Sstevel@tonic-gate if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 39247c478bd9Sstevel@tonic-gate r = 1; 39257c478bd9Sstevel@tonic-gate if (++pp->p_lckcnt == 39267c478bd9Sstevel@tonic-gate (ushort_t)PAGE_LOCK_MAXIMUM) { 39277c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Page lock limit " 39287c478bd9Sstevel@tonic-gate "reached on pfn 0x%lx", 39297c478bd9Sstevel@tonic-gate page_pptonum(pp)); 39307c478bd9Sstevel@tonic-gate } 39317c478bd9Sstevel@tonic-gate } 39327c478bd9Sstevel@tonic-gate } else { 39337c478bd9Sstevel@tonic-gate if (kernel) { 39347c478bd9Sstevel@tonic-gate /* availrmem accounting done by caller */ 39357c478bd9Sstevel@tonic-gate ++pp->p_lckcnt; 39367c478bd9Sstevel@tonic-gate r = 1; 39377c478bd9Sstevel@tonic-gate } else { 39387c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 39397c478bd9Sstevel@tonic-gate if (availrmem > pages_pp_maximum) { 39407c478bd9Sstevel@tonic-gate availrmem--; 39417c478bd9Sstevel@tonic-gate pages_locked++; 39427c478bd9Sstevel@tonic-gate ++pp->p_lckcnt; 39437c478bd9Sstevel@tonic-gate r = 1; 39447c478bd9Sstevel@tonic-gate } 39457c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 39467c478bd9Sstevel@tonic-gate } 39477c478bd9Sstevel@tonic-gate } 39487c478bd9Sstevel@tonic-gate } 39497c478bd9Sstevel@tonic-gate page_struct_unlock(pp); 39507c478bd9Sstevel@tonic-gate return (r); 39517c478bd9Sstevel@tonic-gate } 39527c478bd9Sstevel@tonic-gate 39537c478bd9Sstevel@tonic-gate /* 39547c478bd9Sstevel@tonic-gate * Decommit a lock on a physical page frame. Account for cow locks if 39557c478bd9Sstevel@tonic-gate * appropriate. 39567c478bd9Sstevel@tonic-gate */ 39577c478bd9Sstevel@tonic-gate void 39587c478bd9Sstevel@tonic-gate page_pp_unlock( 39597c478bd9Sstevel@tonic-gate page_t *pp, /* page to be unlocked */ 39607c478bd9Sstevel@tonic-gate int cow, /* expect cow lock */ 39617c478bd9Sstevel@tonic-gate int kernel) /* this was a kernel lock */ 39627c478bd9Sstevel@tonic-gate { 39637c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 39647c478bd9Sstevel@tonic-gate 39657c478bd9Sstevel@tonic-gate page_struct_lock(pp); 39667c478bd9Sstevel@tonic-gate /* 39677c478bd9Sstevel@tonic-gate * Acquire the "freemem_lock" for availrmem. 39687c478bd9Sstevel@tonic-gate * If cowcnt or lcknt is already 0 do nothing; i.e., we 39697c478bd9Sstevel@tonic-gate * could be called to unlock even if nothing is locked. This could 39707c478bd9Sstevel@tonic-gate * happen if locked file pages were truncated (removing the lock) 39717c478bd9Sstevel@tonic-gate * and the file was grown again and new pages faulted in; the new 39727c478bd9Sstevel@tonic-gate * pages are unlocked but the segment still thinks they're locked. 39737c478bd9Sstevel@tonic-gate */ 39747c478bd9Sstevel@tonic-gate if (cow) { 39757c478bd9Sstevel@tonic-gate if (pp->p_cowcnt) { 39767c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 39777c478bd9Sstevel@tonic-gate pp->p_cowcnt--; 39787c478bd9Sstevel@tonic-gate availrmem++; 39797c478bd9Sstevel@tonic-gate pages_locked--; 39807c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 39817c478bd9Sstevel@tonic-gate } 39827c478bd9Sstevel@tonic-gate } else { 39837c478bd9Sstevel@tonic-gate if (pp->p_lckcnt && --pp->p_lckcnt == 0) { 39847c478bd9Sstevel@tonic-gate if (!kernel) { 39857c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 39867c478bd9Sstevel@tonic-gate availrmem++; 39877c478bd9Sstevel@tonic-gate pages_locked--; 39887c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 39897c478bd9Sstevel@tonic-gate } 39907c478bd9Sstevel@tonic-gate } 39917c478bd9Sstevel@tonic-gate } 39927c478bd9Sstevel@tonic-gate page_struct_unlock(pp); 39937c478bd9Sstevel@tonic-gate } 39947c478bd9Sstevel@tonic-gate 39957c478bd9Sstevel@tonic-gate /* 39967c478bd9Sstevel@tonic-gate * This routine reserves availrmem for npages; 39977c478bd9Sstevel@tonic-gate * flags: KM_NOSLEEP or KM_SLEEP 39987c478bd9Sstevel@tonic-gate * returns 1 on success or 0 on failure 39997c478bd9Sstevel@tonic-gate */ 40007c478bd9Sstevel@tonic-gate int 40017c478bd9Sstevel@tonic-gate page_resv(pgcnt_t npages, uint_t flags) 40027c478bd9Sstevel@tonic-gate { 40037c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 40047c478bd9Sstevel@tonic-gate while (availrmem < tune.t_minarmem + npages) { 40057c478bd9Sstevel@tonic-gate if (flags & KM_NOSLEEP) { 40067c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 40077c478bd9Sstevel@tonic-gate return (0); 40087c478bd9Sstevel@tonic-gate } 40097c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 40107c478bd9Sstevel@tonic-gate page_needfree(npages); 40117c478bd9Sstevel@tonic-gate kmem_reap(); 40127c478bd9Sstevel@tonic-gate delay(hz >> 2); 40137c478bd9Sstevel@tonic-gate page_needfree(-(spgcnt_t)npages); 40147c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 40157c478bd9Sstevel@tonic-gate } 40167c478bd9Sstevel@tonic-gate availrmem -= npages; 40177c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 40187c478bd9Sstevel@tonic-gate return (1); 40197c478bd9Sstevel@tonic-gate } 40207c478bd9Sstevel@tonic-gate 40217c478bd9Sstevel@tonic-gate /* 40227c478bd9Sstevel@tonic-gate * This routine unreserves availrmem for npages; 40237c478bd9Sstevel@tonic-gate */ 40247c478bd9Sstevel@tonic-gate void 40257c478bd9Sstevel@tonic-gate page_unresv(pgcnt_t npages) 40267c478bd9Sstevel@tonic-gate { 40277c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 40287c478bd9Sstevel@tonic-gate availrmem += npages; 40297c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 40307c478bd9Sstevel@tonic-gate } 40317c478bd9Sstevel@tonic-gate 40327c478bd9Sstevel@tonic-gate /* 40337c478bd9Sstevel@tonic-gate * See Statement at the beginning of segvn_lockop() regarding 40347c478bd9Sstevel@tonic-gate * the way we handle cowcnts and lckcnts. 40357c478bd9Sstevel@tonic-gate * 40367c478bd9Sstevel@tonic-gate * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage 40377c478bd9Sstevel@tonic-gate * that breaks COW has PROT_WRITE. 40387c478bd9Sstevel@tonic-gate * 40397c478bd9Sstevel@tonic-gate * Note that, we may also break COW in case we are softlocking 40407c478bd9Sstevel@tonic-gate * on read access during physio; 40417c478bd9Sstevel@tonic-gate * in this softlock case, the vpage may not have PROT_WRITE. 40427c478bd9Sstevel@tonic-gate * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp' 40437c478bd9Sstevel@tonic-gate * if the vpage doesn't have PROT_WRITE. 40447c478bd9Sstevel@tonic-gate * 40457c478bd9Sstevel@tonic-gate * This routine is never called if we are stealing a page 40467c478bd9Sstevel@tonic-gate * in anon_private. 40477c478bd9Sstevel@tonic-gate * 40487c478bd9Sstevel@tonic-gate * The caller subtracted from availrmem for read only mapping. 40497c478bd9Sstevel@tonic-gate * if lckcnt is 1 increment availrmem. 40507c478bd9Sstevel@tonic-gate */ 40517c478bd9Sstevel@tonic-gate void 40527c478bd9Sstevel@tonic-gate page_pp_useclaim( 40537c478bd9Sstevel@tonic-gate page_t *opp, /* original page frame losing lock */ 40547c478bd9Sstevel@tonic-gate page_t *npp, /* new page frame gaining lock */ 40557c478bd9Sstevel@tonic-gate uint_t write_perm) /* set if vpage has PROT_WRITE */ 40567c478bd9Sstevel@tonic-gate { 40577c478bd9Sstevel@tonic-gate int payback = 0; 40587c478bd9Sstevel@tonic-gate 40597c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(opp)); 40607c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(npp)); 40617c478bd9Sstevel@tonic-gate 40627c478bd9Sstevel@tonic-gate page_struct_lock(opp); 40637c478bd9Sstevel@tonic-gate 40647c478bd9Sstevel@tonic-gate ASSERT(npp->p_cowcnt == 0); 40657c478bd9Sstevel@tonic-gate ASSERT(npp->p_lckcnt == 0); 40667c478bd9Sstevel@tonic-gate 40677c478bd9Sstevel@tonic-gate /* Don't use claim if nothing is locked (see page_pp_unlock above) */ 40687c478bd9Sstevel@tonic-gate if ((write_perm && opp->p_cowcnt != 0) || 40697c478bd9Sstevel@tonic-gate (!write_perm && opp->p_lckcnt != 0)) { 40707c478bd9Sstevel@tonic-gate 40717c478bd9Sstevel@tonic-gate if (write_perm) { 40727c478bd9Sstevel@tonic-gate npp->p_cowcnt++; 40737c478bd9Sstevel@tonic-gate ASSERT(opp->p_cowcnt != 0); 40747c478bd9Sstevel@tonic-gate opp->p_cowcnt--; 40757c478bd9Sstevel@tonic-gate } else { 40767c478bd9Sstevel@tonic-gate 40777c478bd9Sstevel@tonic-gate ASSERT(opp->p_lckcnt != 0); 40787c478bd9Sstevel@tonic-gate 40797c478bd9Sstevel@tonic-gate /* 40807c478bd9Sstevel@tonic-gate * We didn't need availrmem decremented if p_lckcnt on 40817c478bd9Sstevel@tonic-gate * original page is 1. Here, we are unlocking 40827c478bd9Sstevel@tonic-gate * read-only copy belonging to original page and 40837c478bd9Sstevel@tonic-gate * are locking a copy belonging to new page. 40847c478bd9Sstevel@tonic-gate */ 40857c478bd9Sstevel@tonic-gate if (opp->p_lckcnt == 1) 40867c478bd9Sstevel@tonic-gate payback = 1; 40877c478bd9Sstevel@tonic-gate 40887c478bd9Sstevel@tonic-gate npp->p_lckcnt++; 40897c478bd9Sstevel@tonic-gate opp->p_lckcnt--; 40907c478bd9Sstevel@tonic-gate } 40917c478bd9Sstevel@tonic-gate } 40927c478bd9Sstevel@tonic-gate if (payback) { 40937c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 40947c478bd9Sstevel@tonic-gate availrmem++; 40957c478bd9Sstevel@tonic-gate pages_useclaim--; 40967c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 40977c478bd9Sstevel@tonic-gate } 40987c478bd9Sstevel@tonic-gate page_struct_unlock(opp); 40997c478bd9Sstevel@tonic-gate } 41007c478bd9Sstevel@tonic-gate 41017c478bd9Sstevel@tonic-gate /* 41027c478bd9Sstevel@tonic-gate * Simple claim adjust functions -- used to support changes in 41037c478bd9Sstevel@tonic-gate * claims due to changes in access permissions. Used by segvn_setprot(). 41047c478bd9Sstevel@tonic-gate */ 41057c478bd9Sstevel@tonic-gate int 41067c478bd9Sstevel@tonic-gate page_addclaim(page_t *pp) 41077c478bd9Sstevel@tonic-gate { 41087c478bd9Sstevel@tonic-gate int r = 0; /* result */ 41097c478bd9Sstevel@tonic-gate 41107c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 41117c478bd9Sstevel@tonic-gate 41127c478bd9Sstevel@tonic-gate page_struct_lock(pp); 41137c478bd9Sstevel@tonic-gate ASSERT(pp->p_lckcnt != 0); 41147c478bd9Sstevel@tonic-gate 41157c478bd9Sstevel@tonic-gate if (pp->p_lckcnt == 1) { 41167c478bd9Sstevel@tonic-gate if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 41177c478bd9Sstevel@tonic-gate --pp->p_lckcnt; 41187c478bd9Sstevel@tonic-gate r = 1; 41197c478bd9Sstevel@tonic-gate if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 41207c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 41217c478bd9Sstevel@tonic-gate "COW lock limit reached on pfn 0x%lx", 41227c478bd9Sstevel@tonic-gate page_pptonum(pp)); 41237c478bd9Sstevel@tonic-gate } 41247c478bd9Sstevel@tonic-gate } 41257c478bd9Sstevel@tonic-gate } else { 41267c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 41277c478bd9Sstevel@tonic-gate if ((availrmem > pages_pp_maximum) && 41287c478bd9Sstevel@tonic-gate (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) { 41297c478bd9Sstevel@tonic-gate --availrmem; 41307c478bd9Sstevel@tonic-gate ++pages_claimed; 41317c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 41327c478bd9Sstevel@tonic-gate --pp->p_lckcnt; 41337c478bd9Sstevel@tonic-gate r = 1; 41347c478bd9Sstevel@tonic-gate if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 41357c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 41367c478bd9Sstevel@tonic-gate "COW lock limit reached on pfn 0x%lx", 41377c478bd9Sstevel@tonic-gate page_pptonum(pp)); 41387c478bd9Sstevel@tonic-gate } 41397c478bd9Sstevel@tonic-gate } else 41407c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 41417c478bd9Sstevel@tonic-gate } 41427c478bd9Sstevel@tonic-gate page_struct_unlock(pp); 41437c478bd9Sstevel@tonic-gate return (r); 41447c478bd9Sstevel@tonic-gate } 41457c478bd9Sstevel@tonic-gate 41467c478bd9Sstevel@tonic-gate int 41477c478bd9Sstevel@tonic-gate page_subclaim(page_t *pp) 41487c478bd9Sstevel@tonic-gate { 41497c478bd9Sstevel@tonic-gate int r = 0; 41507c478bd9Sstevel@tonic-gate 41517c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 41527c478bd9Sstevel@tonic-gate 41537c478bd9Sstevel@tonic-gate page_struct_lock(pp); 41547c478bd9Sstevel@tonic-gate ASSERT(pp->p_cowcnt != 0); 41557c478bd9Sstevel@tonic-gate 41567c478bd9Sstevel@tonic-gate if (pp->p_lckcnt) { 41577c478bd9Sstevel@tonic-gate if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) { 41587c478bd9Sstevel@tonic-gate r = 1; 41597c478bd9Sstevel@tonic-gate /* 41607c478bd9Sstevel@tonic-gate * for availrmem 41617c478bd9Sstevel@tonic-gate */ 41627c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 41637c478bd9Sstevel@tonic-gate availrmem++; 41647c478bd9Sstevel@tonic-gate pages_claimed--; 41657c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 41667c478bd9Sstevel@tonic-gate 41677c478bd9Sstevel@tonic-gate pp->p_cowcnt--; 41687c478bd9Sstevel@tonic-gate 41697c478bd9Sstevel@tonic-gate if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 41707c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 41717c478bd9Sstevel@tonic-gate "Page lock limit reached on pfn 0x%lx", 41727c478bd9Sstevel@tonic-gate page_pptonum(pp)); 41737c478bd9Sstevel@tonic-gate } 41747c478bd9Sstevel@tonic-gate } 41757c478bd9Sstevel@tonic-gate } else { 41767c478bd9Sstevel@tonic-gate r = 1; 41777c478bd9Sstevel@tonic-gate pp->p_cowcnt--; 41787c478bd9Sstevel@tonic-gate pp->p_lckcnt++; 41797c478bd9Sstevel@tonic-gate } 41807c478bd9Sstevel@tonic-gate page_struct_unlock(pp); 41817c478bd9Sstevel@tonic-gate return (r); 41827c478bd9Sstevel@tonic-gate } 41837c478bd9Sstevel@tonic-gate 41847c478bd9Sstevel@tonic-gate int 41857c478bd9Sstevel@tonic-gate page_addclaim_pages(page_t **ppa) 41867c478bd9Sstevel@tonic-gate { 41877c478bd9Sstevel@tonic-gate 41887c478bd9Sstevel@tonic-gate pgcnt_t lckpgs = 0, pg_idx; 41897c478bd9Sstevel@tonic-gate 41907c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_addclaim_pages); 41917c478bd9Sstevel@tonic-gate 41927c478bd9Sstevel@tonic-gate mutex_enter(&page_llock); 41937c478bd9Sstevel@tonic-gate for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 41947c478bd9Sstevel@tonic-gate 41957c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(ppa[pg_idx])); 41967c478bd9Sstevel@tonic-gate ASSERT(ppa[pg_idx]->p_lckcnt != 0); 41977c478bd9Sstevel@tonic-gate if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 41987c478bd9Sstevel@tonic-gate mutex_exit(&page_llock); 41997c478bd9Sstevel@tonic-gate return (0); 42007c478bd9Sstevel@tonic-gate } 42017c478bd9Sstevel@tonic-gate if (ppa[pg_idx]->p_lckcnt > 1) 42027c478bd9Sstevel@tonic-gate lckpgs++; 42037c478bd9Sstevel@tonic-gate } 42047c478bd9Sstevel@tonic-gate 42057c478bd9Sstevel@tonic-gate if (lckpgs != 0) { 42067c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 42077c478bd9Sstevel@tonic-gate if (availrmem >= pages_pp_maximum + lckpgs) { 42087c478bd9Sstevel@tonic-gate availrmem -= lckpgs; 42097c478bd9Sstevel@tonic-gate pages_claimed += lckpgs; 42107c478bd9Sstevel@tonic-gate } else { 42117c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 42127c478bd9Sstevel@tonic-gate mutex_exit(&page_llock); 42137c478bd9Sstevel@tonic-gate return (0); 42147c478bd9Sstevel@tonic-gate } 42157c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 42167c478bd9Sstevel@tonic-gate } 42177c478bd9Sstevel@tonic-gate 42187c478bd9Sstevel@tonic-gate for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 42197c478bd9Sstevel@tonic-gate ppa[pg_idx]->p_lckcnt--; 42207c478bd9Sstevel@tonic-gate ppa[pg_idx]->p_cowcnt++; 42217c478bd9Sstevel@tonic-gate } 42227c478bd9Sstevel@tonic-gate mutex_exit(&page_llock); 42237c478bd9Sstevel@tonic-gate return (1); 42247c478bd9Sstevel@tonic-gate } 42257c478bd9Sstevel@tonic-gate 42267c478bd9Sstevel@tonic-gate int 42277c478bd9Sstevel@tonic-gate page_subclaim_pages(page_t **ppa) 42287c478bd9Sstevel@tonic-gate { 42297c478bd9Sstevel@tonic-gate pgcnt_t ulckpgs = 0, pg_idx; 42307c478bd9Sstevel@tonic-gate 42317c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_subclaim_pages); 42327c478bd9Sstevel@tonic-gate 42337c478bd9Sstevel@tonic-gate mutex_enter(&page_llock); 42347c478bd9Sstevel@tonic-gate for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 42357c478bd9Sstevel@tonic-gate 42367c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(ppa[pg_idx])); 42377c478bd9Sstevel@tonic-gate ASSERT(ppa[pg_idx]->p_cowcnt != 0); 42387c478bd9Sstevel@tonic-gate if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) { 42397c478bd9Sstevel@tonic-gate mutex_exit(&page_llock); 42407c478bd9Sstevel@tonic-gate return (0); 42417c478bd9Sstevel@tonic-gate } 42427c478bd9Sstevel@tonic-gate if (ppa[pg_idx]->p_lckcnt != 0) 42437c478bd9Sstevel@tonic-gate ulckpgs++; 42447c478bd9Sstevel@tonic-gate } 42457c478bd9Sstevel@tonic-gate 42467c478bd9Sstevel@tonic-gate if (ulckpgs != 0) { 42477c478bd9Sstevel@tonic-gate mutex_enter(&freemem_lock); 42487c478bd9Sstevel@tonic-gate availrmem += ulckpgs; 42497c478bd9Sstevel@tonic-gate pages_claimed -= ulckpgs; 42507c478bd9Sstevel@tonic-gate mutex_exit(&freemem_lock); 42517c478bd9Sstevel@tonic-gate } 42527c478bd9Sstevel@tonic-gate 42537c478bd9Sstevel@tonic-gate for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) { 42547c478bd9Sstevel@tonic-gate ppa[pg_idx]->p_cowcnt--; 42557c478bd9Sstevel@tonic-gate ppa[pg_idx]->p_lckcnt++; 42567c478bd9Sstevel@tonic-gate 42577c478bd9Sstevel@tonic-gate } 42587c478bd9Sstevel@tonic-gate mutex_exit(&page_llock); 42597c478bd9Sstevel@tonic-gate return (1); 42607c478bd9Sstevel@tonic-gate } 42617c478bd9Sstevel@tonic-gate 42627c478bd9Sstevel@tonic-gate page_t * 42637c478bd9Sstevel@tonic-gate page_numtopp(pfn_t pfnum, se_t se) 42647c478bd9Sstevel@tonic-gate { 42657c478bd9Sstevel@tonic-gate page_t *pp; 42667c478bd9Sstevel@tonic-gate 42677c478bd9Sstevel@tonic-gate retry: 42687c478bd9Sstevel@tonic-gate pp = page_numtopp_nolock(pfnum); 42697c478bd9Sstevel@tonic-gate if (pp == NULL) { 42707c478bd9Sstevel@tonic-gate return ((page_t *)NULL); 42717c478bd9Sstevel@tonic-gate } 42727c478bd9Sstevel@tonic-gate 42737c478bd9Sstevel@tonic-gate /* 42747c478bd9Sstevel@tonic-gate * Acquire the appropriate lock on the page. 42757c478bd9Sstevel@tonic-gate */ 42767c478bd9Sstevel@tonic-gate while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) { 42777c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfnum) 42787c478bd9Sstevel@tonic-gate goto retry; 42797c478bd9Sstevel@tonic-gate continue; 42807c478bd9Sstevel@tonic-gate } 42817c478bd9Sstevel@tonic-gate 42827c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfnum) { 42837c478bd9Sstevel@tonic-gate page_unlock(pp); 42847c478bd9Sstevel@tonic-gate goto retry; 42857c478bd9Sstevel@tonic-gate } 42867c478bd9Sstevel@tonic-gate 42877c478bd9Sstevel@tonic-gate return (pp); 42887c478bd9Sstevel@tonic-gate } 42897c478bd9Sstevel@tonic-gate 42907c478bd9Sstevel@tonic-gate page_t * 42917c478bd9Sstevel@tonic-gate page_numtopp_noreclaim(pfn_t pfnum, se_t se) 42927c478bd9Sstevel@tonic-gate { 42937c478bd9Sstevel@tonic-gate page_t *pp; 42947c478bd9Sstevel@tonic-gate 42957c478bd9Sstevel@tonic-gate retry: 42967c478bd9Sstevel@tonic-gate pp = page_numtopp_nolock(pfnum); 42977c478bd9Sstevel@tonic-gate if (pp == NULL) { 42987c478bd9Sstevel@tonic-gate return ((page_t *)NULL); 42997c478bd9Sstevel@tonic-gate } 43007c478bd9Sstevel@tonic-gate 43017c478bd9Sstevel@tonic-gate /* 43027c478bd9Sstevel@tonic-gate * Acquire the appropriate lock on the page. 43037c478bd9Sstevel@tonic-gate */ 43047c478bd9Sstevel@tonic-gate while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) { 43057c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfnum) 43067c478bd9Sstevel@tonic-gate goto retry; 43077c478bd9Sstevel@tonic-gate continue; 43087c478bd9Sstevel@tonic-gate } 43097c478bd9Sstevel@tonic-gate 43107c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfnum) { 43117c478bd9Sstevel@tonic-gate page_unlock(pp); 43127c478bd9Sstevel@tonic-gate goto retry; 43137c478bd9Sstevel@tonic-gate } 43147c478bd9Sstevel@tonic-gate 43157c478bd9Sstevel@tonic-gate return (pp); 43167c478bd9Sstevel@tonic-gate } 43177c478bd9Sstevel@tonic-gate 43187c478bd9Sstevel@tonic-gate /* 43197c478bd9Sstevel@tonic-gate * This routine is like page_numtopp, but will only return page structs 43207c478bd9Sstevel@tonic-gate * for pages which are ok for loading into hardware using the page struct. 43217c478bd9Sstevel@tonic-gate */ 43227c478bd9Sstevel@tonic-gate page_t * 43237c478bd9Sstevel@tonic-gate page_numtopp_nowait(pfn_t pfnum, se_t se) 43247c478bd9Sstevel@tonic-gate { 43257c478bd9Sstevel@tonic-gate page_t *pp; 43267c478bd9Sstevel@tonic-gate 43277c478bd9Sstevel@tonic-gate retry: 43287c478bd9Sstevel@tonic-gate pp = page_numtopp_nolock(pfnum); 43297c478bd9Sstevel@tonic-gate if (pp == NULL) { 43307c478bd9Sstevel@tonic-gate return ((page_t *)NULL); 43317c478bd9Sstevel@tonic-gate } 43327c478bd9Sstevel@tonic-gate 43337c478bd9Sstevel@tonic-gate /* 43347c478bd9Sstevel@tonic-gate * Try to acquire the appropriate lock on the page. 43357c478bd9Sstevel@tonic-gate */ 43367c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) 43377c478bd9Sstevel@tonic-gate pp = NULL; 43387c478bd9Sstevel@tonic-gate else { 43397c478bd9Sstevel@tonic-gate if (!page_trylock(pp, se)) 43407c478bd9Sstevel@tonic-gate pp = NULL; 43417c478bd9Sstevel@tonic-gate else { 43427c478bd9Sstevel@tonic-gate if (page_pptonum(pp) != pfnum) { 43437c478bd9Sstevel@tonic-gate page_unlock(pp); 43447c478bd9Sstevel@tonic-gate goto retry; 43457c478bd9Sstevel@tonic-gate } 43467c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 43477c478bd9Sstevel@tonic-gate page_unlock(pp); 43487c478bd9Sstevel@tonic-gate pp = NULL; 43497c478bd9Sstevel@tonic-gate } 43507c478bd9Sstevel@tonic-gate } 43517c478bd9Sstevel@tonic-gate } 43527c478bd9Sstevel@tonic-gate return (pp); 43537c478bd9Sstevel@tonic-gate } 43547c478bd9Sstevel@tonic-gate 43557c478bd9Sstevel@tonic-gate /* 43567c478bd9Sstevel@tonic-gate * Returns a count of dirty pages that are in the process 43577c478bd9Sstevel@tonic-gate * of being written out. If 'cleanit' is set, try to push the page. 43587c478bd9Sstevel@tonic-gate */ 43597c478bd9Sstevel@tonic-gate pgcnt_t 43607c478bd9Sstevel@tonic-gate page_busy(int cleanit) 43617c478bd9Sstevel@tonic-gate { 43627c478bd9Sstevel@tonic-gate page_t *page0 = page_first(); 43637c478bd9Sstevel@tonic-gate page_t *pp = page0; 43647c478bd9Sstevel@tonic-gate pgcnt_t nppbusy = 0; 43657c478bd9Sstevel@tonic-gate u_offset_t off; 43667c478bd9Sstevel@tonic-gate 43677c478bd9Sstevel@tonic-gate do { 43687c478bd9Sstevel@tonic-gate vnode_t *vp = pp->p_vnode; 43697c478bd9Sstevel@tonic-gate 43707c478bd9Sstevel@tonic-gate /* 43717c478bd9Sstevel@tonic-gate * A page is a candidate for syncing if it is: 43727c478bd9Sstevel@tonic-gate * 43737c478bd9Sstevel@tonic-gate * (a) On neither the freelist nor the cachelist 43747c478bd9Sstevel@tonic-gate * (b) Hashed onto a vnode 43757c478bd9Sstevel@tonic-gate * (c) Not a kernel page 43767c478bd9Sstevel@tonic-gate * (d) Dirty 43777c478bd9Sstevel@tonic-gate * (e) Not part of a swapfile 43787c478bd9Sstevel@tonic-gate * (f) a page which belongs to a real vnode; eg has a non-null 43797c478bd9Sstevel@tonic-gate * v_vfsp pointer. 43807c478bd9Sstevel@tonic-gate * (g) Backed by a filesystem which doesn't have a 43817c478bd9Sstevel@tonic-gate * stubbed-out sync operation 43827c478bd9Sstevel@tonic-gate */ 4383ad23a2dbSjohansen if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) && 43847c478bd9Sstevel@tonic-gate hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL && 43857c478bd9Sstevel@tonic-gate vfs_can_sync(vp->v_vfsp)) { 43867c478bd9Sstevel@tonic-gate nppbusy++; 43877c478bd9Sstevel@tonic-gate vfs_syncprogress(); 43887c478bd9Sstevel@tonic-gate 43897c478bd9Sstevel@tonic-gate if (!cleanit) 43907c478bd9Sstevel@tonic-gate continue; 43917c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) 43927c478bd9Sstevel@tonic-gate continue; 43937c478bd9Sstevel@tonic-gate 43947c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) || 43957c478bd9Sstevel@tonic-gate pp->p_lckcnt != 0 || pp->p_cowcnt != 0 || 43967c478bd9Sstevel@tonic-gate !(hat_pagesync(pp, 43977c478bd9Sstevel@tonic-gate HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) { 43987c478bd9Sstevel@tonic-gate page_unlock(pp); 43997c478bd9Sstevel@tonic-gate continue; 44007c478bd9Sstevel@tonic-gate } 44017c478bd9Sstevel@tonic-gate off = pp->p_offset; 44027c478bd9Sstevel@tonic-gate VN_HOLD(vp); 44037c478bd9Sstevel@tonic-gate page_unlock(pp); 44047c478bd9Sstevel@tonic-gate (void) VOP_PUTPAGE(vp, off, PAGESIZE, 44057c478bd9Sstevel@tonic-gate B_ASYNC | B_FREE, kcred); 44067c478bd9Sstevel@tonic-gate VN_RELE(vp); 44077c478bd9Sstevel@tonic-gate } 44087c478bd9Sstevel@tonic-gate } while ((pp = page_next(pp)) != page0); 44097c478bd9Sstevel@tonic-gate 44107c478bd9Sstevel@tonic-gate return (nppbusy); 44117c478bd9Sstevel@tonic-gate } 44127c478bd9Sstevel@tonic-gate 44137c478bd9Sstevel@tonic-gate void page_invalidate_pages(void); 44147c478bd9Sstevel@tonic-gate 44157c478bd9Sstevel@tonic-gate /* 44167c478bd9Sstevel@tonic-gate * callback handler to vm sub-system 44177c478bd9Sstevel@tonic-gate * 44187c478bd9Sstevel@tonic-gate * callers make sure no recursive entries to this func. 44197c478bd9Sstevel@tonic-gate */ 44207c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 44217c478bd9Sstevel@tonic-gate boolean_t 44227c478bd9Sstevel@tonic-gate callb_vm_cpr(void *arg, int code) 44237c478bd9Sstevel@tonic-gate { 44247c478bd9Sstevel@tonic-gate if (code == CB_CODE_CPR_CHKPT) 44257c478bd9Sstevel@tonic-gate page_invalidate_pages(); 44267c478bd9Sstevel@tonic-gate return (B_TRUE); 44277c478bd9Sstevel@tonic-gate } 44287c478bd9Sstevel@tonic-gate 44297c478bd9Sstevel@tonic-gate /* 44307c478bd9Sstevel@tonic-gate * Invalidate all pages of the system. 44317c478bd9Sstevel@tonic-gate * It shouldn't be called until all user page activities are all stopped. 44327c478bd9Sstevel@tonic-gate */ 44337c478bd9Sstevel@tonic-gate void 44347c478bd9Sstevel@tonic-gate page_invalidate_pages() 44357c478bd9Sstevel@tonic-gate { 44367c478bd9Sstevel@tonic-gate page_t *pp; 44377c478bd9Sstevel@tonic-gate page_t *page0; 44387c478bd9Sstevel@tonic-gate pgcnt_t nbusypages; 44397c478bd9Sstevel@tonic-gate int retry = 0; 44407c478bd9Sstevel@tonic-gate const int MAXRETRIES = 4; 44417c478bd9Sstevel@tonic-gate #if defined(__sparc) 44427c478bd9Sstevel@tonic-gate extern struct vnode prom_ppages; 44437c478bd9Sstevel@tonic-gate #endif /* __sparc */ 44447c478bd9Sstevel@tonic-gate 44457c478bd9Sstevel@tonic-gate top: 44467c478bd9Sstevel@tonic-gate /* 44478b464eb8Smec * Flush dirty pages and destroy the clean ones. 44487c478bd9Sstevel@tonic-gate */ 44497c478bd9Sstevel@tonic-gate nbusypages = 0; 44507c478bd9Sstevel@tonic-gate 44517c478bd9Sstevel@tonic-gate pp = page0 = page_first(); 44527c478bd9Sstevel@tonic-gate do { 44537c478bd9Sstevel@tonic-gate struct vnode *vp; 44547c478bd9Sstevel@tonic-gate u_offset_t offset; 44557c478bd9Sstevel@tonic-gate int mod; 44567c478bd9Sstevel@tonic-gate 44577c478bd9Sstevel@tonic-gate /* 44587c478bd9Sstevel@tonic-gate * skip the page if it has no vnode or the page associated 44597c478bd9Sstevel@tonic-gate * with the kernel vnode or prom allocated kernel mem. 44607c478bd9Sstevel@tonic-gate */ 44617c478bd9Sstevel@tonic-gate #if defined(__sparc) 4462ad23a2dbSjohansen if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp) || 44637c478bd9Sstevel@tonic-gate vp == &prom_ppages) 44647c478bd9Sstevel@tonic-gate #else /* x86 doesn't have prom or prom_ppage */ 4465ad23a2dbSjohansen if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp)) 44667c478bd9Sstevel@tonic-gate #endif /* __sparc */ 44677c478bd9Sstevel@tonic-gate continue; 44687c478bd9Sstevel@tonic-gate 44697c478bd9Sstevel@tonic-gate /* 44707c478bd9Sstevel@tonic-gate * skip the page which is already free invalidated. 44717c478bd9Sstevel@tonic-gate */ 44727c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp) && PP_ISAGED(pp)) 44737c478bd9Sstevel@tonic-gate continue; 44747c478bd9Sstevel@tonic-gate 44757c478bd9Sstevel@tonic-gate /* 44767c478bd9Sstevel@tonic-gate * skip pages that are already locked or can't be "exclusively" 44777c478bd9Sstevel@tonic-gate * locked or are already free. After we lock the page, check 44787c478bd9Sstevel@tonic-gate * the free and age bits again to be sure it's not destroied 44797c478bd9Sstevel@tonic-gate * yet. 44807c478bd9Sstevel@tonic-gate * To achieve max. parallelization, we use page_trylock instead 44817c478bd9Sstevel@tonic-gate * of page_lock so that we don't get block on individual pages 44827c478bd9Sstevel@tonic-gate * while we have thousands of other pages to process. 44837c478bd9Sstevel@tonic-gate */ 44847c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 44857c478bd9Sstevel@tonic-gate nbusypages++; 44867c478bd9Sstevel@tonic-gate continue; 44877c478bd9Sstevel@tonic-gate } else if (PP_ISFREE(pp)) { 44887c478bd9Sstevel@tonic-gate if (!PP_ISAGED(pp)) { 44897c478bd9Sstevel@tonic-gate page_destroy_free(pp); 44907c478bd9Sstevel@tonic-gate } else { 44917c478bd9Sstevel@tonic-gate page_unlock(pp); 44927c478bd9Sstevel@tonic-gate } 44937c478bd9Sstevel@tonic-gate continue; 44947c478bd9Sstevel@tonic-gate } 44957c478bd9Sstevel@tonic-gate /* 44967c478bd9Sstevel@tonic-gate * Is this page involved in some I/O? shared? 44977c478bd9Sstevel@tonic-gate * 44987c478bd9Sstevel@tonic-gate * The page_struct_lock need not be acquired to 44997c478bd9Sstevel@tonic-gate * examine these fields since the page has an 45007c478bd9Sstevel@tonic-gate * "exclusive" lock. 45017c478bd9Sstevel@tonic-gate */ 45027c478bd9Sstevel@tonic-gate if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) { 45037c478bd9Sstevel@tonic-gate page_unlock(pp); 45047c478bd9Sstevel@tonic-gate continue; 45057c478bd9Sstevel@tonic-gate } 45067c478bd9Sstevel@tonic-gate 45077c478bd9Sstevel@tonic-gate if (vp->v_type == VCHR) { 45087c478bd9Sstevel@tonic-gate panic("vp->v_type == VCHR"); 45097c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 45107c478bd9Sstevel@tonic-gate } 45117c478bd9Sstevel@tonic-gate 45127c478bd9Sstevel@tonic-gate if (!page_try_demote_pages(pp)) { 45137c478bd9Sstevel@tonic-gate page_unlock(pp); 45147c478bd9Sstevel@tonic-gate continue; 45157c478bd9Sstevel@tonic-gate } 45167c478bd9Sstevel@tonic-gate 45177c478bd9Sstevel@tonic-gate /* 45187c478bd9Sstevel@tonic-gate * Check the modified bit. Leave the bits alone in hardware 45197c478bd9Sstevel@tonic-gate * (they will be modified if we do the putpage). 45207c478bd9Sstevel@tonic-gate */ 45217c478bd9Sstevel@tonic-gate mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) 45227c478bd9Sstevel@tonic-gate & P_MOD); 45237c478bd9Sstevel@tonic-gate if (mod) { 45247c478bd9Sstevel@tonic-gate offset = pp->p_offset; 45257c478bd9Sstevel@tonic-gate /* 45267c478bd9Sstevel@tonic-gate * Hold the vnode before releasing the page lock 45277c478bd9Sstevel@tonic-gate * to prevent it from being freed and re-used by 45287c478bd9Sstevel@tonic-gate * some other thread. 45297c478bd9Sstevel@tonic-gate */ 45307c478bd9Sstevel@tonic-gate VN_HOLD(vp); 45317c478bd9Sstevel@tonic-gate page_unlock(pp); 45327c478bd9Sstevel@tonic-gate /* 45337c478bd9Sstevel@tonic-gate * No error return is checked here. Callers such as 45347c478bd9Sstevel@tonic-gate * cpr deals with the dirty pages at the dump time 45357c478bd9Sstevel@tonic-gate * if this putpage fails. 45367c478bd9Sstevel@tonic-gate */ 45377c478bd9Sstevel@tonic-gate (void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL, 45387c478bd9Sstevel@tonic-gate kcred); 45397c478bd9Sstevel@tonic-gate VN_RELE(vp); 45407c478bd9Sstevel@tonic-gate } else { 45417c478bd9Sstevel@tonic-gate page_destroy(pp, 0); 45427c478bd9Sstevel@tonic-gate } 45437c478bd9Sstevel@tonic-gate } while ((pp = page_next(pp)) != page0); 45447c478bd9Sstevel@tonic-gate if (nbusypages && retry++ < MAXRETRIES) { 45457c478bd9Sstevel@tonic-gate delay(1); 45467c478bd9Sstevel@tonic-gate goto top; 45477c478bd9Sstevel@tonic-gate } 45487c478bd9Sstevel@tonic-gate } 45497c478bd9Sstevel@tonic-gate 45507c478bd9Sstevel@tonic-gate /* 45517c478bd9Sstevel@tonic-gate * Replace the page "old" with the page "new" on the page hash and vnode lists 45527c478bd9Sstevel@tonic-gate * 45537c478bd9Sstevel@tonic-gate * the replacemnt must be done in place, ie the equivalent sequence: 45547c478bd9Sstevel@tonic-gate * 45557c478bd9Sstevel@tonic-gate * vp = old->p_vnode; 45567c478bd9Sstevel@tonic-gate * off = old->p_offset; 45577c478bd9Sstevel@tonic-gate * page_do_hashout(old) 45587c478bd9Sstevel@tonic-gate * page_do_hashin(new, vp, off) 45597c478bd9Sstevel@tonic-gate * 45607c478bd9Sstevel@tonic-gate * doesn't work, since 45617c478bd9Sstevel@tonic-gate * 1) if old is the only page on the vnode, the v_pages list has a window 45627c478bd9Sstevel@tonic-gate * where it looks empty. This will break file system assumptions. 45637c478bd9Sstevel@tonic-gate * and 45647c478bd9Sstevel@tonic-gate * 2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list. 45657c478bd9Sstevel@tonic-gate */ 45667c478bd9Sstevel@tonic-gate static void 45677c478bd9Sstevel@tonic-gate page_do_relocate_hash(page_t *new, page_t *old) 45687c478bd9Sstevel@tonic-gate { 45697c478bd9Sstevel@tonic-gate page_t **hash_list; 45707c478bd9Sstevel@tonic-gate vnode_t *vp = old->p_vnode; 45717c478bd9Sstevel@tonic-gate kmutex_t *sep; 45727c478bd9Sstevel@tonic-gate 45737c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(old)); 45747c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(new)); 45757c478bd9Sstevel@tonic-gate ASSERT(vp != NULL); 45767c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(page_vnode_mutex(vp))); 45777c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset)))); 45787c478bd9Sstevel@tonic-gate 45797c478bd9Sstevel@tonic-gate /* 45807c478bd9Sstevel@tonic-gate * First find old page on the page hash list 45817c478bd9Sstevel@tonic-gate */ 45827c478bd9Sstevel@tonic-gate hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)]; 45837c478bd9Sstevel@tonic-gate 45847c478bd9Sstevel@tonic-gate for (;;) { 45857c478bd9Sstevel@tonic-gate if (*hash_list == old) 45867c478bd9Sstevel@tonic-gate break; 45877c478bd9Sstevel@tonic-gate if (*hash_list == NULL) { 45887c478bd9Sstevel@tonic-gate panic("page_do_hashout"); 45897c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 45907c478bd9Sstevel@tonic-gate } 45917c478bd9Sstevel@tonic-gate hash_list = &(*hash_list)->p_hash; 45927c478bd9Sstevel@tonic-gate } 45937c478bd9Sstevel@tonic-gate 45947c478bd9Sstevel@tonic-gate /* 45957c478bd9Sstevel@tonic-gate * update new and replace old with new on the page hash list 45967c478bd9Sstevel@tonic-gate */ 45977c478bd9Sstevel@tonic-gate new->p_vnode = old->p_vnode; 45987c478bd9Sstevel@tonic-gate new->p_offset = old->p_offset; 45997c478bd9Sstevel@tonic-gate new->p_hash = old->p_hash; 46007c478bd9Sstevel@tonic-gate *hash_list = new; 46017c478bd9Sstevel@tonic-gate 46027c478bd9Sstevel@tonic-gate if ((new->p_vnode->v_flag & VISSWAP) != 0) 46037c478bd9Sstevel@tonic-gate PP_SETSWAP(new); 46047c478bd9Sstevel@tonic-gate 46057c478bd9Sstevel@tonic-gate /* 46067c478bd9Sstevel@tonic-gate * replace old with new on the vnode's page list 46077c478bd9Sstevel@tonic-gate */ 46087c478bd9Sstevel@tonic-gate if (old->p_vpnext == old) { 46097c478bd9Sstevel@tonic-gate new->p_vpnext = new; 46107c478bd9Sstevel@tonic-gate new->p_vpprev = new; 46117c478bd9Sstevel@tonic-gate } else { 46127c478bd9Sstevel@tonic-gate new->p_vpnext = old->p_vpnext; 46137c478bd9Sstevel@tonic-gate new->p_vpprev = old->p_vpprev; 46147c478bd9Sstevel@tonic-gate new->p_vpnext->p_vpprev = new; 46157c478bd9Sstevel@tonic-gate new->p_vpprev->p_vpnext = new; 46167c478bd9Sstevel@tonic-gate } 46177c478bd9Sstevel@tonic-gate if (vp->v_pages == old) 46187c478bd9Sstevel@tonic-gate vp->v_pages = new; 46197c478bd9Sstevel@tonic-gate 46207c478bd9Sstevel@tonic-gate /* 46217c478bd9Sstevel@tonic-gate * clear out the old page 46227c478bd9Sstevel@tonic-gate */ 46237c478bd9Sstevel@tonic-gate old->p_hash = NULL; 46247c478bd9Sstevel@tonic-gate old->p_vpnext = NULL; 46257c478bd9Sstevel@tonic-gate old->p_vpprev = NULL; 46267c478bd9Sstevel@tonic-gate old->p_vnode = NULL; 46277c478bd9Sstevel@tonic-gate PP_CLRSWAP(old); 46287c478bd9Sstevel@tonic-gate old->p_offset = (u_offset_t)-1; 46297c478bd9Sstevel@tonic-gate page_clr_all_props(old); 46307c478bd9Sstevel@tonic-gate 46317c478bd9Sstevel@tonic-gate /* 46327c478bd9Sstevel@tonic-gate * Wake up processes waiting for this page. The page's 46337c478bd9Sstevel@tonic-gate * identity has been changed, and is probably not the 46347c478bd9Sstevel@tonic-gate * desired page any longer. 46357c478bd9Sstevel@tonic-gate */ 46367c478bd9Sstevel@tonic-gate sep = page_se_mutex(old); 46377c478bd9Sstevel@tonic-gate mutex_enter(sep); 463842787a71Sstans old->p_selock &= ~SE_EWANTED; 46397c478bd9Sstevel@tonic-gate if (CV_HAS_WAITERS(&old->p_cv)) 46407c478bd9Sstevel@tonic-gate cv_broadcast(&old->p_cv); 46417c478bd9Sstevel@tonic-gate mutex_exit(sep); 46427c478bd9Sstevel@tonic-gate } 46437c478bd9Sstevel@tonic-gate 46447c478bd9Sstevel@tonic-gate /* 46457c478bd9Sstevel@tonic-gate * This function moves the identity of page "pp_old" to page "pp_new". 46467c478bd9Sstevel@tonic-gate * Both pages must be locked on entry. "pp_new" is free, has no identity, 46477c478bd9Sstevel@tonic-gate * and need not be hashed out from anywhere. 46487c478bd9Sstevel@tonic-gate */ 46497c478bd9Sstevel@tonic-gate void 46507c478bd9Sstevel@tonic-gate page_relocate_hash(page_t *pp_new, page_t *pp_old) 46517c478bd9Sstevel@tonic-gate { 46527c478bd9Sstevel@tonic-gate vnode_t *vp = pp_old->p_vnode; 46537c478bd9Sstevel@tonic-gate u_offset_t off = pp_old->p_offset; 46547c478bd9Sstevel@tonic-gate kmutex_t *phm, *vphm; 46557c478bd9Sstevel@tonic-gate 46567c478bd9Sstevel@tonic-gate /* 46577c478bd9Sstevel@tonic-gate * Rehash two pages 46587c478bd9Sstevel@tonic-gate */ 46597c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp_old)); 46607c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp_new)); 46617c478bd9Sstevel@tonic-gate ASSERT(vp != NULL); 46627c478bd9Sstevel@tonic-gate ASSERT(pp_new->p_vnode == NULL); 46637c478bd9Sstevel@tonic-gate 46647c478bd9Sstevel@tonic-gate /* 46657c478bd9Sstevel@tonic-gate * hashout then hashin while holding the mutexes 46667c478bd9Sstevel@tonic-gate */ 46677c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off)); 46687c478bd9Sstevel@tonic-gate mutex_enter(phm); 46697c478bd9Sstevel@tonic-gate vphm = page_vnode_mutex(vp); 46707c478bd9Sstevel@tonic-gate mutex_enter(vphm); 46717c478bd9Sstevel@tonic-gate 46727c478bd9Sstevel@tonic-gate page_do_relocate_hash(pp_new, pp_old); 46737c478bd9Sstevel@tonic-gate 46747c478bd9Sstevel@tonic-gate mutex_exit(vphm); 46757c478bd9Sstevel@tonic-gate mutex_exit(phm); 46767c478bd9Sstevel@tonic-gate 46777c478bd9Sstevel@tonic-gate /* 46787c478bd9Sstevel@tonic-gate * The page_struct_lock need not be acquired for lckcnt and 46797c478bd9Sstevel@tonic-gate * cowcnt since the page has an "exclusive" lock. 46807c478bd9Sstevel@tonic-gate */ 46817c478bd9Sstevel@tonic-gate ASSERT(pp_new->p_lckcnt == 0); 46827c478bd9Sstevel@tonic-gate ASSERT(pp_new->p_cowcnt == 0); 46837c478bd9Sstevel@tonic-gate pp_new->p_lckcnt = pp_old->p_lckcnt; 46847c478bd9Sstevel@tonic-gate pp_new->p_cowcnt = pp_old->p_cowcnt; 46857c478bd9Sstevel@tonic-gate pp_old->p_lckcnt = pp_old->p_cowcnt = 0; 46867c478bd9Sstevel@tonic-gate 46877c478bd9Sstevel@tonic-gate /* The following comment preserved from page_flip(). */ 46887c478bd9Sstevel@tonic-gate /* XXX - Do we need to protect fsdata? */ 46897c478bd9Sstevel@tonic-gate pp_new->p_fsdata = pp_old->p_fsdata; 46907c478bd9Sstevel@tonic-gate } 46917c478bd9Sstevel@tonic-gate 46927c478bd9Sstevel@tonic-gate /* 46937c478bd9Sstevel@tonic-gate * Helper routine used to lock all remaining members of a 46947c478bd9Sstevel@tonic-gate * large page. The caller is responsible for passing in a locked 46957c478bd9Sstevel@tonic-gate * pp. If pp is a large page, then it succeeds in locking all the 46967c478bd9Sstevel@tonic-gate * remaining constituent pages or it returns with only the 46977c478bd9Sstevel@tonic-gate * original page locked. 46987c478bd9Sstevel@tonic-gate * 46997c478bd9Sstevel@tonic-gate * Returns 1 on success, 0 on failure. 47007c478bd9Sstevel@tonic-gate * 47017c478bd9Sstevel@tonic-gate * If success is returned this routine gurantees p_szc for all constituent 47027c478bd9Sstevel@tonic-gate * pages of a large page pp belongs to can't change. To achieve this we 47037c478bd9Sstevel@tonic-gate * recheck szc of pp after locking all constituent pages and retry if szc 47047c478bd9Sstevel@tonic-gate * changed (it could only decrease). Since hat_page_demote() needs an EXCL 47057c478bd9Sstevel@tonic-gate * lock on one of constituent pages it can't be running after all constituent 47067c478bd9Sstevel@tonic-gate * pages are locked. hat_page_demote() with a lock on a constituent page 47077c478bd9Sstevel@tonic-gate * outside of this large page (i.e. pp belonged to a larger large page) is 47087c478bd9Sstevel@tonic-gate * already done with all constituent pages of pp since the root's p_szc is 47097c478bd9Sstevel@tonic-gate * changed last. Thefore no need to synchronize with hat_page_demote() that 47107c478bd9Sstevel@tonic-gate * locked a constituent page outside of pp's current large page. 47117c478bd9Sstevel@tonic-gate */ 47127c478bd9Sstevel@tonic-gate #ifdef DEBUG 47137c478bd9Sstevel@tonic-gate uint32_t gpg_trylock_mtbf = 0; 47147c478bd9Sstevel@tonic-gate #endif 47157c478bd9Sstevel@tonic-gate 47167c478bd9Sstevel@tonic-gate int 47177c478bd9Sstevel@tonic-gate group_page_trylock(page_t *pp, se_t se) 47187c478bd9Sstevel@tonic-gate { 47197c478bd9Sstevel@tonic-gate page_t *tpp; 47207c478bd9Sstevel@tonic-gate pgcnt_t npgs, i, j; 47217c478bd9Sstevel@tonic-gate uint_t pszc = pp->p_szc; 47227c478bd9Sstevel@tonic-gate 47237c478bd9Sstevel@tonic-gate #ifdef DEBUG 47247c478bd9Sstevel@tonic-gate if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) { 47257c478bd9Sstevel@tonic-gate return (0); 47267c478bd9Sstevel@tonic-gate } 47277c478bd9Sstevel@tonic-gate #endif 47287c478bd9Sstevel@tonic-gate 47297c478bd9Sstevel@tonic-gate if (pp != PP_GROUPLEADER(pp, pszc)) { 47307c478bd9Sstevel@tonic-gate return (0); 47317c478bd9Sstevel@tonic-gate } 47327c478bd9Sstevel@tonic-gate 47337c478bd9Sstevel@tonic-gate retry: 47347c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED_SE(pp, se)); 47357c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(pp)); 47367c478bd9Sstevel@tonic-gate if (pszc == 0) { 47377c478bd9Sstevel@tonic-gate return (1); 47387c478bd9Sstevel@tonic-gate } 47397c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pszc); 47407c478bd9Sstevel@tonic-gate tpp = pp + 1; 47417c478bd9Sstevel@tonic-gate for (i = 1; i < npgs; i++, tpp++) { 47427c478bd9Sstevel@tonic-gate if (!page_trylock(tpp, se)) { 47437c478bd9Sstevel@tonic-gate tpp = pp + 1; 47447c478bd9Sstevel@tonic-gate for (j = 1; j < i; j++, tpp++) { 47457c478bd9Sstevel@tonic-gate page_unlock(tpp); 47467c478bd9Sstevel@tonic-gate } 47477c478bd9Sstevel@tonic-gate return (0); 47487c478bd9Sstevel@tonic-gate } 47497c478bd9Sstevel@tonic-gate } 47507c478bd9Sstevel@tonic-gate if (pp->p_szc != pszc) { 47517c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < pszc); 4752ad23a2dbSjohansen ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) && 47537c478bd9Sstevel@tonic-gate !IS_SWAPFSVP(pp->p_vnode)); 47547c478bd9Sstevel@tonic-gate tpp = pp + 1; 47557c478bd9Sstevel@tonic-gate for (i = 1; i < npgs; i++, tpp++) { 47567c478bd9Sstevel@tonic-gate page_unlock(tpp); 47577c478bd9Sstevel@tonic-gate } 47587c478bd9Sstevel@tonic-gate pszc = pp->p_szc; 47597c478bd9Sstevel@tonic-gate goto retry; 47607c478bd9Sstevel@tonic-gate } 47617c478bd9Sstevel@tonic-gate return (1); 47627c478bd9Sstevel@tonic-gate } 47637c478bd9Sstevel@tonic-gate 47647c478bd9Sstevel@tonic-gate void 47657c478bd9Sstevel@tonic-gate group_page_unlock(page_t *pp) 47667c478bd9Sstevel@tonic-gate { 47677c478bd9Sstevel@tonic-gate page_t *tpp; 47687c478bd9Sstevel@tonic-gate pgcnt_t npgs, i; 47697c478bd9Sstevel@tonic-gate 47707c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 47717c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(pp)); 47727c478bd9Sstevel@tonic-gate ASSERT(pp == PP_PAGEROOT(pp)); 47737c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 47747c478bd9Sstevel@tonic-gate for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) { 47757c478bd9Sstevel@tonic-gate page_unlock(tpp); 47767c478bd9Sstevel@tonic-gate } 47777c478bd9Sstevel@tonic-gate } 47787c478bd9Sstevel@tonic-gate 47797c478bd9Sstevel@tonic-gate /* 47807c478bd9Sstevel@tonic-gate * returns 47817c478bd9Sstevel@tonic-gate * 0 : on success and *nrelocp is number of relocated PAGESIZE pages 47827c478bd9Sstevel@tonic-gate * ERANGE : this is not a base page 47837c478bd9Sstevel@tonic-gate * EBUSY : failure to get locks on the page/pages 47847c478bd9Sstevel@tonic-gate * ENOMEM : failure to obtain replacement pages 47857c478bd9Sstevel@tonic-gate * EAGAIN : OBP has not yet completed its boot-time handoff to the kernel 47868b464eb8Smec * EIO : An error occurred while trying to copy the page data 47877c478bd9Sstevel@tonic-gate * 47887c478bd9Sstevel@tonic-gate * Return with all constituent members of target and replacement 47897c478bd9Sstevel@tonic-gate * SE_EXCL locked. It is the callers responsibility to drop the 47907c478bd9Sstevel@tonic-gate * locks. 47917c478bd9Sstevel@tonic-gate */ 47927c478bd9Sstevel@tonic-gate int 47937c478bd9Sstevel@tonic-gate do_page_relocate( 47947c478bd9Sstevel@tonic-gate page_t **target, 47957c478bd9Sstevel@tonic-gate page_t **replacement, 47967c478bd9Sstevel@tonic-gate int grouplock, 47977c478bd9Sstevel@tonic-gate spgcnt_t *nrelocp, 47987c478bd9Sstevel@tonic-gate lgrp_t *lgrp) 47997c478bd9Sstevel@tonic-gate { 48007c478bd9Sstevel@tonic-gate page_t *first_repl; 48017c478bd9Sstevel@tonic-gate page_t *repl; 48027c478bd9Sstevel@tonic-gate page_t *targ; 48037c478bd9Sstevel@tonic-gate page_t *pl = NULL; 48047c478bd9Sstevel@tonic-gate uint_t ppattr; 48057c478bd9Sstevel@tonic-gate pfn_t pfn, repl_pfn; 48067c478bd9Sstevel@tonic-gate uint_t szc; 48077c478bd9Sstevel@tonic-gate spgcnt_t npgs, i; 48087c478bd9Sstevel@tonic-gate int repl_contig = 0; 48097c478bd9Sstevel@tonic-gate uint_t flags = 0; 48107c478bd9Sstevel@tonic-gate spgcnt_t dofree = 0; 48117c478bd9Sstevel@tonic-gate 48127c478bd9Sstevel@tonic-gate *nrelocp = 0; 48137c478bd9Sstevel@tonic-gate 48147c478bd9Sstevel@tonic-gate #if defined(__sparc) 48157c478bd9Sstevel@tonic-gate /* 48167c478bd9Sstevel@tonic-gate * We need to wait till OBP has completed 48177c478bd9Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel 48187c478bd9Sstevel@tonic-gate * before we allow page relocation 48197c478bd9Sstevel@tonic-gate */ 48207c478bd9Sstevel@tonic-gate if (page_relocate_ready == 0) { 48217c478bd9Sstevel@tonic-gate return (EAGAIN); 48227c478bd9Sstevel@tonic-gate } 48237c478bd9Sstevel@tonic-gate #endif 48247c478bd9Sstevel@tonic-gate 48257c478bd9Sstevel@tonic-gate /* 48267c478bd9Sstevel@tonic-gate * If this is not a base page, 48277c478bd9Sstevel@tonic-gate * just return with 0x0 pages relocated. 48287c478bd9Sstevel@tonic-gate */ 48297c478bd9Sstevel@tonic-gate targ = *target; 48307c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(targ)); 48317c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(targ)); 48327c478bd9Sstevel@tonic-gate szc = targ->p_szc; 48337c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 48347c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 48357c478bd9Sstevel@tonic-gate pfn = targ->p_pagenum; 48367c478bd9Sstevel@tonic-gate if (pfn != PFN_BASE(pfn, szc)) { 48377c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]); 48387c478bd9Sstevel@tonic-gate return (ERANGE); 48397c478bd9Sstevel@tonic-gate } 48407c478bd9Sstevel@tonic-gate 48417c478bd9Sstevel@tonic-gate if ((repl = *replacement) != NULL && repl->p_szc >= szc) { 48427c478bd9Sstevel@tonic-gate repl_pfn = repl->p_pagenum; 48437c478bd9Sstevel@tonic-gate if (repl_pfn != PFN_BASE(repl_pfn, szc)) { 48447c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]); 48457c478bd9Sstevel@tonic-gate return (ERANGE); 48467c478bd9Sstevel@tonic-gate } 48477c478bd9Sstevel@tonic-gate repl_contig = 1; 48487c478bd9Sstevel@tonic-gate } 48497c478bd9Sstevel@tonic-gate 48507c478bd9Sstevel@tonic-gate /* 48517c478bd9Sstevel@tonic-gate * We must lock all members of this large page or we cannot 48527c478bd9Sstevel@tonic-gate * relocate any part of it. 48537c478bd9Sstevel@tonic-gate */ 48547c478bd9Sstevel@tonic-gate if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) { 48557c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]); 48567c478bd9Sstevel@tonic-gate return (EBUSY); 48577c478bd9Sstevel@tonic-gate } 48587c478bd9Sstevel@tonic-gate 48597c478bd9Sstevel@tonic-gate /* 48607c478bd9Sstevel@tonic-gate * reread szc it could have been decreased before 48617c478bd9Sstevel@tonic-gate * group_page_trylock() was done. 48627c478bd9Sstevel@tonic-gate */ 48637c478bd9Sstevel@tonic-gate szc = targ->p_szc; 48647c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 48657c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]); 48667c478bd9Sstevel@tonic-gate ASSERT(pfn == PFN_BASE(pfn, szc)); 48677c478bd9Sstevel@tonic-gate 48687c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(targ->p_szc); 48697c478bd9Sstevel@tonic-gate 48707c478bd9Sstevel@tonic-gate if (repl == NULL) { 48717c478bd9Sstevel@tonic-gate dofree = npgs; /* Size of target page in MMU pages */ 48727c478bd9Sstevel@tonic-gate if (!page_create_wait(dofree, 0)) { 48737c478bd9Sstevel@tonic-gate if (grouplock != 0) { 48747c478bd9Sstevel@tonic-gate group_page_unlock(targ); 48757c478bd9Sstevel@tonic-gate } 48767c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 48777c478bd9Sstevel@tonic-gate return (ENOMEM); 48787c478bd9Sstevel@tonic-gate } 48797c478bd9Sstevel@tonic-gate 48807c478bd9Sstevel@tonic-gate /* 48817c478bd9Sstevel@tonic-gate * seg kmem pages require that the target and replacement 48827c478bd9Sstevel@tonic-gate * page be the same pagesize. 48837c478bd9Sstevel@tonic-gate */ 4884ad23a2dbSjohansen flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0; 48857c478bd9Sstevel@tonic-gate repl = page_get_replacement_page(targ, lgrp, flags); 48867c478bd9Sstevel@tonic-gate if (repl == NULL) { 48877c478bd9Sstevel@tonic-gate if (grouplock != 0) { 48887c478bd9Sstevel@tonic-gate group_page_unlock(targ); 48897c478bd9Sstevel@tonic-gate } 48907c478bd9Sstevel@tonic-gate page_create_putback(dofree); 48917c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]); 48927c478bd9Sstevel@tonic-gate return (ENOMEM); 48937c478bd9Sstevel@tonic-gate } 48947c478bd9Sstevel@tonic-gate } 48957c478bd9Sstevel@tonic-gate #ifdef DEBUG 48967c478bd9Sstevel@tonic-gate else { 48977c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(repl)); 48987c478bd9Sstevel@tonic-gate } 48997c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 49007c478bd9Sstevel@tonic-gate 49017c478bd9Sstevel@tonic-gate #if defined(__sparc) 49027c478bd9Sstevel@tonic-gate /* 49037c478bd9Sstevel@tonic-gate * Let hat_page_relocate() complete the relocation if it's kernel page 49047c478bd9Sstevel@tonic-gate */ 4905ad23a2dbSjohansen if (VN_ISKAS(targ->p_vnode)) { 49067c478bd9Sstevel@tonic-gate *replacement = repl; 49077c478bd9Sstevel@tonic-gate if (hat_page_relocate(target, replacement, nrelocp) != 0) { 49087c478bd9Sstevel@tonic-gate if (grouplock != 0) { 49097c478bd9Sstevel@tonic-gate group_page_unlock(targ); 49107c478bd9Sstevel@tonic-gate } 49117c478bd9Sstevel@tonic-gate if (dofree) { 49127c478bd9Sstevel@tonic-gate *replacement = NULL; 49137c478bd9Sstevel@tonic-gate page_free_replacement_page(repl); 49147c478bd9Sstevel@tonic-gate page_create_putback(dofree); 49157c478bd9Sstevel@tonic-gate } 49167c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]); 49177c478bd9Sstevel@tonic-gate return (EAGAIN); 49187c478bd9Sstevel@tonic-gate } 49197c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 49207c478bd9Sstevel@tonic-gate return (0); 49217c478bd9Sstevel@tonic-gate } 49227c478bd9Sstevel@tonic-gate #else 49237c478bd9Sstevel@tonic-gate #if defined(lint) 49247c478bd9Sstevel@tonic-gate dofree = dofree; 49257c478bd9Sstevel@tonic-gate #endif 49267c478bd9Sstevel@tonic-gate #endif 49277c478bd9Sstevel@tonic-gate 49287c478bd9Sstevel@tonic-gate first_repl = repl; 49297c478bd9Sstevel@tonic-gate 49307c478bd9Sstevel@tonic-gate for (i = 0; i < npgs; i++) { 49317c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(targ)); 493207b65a64Saguzovsk ASSERT(targ->p_slckcnt == 0); 493307b65a64Saguzovsk ASSERT(repl->p_slckcnt == 0); 49347c478bd9Sstevel@tonic-gate 49357c478bd9Sstevel@tonic-gate (void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD); 49367c478bd9Sstevel@tonic-gate 49377c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(targ) == 0); 49387c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(targ)); 49397c478bd9Sstevel@tonic-gate ASSERT(targ->p_pagenum == (pfn + i)); 49407c478bd9Sstevel@tonic-gate ASSERT(repl_contig == 0 || 49417c478bd9Sstevel@tonic-gate repl->p_pagenum == (repl_pfn + i)); 49427c478bd9Sstevel@tonic-gate 49437c478bd9Sstevel@tonic-gate /* 49447c478bd9Sstevel@tonic-gate * Copy the page contents and attributes then 49457c478bd9Sstevel@tonic-gate * relocate the page in the page hash. 49467c478bd9Sstevel@tonic-gate */ 49478b464eb8Smec if (ppcopy(targ, repl) == 0) { 49488b464eb8Smec targ = *target; 49498b464eb8Smec repl = first_repl; 49508b464eb8Smec VM_STAT_ADD(vmm_vmstats.ppr_copyfail); 49518b464eb8Smec if (grouplock != 0) { 49528b464eb8Smec group_page_unlock(targ); 49538b464eb8Smec } 49548b464eb8Smec if (dofree) { 49558b464eb8Smec *replacement = NULL; 49568b464eb8Smec page_free_replacement_page(repl); 49578b464eb8Smec page_create_putback(dofree); 49588b464eb8Smec } 49598b464eb8Smec return (EIO); 49608b464eb8Smec } 49618b464eb8Smec 49628b464eb8Smec targ++; 49638b464eb8Smec if (repl_contig != 0) { 49648b464eb8Smec repl++; 49658b464eb8Smec } else { 49668b464eb8Smec repl = repl->p_next; 49678b464eb8Smec } 49688b464eb8Smec } 49698b464eb8Smec 49708b464eb8Smec repl = first_repl; 49718b464eb8Smec targ = *target; 49728b464eb8Smec 49738b464eb8Smec for (i = 0; i < npgs; i++) { 49747c478bd9Sstevel@tonic-gate ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO)); 49757c478bd9Sstevel@tonic-gate page_clr_all_props(repl); 49767c478bd9Sstevel@tonic-gate page_set_props(repl, ppattr); 49777c478bd9Sstevel@tonic-gate page_relocate_hash(repl, targ); 49787c478bd9Sstevel@tonic-gate 49797c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(targ) == 0); 49807c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(repl) == 0); 49817c478bd9Sstevel@tonic-gate /* 49827c478bd9Sstevel@tonic-gate * Now clear the props on targ, after the 49837c478bd9Sstevel@tonic-gate * page_relocate_hash(), they no longer 49847c478bd9Sstevel@tonic-gate * have any meaning. 49857c478bd9Sstevel@tonic-gate */ 49867c478bd9Sstevel@tonic-gate page_clr_all_props(targ); 49877c478bd9Sstevel@tonic-gate ASSERT(targ->p_next == targ); 49887c478bd9Sstevel@tonic-gate ASSERT(targ->p_prev == targ); 49897c478bd9Sstevel@tonic-gate page_list_concat(&pl, &targ); 49907c478bd9Sstevel@tonic-gate 49917c478bd9Sstevel@tonic-gate targ++; 49927c478bd9Sstevel@tonic-gate if (repl_contig != 0) { 49937c478bd9Sstevel@tonic-gate repl++; 49947c478bd9Sstevel@tonic-gate } else { 49957c478bd9Sstevel@tonic-gate repl = repl->p_next; 49967c478bd9Sstevel@tonic-gate } 49977c478bd9Sstevel@tonic-gate } 49987c478bd9Sstevel@tonic-gate /* assert that we have come full circle with repl */ 49997c478bd9Sstevel@tonic-gate ASSERT(repl_contig == 1 || first_repl == repl); 50007c478bd9Sstevel@tonic-gate 50017c478bd9Sstevel@tonic-gate *target = pl; 50027c478bd9Sstevel@tonic-gate if (*replacement == NULL) { 50037c478bd9Sstevel@tonic-gate ASSERT(first_repl == repl); 50047c478bd9Sstevel@tonic-gate *replacement = repl; 50057c478bd9Sstevel@tonic-gate } 50067c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]); 50077c478bd9Sstevel@tonic-gate *nrelocp = npgs; 50087c478bd9Sstevel@tonic-gate return (0); 50097c478bd9Sstevel@tonic-gate } 50107c478bd9Sstevel@tonic-gate /* 50117c478bd9Sstevel@tonic-gate * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated. 50127c478bd9Sstevel@tonic-gate */ 50137c478bd9Sstevel@tonic-gate int 50147c478bd9Sstevel@tonic-gate page_relocate( 50157c478bd9Sstevel@tonic-gate page_t **target, 50167c478bd9Sstevel@tonic-gate page_t **replacement, 50177c478bd9Sstevel@tonic-gate int grouplock, 50187c478bd9Sstevel@tonic-gate int freetarget, 50197c478bd9Sstevel@tonic-gate spgcnt_t *nrelocp, 50207c478bd9Sstevel@tonic-gate lgrp_t *lgrp) 50217c478bd9Sstevel@tonic-gate { 50227c478bd9Sstevel@tonic-gate spgcnt_t ret; 50237c478bd9Sstevel@tonic-gate 50247c478bd9Sstevel@tonic-gate /* do_page_relocate returns 0 on success or errno value */ 50257c478bd9Sstevel@tonic-gate ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp); 50267c478bd9Sstevel@tonic-gate 50277c478bd9Sstevel@tonic-gate if (ret != 0 || freetarget == 0) { 50287c478bd9Sstevel@tonic-gate return (ret); 50297c478bd9Sstevel@tonic-gate } 50307c478bd9Sstevel@tonic-gate if (*nrelocp == 1) { 50317c478bd9Sstevel@tonic-gate ASSERT(*target != NULL); 50327c478bd9Sstevel@tonic-gate page_free(*target, 1); 50337c478bd9Sstevel@tonic-gate } else { 50347c478bd9Sstevel@tonic-gate page_t *tpp = *target; 50357c478bd9Sstevel@tonic-gate uint_t szc = tpp->p_szc; 50367c478bd9Sstevel@tonic-gate pgcnt_t npgs = page_get_pagecnt(szc); 50377c478bd9Sstevel@tonic-gate ASSERT(npgs > 1); 50387c478bd9Sstevel@tonic-gate ASSERT(szc != 0); 50397c478bd9Sstevel@tonic-gate do { 50407c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(tpp)); 50417c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(tpp)); 50427c478bd9Sstevel@tonic-gate ASSERT(tpp->p_szc == szc); 50437c478bd9Sstevel@tonic-gate PP_SETFREE(tpp); 50447c478bd9Sstevel@tonic-gate PP_SETAGED(tpp); 50457c478bd9Sstevel@tonic-gate npgs--; 50467c478bd9Sstevel@tonic-gate } while ((tpp = tpp->p_next) != *target); 50477c478bd9Sstevel@tonic-gate ASSERT(npgs == 0); 50487c478bd9Sstevel@tonic-gate page_list_add_pages(*target, 0); 50497c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 50507c478bd9Sstevel@tonic-gate page_create_putback(npgs); 50517c478bd9Sstevel@tonic-gate } 50527c478bd9Sstevel@tonic-gate return (ret); 50537c478bd9Sstevel@tonic-gate } 50547c478bd9Sstevel@tonic-gate 50557c478bd9Sstevel@tonic-gate /* 50567c478bd9Sstevel@tonic-gate * it is up to the caller to deal with pcf accounting. 50577c478bd9Sstevel@tonic-gate */ 50587c478bd9Sstevel@tonic-gate void 50597c478bd9Sstevel@tonic-gate page_free_replacement_page(page_t *pplist) 50607c478bd9Sstevel@tonic-gate { 50617c478bd9Sstevel@tonic-gate page_t *pp; 50627c478bd9Sstevel@tonic-gate 50637c478bd9Sstevel@tonic-gate while (pplist != NULL) { 50647c478bd9Sstevel@tonic-gate /* 50657c478bd9Sstevel@tonic-gate * pp_targ is a linked list. 50667c478bd9Sstevel@tonic-gate */ 50677c478bd9Sstevel@tonic-gate pp = pplist; 50687c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 50697c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 50707c478bd9Sstevel@tonic-gate page_clr_all_props(pp); 50717c478bd9Sstevel@tonic-gate PP_SETFREE(pp); 50727c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 50737c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 50747c478bd9Sstevel@tonic-gate page_unlock(pp); 50757c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]); 50767c478bd9Sstevel@tonic-gate } else { 50777c478bd9Sstevel@tonic-gate spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc); 50787c478bd9Sstevel@tonic-gate page_t *tpp; 50797c478bd9Sstevel@tonic-gate page_list_break(&pp, &pplist, curnpgs); 50807c478bd9Sstevel@tonic-gate tpp = pp; 50817c478bd9Sstevel@tonic-gate do { 50827c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(tpp)); 50837c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(tpp)); 50847c478bd9Sstevel@tonic-gate page_clr_all_props(pp); 50857c478bd9Sstevel@tonic-gate PP_SETFREE(tpp); 50867c478bd9Sstevel@tonic-gate PP_SETAGED(tpp); 50877c478bd9Sstevel@tonic-gate } while ((tpp = tpp->p_next) != pp); 50887c478bd9Sstevel@tonic-gate page_list_add_pages(pp, 0); 50897c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]); 50907c478bd9Sstevel@tonic-gate } 50917c478bd9Sstevel@tonic-gate } 50927c478bd9Sstevel@tonic-gate } 50937c478bd9Sstevel@tonic-gate 50947c478bd9Sstevel@tonic-gate /* 50957c478bd9Sstevel@tonic-gate * Relocate target to non-relocatable replacement page. 50967c478bd9Sstevel@tonic-gate */ 50977c478bd9Sstevel@tonic-gate int 50987c478bd9Sstevel@tonic-gate page_relocate_cage(page_t **target, page_t **replacement) 50997c478bd9Sstevel@tonic-gate { 51007c478bd9Sstevel@tonic-gate page_t *tpp, *rpp; 51017c478bd9Sstevel@tonic-gate spgcnt_t pgcnt, npgs; 51027c478bd9Sstevel@tonic-gate int result; 51037c478bd9Sstevel@tonic-gate 51047c478bd9Sstevel@tonic-gate tpp = *target; 51057c478bd9Sstevel@tonic-gate 51067c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(tpp)); 51077c478bd9Sstevel@tonic-gate ASSERT(tpp->p_szc == 0); 51087c478bd9Sstevel@tonic-gate 51097c478bd9Sstevel@tonic-gate pgcnt = btop(page_get_pagesize(tpp->p_szc)); 51107c478bd9Sstevel@tonic-gate 51117c478bd9Sstevel@tonic-gate do { 51127c478bd9Sstevel@tonic-gate (void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC); 51137c478bd9Sstevel@tonic-gate rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC); 51147c478bd9Sstevel@tonic-gate if (rpp == NULL) { 51157c478bd9Sstevel@tonic-gate page_create_putback(pgcnt); 51167c478bd9Sstevel@tonic-gate kcage_cageout_wakeup(); 51177c478bd9Sstevel@tonic-gate } 51187c478bd9Sstevel@tonic-gate } while (rpp == NULL); 51197c478bd9Sstevel@tonic-gate 51207c478bd9Sstevel@tonic-gate ASSERT(PP_ISNORELOC(rpp)); 51217c478bd9Sstevel@tonic-gate 51227c478bd9Sstevel@tonic-gate result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL); 51237c478bd9Sstevel@tonic-gate 51247c478bd9Sstevel@tonic-gate if (result == 0) { 51257c478bd9Sstevel@tonic-gate *replacement = rpp; 51267c478bd9Sstevel@tonic-gate if (pgcnt != npgs) 51277c478bd9Sstevel@tonic-gate panic("page_relocate_cage: partial relocation"); 51287c478bd9Sstevel@tonic-gate } 51297c478bd9Sstevel@tonic-gate 51307c478bd9Sstevel@tonic-gate return (result); 51317c478bd9Sstevel@tonic-gate } 51327c478bd9Sstevel@tonic-gate 51337c478bd9Sstevel@tonic-gate /* 51347c478bd9Sstevel@tonic-gate * Release the page lock on a page, place on cachelist 51357c478bd9Sstevel@tonic-gate * tail if no longer mapped. Caller can let us know if 51367c478bd9Sstevel@tonic-gate * the page is known to be clean. 51377c478bd9Sstevel@tonic-gate */ 51387c478bd9Sstevel@tonic-gate int 51397c478bd9Sstevel@tonic-gate page_release(page_t *pp, int checkmod) 51407c478bd9Sstevel@tonic-gate { 51417c478bd9Sstevel@tonic-gate int status; 51427c478bd9Sstevel@tonic-gate 51437c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) && 51447c478bd9Sstevel@tonic-gate (pp->p_vnode != NULL)); 51457c478bd9Sstevel@tonic-gate 51467c478bd9Sstevel@tonic-gate if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) && 51477c478bd9Sstevel@tonic-gate ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) && 51487c478bd9Sstevel@tonic-gate pp->p_lckcnt == 0 && pp->p_cowcnt == 0 && 51497c478bd9Sstevel@tonic-gate !hat_page_is_mapped(pp)) { 51507c478bd9Sstevel@tonic-gate 51517c478bd9Sstevel@tonic-gate /* 51527c478bd9Sstevel@tonic-gate * If page is modified, unlock it 51537c478bd9Sstevel@tonic-gate * 51547c478bd9Sstevel@tonic-gate * (p_nrm & P_MOD) bit has the latest stuff because: 51557c478bd9Sstevel@tonic-gate * (1) We found that this page doesn't have any mappings 51567c478bd9Sstevel@tonic-gate * _after_ holding SE_EXCL and 51577c478bd9Sstevel@tonic-gate * (2) We didn't drop SE_EXCL lock after the check in (1) 51587c478bd9Sstevel@tonic-gate */ 51597c478bd9Sstevel@tonic-gate if (checkmod && hat_ismod(pp)) { 51607c478bd9Sstevel@tonic-gate page_unlock(pp); 51617c478bd9Sstevel@tonic-gate status = PGREL_MOD; 51627c478bd9Sstevel@tonic-gate } else { 51637c478bd9Sstevel@tonic-gate /*LINTED: constant in conditional context*/ 51647c478bd9Sstevel@tonic-gate VN_DISPOSE(pp, B_FREE, 0, kcred); 51657c478bd9Sstevel@tonic-gate status = PGREL_CLEAN; 51667c478bd9Sstevel@tonic-gate } 51677c478bd9Sstevel@tonic-gate } else { 51687c478bd9Sstevel@tonic-gate page_unlock(pp); 51697c478bd9Sstevel@tonic-gate status = PGREL_NOTREL; 51707c478bd9Sstevel@tonic-gate } 51717c478bd9Sstevel@tonic-gate return (status); 51727c478bd9Sstevel@tonic-gate } 51737c478bd9Sstevel@tonic-gate 5174db874c57Selowe /* 5175db874c57Selowe * Given a constituent page, try to demote the large page on the freelist. 5176db874c57Selowe * 5177db874c57Selowe * Returns nonzero if the page could be demoted successfully. Returns with 5178db874c57Selowe * the constituent page still locked. 5179db874c57Selowe */ 5180db874c57Selowe int 5181db874c57Selowe page_try_demote_free_pages(page_t *pp) 5182db874c57Selowe { 5183db874c57Selowe page_t *rootpp = pp; 5184db874c57Selowe pfn_t pfn = page_pptonum(pp); 5185db874c57Selowe spgcnt_t npgs; 5186db874c57Selowe uint_t szc = pp->p_szc; 5187db874c57Selowe 5188db874c57Selowe ASSERT(PP_ISFREE(pp)); 5189db874c57Selowe ASSERT(PAGE_EXCL(pp)); 5190db874c57Selowe 5191db874c57Selowe /* 5192db874c57Selowe * Adjust rootpp and lock it, if `pp' is not the base 5193db874c57Selowe * constituent page. 5194db874c57Selowe */ 5195db874c57Selowe npgs = page_get_pagecnt(pp->p_szc); 5196db874c57Selowe if (npgs == 1) { 5197db874c57Selowe return (0); 5198db874c57Selowe } 5199db874c57Selowe 5200db874c57Selowe if (!IS_P2ALIGNED(pfn, npgs)) { 5201db874c57Selowe pfn = P2ALIGN(pfn, npgs); 5202db874c57Selowe rootpp = page_numtopp_nolock(pfn); 5203db874c57Selowe } 5204db874c57Selowe 5205db874c57Selowe if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) { 5206db874c57Selowe return (0); 5207db874c57Selowe } 5208db874c57Selowe 5209db874c57Selowe if (rootpp->p_szc != szc) { 5210db874c57Selowe if (pp != rootpp) 5211db874c57Selowe page_unlock(rootpp); 5212db874c57Selowe return (0); 5213db874c57Selowe } 5214db874c57Selowe 5215db874c57Selowe page_demote_free_pages(rootpp); 5216db874c57Selowe 5217db874c57Selowe if (pp != rootpp) 5218db874c57Selowe page_unlock(rootpp); 5219db874c57Selowe 5220db874c57Selowe ASSERT(PP_ISFREE(pp)); 5221db874c57Selowe ASSERT(PAGE_EXCL(pp)); 5222db874c57Selowe return (1); 5223db874c57Selowe } 5224db874c57Selowe 5225db874c57Selowe /* 5226db874c57Selowe * Given a constituent page, try to demote the large page. 5227db874c57Selowe * 5228db874c57Selowe * Returns nonzero if the page could be demoted successfully. Returns with 5229db874c57Selowe * the constituent page still locked. 5230db874c57Selowe */ 52317c478bd9Sstevel@tonic-gate int 52327c478bd9Sstevel@tonic-gate page_try_demote_pages(page_t *pp) 52337c478bd9Sstevel@tonic-gate { 52347c478bd9Sstevel@tonic-gate page_t *tpp, *rootpp = pp; 52357c478bd9Sstevel@tonic-gate pfn_t pfn = page_pptonum(pp); 52367c478bd9Sstevel@tonic-gate spgcnt_t i, npgs; 52377c478bd9Sstevel@tonic-gate uint_t szc = pp->p_szc; 52387c478bd9Sstevel@tonic-gate vnode_t *vp = pp->p_vnode; 52397c478bd9Sstevel@tonic-gate 5240db874c57Selowe ASSERT(PAGE_EXCL(pp)); 52417c478bd9Sstevel@tonic-gate 52427c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]); 52437c478bd9Sstevel@tonic-gate 5244db874c57Selowe if (pp->p_szc == 0) { 52457c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]); 52467c478bd9Sstevel@tonic-gate return (1); 52477c478bd9Sstevel@tonic-gate } 52487c478bd9Sstevel@tonic-gate 5249ad23a2dbSjohansen if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) { 52507c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]); 5251db874c57Selowe page_demote_vp_pages(pp); 52527c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 52537c478bd9Sstevel@tonic-gate return (1); 52547c478bd9Sstevel@tonic-gate } 52557c478bd9Sstevel@tonic-gate 52567c478bd9Sstevel@tonic-gate /* 5257db874c57Selowe * Adjust rootpp if passed in is not the base 52587c478bd9Sstevel@tonic-gate * constituent page. 52597c478bd9Sstevel@tonic-gate */ 5260db874c57Selowe npgs = page_get_pagecnt(pp->p_szc); 52617c478bd9Sstevel@tonic-gate ASSERT(npgs > 1); 52627c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(pfn, npgs)) { 52637c478bd9Sstevel@tonic-gate pfn = P2ALIGN(pfn, npgs); 52647c478bd9Sstevel@tonic-gate rootpp = page_numtopp_nolock(pfn); 52657c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]); 52667c478bd9Sstevel@tonic-gate ASSERT(rootpp->p_vnode != NULL); 52677c478bd9Sstevel@tonic-gate ASSERT(rootpp->p_szc == szc); 52687c478bd9Sstevel@tonic-gate } 52697c478bd9Sstevel@tonic-gate 52707c478bd9Sstevel@tonic-gate /* 52717c478bd9Sstevel@tonic-gate * We can't demote kernel pages since we can't hat_unload() 52727c478bd9Sstevel@tonic-gate * the mappings. 52737c478bd9Sstevel@tonic-gate */ 5274ad23a2dbSjohansen if (VN_ISKAS(rootpp->p_vnode)) 52757c478bd9Sstevel@tonic-gate return (0); 52767c478bd9Sstevel@tonic-gate 52777c478bd9Sstevel@tonic-gate /* 52787c478bd9Sstevel@tonic-gate * Attempt to lock all constituent pages except the page passed 52797c478bd9Sstevel@tonic-gate * in since it's already locked. 52807c478bd9Sstevel@tonic-gate */ 5281affbd3ccSkchow for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 52827c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(tpp)); 52837c478bd9Sstevel@tonic-gate ASSERT(tpp->p_vnode != NULL); 52847c478bd9Sstevel@tonic-gate 52857c478bd9Sstevel@tonic-gate if (tpp != pp && !page_trylock(tpp, SE_EXCL)) 52867c478bd9Sstevel@tonic-gate break; 52877c478bd9Sstevel@tonic-gate ASSERT(tpp->p_szc == rootpp->p_szc); 52887c478bd9Sstevel@tonic-gate ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i); 52897c478bd9Sstevel@tonic-gate } 52907c478bd9Sstevel@tonic-gate 52917c478bd9Sstevel@tonic-gate /* 5292db874c57Selowe * If we failed to lock them all then unlock what we have 5293db874c57Selowe * locked so far and bail. 52947c478bd9Sstevel@tonic-gate */ 52957c478bd9Sstevel@tonic-gate if (i < npgs) { 52967c478bd9Sstevel@tonic-gate tpp = rootpp; 52977c478bd9Sstevel@tonic-gate while (i-- > 0) { 52987c478bd9Sstevel@tonic-gate if (tpp != pp) 52997c478bd9Sstevel@tonic-gate page_unlock(tpp); 5300affbd3ccSkchow tpp++; 53017c478bd9Sstevel@tonic-gate } 53027c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]); 53037c478bd9Sstevel@tonic-gate return (0); 53047c478bd9Sstevel@tonic-gate } 53057c478bd9Sstevel@tonic-gate 5306affbd3ccSkchow for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 53077c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(tpp)); 530807b65a64Saguzovsk ASSERT(tpp->p_slckcnt == 0); 5309db874c57Selowe (void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD); 53107c478bd9Sstevel@tonic-gate tpp->p_szc = 0; 53117c478bd9Sstevel@tonic-gate } 53127c478bd9Sstevel@tonic-gate 53137c478bd9Sstevel@tonic-gate /* 53147c478bd9Sstevel@tonic-gate * Unlock all pages except the page passed in. 53157c478bd9Sstevel@tonic-gate */ 5316affbd3ccSkchow for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) { 53177c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(tpp)); 53187c478bd9Sstevel@tonic-gate if (tpp != pp) 53197c478bd9Sstevel@tonic-gate page_unlock(tpp); 53207c478bd9Sstevel@tonic-gate } 5321db874c57Selowe 53227c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]); 53237c478bd9Sstevel@tonic-gate return (1); 53247c478bd9Sstevel@tonic-gate } 53257c478bd9Sstevel@tonic-gate 53267c478bd9Sstevel@tonic-gate /* 53277c478bd9Sstevel@tonic-gate * Called by page_free() and page_destroy() to demote the page size code 53287c478bd9Sstevel@tonic-gate * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero 53297c478bd9Sstevel@tonic-gate * p_szc on free list, neither can we just clear p_szc of a single page_t 53307c478bd9Sstevel@tonic-gate * within a large page since it will break other code that relies on p_szc 53317c478bd9Sstevel@tonic-gate * being the same for all page_t's of a large page). Anonymous pages should 53327c478bd9Sstevel@tonic-gate * never end up here because anon_map_getpages() cannot deal with p_szc 53337c478bd9Sstevel@tonic-gate * changes after a single constituent page is locked. While anonymous or 53347c478bd9Sstevel@tonic-gate * kernel large pages are demoted or freed the entire large page at a time 53357c478bd9Sstevel@tonic-gate * with all constituent pages locked EXCL for the file system pages we 53367c478bd9Sstevel@tonic-gate * have to be able to demote a large page (i.e. decrease all constituent pages 53377c478bd9Sstevel@tonic-gate * p_szc) with only just an EXCL lock on one of constituent pages. The reason 53387c478bd9Sstevel@tonic-gate * we can easily deal with anonymous page demotion the entire large page at a 53397c478bd9Sstevel@tonic-gate * time is that those operation originate at address space level and concern 53407c478bd9Sstevel@tonic-gate * the entire large page region with actual demotion only done when pages are 53417c478bd9Sstevel@tonic-gate * not shared with any other processes (therefore we can always get EXCL lock 53427c478bd9Sstevel@tonic-gate * on all anonymous constituent pages after clearing segment page 53437c478bd9Sstevel@tonic-gate * cache). However file system pages can be truncated or invalidated at a 53447c478bd9Sstevel@tonic-gate * PAGESIZE level from the file system side and end up in page_free() or 53457c478bd9Sstevel@tonic-gate * page_destroy() (we also allow only part of the large page to be SOFTLOCKed 53467c478bd9Sstevel@tonic-gate * and therfore pageout should be able to demote a large page by EXCL locking 53477c478bd9Sstevel@tonic-gate * any constituent page that is not under SOFTLOCK). In those cases we cannot 53487c478bd9Sstevel@tonic-gate * rely on being able to lock EXCL all constituent pages. 53497c478bd9Sstevel@tonic-gate * 53507c478bd9Sstevel@tonic-gate * To prevent szc changes on file system pages one has to lock all constituent 53517c478bd9Sstevel@tonic-gate * pages at least SHARED (or call page_szc_lock()). The only subsystem that 53527c478bd9Sstevel@tonic-gate * doesn't rely on locking all constituent pages (or using page_szc_lock()) to 53537c478bd9Sstevel@tonic-gate * prevent szc changes is hat layer that uses its own page level mlist 53547c478bd9Sstevel@tonic-gate * locks. hat assumes that szc doesn't change after mlist lock for a page is 53557c478bd9Sstevel@tonic-gate * taken. Therefore we need to change szc under hat level locks if we only 53567c478bd9Sstevel@tonic-gate * have an EXCL lock on a single constituent page and hat still references any 53577c478bd9Sstevel@tonic-gate * of constituent pages. (Note we can't "ignore" hat layer by simply 53587c478bd9Sstevel@tonic-gate * hat_pageunload() all constituent pages without having EXCL locks on all of 53597c478bd9Sstevel@tonic-gate * constituent pages). We use hat_page_demote() call to safely demote szc of 53607c478bd9Sstevel@tonic-gate * all constituent pages under hat locks when we only have an EXCL lock on one 53617c478bd9Sstevel@tonic-gate * of constituent pages. 53627c478bd9Sstevel@tonic-gate * 53637c478bd9Sstevel@tonic-gate * This routine calls page_szc_lock() before calling hat_page_demote() to 53647c478bd9Sstevel@tonic-gate * allow segvn in one special case not to lock all constituent pages SHARED 53657c478bd9Sstevel@tonic-gate * before calling hat_memload_array() that relies on p_szc not changeing even 53667c478bd9Sstevel@tonic-gate * before hat level mlist lock is taken. In that case segvn uses 53677c478bd9Sstevel@tonic-gate * page_szc_lock() to prevent hat_page_demote() changeing p_szc values. 53687c478bd9Sstevel@tonic-gate * 53697c478bd9Sstevel@tonic-gate * Anonymous or kernel page demotion still has to lock all pages exclusively 53707c478bd9Sstevel@tonic-gate * and do hat_pageunload() on all constituent pages before demoting the page 53717c478bd9Sstevel@tonic-gate * therefore there's no need for anonymous or kernel page demotion to use 53727c478bd9Sstevel@tonic-gate * hat_page_demote() mechanism. 53737c478bd9Sstevel@tonic-gate * 53747c478bd9Sstevel@tonic-gate * hat_page_demote() removes all large mappings that map pp and then decreases 53757c478bd9Sstevel@tonic-gate * p_szc starting from the last constituent page of the large page. By working 53767c478bd9Sstevel@tonic-gate * from the tail of a large page in pfn decreasing order allows one looking at 53777c478bd9Sstevel@tonic-gate * the root page to know that hat_page_demote() is done for root's szc area. 53787c478bd9Sstevel@tonic-gate * e.g. if a root page has szc 1 one knows it only has to lock all constituent 53797c478bd9Sstevel@tonic-gate * pages within szc 1 area to prevent szc changes because hat_page_demote() 53807c478bd9Sstevel@tonic-gate * that started on this page when it had szc > 1 is done for this szc 1 area. 53817c478bd9Sstevel@tonic-gate * 53827c478bd9Sstevel@tonic-gate * We are guranteed that all constituent pages of pp's large page belong to 53837c478bd9Sstevel@tonic-gate * the same vnode with the consecutive offsets increasing in the direction of 53847c478bd9Sstevel@tonic-gate * the pfn i.e. the identity of constituent pages can't change until their 53857c478bd9Sstevel@tonic-gate * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove 53867c478bd9Sstevel@tonic-gate * large mappings to pp even though we don't lock any constituent page except 53877c478bd9Sstevel@tonic-gate * pp (i.e. we won't unload e.g. kernel locked page). 53887c478bd9Sstevel@tonic-gate */ 53897c478bd9Sstevel@tonic-gate static void 53907c478bd9Sstevel@tonic-gate page_demote_vp_pages(page_t *pp) 53917c478bd9Sstevel@tonic-gate { 53927c478bd9Sstevel@tonic-gate kmutex_t *mtx; 53937c478bd9Sstevel@tonic-gate 53947c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 53957c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(pp)); 53967c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode != NULL); 53977c478bd9Sstevel@tonic-gate ASSERT(!IS_SWAPFSVP(pp->p_vnode)); 5398ad23a2dbSjohansen ASSERT(!PP_ISKAS(pp)); 53997c478bd9Sstevel@tonic-gate 54007c478bd9Sstevel@tonic-gate VM_STAT_ADD(pagecnt.pc_demote_pages[0]); 54017c478bd9Sstevel@tonic-gate 54027c478bd9Sstevel@tonic-gate mtx = page_szc_lock(pp); 54037c478bd9Sstevel@tonic-gate if (mtx != NULL) { 54047c478bd9Sstevel@tonic-gate hat_page_demote(pp); 54057c478bd9Sstevel@tonic-gate mutex_exit(mtx); 54067c478bd9Sstevel@tonic-gate } 54077c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 54087c478bd9Sstevel@tonic-gate } 54097c478bd9Sstevel@tonic-gate 54107c478bd9Sstevel@tonic-gate /* 54117c478bd9Sstevel@tonic-gate * Mark any existing pages for migration in the given range 54127c478bd9Sstevel@tonic-gate */ 54137c478bd9Sstevel@tonic-gate void 54147c478bd9Sstevel@tonic-gate page_mark_migrate(struct seg *seg, caddr_t addr, size_t len, 54157c478bd9Sstevel@tonic-gate struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 54167c478bd9Sstevel@tonic-gate u_offset_t vnoff, int rflag) 54177c478bd9Sstevel@tonic-gate { 54187c478bd9Sstevel@tonic-gate struct anon *ap; 54197c478bd9Sstevel@tonic-gate vnode_t *curvp; 54207c478bd9Sstevel@tonic-gate lgrp_t *from; 54217c478bd9Sstevel@tonic-gate pgcnt_t i; 54227c478bd9Sstevel@tonic-gate pgcnt_t nlocked; 54237c478bd9Sstevel@tonic-gate u_offset_t off; 54247c478bd9Sstevel@tonic-gate pfn_t pfn; 54257c478bd9Sstevel@tonic-gate size_t pgsz; 54267c478bd9Sstevel@tonic-gate size_t segpgsz; 54277c478bd9Sstevel@tonic-gate pgcnt_t pages; 54287c478bd9Sstevel@tonic-gate uint_t pszc; 54297c478bd9Sstevel@tonic-gate page_t **ppa; 54307c478bd9Sstevel@tonic-gate pgcnt_t ppa_nentries; 54317c478bd9Sstevel@tonic-gate page_t *pp; 54327c478bd9Sstevel@tonic-gate caddr_t va; 54337c478bd9Sstevel@tonic-gate ulong_t an_idx; 54347c478bd9Sstevel@tonic-gate anon_sync_obj_t cookie; 54357c478bd9Sstevel@tonic-gate 54367c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 54377c478bd9Sstevel@tonic-gate 54387c478bd9Sstevel@tonic-gate /* 54397c478bd9Sstevel@tonic-gate * Don't do anything if don't need to do lgroup optimizations 54407c478bd9Sstevel@tonic-gate * on this system 54417c478bd9Sstevel@tonic-gate */ 54427c478bd9Sstevel@tonic-gate if (!lgrp_optimizations()) 54437c478bd9Sstevel@tonic-gate return; 54447c478bd9Sstevel@tonic-gate 54457c478bd9Sstevel@tonic-gate /* 54467c478bd9Sstevel@tonic-gate * Align address and length to (potentially large) page boundary 54477c478bd9Sstevel@tonic-gate */ 54487c478bd9Sstevel@tonic-gate segpgsz = page_get_pagesize(seg->s_szc); 54497c478bd9Sstevel@tonic-gate addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz); 54507c478bd9Sstevel@tonic-gate if (rflag) 54517c478bd9Sstevel@tonic-gate len = P2ROUNDUP(len, segpgsz); 54527c478bd9Sstevel@tonic-gate 54537c478bd9Sstevel@tonic-gate /* 54547c478bd9Sstevel@tonic-gate * Allocate page array to accomodate largest page size 54557c478bd9Sstevel@tonic-gate */ 54567c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(page_num_pagesizes() - 1); 54577c478bd9Sstevel@tonic-gate ppa_nentries = btop(pgsz); 54587c478bd9Sstevel@tonic-gate ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP); 54597c478bd9Sstevel@tonic-gate 54607c478bd9Sstevel@tonic-gate /* 54617c478bd9Sstevel@tonic-gate * Do one (large) page at a time 54627c478bd9Sstevel@tonic-gate */ 54637c478bd9Sstevel@tonic-gate va = addr; 54647c478bd9Sstevel@tonic-gate while (va < addr + len) { 54657c478bd9Sstevel@tonic-gate /* 54667c478bd9Sstevel@tonic-gate * Lookup (root) page for vnode and offset corresponding to 54677c478bd9Sstevel@tonic-gate * this virtual address 54687c478bd9Sstevel@tonic-gate * Try anonmap first since there may be copy-on-write 54697c478bd9Sstevel@tonic-gate * pages, but initialize vnode pointer and offset using 54707c478bd9Sstevel@tonic-gate * vnode arguments just in case there isn't an amp. 54717c478bd9Sstevel@tonic-gate */ 54727c478bd9Sstevel@tonic-gate curvp = vp; 54737c478bd9Sstevel@tonic-gate off = vnoff + va - seg->s_base; 54747c478bd9Sstevel@tonic-gate if (amp) { 54757c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 54767c478bd9Sstevel@tonic-gate an_idx = anon_index + seg_page(seg, va); 54777c478bd9Sstevel@tonic-gate anon_array_enter(amp, an_idx, &cookie); 54787c478bd9Sstevel@tonic-gate ap = anon_get_ptr(amp->ahp, an_idx); 54797c478bd9Sstevel@tonic-gate if (ap) 54807c478bd9Sstevel@tonic-gate swap_xlate(ap, &curvp, &off); 54817c478bd9Sstevel@tonic-gate anon_array_exit(&cookie); 54827c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 54837c478bd9Sstevel@tonic-gate } 54847c478bd9Sstevel@tonic-gate 54857c478bd9Sstevel@tonic-gate pp = NULL; 54867c478bd9Sstevel@tonic-gate if (curvp) 54877c478bd9Sstevel@tonic-gate pp = page_lookup(curvp, off, SE_SHARED); 54887c478bd9Sstevel@tonic-gate 54897c478bd9Sstevel@tonic-gate /* 54907c478bd9Sstevel@tonic-gate * If there isn't a page at this virtual address, 54917c478bd9Sstevel@tonic-gate * skip to next page 54927c478bd9Sstevel@tonic-gate */ 54937c478bd9Sstevel@tonic-gate if (pp == NULL) { 54947c478bd9Sstevel@tonic-gate va += PAGESIZE; 54957c478bd9Sstevel@tonic-gate continue; 54967c478bd9Sstevel@tonic-gate } 54977c478bd9Sstevel@tonic-gate 54987c478bd9Sstevel@tonic-gate /* 54997c478bd9Sstevel@tonic-gate * Figure out which lgroup this page is in for kstats 55007c478bd9Sstevel@tonic-gate */ 55017c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 55027c478bd9Sstevel@tonic-gate from = lgrp_pfn_to_lgrp(pfn); 55037c478bd9Sstevel@tonic-gate 55047c478bd9Sstevel@tonic-gate /* 55057c478bd9Sstevel@tonic-gate * Get page size, and round up and skip to next page boundary 55067c478bd9Sstevel@tonic-gate * if unaligned address 55077c478bd9Sstevel@tonic-gate */ 55087c478bd9Sstevel@tonic-gate pszc = pp->p_szc; 55097c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(pszc); 55107c478bd9Sstevel@tonic-gate pages = btop(pgsz); 55117c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(va, pgsz) || 55127c478bd9Sstevel@tonic-gate !IS_P2ALIGNED(pfn, pages) || 55137c478bd9Sstevel@tonic-gate pgsz > segpgsz) { 55147c478bd9Sstevel@tonic-gate pgsz = MIN(pgsz, segpgsz); 55157c478bd9Sstevel@tonic-gate page_unlock(pp); 55167c478bd9Sstevel@tonic-gate i = btop(P2END((uintptr_t)va, pgsz) - 55177c478bd9Sstevel@tonic-gate (uintptr_t)va); 55187c478bd9Sstevel@tonic-gate va = (caddr_t)P2END((uintptr_t)va, pgsz); 55197c478bd9Sstevel@tonic-gate lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i); 55207c478bd9Sstevel@tonic-gate continue; 55217c478bd9Sstevel@tonic-gate } 55227c478bd9Sstevel@tonic-gate 55237c478bd9Sstevel@tonic-gate /* 55247c478bd9Sstevel@tonic-gate * Upgrade to exclusive lock on page 55257c478bd9Sstevel@tonic-gate */ 55267c478bd9Sstevel@tonic-gate if (!page_tryupgrade(pp)) { 55277c478bd9Sstevel@tonic-gate page_unlock(pp); 55287c478bd9Sstevel@tonic-gate va += pgsz; 55297c478bd9Sstevel@tonic-gate lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 55307c478bd9Sstevel@tonic-gate btop(pgsz)); 55317c478bd9Sstevel@tonic-gate continue; 55327c478bd9Sstevel@tonic-gate } 55337c478bd9Sstevel@tonic-gate 55347c478bd9Sstevel@tonic-gate /* 55357c478bd9Sstevel@tonic-gate * Remember pages locked exclusively and how many 55367c478bd9Sstevel@tonic-gate */ 55377c478bd9Sstevel@tonic-gate ppa[0] = pp; 55387c478bd9Sstevel@tonic-gate nlocked = 1; 55397c478bd9Sstevel@tonic-gate 55407c478bd9Sstevel@tonic-gate /* 55417c478bd9Sstevel@tonic-gate * Lock constituent pages if this is large page 55427c478bd9Sstevel@tonic-gate */ 55437c478bd9Sstevel@tonic-gate if (pages > 1) { 55447c478bd9Sstevel@tonic-gate /* 55457c478bd9Sstevel@tonic-gate * Lock all constituents except root page, since it 55467c478bd9Sstevel@tonic-gate * should be locked already. 55477c478bd9Sstevel@tonic-gate */ 55487c478bd9Sstevel@tonic-gate for (i = 1; i < pages; i++) { 5549affbd3ccSkchow pp++; 55507c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 55517c478bd9Sstevel@tonic-gate break; 55527c478bd9Sstevel@tonic-gate } 55537c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp) || 55547c478bd9Sstevel@tonic-gate pp->p_szc != pszc) { 55557c478bd9Sstevel@tonic-gate /* 55567c478bd9Sstevel@tonic-gate * hat_page_demote() raced in with us. 55577c478bd9Sstevel@tonic-gate */ 55587c478bd9Sstevel@tonic-gate ASSERT(!IS_SWAPFSVP(curvp)); 55597c478bd9Sstevel@tonic-gate page_unlock(pp); 55607c478bd9Sstevel@tonic-gate break; 55617c478bd9Sstevel@tonic-gate } 55627c478bd9Sstevel@tonic-gate ppa[nlocked] = pp; 55637c478bd9Sstevel@tonic-gate nlocked++; 55647c478bd9Sstevel@tonic-gate } 55657c478bd9Sstevel@tonic-gate } 55667c478bd9Sstevel@tonic-gate 55677c478bd9Sstevel@tonic-gate /* 55687c478bd9Sstevel@tonic-gate * If all constituent pages couldn't be locked, 55697c478bd9Sstevel@tonic-gate * unlock pages locked so far and skip to next page. 55707c478bd9Sstevel@tonic-gate */ 55717c478bd9Sstevel@tonic-gate if (nlocked != pages) { 55727c478bd9Sstevel@tonic-gate for (i = 0; i < nlocked; i++) 55737c478bd9Sstevel@tonic-gate page_unlock(ppa[i]); 55747c478bd9Sstevel@tonic-gate va += pgsz; 55757c478bd9Sstevel@tonic-gate lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, 55767c478bd9Sstevel@tonic-gate btop(pgsz)); 55777c478bd9Sstevel@tonic-gate continue; 55787c478bd9Sstevel@tonic-gate } 55797c478bd9Sstevel@tonic-gate 55807c478bd9Sstevel@tonic-gate /* 55817c478bd9Sstevel@tonic-gate * hat_page_demote() can no longer happen 55827c478bd9Sstevel@tonic-gate * since last cons page had the right p_szc after 55837c478bd9Sstevel@tonic-gate * all cons pages were locked. all cons pages 55847c478bd9Sstevel@tonic-gate * should now have the same p_szc. 55857c478bd9Sstevel@tonic-gate */ 55867c478bd9Sstevel@tonic-gate 55877c478bd9Sstevel@tonic-gate /* 55887c478bd9Sstevel@tonic-gate * All constituent pages locked successfully, so mark 55897c478bd9Sstevel@tonic-gate * large page for migration and unload the mappings of 55907c478bd9Sstevel@tonic-gate * constituent pages, so a fault will occur on any part of the 55917c478bd9Sstevel@tonic-gate * large page 55927c478bd9Sstevel@tonic-gate */ 55937c478bd9Sstevel@tonic-gate PP_SETMIGRATE(ppa[0]); 55947c478bd9Sstevel@tonic-gate for (i = 0; i < nlocked; i++) { 55957c478bd9Sstevel@tonic-gate pp = ppa[i]; 55967c478bd9Sstevel@tonic-gate (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 55977c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(pp) == 0); 55987c478bd9Sstevel@tonic-gate page_unlock(pp); 55997c478bd9Sstevel@tonic-gate } 56007c478bd9Sstevel@tonic-gate lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked); 56017c478bd9Sstevel@tonic-gate 56027c478bd9Sstevel@tonic-gate va += pgsz; 56037c478bd9Sstevel@tonic-gate } 56047c478bd9Sstevel@tonic-gate kmem_free(ppa, ppa_nentries * sizeof (page_t *)); 56057c478bd9Sstevel@tonic-gate } 56067c478bd9Sstevel@tonic-gate 56077c478bd9Sstevel@tonic-gate /* 56087c478bd9Sstevel@tonic-gate * Migrate any pages that have been marked for migration in the given range 56097c478bd9Sstevel@tonic-gate */ 56107c478bd9Sstevel@tonic-gate void 56117c478bd9Sstevel@tonic-gate page_migrate( 56127c478bd9Sstevel@tonic-gate struct seg *seg, 56137c478bd9Sstevel@tonic-gate caddr_t addr, 56147c478bd9Sstevel@tonic-gate page_t **ppa, 56157c478bd9Sstevel@tonic-gate pgcnt_t npages) 56167c478bd9Sstevel@tonic-gate { 56177c478bd9Sstevel@tonic-gate lgrp_t *from; 56187c478bd9Sstevel@tonic-gate lgrp_t *to; 56197c478bd9Sstevel@tonic-gate page_t *newpp; 56207c478bd9Sstevel@tonic-gate page_t *pp; 56217c478bd9Sstevel@tonic-gate pfn_t pfn; 56227c478bd9Sstevel@tonic-gate size_t pgsz; 56237c478bd9Sstevel@tonic-gate spgcnt_t page_cnt; 56247c478bd9Sstevel@tonic-gate spgcnt_t i; 56257c478bd9Sstevel@tonic-gate uint_t pszc; 56267c478bd9Sstevel@tonic-gate 56277c478bd9Sstevel@tonic-gate ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock)); 56287c478bd9Sstevel@tonic-gate 56297c478bd9Sstevel@tonic-gate while (npages > 0) { 56307c478bd9Sstevel@tonic-gate pp = *ppa; 56317c478bd9Sstevel@tonic-gate pszc = pp->p_szc; 56327c478bd9Sstevel@tonic-gate pgsz = page_get_pagesize(pszc); 56337c478bd9Sstevel@tonic-gate page_cnt = btop(pgsz); 56347c478bd9Sstevel@tonic-gate 56357c478bd9Sstevel@tonic-gate /* 56367c478bd9Sstevel@tonic-gate * Check to see whether this page is marked for migration 56377c478bd9Sstevel@tonic-gate * 56387c478bd9Sstevel@tonic-gate * Assume that root page of large page is marked for 56397c478bd9Sstevel@tonic-gate * migration and none of the other constituent pages 56407c478bd9Sstevel@tonic-gate * are marked. This really simplifies clearing the 56417c478bd9Sstevel@tonic-gate * migrate bit by not having to clear it from each 56427c478bd9Sstevel@tonic-gate * constituent page. 56437c478bd9Sstevel@tonic-gate * 56447c478bd9Sstevel@tonic-gate * note we don't want to relocate an entire large page if 56457c478bd9Sstevel@tonic-gate * someone is only using one subpage. 56467c478bd9Sstevel@tonic-gate */ 56477c478bd9Sstevel@tonic-gate if (npages < page_cnt) 56487c478bd9Sstevel@tonic-gate break; 56497c478bd9Sstevel@tonic-gate 56507c478bd9Sstevel@tonic-gate /* 56517c478bd9Sstevel@tonic-gate * Is it marked for migration? 56527c478bd9Sstevel@tonic-gate */ 56537c478bd9Sstevel@tonic-gate if (!PP_ISMIGRATE(pp)) 56547c478bd9Sstevel@tonic-gate goto next; 56557c478bd9Sstevel@tonic-gate 56567c478bd9Sstevel@tonic-gate /* 56577c478bd9Sstevel@tonic-gate * Determine lgroups that page is being migrated between 56587c478bd9Sstevel@tonic-gate */ 56597c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 56607c478bd9Sstevel@tonic-gate if (!IS_P2ALIGNED(pfn, page_cnt)) { 56617c478bd9Sstevel@tonic-gate break; 56627c478bd9Sstevel@tonic-gate } 56637c478bd9Sstevel@tonic-gate from = lgrp_pfn_to_lgrp(pfn); 56647c478bd9Sstevel@tonic-gate to = lgrp_mem_choose(seg, addr, pgsz); 56657c478bd9Sstevel@tonic-gate 56667c478bd9Sstevel@tonic-gate /* 56677c478bd9Sstevel@tonic-gate * Check to see whether we are trying to migrate page to lgroup 56687c478bd9Sstevel@tonic-gate * where it is allocated already 56697c478bd9Sstevel@tonic-gate */ 56707c478bd9Sstevel@tonic-gate if (to == from) { 56717c478bd9Sstevel@tonic-gate PP_CLRMIGRATE(pp); 56727c478bd9Sstevel@tonic-gate goto next; 56737c478bd9Sstevel@tonic-gate } 56747c478bd9Sstevel@tonic-gate 56757c478bd9Sstevel@tonic-gate /* 56767c478bd9Sstevel@tonic-gate * Need to get exclusive lock's to migrate 56777c478bd9Sstevel@tonic-gate */ 56787c478bd9Sstevel@tonic-gate for (i = 0; i < page_cnt; i++) { 56797c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(ppa[i])); 56807c478bd9Sstevel@tonic-gate if (page_pptonum(ppa[i]) != pfn + i || 56817c478bd9Sstevel@tonic-gate ppa[i]->p_szc != pszc) { 56827c478bd9Sstevel@tonic-gate break; 56837c478bd9Sstevel@tonic-gate } 56847c478bd9Sstevel@tonic-gate if (!page_tryupgrade(ppa[i])) { 56857c478bd9Sstevel@tonic-gate lgrp_stat_add(from->lgrp_id, 56867c478bd9Sstevel@tonic-gate LGRP_PM_FAIL_LOCK_PGS, 56877c478bd9Sstevel@tonic-gate page_cnt); 56887c478bd9Sstevel@tonic-gate break; 56897c478bd9Sstevel@tonic-gate } 56907c478bd9Sstevel@tonic-gate } 56917c478bd9Sstevel@tonic-gate if (i != page_cnt) { 56927c478bd9Sstevel@tonic-gate while (--i != -1) { 56937c478bd9Sstevel@tonic-gate page_downgrade(ppa[i]); 56947c478bd9Sstevel@tonic-gate } 56957c478bd9Sstevel@tonic-gate goto next; 56967c478bd9Sstevel@tonic-gate } 56977c478bd9Sstevel@tonic-gate 56987c478bd9Sstevel@tonic-gate (void) page_create_wait(page_cnt, PG_WAIT); 56997c478bd9Sstevel@tonic-gate newpp = page_get_replacement_page(pp, to, PGR_SAMESZC); 57007c478bd9Sstevel@tonic-gate if (newpp == NULL) { 57017c478bd9Sstevel@tonic-gate page_create_putback(page_cnt); 57027c478bd9Sstevel@tonic-gate for (i = 0; i < page_cnt; i++) { 57037c478bd9Sstevel@tonic-gate page_downgrade(ppa[i]); 57047c478bd9Sstevel@tonic-gate } 57057c478bd9Sstevel@tonic-gate lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS, 57067c478bd9Sstevel@tonic-gate page_cnt); 57077c478bd9Sstevel@tonic-gate goto next; 57087c478bd9Sstevel@tonic-gate } 57097c478bd9Sstevel@tonic-gate ASSERT(newpp->p_szc == pszc); 57107c478bd9Sstevel@tonic-gate /* 57117c478bd9Sstevel@tonic-gate * Clear migrate bit and relocate page 57127c478bd9Sstevel@tonic-gate */ 57137c478bd9Sstevel@tonic-gate PP_CLRMIGRATE(pp); 57147c478bd9Sstevel@tonic-gate if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) { 57157c478bd9Sstevel@tonic-gate panic("page_migrate: page_relocate failed"); 57167c478bd9Sstevel@tonic-gate } 57177c478bd9Sstevel@tonic-gate ASSERT(page_cnt * PAGESIZE == pgsz); 57187c478bd9Sstevel@tonic-gate 57197c478bd9Sstevel@tonic-gate /* 57207c478bd9Sstevel@tonic-gate * Keep stats for number of pages migrated from and to 57217c478bd9Sstevel@tonic-gate * each lgroup 57227c478bd9Sstevel@tonic-gate */ 57237c478bd9Sstevel@tonic-gate lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt); 57247c478bd9Sstevel@tonic-gate lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt); 57257c478bd9Sstevel@tonic-gate /* 57267c478bd9Sstevel@tonic-gate * update the page_t array we were passed in and 57277c478bd9Sstevel@tonic-gate * unlink constituent pages of a large page. 57287c478bd9Sstevel@tonic-gate */ 57297c478bd9Sstevel@tonic-gate for (i = 0; i < page_cnt; ++i, ++pp) { 57307c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(newpp)); 57317c478bd9Sstevel@tonic-gate ASSERT(newpp->p_szc == pszc); 57327c478bd9Sstevel@tonic-gate ppa[i] = newpp; 57337c478bd9Sstevel@tonic-gate pp = newpp; 57347c478bd9Sstevel@tonic-gate page_sub(&newpp, pp); 57357c478bd9Sstevel@tonic-gate page_downgrade(pp); 57367c478bd9Sstevel@tonic-gate } 57377c478bd9Sstevel@tonic-gate ASSERT(newpp == NULL); 57387c478bd9Sstevel@tonic-gate next: 57397c478bd9Sstevel@tonic-gate addr += pgsz; 57407c478bd9Sstevel@tonic-gate ppa += page_cnt; 57417c478bd9Sstevel@tonic-gate npages -= page_cnt; 57427c478bd9Sstevel@tonic-gate } 57437c478bd9Sstevel@tonic-gate } 57447c478bd9Sstevel@tonic-gate 57457c478bd9Sstevel@tonic-gate ulong_t mem_waiters = 0; 57467c478bd9Sstevel@tonic-gate ulong_t max_count = 20; 57477c478bd9Sstevel@tonic-gate #define MAX_DELAY 0x1ff 57487c478bd9Sstevel@tonic-gate 57497c478bd9Sstevel@tonic-gate /* 57507c478bd9Sstevel@tonic-gate * Check if enough memory is available to proceed. 57517c478bd9Sstevel@tonic-gate * Depending on system configuration and how much memory is 57527c478bd9Sstevel@tonic-gate * reserved for swap we need to check against two variables. 57537c478bd9Sstevel@tonic-gate * e.g. on systems with little physical swap availrmem can be 57547c478bd9Sstevel@tonic-gate * more reliable indicator of how much memory is available. 57557c478bd9Sstevel@tonic-gate * On systems with large phys swap freemem can be better indicator. 57567c478bd9Sstevel@tonic-gate * If freemem drops below threshold level don't return an error 57577c478bd9Sstevel@tonic-gate * immediately but wake up pageout to free memory and block. 57587c478bd9Sstevel@tonic-gate * This is done number of times. If pageout is not able to free 57597c478bd9Sstevel@tonic-gate * memory within certain time return an error. 57607c478bd9Sstevel@tonic-gate * The same applies for availrmem but kmem_reap is used to 57617c478bd9Sstevel@tonic-gate * free memory. 57627c478bd9Sstevel@tonic-gate */ 57637c478bd9Sstevel@tonic-gate int 57647c478bd9Sstevel@tonic-gate page_mem_avail(pgcnt_t npages) 57657c478bd9Sstevel@tonic-gate { 57667c478bd9Sstevel@tonic-gate ulong_t count; 57677c478bd9Sstevel@tonic-gate 57687c478bd9Sstevel@tonic-gate #if defined(__i386) 57697c478bd9Sstevel@tonic-gate if (freemem > desfree + npages && 57707c478bd9Sstevel@tonic-gate availrmem > swapfs_reserve + npages && 57717c478bd9Sstevel@tonic-gate btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem + 57727c478bd9Sstevel@tonic-gate npages) 57737c478bd9Sstevel@tonic-gate return (1); 57747c478bd9Sstevel@tonic-gate #else 57757c478bd9Sstevel@tonic-gate if (freemem > desfree + npages && 57767c478bd9Sstevel@tonic-gate availrmem > swapfs_reserve + npages) 57777c478bd9Sstevel@tonic-gate return (1); 57787c478bd9Sstevel@tonic-gate #endif 57797c478bd9Sstevel@tonic-gate 57807c478bd9Sstevel@tonic-gate count = max_count; 57817c478bd9Sstevel@tonic-gate atomic_add_long(&mem_waiters, 1); 57827c478bd9Sstevel@tonic-gate 57837c478bd9Sstevel@tonic-gate while (freemem < desfree + npages && --count) { 57847c478bd9Sstevel@tonic-gate cv_signal(&proc_pageout->p_cv); 57857c478bd9Sstevel@tonic-gate if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 57867c478bd9Sstevel@tonic-gate atomic_add_long(&mem_waiters, -1); 57877c478bd9Sstevel@tonic-gate return (0); 57887c478bd9Sstevel@tonic-gate } 57897c478bd9Sstevel@tonic-gate } 57907c478bd9Sstevel@tonic-gate if (count == 0) { 57917c478bd9Sstevel@tonic-gate atomic_add_long(&mem_waiters, -1); 57927c478bd9Sstevel@tonic-gate return (0); 57937c478bd9Sstevel@tonic-gate } 57947c478bd9Sstevel@tonic-gate 57957c478bd9Sstevel@tonic-gate count = max_count; 57967c478bd9Sstevel@tonic-gate while (availrmem < swapfs_reserve + npages && --count) { 57977c478bd9Sstevel@tonic-gate kmem_reap(); 57987c478bd9Sstevel@tonic-gate if (delay_sig(hz + (mem_waiters & MAX_DELAY))) { 57997c478bd9Sstevel@tonic-gate atomic_add_long(&mem_waiters, -1); 58007c478bd9Sstevel@tonic-gate return (0); 58017c478bd9Sstevel@tonic-gate } 58027c478bd9Sstevel@tonic-gate } 58037c478bd9Sstevel@tonic-gate atomic_add_long(&mem_waiters, -1); 58047c478bd9Sstevel@tonic-gate if (count == 0) 58057c478bd9Sstevel@tonic-gate return (0); 58067c478bd9Sstevel@tonic-gate 58077c478bd9Sstevel@tonic-gate #if defined(__i386) 58087c478bd9Sstevel@tonic-gate if (btop(vmem_size(heap_arena, VMEM_FREE)) < 58097c478bd9Sstevel@tonic-gate tune.t_minarmem + npages) 58107c478bd9Sstevel@tonic-gate return (0); 58117c478bd9Sstevel@tonic-gate #endif 58127c478bd9Sstevel@tonic-gate return (1); 58137c478bd9Sstevel@tonic-gate } 58147c478bd9Sstevel@tonic-gate 58153cff2f43Sstans #define MAX_CNT 60 /* max num of iterations */ 58163cff2f43Sstans /* 58173cff2f43Sstans * Reclaim/reserve availrmem for npages. 58183cff2f43Sstans * If there is not enough memory start reaping seg, kmem caches. 58193cff2f43Sstans * Start pageout scanner (via page_needfree()). 58203cff2f43Sstans * Exit after ~ MAX_CNT s regardless of how much memory has been released. 58213cff2f43Sstans * Note: There is no guarantee that any availrmem will be freed as 58223cff2f43Sstans * this memory typically is locked (kernel heap) or reserved for swap. 58233cff2f43Sstans * Also due to memory fragmentation kmem allocator may not be able 58243cff2f43Sstans * to free any memory (single user allocated buffer will prevent 58253cff2f43Sstans * freeing slab or a page). 58263cff2f43Sstans */ 58273cff2f43Sstans int 58283cff2f43Sstans page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust) 58293cff2f43Sstans { 58303cff2f43Sstans int i = 0; 58313cff2f43Sstans int ret = 0; 58323cff2f43Sstans pgcnt_t deficit; 58333cff2f43Sstans pgcnt_t old_availrmem; 58343cff2f43Sstans 58353cff2f43Sstans mutex_enter(&freemem_lock); 58363cff2f43Sstans old_availrmem = availrmem - 1; 58373cff2f43Sstans while ((availrmem < tune.t_minarmem + npages + epages) && 58383cff2f43Sstans (old_availrmem < availrmem) && (i++ < MAX_CNT)) { 58393cff2f43Sstans old_availrmem = availrmem; 58403cff2f43Sstans deficit = tune.t_minarmem + npages + epages - availrmem; 58413cff2f43Sstans mutex_exit(&freemem_lock); 58423cff2f43Sstans page_needfree(deficit); 58433cff2f43Sstans seg_preap(); 58443cff2f43Sstans kmem_reap(); 58453cff2f43Sstans delay(hz); 58463cff2f43Sstans page_needfree(-(spgcnt_t)deficit); 58473cff2f43Sstans mutex_enter(&freemem_lock); 58483cff2f43Sstans } 58493cff2f43Sstans 58503cff2f43Sstans if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) { 58513cff2f43Sstans availrmem -= npages; 58523cff2f43Sstans ret = 1; 58533cff2f43Sstans } 58543cff2f43Sstans 58553cff2f43Sstans mutex_exit(&freemem_lock); 58563cff2f43Sstans 58573cff2f43Sstans return (ret); 58583cff2f43Sstans } 58597c478bd9Sstevel@tonic-gate 58607c478bd9Sstevel@tonic-gate /* 58617c478bd9Sstevel@tonic-gate * Search the memory segments to locate the desired page. Within a 58627c478bd9Sstevel@tonic-gate * segment, pages increase linearly with one page structure per 58637c478bd9Sstevel@tonic-gate * physical page frame (size PAGESIZE). The search begins 58647c478bd9Sstevel@tonic-gate * with the segment that was accessed last, to take advantage of locality. 58657c478bd9Sstevel@tonic-gate * If the hint misses, we start from the beginning of the sorted memseg list 58667c478bd9Sstevel@tonic-gate */ 58677c478bd9Sstevel@tonic-gate 58687c478bd9Sstevel@tonic-gate 58697c478bd9Sstevel@tonic-gate /* 58707c478bd9Sstevel@tonic-gate * Some data structures for pfn to pp lookup. 58717c478bd9Sstevel@tonic-gate */ 58727c478bd9Sstevel@tonic-gate ulong_t mhash_per_slot; 58737c478bd9Sstevel@tonic-gate struct memseg *memseg_hash[N_MEM_SLOTS]; 58747c478bd9Sstevel@tonic-gate 58757c478bd9Sstevel@tonic-gate page_t * 58767c478bd9Sstevel@tonic-gate page_numtopp_nolock(pfn_t pfnum) 58777c478bd9Sstevel@tonic-gate { 58787c478bd9Sstevel@tonic-gate struct memseg *seg; 58797c478bd9Sstevel@tonic-gate page_t *pp; 5880affbd3ccSkchow vm_cpu_data_t *vc = CPU->cpu_vm_data; 58817c478bd9Sstevel@tonic-gate 5882affbd3ccSkchow ASSERT(vc != NULL); 58837c478bd9Sstevel@tonic-gate 58847c478bd9Sstevel@tonic-gate MEMSEG_STAT_INCR(nsearch); 58857c478bd9Sstevel@tonic-gate 58867c478bd9Sstevel@tonic-gate /* Try last winner first */ 5887affbd3ccSkchow if (((seg = vc->vc_pnum_memseg) != NULL) && 58887c478bd9Sstevel@tonic-gate (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 58897c478bd9Sstevel@tonic-gate MEMSEG_STAT_INCR(nlastwon); 58907c478bd9Sstevel@tonic-gate pp = seg->pages + (pfnum - seg->pages_base); 58917c478bd9Sstevel@tonic-gate if (pp->p_pagenum == pfnum) 58927c478bd9Sstevel@tonic-gate return ((page_t *)pp); 58937c478bd9Sstevel@tonic-gate } 58947c478bd9Sstevel@tonic-gate 58957c478bd9Sstevel@tonic-gate /* Else Try hash */ 58967c478bd9Sstevel@tonic-gate if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 58977c478bd9Sstevel@tonic-gate (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 58987c478bd9Sstevel@tonic-gate MEMSEG_STAT_INCR(nhashwon); 5899affbd3ccSkchow vc->vc_pnum_memseg = seg; 59007c478bd9Sstevel@tonic-gate pp = seg->pages + (pfnum - seg->pages_base); 59017c478bd9Sstevel@tonic-gate if (pp->p_pagenum == pfnum) 59027c478bd9Sstevel@tonic-gate return ((page_t *)pp); 59037c478bd9Sstevel@tonic-gate } 59047c478bd9Sstevel@tonic-gate 59057c478bd9Sstevel@tonic-gate /* Else Brute force */ 59067c478bd9Sstevel@tonic-gate for (seg = memsegs; seg != NULL; seg = seg->next) { 59077c478bd9Sstevel@tonic-gate if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 5908affbd3ccSkchow vc->vc_pnum_memseg = seg; 59097c478bd9Sstevel@tonic-gate pp = seg->pages + (pfnum - seg->pages_base); 59107c478bd9Sstevel@tonic-gate return ((page_t *)pp); 59117c478bd9Sstevel@tonic-gate } 59127c478bd9Sstevel@tonic-gate } 5913affbd3ccSkchow vc->vc_pnum_memseg = NULL; 59147c478bd9Sstevel@tonic-gate MEMSEG_STAT_INCR(nnotfound); 59157c478bd9Sstevel@tonic-gate return ((page_t *)NULL); 59167c478bd9Sstevel@tonic-gate 59177c478bd9Sstevel@tonic-gate } 59187c478bd9Sstevel@tonic-gate 59197c478bd9Sstevel@tonic-gate struct memseg * 59207c478bd9Sstevel@tonic-gate page_numtomemseg_nolock(pfn_t pfnum) 59217c478bd9Sstevel@tonic-gate { 59227c478bd9Sstevel@tonic-gate struct memseg *seg; 59237c478bd9Sstevel@tonic-gate page_t *pp; 59247c478bd9Sstevel@tonic-gate 59257c478bd9Sstevel@tonic-gate /* Try hash */ 59267c478bd9Sstevel@tonic-gate if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) && 59277c478bd9Sstevel@tonic-gate (pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) { 59287c478bd9Sstevel@tonic-gate pp = seg->pages + (pfnum - seg->pages_base); 59297c478bd9Sstevel@tonic-gate if (pp->p_pagenum == pfnum) 59307c478bd9Sstevel@tonic-gate return (seg); 59317c478bd9Sstevel@tonic-gate } 59327c478bd9Sstevel@tonic-gate 59337c478bd9Sstevel@tonic-gate /* Else Brute force */ 59347c478bd9Sstevel@tonic-gate for (seg = memsegs; seg != NULL; seg = seg->next) { 59357c478bd9Sstevel@tonic-gate if (pfnum >= seg->pages_base && pfnum < seg->pages_end) { 59367c478bd9Sstevel@tonic-gate return (seg); 59377c478bd9Sstevel@tonic-gate } 59387c478bd9Sstevel@tonic-gate } 59397c478bd9Sstevel@tonic-gate return ((struct memseg *)NULL); 59407c478bd9Sstevel@tonic-gate } 59417c478bd9Sstevel@tonic-gate 59427c478bd9Sstevel@tonic-gate /* 59437c478bd9Sstevel@tonic-gate * Given a page and a count return the page struct that is 59447c478bd9Sstevel@tonic-gate * n structs away from the current one in the global page 59457c478bd9Sstevel@tonic-gate * list. 59467c478bd9Sstevel@tonic-gate * 59477c478bd9Sstevel@tonic-gate * This function wraps to the first page upon 59487c478bd9Sstevel@tonic-gate * reaching the end of the memseg list. 59497c478bd9Sstevel@tonic-gate */ 59507c478bd9Sstevel@tonic-gate page_t * 59517c478bd9Sstevel@tonic-gate page_nextn(page_t *pp, ulong_t n) 59527c478bd9Sstevel@tonic-gate { 59537c478bd9Sstevel@tonic-gate struct memseg *seg; 59547c478bd9Sstevel@tonic-gate page_t *ppn; 5955affbd3ccSkchow vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data; 59567c478bd9Sstevel@tonic-gate 5957affbd3ccSkchow ASSERT(vc != NULL); 5958affbd3ccSkchow 5959affbd3ccSkchow if (((seg = vc->vc_pnext_memseg) == NULL) || 59607c478bd9Sstevel@tonic-gate (seg->pages_base == seg->pages_end) || 59617c478bd9Sstevel@tonic-gate !(pp >= seg->pages && pp < seg->epages)) { 59627c478bd9Sstevel@tonic-gate 59637c478bd9Sstevel@tonic-gate for (seg = memsegs; seg; seg = seg->next) { 59647c478bd9Sstevel@tonic-gate if (pp >= seg->pages && pp < seg->epages) 59657c478bd9Sstevel@tonic-gate break; 59667c478bd9Sstevel@tonic-gate } 59677c478bd9Sstevel@tonic-gate 59687c478bd9Sstevel@tonic-gate if (seg == NULL) { 59697c478bd9Sstevel@tonic-gate /* Memory delete got in, return something valid. */ 59707c478bd9Sstevel@tonic-gate /* TODO: fix me. */ 59717c478bd9Sstevel@tonic-gate seg = memsegs; 59727c478bd9Sstevel@tonic-gate pp = seg->pages; 59737c478bd9Sstevel@tonic-gate } 59747c478bd9Sstevel@tonic-gate } 59757c478bd9Sstevel@tonic-gate 59767c478bd9Sstevel@tonic-gate /* check for wraparound - possible if n is large */ 59777c478bd9Sstevel@tonic-gate while ((ppn = (pp + n)) >= seg->epages || ppn < pp) { 59787c478bd9Sstevel@tonic-gate n -= seg->epages - pp; 59797c478bd9Sstevel@tonic-gate seg = seg->next; 59807c478bd9Sstevel@tonic-gate if (seg == NULL) 59817c478bd9Sstevel@tonic-gate seg = memsegs; 59827c478bd9Sstevel@tonic-gate pp = seg->pages; 59837c478bd9Sstevel@tonic-gate } 5984affbd3ccSkchow vc->vc_pnext_memseg = seg; 59857c478bd9Sstevel@tonic-gate return (ppn); 59867c478bd9Sstevel@tonic-gate } 59877c478bd9Sstevel@tonic-gate 59887c478bd9Sstevel@tonic-gate /* 59897c478bd9Sstevel@tonic-gate * Initialize for a loop using page_next_scan_large(). 59907c478bd9Sstevel@tonic-gate */ 59917c478bd9Sstevel@tonic-gate page_t * 59927c478bd9Sstevel@tonic-gate page_next_scan_init(void **cookie) 59937c478bd9Sstevel@tonic-gate { 59947c478bd9Sstevel@tonic-gate ASSERT(cookie != NULL); 59957c478bd9Sstevel@tonic-gate *cookie = (void *)memsegs; 59967c478bd9Sstevel@tonic-gate return ((page_t *)memsegs->pages); 59977c478bd9Sstevel@tonic-gate } 59987c478bd9Sstevel@tonic-gate 59997c478bd9Sstevel@tonic-gate /* 60007c478bd9Sstevel@tonic-gate * Return the next page in a scan of page_t's, assuming we want 60017c478bd9Sstevel@tonic-gate * to skip over sub-pages within larger page sizes. 60027c478bd9Sstevel@tonic-gate * 60037c478bd9Sstevel@tonic-gate * The cookie is used to keep track of the current memseg. 60047c478bd9Sstevel@tonic-gate */ 60057c478bd9Sstevel@tonic-gate page_t * 60067c478bd9Sstevel@tonic-gate page_next_scan_large( 60077c478bd9Sstevel@tonic-gate page_t *pp, 60087c478bd9Sstevel@tonic-gate ulong_t *n, 60097c478bd9Sstevel@tonic-gate void **cookie) 60107c478bd9Sstevel@tonic-gate { 60117c478bd9Sstevel@tonic-gate struct memseg *seg = (struct memseg *)*cookie; 60127c478bd9Sstevel@tonic-gate page_t *new_pp; 60137c478bd9Sstevel@tonic-gate ulong_t cnt; 60147c478bd9Sstevel@tonic-gate pfn_t pfn; 60157c478bd9Sstevel@tonic-gate 60167c478bd9Sstevel@tonic-gate 60177c478bd9Sstevel@tonic-gate /* 60187c478bd9Sstevel@tonic-gate * get the count of page_t's to skip based on the page size 60197c478bd9Sstevel@tonic-gate */ 60207c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 60217c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 60227c478bd9Sstevel@tonic-gate cnt = 1; 60237c478bd9Sstevel@tonic-gate } else { 60247c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 60257c478bd9Sstevel@tonic-gate cnt = page_get_pagecnt(pp->p_szc); 60267c478bd9Sstevel@tonic-gate cnt -= pfn & (cnt - 1); 60277c478bd9Sstevel@tonic-gate } 60287c478bd9Sstevel@tonic-gate *n += cnt; 60297c478bd9Sstevel@tonic-gate new_pp = pp + cnt; 60307c478bd9Sstevel@tonic-gate 60317c478bd9Sstevel@tonic-gate /* 60327c478bd9Sstevel@tonic-gate * Catch if we went past the end of the current memory segment. If so, 60337c478bd9Sstevel@tonic-gate * just move to the next segment with pages. 60347c478bd9Sstevel@tonic-gate */ 60357c478bd9Sstevel@tonic-gate if (new_pp >= seg->epages) { 60367c478bd9Sstevel@tonic-gate do { 60377c478bd9Sstevel@tonic-gate seg = seg->next; 60387c478bd9Sstevel@tonic-gate if (seg == NULL) 60397c478bd9Sstevel@tonic-gate seg = memsegs; 60407c478bd9Sstevel@tonic-gate } while (seg->pages == seg->epages); 60417c478bd9Sstevel@tonic-gate new_pp = seg->pages; 60427c478bd9Sstevel@tonic-gate *cookie = (void *)seg; 60437c478bd9Sstevel@tonic-gate } 60447c478bd9Sstevel@tonic-gate 60457c478bd9Sstevel@tonic-gate return (new_pp); 60467c478bd9Sstevel@tonic-gate } 60477c478bd9Sstevel@tonic-gate 60487c478bd9Sstevel@tonic-gate 60497c478bd9Sstevel@tonic-gate /* 60507c478bd9Sstevel@tonic-gate * Returns next page in list. Note: this function wraps 60517c478bd9Sstevel@tonic-gate * to the first page in the list upon reaching the end 60527c478bd9Sstevel@tonic-gate * of the list. Callers should be aware of this fact. 60537c478bd9Sstevel@tonic-gate */ 60547c478bd9Sstevel@tonic-gate 60557c478bd9Sstevel@tonic-gate /* We should change this be a #define */ 60567c478bd9Sstevel@tonic-gate 60577c478bd9Sstevel@tonic-gate page_t * 60587c478bd9Sstevel@tonic-gate page_next(page_t *pp) 60597c478bd9Sstevel@tonic-gate { 60607c478bd9Sstevel@tonic-gate return (page_nextn(pp, 1)); 60617c478bd9Sstevel@tonic-gate } 60627c478bd9Sstevel@tonic-gate 60637c478bd9Sstevel@tonic-gate page_t * 60647c478bd9Sstevel@tonic-gate page_first() 60657c478bd9Sstevel@tonic-gate { 60667c478bd9Sstevel@tonic-gate return ((page_t *)memsegs->pages); 60677c478bd9Sstevel@tonic-gate } 60687c478bd9Sstevel@tonic-gate 60697c478bd9Sstevel@tonic-gate 60707c478bd9Sstevel@tonic-gate /* 60717c478bd9Sstevel@tonic-gate * This routine is called at boot with the initial memory configuration 60727c478bd9Sstevel@tonic-gate * and when memory is added or removed. 60737c478bd9Sstevel@tonic-gate */ 60747c478bd9Sstevel@tonic-gate void 60757c478bd9Sstevel@tonic-gate build_pfn_hash() 60767c478bd9Sstevel@tonic-gate { 60777c478bd9Sstevel@tonic-gate pfn_t cur; 60787c478bd9Sstevel@tonic-gate pgcnt_t index; 60797c478bd9Sstevel@tonic-gate struct memseg *pseg; 60807c478bd9Sstevel@tonic-gate int i; 60817c478bd9Sstevel@tonic-gate 60827c478bd9Sstevel@tonic-gate /* 60837c478bd9Sstevel@tonic-gate * Clear memseg_hash array. 60847c478bd9Sstevel@tonic-gate * Since memory add/delete is designed to operate concurrently 60857c478bd9Sstevel@tonic-gate * with normal operation, the hash rebuild must be able to run 60867c478bd9Sstevel@tonic-gate * concurrently with page_numtopp_nolock(). To support this 60877c478bd9Sstevel@tonic-gate * functionality, assignments to memseg_hash array members must 60887c478bd9Sstevel@tonic-gate * be done atomically. 60897c478bd9Sstevel@tonic-gate * 60907c478bd9Sstevel@tonic-gate * NOTE: bzero() does not currently guarantee this for kernel 60917c478bd9Sstevel@tonic-gate * threads, and cannot be used here. 60927c478bd9Sstevel@tonic-gate */ 60937c478bd9Sstevel@tonic-gate for (i = 0; i < N_MEM_SLOTS; i++) 60947c478bd9Sstevel@tonic-gate memseg_hash[i] = NULL; 60957c478bd9Sstevel@tonic-gate 60967c478bd9Sstevel@tonic-gate hat_kpm_mseghash_clear(N_MEM_SLOTS); 60977c478bd9Sstevel@tonic-gate 60987c478bd9Sstevel@tonic-gate /* 60997c478bd9Sstevel@tonic-gate * Physmax is the last valid pfn. 61007c478bd9Sstevel@tonic-gate */ 61017c478bd9Sstevel@tonic-gate mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT; 61027c478bd9Sstevel@tonic-gate for (pseg = memsegs; pseg != NULL; pseg = pseg->next) { 61037c478bd9Sstevel@tonic-gate index = MEMSEG_PFN_HASH(pseg->pages_base); 61047c478bd9Sstevel@tonic-gate cur = pseg->pages_base; 61057c478bd9Sstevel@tonic-gate do { 61067c478bd9Sstevel@tonic-gate if (index >= N_MEM_SLOTS) 61077c478bd9Sstevel@tonic-gate index = MEMSEG_PFN_HASH(cur); 61087c478bd9Sstevel@tonic-gate 61097c478bd9Sstevel@tonic-gate if (memseg_hash[index] == NULL || 61107c478bd9Sstevel@tonic-gate memseg_hash[index]->pages_base > pseg->pages_base) { 61117c478bd9Sstevel@tonic-gate memseg_hash[index] = pseg; 61127c478bd9Sstevel@tonic-gate hat_kpm_mseghash_update(index, pseg); 61137c478bd9Sstevel@tonic-gate } 61147c478bd9Sstevel@tonic-gate cur += mhash_per_slot; 61157c478bd9Sstevel@tonic-gate index++; 61167c478bd9Sstevel@tonic-gate } while (cur < pseg->pages_end); 61177c478bd9Sstevel@tonic-gate } 61187c478bd9Sstevel@tonic-gate } 61197c478bd9Sstevel@tonic-gate 61207c478bd9Sstevel@tonic-gate /* 61217c478bd9Sstevel@tonic-gate * Return the pagenum for the pp 61227c478bd9Sstevel@tonic-gate */ 61237c478bd9Sstevel@tonic-gate pfn_t 61247c478bd9Sstevel@tonic-gate page_pptonum(page_t *pp) 61257c478bd9Sstevel@tonic-gate { 61267c478bd9Sstevel@tonic-gate return (pp->p_pagenum); 61277c478bd9Sstevel@tonic-gate } 61287c478bd9Sstevel@tonic-gate 61297c478bd9Sstevel@tonic-gate /* 61307c478bd9Sstevel@tonic-gate * interface to the referenced and modified etc bits 61317c478bd9Sstevel@tonic-gate * in the PSM part of the page struct 61327c478bd9Sstevel@tonic-gate * when no locking is desired. 61337c478bd9Sstevel@tonic-gate */ 61347c478bd9Sstevel@tonic-gate void 61357c478bd9Sstevel@tonic-gate page_set_props(page_t *pp, uint_t flags) 61367c478bd9Sstevel@tonic-gate { 61377c478bd9Sstevel@tonic-gate ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0); 61387c478bd9Sstevel@tonic-gate pp->p_nrm |= (uchar_t)flags; 61397c478bd9Sstevel@tonic-gate } 61407c478bd9Sstevel@tonic-gate 61417c478bd9Sstevel@tonic-gate void 61427c478bd9Sstevel@tonic-gate page_clr_all_props(page_t *pp) 61437c478bd9Sstevel@tonic-gate { 61447c478bd9Sstevel@tonic-gate pp->p_nrm = 0; 61457c478bd9Sstevel@tonic-gate } 61467c478bd9Sstevel@tonic-gate 61477c478bd9Sstevel@tonic-gate /* 6148db874c57Selowe * Clear p_lckcnt and p_cowcnt, adjusting freemem if required. 61497c478bd9Sstevel@tonic-gate */ 61507c478bd9Sstevel@tonic-gate int 6151db874c57Selowe page_clear_lck_cow(page_t *pp, int adjust) 61527c478bd9Sstevel@tonic-gate { 6153db874c57Selowe int f_amount; 61547c478bd9Sstevel@tonic-gate 6155db874c57Selowe ASSERT(PAGE_EXCL(pp)); 61567c478bd9Sstevel@tonic-gate 61577c478bd9Sstevel@tonic-gate /* 6158db874c57Selowe * The page_struct_lock need not be acquired here since 6159db874c57Selowe * we require the caller hold the page exclusively locked. 61607c478bd9Sstevel@tonic-gate */ 6161db874c57Selowe f_amount = 0; 6162db874c57Selowe if (pp->p_lckcnt) { 6163db874c57Selowe f_amount = 1; 6164db874c57Selowe pp->p_lckcnt = 0; 6165db874c57Selowe } 6166db874c57Selowe if (pp->p_cowcnt) { 6167db874c57Selowe f_amount += pp->p_cowcnt; 6168db874c57Selowe pp->p_cowcnt = 0; 6169db874c57Selowe } 61707c478bd9Sstevel@tonic-gate 6171db874c57Selowe if (adjust && f_amount) { 6172db874c57Selowe mutex_enter(&freemem_lock); 6173db874c57Selowe availrmem += f_amount; 6174db874c57Selowe mutex_exit(&freemem_lock); 61757c478bd9Sstevel@tonic-gate } 61767c478bd9Sstevel@tonic-gate 6177db874c57Selowe return (f_amount); 61787c478bd9Sstevel@tonic-gate } 61797c478bd9Sstevel@tonic-gate 61807c478bd9Sstevel@tonic-gate /* 6181db874c57Selowe * The following functions is called from free_vp_pages() 6182db874c57Selowe * for an inexact estimate of a newly free'd page... 61837c478bd9Sstevel@tonic-gate */ 6184db874c57Selowe ulong_t 6185db874c57Selowe page_share_cnt(page_t *pp) 6186db874c57Selowe { 6187db874c57Selowe return (hat_page_getshare(pp)); 6188db874c57Selowe } 6189db874c57Selowe 61907c478bd9Sstevel@tonic-gate int 61917c478bd9Sstevel@tonic-gate page_isshared(page_t *pp) 61927c478bd9Sstevel@tonic-gate { 61937c478bd9Sstevel@tonic-gate return (hat_page_getshare(pp) > 1); 61947c478bd9Sstevel@tonic-gate } 61957c478bd9Sstevel@tonic-gate 61967c478bd9Sstevel@tonic-gate int 61977c478bd9Sstevel@tonic-gate page_isfree(page_t *pp) 61987c478bd9Sstevel@tonic-gate { 61997c478bd9Sstevel@tonic-gate return (PP_ISFREE(pp)); 62007c478bd9Sstevel@tonic-gate } 62017c478bd9Sstevel@tonic-gate 62027c478bd9Sstevel@tonic-gate int 62037c478bd9Sstevel@tonic-gate page_isref(page_t *pp) 62047c478bd9Sstevel@tonic-gate { 62057c478bd9Sstevel@tonic-gate return (hat_page_getattr(pp, P_REF)); 62067c478bd9Sstevel@tonic-gate } 62077c478bd9Sstevel@tonic-gate 62087c478bd9Sstevel@tonic-gate int 62097c478bd9Sstevel@tonic-gate page_ismod(page_t *pp) 62107c478bd9Sstevel@tonic-gate { 62117c478bd9Sstevel@tonic-gate return (hat_page_getattr(pp, P_MOD)); 62127c478bd9Sstevel@tonic-gate } 62138b464eb8Smec 62148b464eb8Smec /* 62158b464eb8Smec * Reclaim the given constituent page from the freelist, regardless of it's 62168b464eb8Smec * size. The page will be demoted as required. 62178b464eb8Smec * Returns 1 on success or 0 on failure. 62188b464eb8Smec * 62198b464eb8Smec * The page is unlocked if it can't be reclaimed (when freemem == 0). 62208b464eb8Smec * If `lock' is non-null, it will be dropped and re-acquired if 62218b464eb8Smec * the routine must wait while freemem is 0. 62228b464eb8Smec */ 62238b464eb8Smec int 62248b464eb8Smec page_reclaim_page(page_t *pp, kmutex_t *lock) 62258b464eb8Smec { 62268b464eb8Smec struct pcf *p; 62278b464eb8Smec uint_t pcf_index; 62288b464eb8Smec struct cpu *cpup; 62298b464eb8Smec uint_t i; 62308b464eb8Smec pgcnt_t collected = 0; 62318b464eb8Smec 62328b464eb8Smec ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1); 62338b464eb8Smec ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp)); 62348b464eb8Smec 62358b464eb8Smec /* 62368b464eb8Smec * If `freemem' is 0, we cannot reclaim this page from the 62378b464eb8Smec * freelist, so release every lock we might hold: the page, 62388b464eb8Smec * and the `lock' before blocking. 62398b464eb8Smec * 62408b464eb8Smec * The only way `freemem' can become 0 while there are pages 62418b464eb8Smec * marked free (have their p->p_free bit set) is when the 62428b464eb8Smec * system is low on memory and doing a page_create(). In 62438b464eb8Smec * order to guarantee that once page_create() starts acquiring 62448b464eb8Smec * pages it will be able to get all that it needs since `freemem' 62458b464eb8Smec * was decreased by the requested amount. So, we need to release 62468b464eb8Smec * this page, and let page_create() have it. 62478b464eb8Smec * 62488b464eb8Smec * Since `freemem' being zero is not supposed to happen, just 62498b464eb8Smec * use the usual hash stuff as a starting point. If that bucket 62508b464eb8Smec * is empty, then assume the worst, and start at the beginning 62518b464eb8Smec * of the pcf array. If we always start at the beginning 62528b464eb8Smec * when acquiring more than one pcf lock, there won't be any 62538b464eb8Smec * deadlock problems. 62548b464eb8Smec */ 62558b464eb8Smec 62568b464eb8Smec /* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */ 62578b464eb8Smec 62588b464eb8Smec if (freemem <= throttlefree && !page_create_throttle(1, 0)) { 62598b464eb8Smec pcf_acquire_all(); 62608b464eb8Smec goto page_reclaim_nomem; 62618b464eb8Smec } 62628b464eb8Smec 62638b464eb8Smec pcf_index = PCF_INDEX(); 62648b464eb8Smec p = &pcf[pcf_index]; 62658b464eb8Smec mutex_enter(&p->pcf_lock); 62668b464eb8Smec if (p->pcf_count > 0) { 62678b464eb8Smec collected = 1; 62688b464eb8Smec p->pcf_count -= 1; 62698b464eb8Smec } 62708b464eb8Smec mutex_exit(&p->pcf_lock); 62718b464eb8Smec 62728b464eb8Smec if (!collected) { 62738b464eb8Smec VM_STAT_ADD(page_reclaim_zero); 62748b464eb8Smec /* 62758b464eb8Smec * Check again. Its possible that some other thread 62768b464eb8Smec * could have been right behind us, and added one 62778b464eb8Smec * to a list somewhere. Acquire each of the pcf locks 62788b464eb8Smec * until we find a page. 62798b464eb8Smec */ 62808b464eb8Smec p = pcf; 62818b464eb8Smec for (i = 0; i < PCF_FANOUT; i++) { 62828b464eb8Smec mutex_enter(&p->pcf_lock); 62838b464eb8Smec if (p->pcf_count) { 62848b464eb8Smec if (p->pcf_count > 0) { 62858b464eb8Smec p->pcf_count -= 1; 62868b464eb8Smec collected = 1; 62878b464eb8Smec break; 62888b464eb8Smec } 62898b464eb8Smec } 62908b464eb8Smec p++; 62918b464eb8Smec } 62928b464eb8Smec 62938b464eb8Smec if (!collected) { 62948b464eb8Smec page_reclaim_nomem: 62958b464eb8Smec /* 62968b464eb8Smec * We really can't have page `pp'. 62978b464eb8Smec * Time for the no-memory dance with 62988b464eb8Smec * page_free(). This is just like 62998b464eb8Smec * page_create_wait(). Plus the added 63008b464eb8Smec * attraction of releasing whatever mutex 63018b464eb8Smec * we held when we were called with in `lock'. 63028b464eb8Smec * Page_unlock() will wakeup any thread 63038b464eb8Smec * waiting around for this page. 63048b464eb8Smec */ 63058b464eb8Smec if (lock) { 63068b464eb8Smec VM_STAT_ADD(page_reclaim_zero_locked); 63078b464eb8Smec mutex_exit(lock); 63088b464eb8Smec } 63098b464eb8Smec page_unlock(pp); 63108b464eb8Smec 63118b464eb8Smec /* 63128b464eb8Smec * get this before we drop all the pcf locks. 63138b464eb8Smec */ 63148b464eb8Smec mutex_enter(&new_freemem_lock); 63158b464eb8Smec 63168b464eb8Smec p = pcf; 63178b464eb8Smec for (i = 0; i < PCF_FANOUT; i++) { 63188b464eb8Smec p->pcf_wait++; 63198b464eb8Smec mutex_exit(&p->pcf_lock); 63208b464eb8Smec p++; 63218b464eb8Smec } 63228b464eb8Smec 63238b464eb8Smec freemem_wait++; 63248b464eb8Smec cv_wait(&freemem_cv, &new_freemem_lock); 63258b464eb8Smec freemem_wait--; 63268b464eb8Smec 63278b464eb8Smec mutex_exit(&new_freemem_lock); 63288b464eb8Smec 63298b464eb8Smec if (lock) { 63308b464eb8Smec mutex_enter(lock); 63318b464eb8Smec } 63328b464eb8Smec return (0); 63338b464eb8Smec } 63348b464eb8Smec 63358b464eb8Smec /* 63368b464eb8Smec * We beat the PCF bins over the head until 63378b464eb8Smec * we got the memory that we wanted. 63388b464eb8Smec * The pcf accounting has been done, 63398b464eb8Smec * though none of the pcf_wait flags have been set, 63408b464eb8Smec * drop the locks and continue on. 63418b464eb8Smec */ 63428b464eb8Smec ASSERT(collected == 1); 63438b464eb8Smec while (p >= pcf) { 63448b464eb8Smec mutex_exit(&p->pcf_lock); 63458b464eb8Smec p--; 63468b464eb8Smec } 63478b464eb8Smec } 63488b464eb8Smec 63498b464eb8Smec /* 63508b464eb8Smec * freemem is not protected by any lock. Thus, we cannot 63518b464eb8Smec * have any assertion containing freemem here. 63528b464eb8Smec */ 63538b464eb8Smec freemem -= 1; 63548b464eb8Smec 63558b464eb8Smec VM_STAT_ADD(pagecnt.pc_reclaim); 63568b464eb8Smec if (PP_ISAGED(pp)) { 63578b464eb8Smec page_list_sub(pp, PG_FREE_LIST); 63588b464eb8Smec TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE, 63598b464eb8Smec "page_reclaim_page_free:pp %p", pp); 63608b464eb8Smec } else { 63618b464eb8Smec page_list_sub(pp, PG_CACHE_LIST); 63628b464eb8Smec TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE, 63638b464eb8Smec "page_reclaim_page_cache:pp %p", pp); 63648b464eb8Smec } 63658b464eb8Smec 63668b464eb8Smec /* 63678b464eb8Smec * The page we took off the freelist must be szc 0 as 63688b464eb8Smec * we used page_list_sub which will demote the page if needed. 63698b464eb8Smec */ 63708b464eb8Smec ASSERT(pp->p_szc == 0); 63718b464eb8Smec 63728b464eb8Smec /* 63738b464eb8Smec * clear the p_free & p_age bits since this page is no longer 63748b464eb8Smec * on the free list. Notice that there was a brief time where 63758b464eb8Smec * a page is marked as free, but is not on the list. 63768b464eb8Smec * 63778b464eb8Smec * Set the reference bit to protect against immediate pageout. 63788b464eb8Smec */ 63798b464eb8Smec PP_CLRFREE(pp); 63808b464eb8Smec PP_CLRAGED(pp); 63818b464eb8Smec page_set_props(pp, P_REF); 63828b464eb8Smec 63838b464eb8Smec CPU_STATS_ENTER_K(); 63848b464eb8Smec cpup = CPU; /* get cpup now that CPU cannot change */ 63858b464eb8Smec CPU_STATS_ADDQ(cpup, vm, pgrec, 1); 63868b464eb8Smec CPU_STATS_ADDQ(cpup, vm, pgfrec, 1); 63878b464eb8Smec CPU_STATS_EXIT_K(); 63888b464eb8Smec 63898b464eb8Smec return (1); 63908b464eb8Smec } 63918b464eb8Smec 63928b464eb8Smec /* 63938b464eb8Smec * The following code all currently relates to the page capture logic: 63948b464eb8Smec * 63958b464eb8Smec * This logic is used for cases where there is a desire to claim a certain 63968b464eb8Smec * physical page in the system for the caller. As it may not be possible 63978b464eb8Smec * to capture the page immediately, the p_toxic bits are used in the page 63988b464eb8Smec * structure to indicate that someone wants to capture this page. When the 63998b464eb8Smec * page gets unlocked, the toxic flag will be noted and an attempt to capture 64008b464eb8Smec * the page will be made. If it is successful, the original callers callback 64018b464eb8Smec * will be called with the page to do with it what they please. 64028b464eb8Smec * 64038b464eb8Smec * There is also an async thread which wakes up to attempt to capture 64048b464eb8Smec * pages occasionally which have the capture bit set. All of the pages which 64058b464eb8Smec * need to be captured asynchronously have been inserted into the 64068b464eb8Smec * page_capture_hash and thus this thread walks that hash list. Items in the 64078b464eb8Smec * hash have an expiration time so this thread handles that as well by removing 64088b464eb8Smec * the item from the hash if it has expired. 64098b464eb8Smec * 64108b464eb8Smec * Some important things to note are: 64118b464eb8Smec * - if the PR_CAPTURE bit is set on a page, then the page is in the 64128b464eb8Smec * page_capture_hash. The page_capture_hash_head.pchh_mutex is needed 64138b464eb8Smec * to set and clear this bit, and while the lock is held is the only time 64148b464eb8Smec * you can add or remove an entry from the hash. 64158b464eb8Smec * - the PR_CAPTURE bit can only be set and cleared while holding the 64168b464eb8Smec * page_capture_hash_head.pchh_mutex 64178b464eb8Smec * - the t_flag field of the thread struct is used with the T_CAPTURING 64188b464eb8Smec * flag to prevent recursion while dealing with large pages. 64198b464eb8Smec * - pages which need to be retired never expire on the page_capture_hash. 64208b464eb8Smec */ 64218b464eb8Smec 64228b464eb8Smec static void page_capture_thread(void); 64238b464eb8Smec static kthread_t *pc_thread_id; 64248b464eb8Smec kcondvar_t pc_cv; 64258b464eb8Smec static kmutex_t pc_thread_mutex; 64268b464eb8Smec static clock_t pc_thread_shortwait; 64278b464eb8Smec static clock_t pc_thread_longwait; 6428*cee1d74bSjfrank static int pc_thread_ism_retry; 64298b464eb8Smec 64308b464eb8Smec struct page_capture_callback pc_cb[PC_NUM_CALLBACKS]; 64318b464eb8Smec 64328b464eb8Smec /* Note that this is a circular linked list */ 64338b464eb8Smec typedef struct page_capture_hash_bucket { 64348b464eb8Smec page_t *pp; 64358b464eb8Smec uint_t szc; 64368b464eb8Smec uint_t flags; 64378b464eb8Smec clock_t expires; /* lbolt at which this request expires. */ 64388b464eb8Smec void *datap; /* Cached data passed in for callback */ 64398b464eb8Smec struct page_capture_hash_bucket *next; 64408b464eb8Smec struct page_capture_hash_bucket *prev; 64418b464eb8Smec } page_capture_hash_bucket_t; 64428b464eb8Smec 64438b464eb8Smec /* 64448b464eb8Smec * Each hash bucket will have it's own mutex and two lists which are: 64458b464eb8Smec * active (0): represents requests which have not been processed by 64468b464eb8Smec * the page_capture async thread yet. 64478b464eb8Smec * walked (1): represents requests which have been processed by the 64488b464eb8Smec * page_capture async thread within it's given walk of this bucket. 64498b464eb8Smec * 64508b464eb8Smec * These are all needed so that we can synchronize all async page_capture 64518b464eb8Smec * events. When the async thread moves to a new bucket, it will append the 64528b464eb8Smec * walked list to the active list and walk each item one at a time, moving it 64538b464eb8Smec * from the active list to the walked list. Thus if there is an async request 64548b464eb8Smec * outstanding for a given page, it will always be in one of the two lists. 64558b464eb8Smec * New requests will always be added to the active list. 64568b464eb8Smec * If we were not able to capture a page before the request expired, we'd free 64578b464eb8Smec * up the request structure which would indicate to page_capture that there is 64588b464eb8Smec * no longer a need for the given page, and clear the PR_CAPTURE flag if 64598b464eb8Smec * possible. 64608b464eb8Smec */ 64618b464eb8Smec typedef struct page_capture_hash_head { 64628b464eb8Smec kmutex_t pchh_mutex; 64638b464eb8Smec uint_t num_pages; 64648b464eb8Smec page_capture_hash_bucket_t lists[2]; /* sentinel nodes */ 64658b464eb8Smec } page_capture_hash_head_t; 64668b464eb8Smec 64678b464eb8Smec #ifdef DEBUG 64688b464eb8Smec #define NUM_PAGE_CAPTURE_BUCKETS 4 64698b464eb8Smec #else 64708b464eb8Smec #define NUM_PAGE_CAPTURE_BUCKETS 64 64718b464eb8Smec #endif 64728b464eb8Smec 64738b464eb8Smec page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS]; 64748b464eb8Smec 64758b464eb8Smec /* for now use a very simple hash based upon the size of a page struct */ 64768b464eb8Smec #define PAGE_CAPTURE_HASH(pp) \ 64778b464eb8Smec ((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1))) 64788b464eb8Smec 64798b464eb8Smec extern pgcnt_t swapfs_minfree; 64808b464eb8Smec 64818b464eb8Smec int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap); 64828b464eb8Smec 64838b464eb8Smec /* 64848b464eb8Smec * a callback function is required for page capture requests. 64858b464eb8Smec */ 64868b464eb8Smec void 64878b464eb8Smec page_capture_register_callback(uint_t index, clock_t duration, 64888b464eb8Smec int (*cb_func)(page_t *, void *, uint_t)) 64898b464eb8Smec { 64908b464eb8Smec ASSERT(pc_cb[index].cb_active == 0); 64918b464eb8Smec ASSERT(cb_func != NULL); 64928b464eb8Smec rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 64938b464eb8Smec pc_cb[index].duration = duration; 64948b464eb8Smec pc_cb[index].cb_func = cb_func; 64958b464eb8Smec pc_cb[index].cb_active = 1; 64968b464eb8Smec rw_exit(&pc_cb[index].cb_rwlock); 64978b464eb8Smec } 64988b464eb8Smec 64998b464eb8Smec void 65008b464eb8Smec page_capture_unregister_callback(uint_t index) 65018b464eb8Smec { 65028b464eb8Smec int i, j; 65038b464eb8Smec struct page_capture_hash_bucket *bp1; 65048b464eb8Smec struct page_capture_hash_bucket *bp2; 65058b464eb8Smec struct page_capture_hash_bucket *head = NULL; 65068b464eb8Smec uint_t flags = (1 << index); 65078b464eb8Smec 65088b464eb8Smec rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER); 65098b464eb8Smec ASSERT(pc_cb[index].cb_active == 1); 65108b464eb8Smec pc_cb[index].duration = 0; /* Paranoia */ 65118b464eb8Smec pc_cb[index].cb_func = NULL; /* Paranoia */ 65128b464eb8Smec pc_cb[index].cb_active = 0; 65138b464eb8Smec rw_exit(&pc_cb[index].cb_rwlock); 65148b464eb8Smec 65158b464eb8Smec /* 65168b464eb8Smec * Just move all the entries to a private list which we can walk 65178b464eb8Smec * through without the need to hold any locks. 65188b464eb8Smec * No more requests can get added to the hash lists for this consumer 65198b464eb8Smec * as the cb_active field for the callback has been cleared. 65208b464eb8Smec */ 65218b464eb8Smec for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 65228b464eb8Smec mutex_enter(&page_capture_hash[i].pchh_mutex); 65238b464eb8Smec for (j = 0; j < 2; j++) { 65248b464eb8Smec bp1 = page_capture_hash[i].lists[j].next; 65258b464eb8Smec /* walk through all but first (sentinel) element */ 65268b464eb8Smec while (bp1 != &page_capture_hash[i].lists[j]) { 65278b464eb8Smec bp2 = bp1; 65288b464eb8Smec if (bp2->flags & flags) { 65298b464eb8Smec bp1 = bp2->next; 65308b464eb8Smec bp1->prev = bp2->prev; 65318b464eb8Smec bp2->prev->next = bp1; 65328b464eb8Smec bp2->next = head; 65338b464eb8Smec head = bp2; 65348b464eb8Smec /* 65358b464eb8Smec * Clear the PR_CAPTURE bit as we 65368b464eb8Smec * hold appropriate locks here. 65378b464eb8Smec */ 65388b464eb8Smec page_clrtoxic(head->pp, PR_CAPTURE); 65398b464eb8Smec page_capture_hash[i].num_pages--; 65408b464eb8Smec continue; 65418b464eb8Smec } 65428b464eb8Smec bp1 = bp1->next; 65438b464eb8Smec } 65448b464eb8Smec } 65458b464eb8Smec mutex_exit(&page_capture_hash[i].pchh_mutex); 65468b464eb8Smec } 65478b464eb8Smec 65488b464eb8Smec while (head != NULL) { 65498b464eb8Smec bp1 = head; 65508b464eb8Smec head = head->next; 65518b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 65528b464eb8Smec } 65538b464eb8Smec } 65548b464eb8Smec 65558b464eb8Smec 65568b464eb8Smec /* 65578b464eb8Smec * Find pp in the active list and move it to the walked list if it 65588b464eb8Smec * exists. 65598b464eb8Smec * Note that most often pp should be at the front of the active list 65608b464eb8Smec * as it is currently used and thus there is no other sort of optimization 65618b464eb8Smec * being done here as this is a linked list data structure. 65628b464eb8Smec * Returns 1 on successful move or 0 if page could not be found. 65638b464eb8Smec */ 65648b464eb8Smec static int 65658b464eb8Smec page_capture_move_to_walked(page_t *pp) 65668b464eb8Smec { 65678b464eb8Smec page_capture_hash_bucket_t *bp; 65688b464eb8Smec int index; 65698b464eb8Smec 65708b464eb8Smec index = PAGE_CAPTURE_HASH(pp); 65718b464eb8Smec 65728b464eb8Smec mutex_enter(&page_capture_hash[index].pchh_mutex); 65738b464eb8Smec bp = page_capture_hash[index].lists[0].next; 65748b464eb8Smec while (bp != &page_capture_hash[index].lists[0]) { 65758b464eb8Smec if (bp->pp == pp) { 65768b464eb8Smec /* Remove from old list */ 65778b464eb8Smec bp->next->prev = bp->prev; 65788b464eb8Smec bp->prev->next = bp->next; 65798b464eb8Smec 65808b464eb8Smec /* Add to new list */ 65818b464eb8Smec bp->next = page_capture_hash[index].lists[1].next; 65828b464eb8Smec bp->prev = &page_capture_hash[index].lists[1]; 65838b464eb8Smec page_capture_hash[index].lists[1].next = bp; 65848b464eb8Smec bp->next->prev = bp; 65858b464eb8Smec mutex_exit(&page_capture_hash[index].pchh_mutex); 65868b464eb8Smec 65878b464eb8Smec return (1); 65888b464eb8Smec } 65898b464eb8Smec bp = bp->next; 65908b464eb8Smec } 65918b464eb8Smec mutex_exit(&page_capture_hash[index].pchh_mutex); 65928b464eb8Smec return (0); 65938b464eb8Smec } 65948b464eb8Smec 65958b464eb8Smec /* 65968b464eb8Smec * Add a new entry to the page capture hash. The only case where a new 65978b464eb8Smec * entry is not added is when the page capture consumer is no longer registered. 65988b464eb8Smec * In this case, we'll silently not add the page to the hash. We know that 65998b464eb8Smec * page retire will always be registered for the case where we are currently 66008b464eb8Smec * unretiring a page and thus there are no conflicts. 66018b464eb8Smec */ 66028b464eb8Smec static void 66038b464eb8Smec page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap) 66048b464eb8Smec { 66058b464eb8Smec page_capture_hash_bucket_t *bp1; 66068b464eb8Smec page_capture_hash_bucket_t *bp2; 66078b464eb8Smec int index; 66088b464eb8Smec int cb_index; 66098b464eb8Smec int i; 66108b464eb8Smec #ifdef DEBUG 66118b464eb8Smec page_capture_hash_bucket_t *tp1; 66128b464eb8Smec int l; 66138b464eb8Smec #endif 66148b464eb8Smec 66158b464eb8Smec ASSERT(!(flags & CAPTURE_ASYNC)); 66168b464eb8Smec 66178b464eb8Smec bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP); 66188b464eb8Smec 66198b464eb8Smec bp1->pp = pp; 66208b464eb8Smec bp1->szc = szc; 66218b464eb8Smec bp1->flags = flags; 66228b464eb8Smec bp1->datap = datap; 66238b464eb8Smec 66248b464eb8Smec for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 66258b464eb8Smec if ((flags >> cb_index) & 1) { 66268b464eb8Smec break; 66278b464eb8Smec } 66288b464eb8Smec } 66298b464eb8Smec 66308b464eb8Smec ASSERT(cb_index != PC_NUM_CALLBACKS); 66318b464eb8Smec 66328b464eb8Smec rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 66338b464eb8Smec if (pc_cb[cb_index].cb_active) { 66348b464eb8Smec if (pc_cb[cb_index].duration == -1) { 66358b464eb8Smec bp1->expires = (clock_t)-1; 66368b464eb8Smec } else { 66378b464eb8Smec bp1->expires = lbolt + pc_cb[cb_index].duration; 66388b464eb8Smec } 66398b464eb8Smec } else { 66408b464eb8Smec /* There's no callback registered so don't add to the hash */ 66418b464eb8Smec rw_exit(&pc_cb[cb_index].cb_rwlock); 66428b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 66438b464eb8Smec return; 66448b464eb8Smec } 66458b464eb8Smec 66468b464eb8Smec index = PAGE_CAPTURE_HASH(pp); 66478b464eb8Smec 66488b464eb8Smec /* 66498b464eb8Smec * Only allow capture flag to be modified under this mutex. 66508b464eb8Smec * Prevents multiple entries for same page getting added. 66518b464eb8Smec */ 66528b464eb8Smec mutex_enter(&page_capture_hash[index].pchh_mutex); 66538b464eb8Smec 66548b464eb8Smec /* 66558b464eb8Smec * if not already on the hash, set capture bit and add to the hash 66568b464eb8Smec */ 66578b464eb8Smec if (!(pp->p_toxic & PR_CAPTURE)) { 66588b464eb8Smec #ifdef DEBUG 66598b464eb8Smec /* Check for duplicate entries */ 66608b464eb8Smec for (l = 0; l < 2; l++) { 66618b464eb8Smec tp1 = page_capture_hash[index].lists[l].next; 66628b464eb8Smec while (tp1 != &page_capture_hash[index].lists[l]) { 66638b464eb8Smec if (tp1->pp == pp) { 66648b464eb8Smec panic("page pp 0x%p already on hash " 66658b464eb8Smec "at 0x%p\n", pp, tp1); 66668b464eb8Smec } 66678b464eb8Smec tp1 = tp1->next; 66688b464eb8Smec } 66698b464eb8Smec } 66708b464eb8Smec 66718b464eb8Smec #endif 66728b464eb8Smec page_settoxic(pp, PR_CAPTURE); 66738b464eb8Smec bp1->next = page_capture_hash[index].lists[0].next; 66748b464eb8Smec bp1->prev = &page_capture_hash[index].lists[0]; 66758b464eb8Smec bp1->next->prev = bp1; 66768b464eb8Smec page_capture_hash[index].lists[0].next = bp1; 66778b464eb8Smec page_capture_hash[index].num_pages++; 6678*cee1d74bSjfrank if (flags & CAPTURE_RETIRE) { 6679*cee1d74bSjfrank page_retire_incr_pend_count(); 6680*cee1d74bSjfrank } 66818b464eb8Smec mutex_exit(&page_capture_hash[index].pchh_mutex); 66828b464eb8Smec rw_exit(&pc_cb[cb_index].cb_rwlock); 66838b464eb8Smec cv_signal(&pc_cv); 66848b464eb8Smec return; 66858b464eb8Smec } 66868b464eb8Smec 66878b464eb8Smec /* 66888b464eb8Smec * A page retire request will replace any other request. 66898b464eb8Smec * A second physmem request which is for a different process than 66908b464eb8Smec * the currently registered one will be dropped as there is 66918b464eb8Smec * no way to hold the private data for both calls. 66928b464eb8Smec * In the future, once there are more callers, this will have to 66938b464eb8Smec * be worked out better as there needs to be private storage for 66948b464eb8Smec * at least each type of caller (maybe have datap be an array of 66958b464eb8Smec * *void's so that we can index based upon callers index). 66968b464eb8Smec */ 66978b464eb8Smec 66988b464eb8Smec /* walk hash list to update expire time */ 66998b464eb8Smec for (i = 0; i < 2; i++) { 67008b464eb8Smec bp2 = page_capture_hash[index].lists[i].next; 67018b464eb8Smec while (bp2 != &page_capture_hash[index].lists[i]) { 67028b464eb8Smec if (bp2->pp == pp) { 67038b464eb8Smec if (flags & CAPTURE_RETIRE) { 67048b464eb8Smec if (!(bp2->flags & CAPTURE_RETIRE)) { 6705*cee1d74bSjfrank page_retire_incr_pend_count(); 67068b464eb8Smec bp2->flags = flags; 67078b464eb8Smec bp2->expires = bp1->expires; 67088b464eb8Smec bp2->datap = datap; 67098b464eb8Smec } 67108b464eb8Smec } else { 67118b464eb8Smec ASSERT(flags & CAPTURE_PHYSMEM); 67128b464eb8Smec if (!(bp2->flags & CAPTURE_RETIRE) && 67138b464eb8Smec (datap == bp2->datap)) { 67148b464eb8Smec bp2->expires = bp1->expires; 67158b464eb8Smec } 67168b464eb8Smec } 67178b464eb8Smec mutex_exit(&page_capture_hash[index]. 67188b464eb8Smec pchh_mutex); 67198b464eb8Smec rw_exit(&pc_cb[cb_index].cb_rwlock); 67208b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 67218b464eb8Smec return; 67228b464eb8Smec } 67238b464eb8Smec bp2 = bp2->next; 67248b464eb8Smec } 67258b464eb8Smec } 67268b464eb8Smec 67278b464eb8Smec /* 67288b464eb8Smec * the PR_CAPTURE flag is protected by the page_capture_hash mutexes 67298b464eb8Smec * and thus it either has to be set or not set and can't change 67308b464eb8Smec * while holding the mutex above. 67318b464eb8Smec */ 67328b464eb8Smec panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", pp); 67338b464eb8Smec } 67348b464eb8Smec 67358b464eb8Smec /* 67368b464eb8Smec * We have a page in our hands, lets try and make it ours by turning 67378b464eb8Smec * it into a clean page like it had just come off the freelists. 67388b464eb8Smec * 67398b464eb8Smec * Returns 0 on success, with the page still EXCL locked. 67408b464eb8Smec * On failure, the page will be unlocked, and returns EAGAIN 67418b464eb8Smec */ 67428b464eb8Smec static int 67438b464eb8Smec page_capture_clean_page(page_t *pp) 67448b464eb8Smec { 67458b464eb8Smec page_t *newpp; 67468b464eb8Smec int skip_unlock = 0; 67478b464eb8Smec spgcnt_t count; 67488b464eb8Smec page_t *tpp; 67498b464eb8Smec int ret = 0; 67508b464eb8Smec int extra; 67518b464eb8Smec 67528b464eb8Smec ASSERT(PAGE_EXCL(pp)); 67538b464eb8Smec ASSERT(!PP_RETIRED(pp)); 67548b464eb8Smec ASSERT(curthread->t_flag & T_CAPTURING); 67558b464eb8Smec 67568b464eb8Smec if (PP_ISFREE(pp)) { 67578b464eb8Smec if (!page_reclaim_page(pp, NULL)) { 67588b464eb8Smec skip_unlock = 1; 67598b464eb8Smec ret = EAGAIN; 67608b464eb8Smec goto cleanup; 67618b464eb8Smec } 67628b464eb8Smec if (pp->p_vnode != NULL) { 67638b464eb8Smec /* 67648b464eb8Smec * Since this page came from the 67658b464eb8Smec * cachelist, we must destroy the 67668b464eb8Smec * old vnode association. 67678b464eb8Smec */ 67688b464eb8Smec page_hashout(pp, NULL); 67698b464eb8Smec } 67708b464eb8Smec goto cleanup; 67718b464eb8Smec } 67728b464eb8Smec 67738b464eb8Smec /* 67748b464eb8Smec * If we know page_relocate will fail, skip it 67758b464eb8Smec * It could still fail due to a UE on another page but we 67768b464eb8Smec * can't do anything about that. 67778b464eb8Smec */ 67788b464eb8Smec if (pp->p_toxic & PR_UE) { 67798b464eb8Smec goto skip_relocate; 67808b464eb8Smec } 67818b464eb8Smec 67828b464eb8Smec /* 67838b464eb8Smec * It's possible that pages can not have a vnode as fsflush comes 67848b464eb8Smec * through and cleans up these pages. It's ugly but that's how it is. 67858b464eb8Smec */ 67868b464eb8Smec if (pp->p_vnode == NULL) { 67878b464eb8Smec goto skip_relocate; 67888b464eb8Smec } 67898b464eb8Smec 67908b464eb8Smec /* 67918b464eb8Smec * Page was not free, so lets try to relocate it. 67928b464eb8Smec * page_relocate only works with root pages, so if this is not a root 67938b464eb8Smec * page, we need to demote it to try and relocate it. 67948b464eb8Smec * Unfortunately this is the best we can do right now. 67958b464eb8Smec */ 67968b464eb8Smec newpp = NULL; 67978b464eb8Smec if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) { 67988b464eb8Smec if (page_try_demote_pages(pp) == 0) { 67998b464eb8Smec ret = EAGAIN; 68008b464eb8Smec goto cleanup; 68018b464eb8Smec } 68028b464eb8Smec } 68038b464eb8Smec ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL); 68048b464eb8Smec if (ret == 0) { 68058b464eb8Smec page_t *npp; 68068b464eb8Smec /* unlock the new page(s) */ 68078b464eb8Smec while (count-- > 0) { 68088b464eb8Smec ASSERT(newpp != NULL); 68098b464eb8Smec npp = newpp; 68108b464eb8Smec page_sub(&newpp, npp); 68118b464eb8Smec page_unlock(npp); 68128b464eb8Smec } 68138b464eb8Smec ASSERT(newpp == NULL); 68148b464eb8Smec /* 68158b464eb8Smec * Check to see if the page we have is too large. 68168b464eb8Smec * If so, demote it freeing up the extra pages. 68178b464eb8Smec */ 68188b464eb8Smec if (pp->p_szc > 0) { 68198b464eb8Smec /* For now demote extra pages to szc == 0 */ 68208b464eb8Smec extra = page_get_pagecnt(pp->p_szc) - 1; 68218b464eb8Smec while (extra > 0) { 68228b464eb8Smec tpp = pp->p_next; 68238b464eb8Smec page_sub(&pp, tpp); 68248b464eb8Smec tpp->p_szc = 0; 68258b464eb8Smec page_free(tpp, 1); 68268b464eb8Smec extra--; 68278b464eb8Smec } 68288b464eb8Smec /* Make sure to set our page to szc 0 as well */ 68298b464eb8Smec ASSERT(pp->p_next == pp && pp->p_prev == pp); 68308b464eb8Smec pp->p_szc = 0; 68318b464eb8Smec } 68328b464eb8Smec goto cleanup; 68338b464eb8Smec } else if (ret == EIO) { 68348b464eb8Smec ret = EAGAIN; 68358b464eb8Smec goto cleanup; 68368b464eb8Smec } else { 68378b464eb8Smec /* 68388b464eb8Smec * Need to reset return type as we failed to relocate the page 68398b464eb8Smec * but that does not mean that some of the next steps will not 68408b464eb8Smec * work. 68418b464eb8Smec */ 68428b464eb8Smec ret = 0; 68438b464eb8Smec } 68448b464eb8Smec 68458b464eb8Smec skip_relocate: 68468b464eb8Smec 68478b464eb8Smec if (pp->p_szc > 0) { 68488b464eb8Smec if (page_try_demote_pages(pp) == 0) { 68498b464eb8Smec ret = EAGAIN; 68508b464eb8Smec goto cleanup; 68518b464eb8Smec } 68528b464eb8Smec } 68538b464eb8Smec 68548b464eb8Smec ASSERT(pp->p_szc == 0); 68558b464eb8Smec 68568b464eb8Smec if (hat_ismod(pp)) { 68578b464eb8Smec ret = EAGAIN; 68588b464eb8Smec goto cleanup; 68598b464eb8Smec } 6860ad23a2dbSjohansen if (PP_ISKAS(pp)) { 68618b464eb8Smec ret = EAGAIN; 68628b464eb8Smec goto cleanup; 68638b464eb8Smec } 68648b464eb8Smec if (pp->p_lckcnt || pp->p_cowcnt) { 68658b464eb8Smec ret = EAGAIN; 68668b464eb8Smec goto cleanup; 68678b464eb8Smec } 68688b464eb8Smec 68698b464eb8Smec (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); 68708b464eb8Smec ASSERT(!hat_page_is_mapped(pp)); 68718b464eb8Smec 68728b464eb8Smec if (hat_ismod(pp)) { 68738b464eb8Smec /* 68748b464eb8Smec * This is a semi-odd case as the page is now modified but not 68758b464eb8Smec * mapped as we just unloaded the mappings above. 68768b464eb8Smec */ 68778b464eb8Smec ret = EAGAIN; 68788b464eb8Smec goto cleanup; 68798b464eb8Smec } 68808b464eb8Smec if (pp->p_vnode != NULL) { 68818b464eb8Smec page_hashout(pp, NULL); 68828b464eb8Smec } 68838b464eb8Smec 68848b464eb8Smec /* 68858b464eb8Smec * At this point, the page should be in a clean state and 68868b464eb8Smec * we can do whatever we want with it. 68878b464eb8Smec */ 68888b464eb8Smec 68898b464eb8Smec cleanup: 68908b464eb8Smec if (ret != 0) { 68918b464eb8Smec if (!skip_unlock) { 68928b464eb8Smec page_unlock(pp); 68938b464eb8Smec } 68948b464eb8Smec } else { 68958b464eb8Smec ASSERT(pp->p_szc == 0); 68968b464eb8Smec ASSERT(PAGE_EXCL(pp)); 68978b464eb8Smec 68988b464eb8Smec pp->p_next = pp; 68998b464eb8Smec pp->p_prev = pp; 69008b464eb8Smec } 69018b464eb8Smec return (ret); 69028b464eb8Smec } 69038b464eb8Smec 69048b464eb8Smec /* 69058b464eb8Smec * Various callers of page_trycapture() can have different restrictions upon 69068b464eb8Smec * what memory they have access to. 69078b464eb8Smec * Returns 0 on success, with the following error codes on failure: 69088b464eb8Smec * EPERM - The requested page is long term locked, and thus repeated 69098b464eb8Smec * requests to capture this page will likely fail. 69108b464eb8Smec * ENOMEM - There was not enough free memory in the system to safely 69118b464eb8Smec * map the requested page. 69128b464eb8Smec * ENOENT - The requested page was inside the kernel cage, and the 69138b464eb8Smec * PHYSMEM_CAGE flag was not set. 69148b464eb8Smec */ 69158b464eb8Smec int 69168b464eb8Smec page_capture_pre_checks(page_t *pp, uint_t flags) 69178b464eb8Smec { 69188b464eb8Smec #if defined(__sparc) 69198b464eb8Smec extern struct vnode prom_ppages; 69208b464eb8Smec #endif /* __sparc */ 69218b464eb8Smec 69228b464eb8Smec ASSERT(pp != NULL); 69238b464eb8Smec 69248b464eb8Smec /* only physmem currently has restrictions */ 69258b464eb8Smec if (!(flags & CAPTURE_PHYSMEM)) { 69268b464eb8Smec return (0); 69278b464eb8Smec } 69288b464eb8Smec 69298b464eb8Smec #if defined(__sparc) 69308b464eb8Smec if (pp->p_vnode == &prom_ppages) { 69318b464eb8Smec return (EPERM); 69328b464eb8Smec } 69338b464eb8Smec 69348b464eb8Smec if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE)) { 69358b464eb8Smec return (ENOENT); 69368b464eb8Smec } 69378b464eb8Smec 69388b464eb8Smec if (PP_ISNORELOCKERNEL(pp)) { 69398b464eb8Smec return (EPERM); 69408b464eb8Smec } 69418b464eb8Smec #else 6942ad23a2dbSjohansen if (PP_ISKAS(pp)) { 69438b464eb8Smec return (EPERM); 69448b464eb8Smec } 69458b464eb8Smec #endif /* __sparc */ 69468b464eb8Smec 69478b464eb8Smec if (availrmem < swapfs_minfree) { 69488b464eb8Smec /* 69498b464eb8Smec * We won't try to capture this page as we are 69508b464eb8Smec * running low on memory. 69518b464eb8Smec */ 69528b464eb8Smec return (ENOMEM); 69538b464eb8Smec } 69548b464eb8Smec return (0); 69558b464eb8Smec } 69568b464eb8Smec 69578b464eb8Smec /* 69588b464eb8Smec * Once we have a page in our mits, go ahead and complete the capture 69598b464eb8Smec * operation. 69608b464eb8Smec * Returns 1 on failure where page is no longer needed 69618b464eb8Smec * Returns 0 on success 69628b464eb8Smec * Returns -1 if there was a transient failure. 69638b464eb8Smec * Failure cases must release the SE_EXCL lock on pp (usually via page_free). 69648b464eb8Smec */ 69658b464eb8Smec int 69668b464eb8Smec page_capture_take_action(page_t *pp, uint_t flags, void *datap) 69678b464eb8Smec { 69688b464eb8Smec int cb_index; 69698b464eb8Smec int ret = 0; 69708b464eb8Smec page_capture_hash_bucket_t *bp1; 69718b464eb8Smec page_capture_hash_bucket_t *bp2; 69728b464eb8Smec int index; 69738b464eb8Smec int found = 0; 69748b464eb8Smec int i; 69758b464eb8Smec 69768b464eb8Smec ASSERT(PAGE_EXCL(pp)); 69778b464eb8Smec ASSERT(curthread->t_flag & T_CAPTURING); 69788b464eb8Smec 69798b464eb8Smec for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 69808b464eb8Smec if ((flags >> cb_index) & 1) { 69818b464eb8Smec break; 69828b464eb8Smec } 69838b464eb8Smec } 69848b464eb8Smec ASSERT(cb_index < PC_NUM_CALLBACKS); 69858b464eb8Smec 69868b464eb8Smec /* 69878b464eb8Smec * Remove the entry from the page_capture hash, but don't free it yet 69888b464eb8Smec * as we may need to put it back. 69898b464eb8Smec * Since we own the page at this point in time, we should find it 69908b464eb8Smec * in the hash if this is an ASYNC call. If we don't it's likely 69918b464eb8Smec * that the page_capture_async() thread decided that this request 69928b464eb8Smec * had expired, in which case we just continue on. 69938b464eb8Smec */ 69948b464eb8Smec if (flags & CAPTURE_ASYNC) { 69958b464eb8Smec 69968b464eb8Smec index = PAGE_CAPTURE_HASH(pp); 69978b464eb8Smec 69988b464eb8Smec mutex_enter(&page_capture_hash[index].pchh_mutex); 69998b464eb8Smec for (i = 0; i < 2 && !found; i++) { 70008b464eb8Smec bp1 = page_capture_hash[index].lists[i].next; 70018b464eb8Smec while (bp1 != &page_capture_hash[index].lists[i]) { 70028b464eb8Smec if (bp1->pp == pp) { 70038b464eb8Smec bp1->next->prev = bp1->prev; 70048b464eb8Smec bp1->prev->next = bp1->next; 70058b464eb8Smec page_capture_hash[index].num_pages--; 70068b464eb8Smec page_clrtoxic(pp, PR_CAPTURE); 70078b464eb8Smec found = 1; 70088b464eb8Smec break; 70098b464eb8Smec } 70108b464eb8Smec bp1 = bp1->next; 70118b464eb8Smec } 70128b464eb8Smec } 70138b464eb8Smec mutex_exit(&page_capture_hash[index].pchh_mutex); 70148b464eb8Smec } 70158b464eb8Smec 70168b464eb8Smec /* Synchronize with the unregister func. */ 70178b464eb8Smec rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER); 70188b464eb8Smec if (!pc_cb[cb_index].cb_active) { 70198b464eb8Smec page_free(pp, 1); 70208b464eb8Smec rw_exit(&pc_cb[cb_index].cb_rwlock); 70218b464eb8Smec if (found) { 70228b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 70238b464eb8Smec } 70248b464eb8Smec return (1); 70258b464eb8Smec } 70268b464eb8Smec 70278b464eb8Smec /* 70288b464eb8Smec * We need to remove the entry from the page capture hash and turn off 70298b464eb8Smec * the PR_CAPTURE bit before calling the callback. We'll need to cache 70308b464eb8Smec * the entry here, and then based upon the return value, cleanup 70318b464eb8Smec * appropriately or re-add it to the hash, making sure that someone else 70328b464eb8Smec * hasn't already done so. 70338b464eb8Smec * It should be rare for the callback to fail and thus it's ok for 70348b464eb8Smec * the failure path to be a bit complicated as the success path is 70358b464eb8Smec * cleaner and the locking rules are easier to follow. 70368b464eb8Smec */ 70378b464eb8Smec 70388b464eb8Smec ret = pc_cb[cb_index].cb_func(pp, datap, flags); 70398b464eb8Smec 70408b464eb8Smec rw_exit(&pc_cb[cb_index].cb_rwlock); 70418b464eb8Smec 70428b464eb8Smec /* 70438b464eb8Smec * If this was an ASYNC request, we need to cleanup the hash if the 70448b464eb8Smec * callback was successful or if the request was no longer valid. 70458b464eb8Smec * For non-ASYNC requests, we return failure to map and the caller 70468b464eb8Smec * will take care of adding the request to the hash. 70478b464eb8Smec * Note also that the callback itself is responsible for the page 70488b464eb8Smec * at this point in time in terms of locking ... The most common 70498b464eb8Smec * case for the failure path should just be a page_free. 70508b464eb8Smec */ 70518b464eb8Smec if (ret >= 0) { 70528b464eb8Smec if (found) { 7053*cee1d74bSjfrank if (bp1->flags & CAPTURE_RETIRE) { 7054*cee1d74bSjfrank page_retire_decr_pend_count(); 7055*cee1d74bSjfrank } 70568b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 70578b464eb8Smec } 70588b464eb8Smec return (ret); 70598b464eb8Smec } 70608b464eb8Smec if (!found) { 70618b464eb8Smec return (ret); 70628b464eb8Smec } 70638b464eb8Smec 70648b464eb8Smec ASSERT(flags & CAPTURE_ASYNC); 70658b464eb8Smec 70668b464eb8Smec /* 70678b464eb8Smec * Check for expiration time first as we can just free it up if it's 70688b464eb8Smec * expired. 70698b464eb8Smec */ 70708b464eb8Smec if (lbolt > bp1->expires && bp1->expires != -1) { 70718b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 70728b464eb8Smec return (ret); 70738b464eb8Smec } 70748b464eb8Smec 70758b464eb8Smec /* 70768b464eb8Smec * The callback failed and there used to be an entry in the hash for 70778b464eb8Smec * this page, so we need to add it back to the hash. 70788b464eb8Smec */ 70798b464eb8Smec mutex_enter(&page_capture_hash[index].pchh_mutex); 70808b464eb8Smec if (!(pp->p_toxic & PR_CAPTURE)) { 70818b464eb8Smec /* just add bp1 back to head of walked list */ 70828b464eb8Smec page_settoxic(pp, PR_CAPTURE); 70838b464eb8Smec bp1->next = page_capture_hash[index].lists[1].next; 70848b464eb8Smec bp1->prev = &page_capture_hash[index].lists[1]; 70858b464eb8Smec bp1->next->prev = bp1; 70868b464eb8Smec page_capture_hash[index].lists[1].next = bp1; 70878b464eb8Smec page_capture_hash[index].num_pages++; 70888b464eb8Smec mutex_exit(&page_capture_hash[index].pchh_mutex); 70898b464eb8Smec return (ret); 70908b464eb8Smec } 70918b464eb8Smec 70928b464eb8Smec /* 70938b464eb8Smec * Otherwise there was a new capture request added to list 70948b464eb8Smec * Need to make sure that our original data is represented if 70958b464eb8Smec * appropriate. 70968b464eb8Smec */ 70978b464eb8Smec for (i = 0; i < 2; i++) { 70988b464eb8Smec bp2 = page_capture_hash[index].lists[i].next; 70998b464eb8Smec while (bp2 != &page_capture_hash[index].lists[i]) { 71008b464eb8Smec if (bp2->pp == pp) { 71018b464eb8Smec if (bp1->flags & CAPTURE_RETIRE) { 71028b464eb8Smec if (!(bp2->flags & CAPTURE_RETIRE)) { 71038b464eb8Smec bp2->szc = bp1->szc; 71048b464eb8Smec bp2->flags = bp1->flags; 71058b464eb8Smec bp2->expires = bp1->expires; 71068b464eb8Smec bp2->datap = bp1->datap; 71078b464eb8Smec } 71088b464eb8Smec } else { 71098b464eb8Smec ASSERT(bp1->flags & CAPTURE_PHYSMEM); 71108b464eb8Smec if (!(bp2->flags & CAPTURE_RETIRE)) { 71118b464eb8Smec bp2->szc = bp1->szc; 71128b464eb8Smec bp2->flags = bp1->flags; 71138b464eb8Smec bp2->expires = bp1->expires; 71148b464eb8Smec bp2->datap = bp1->datap; 71158b464eb8Smec } 71168b464eb8Smec } 71178b464eb8Smec mutex_exit(&page_capture_hash[index]. 71188b464eb8Smec pchh_mutex); 71198b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 71208b464eb8Smec return (ret); 71218b464eb8Smec } 71228b464eb8Smec bp2 = bp2->next; 71238b464eb8Smec } 71248b464eb8Smec } 71258b464eb8Smec panic("PR_CAPTURE set but not on hash for pp 0x%p\n", pp); 71268b464eb8Smec /*NOTREACHED*/ 71278b464eb8Smec } 71288b464eb8Smec 71298b464eb8Smec /* 71308b464eb8Smec * Try to capture the given page for the caller specified in the flags 71318b464eb8Smec * parameter. The page will either be captured and handed over to the 71328b464eb8Smec * appropriate callback, or will be queued up in the page capture hash 71338b464eb8Smec * to be captured asynchronously. 71348b464eb8Smec * If the current request is due to an async capture, the page must be 71358b464eb8Smec * exclusively locked before calling this function. 71368b464eb8Smec * Currently szc must be 0 but in the future this should be expandable to 71378b464eb8Smec * other page sizes. 71388b464eb8Smec * Returns 0 on success, with the following error codes on failure: 71398b464eb8Smec * EPERM - The requested page is long term locked, and thus repeated 71408b464eb8Smec * requests to capture this page will likely fail. 71418b464eb8Smec * ENOMEM - There was not enough free memory in the system to safely 71428b464eb8Smec * map the requested page. 71438b464eb8Smec * ENOENT - The requested page was inside the kernel cage, and the 71448b464eb8Smec * CAPTURE_GET_CAGE flag was not set. 71458b464eb8Smec * EAGAIN - The requested page could not be capturead at this point in 71468b464eb8Smec * time but future requests will likely work. 71478b464eb8Smec * EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag 71488b464eb8Smec * was not set. 71498b464eb8Smec */ 71508b464eb8Smec int 71518b464eb8Smec page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 71528b464eb8Smec { 71538b464eb8Smec int ret; 71548b464eb8Smec int cb_index; 71558b464eb8Smec 71568b464eb8Smec if (flags & CAPTURE_ASYNC) { 71578b464eb8Smec ASSERT(PAGE_EXCL(pp)); 71588b464eb8Smec goto async; 71598b464eb8Smec } 71608b464eb8Smec 71618b464eb8Smec /* Make sure there's enough availrmem ... */ 71628b464eb8Smec ret = page_capture_pre_checks(pp, flags); 71638b464eb8Smec if (ret != 0) { 71648b464eb8Smec return (ret); 71658b464eb8Smec } 71668b464eb8Smec 71678b464eb8Smec if (!page_trylock(pp, SE_EXCL)) { 71688b464eb8Smec for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) { 71698b464eb8Smec if ((flags >> cb_index) & 1) { 71708b464eb8Smec break; 71718b464eb8Smec } 71728b464eb8Smec } 71738b464eb8Smec ASSERT(cb_index < PC_NUM_CALLBACKS); 71748b464eb8Smec ret = EAGAIN; 71758b464eb8Smec /* Special case for retired pages */ 71768b464eb8Smec if (PP_RETIRED(pp)) { 71778b464eb8Smec if (flags & CAPTURE_GET_RETIRED) { 71788b464eb8Smec if (!page_unretire_pp(pp, PR_UNR_TEMP)) { 71798b464eb8Smec /* 71808b464eb8Smec * Need to set capture bit and add to 71818b464eb8Smec * hash so that the page will be 71828b464eb8Smec * retired when freed. 71838b464eb8Smec */ 71848b464eb8Smec page_capture_add_hash(pp, szc, 71858b464eb8Smec CAPTURE_RETIRE, NULL); 71868b464eb8Smec ret = 0; 71878b464eb8Smec goto own_page; 71888b464eb8Smec } 71898b464eb8Smec } else { 71908b464eb8Smec return (EBUSY); 71918b464eb8Smec } 71928b464eb8Smec } 71938b464eb8Smec page_capture_add_hash(pp, szc, flags, datap); 71948b464eb8Smec return (ret); 71958b464eb8Smec } 71968b464eb8Smec 71978b464eb8Smec async: 71988b464eb8Smec ASSERT(PAGE_EXCL(pp)); 71998b464eb8Smec 72008b464eb8Smec /* Need to check for physmem async requests that availrmem is sane */ 72018b464eb8Smec if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) == 72028b464eb8Smec (CAPTURE_ASYNC | CAPTURE_PHYSMEM) && 72038b464eb8Smec (availrmem < swapfs_minfree)) { 72048b464eb8Smec page_unlock(pp); 72058b464eb8Smec return (ENOMEM); 72068b464eb8Smec } 72078b464eb8Smec 72088b464eb8Smec ret = page_capture_clean_page(pp); 72098b464eb8Smec 72108b464eb8Smec if (ret != 0) { 72118b464eb8Smec /* We failed to get the page, so lets add it to the hash */ 72128b464eb8Smec if (!(flags & CAPTURE_ASYNC)) { 72138b464eb8Smec page_capture_add_hash(pp, szc, flags, datap); 72148b464eb8Smec } 72158b464eb8Smec return (ret); 72168b464eb8Smec } 72178b464eb8Smec 72188b464eb8Smec own_page: 72198b464eb8Smec ASSERT(PAGE_EXCL(pp)); 72208b464eb8Smec ASSERT(pp->p_szc == 0); 72218b464eb8Smec 72228b464eb8Smec /* Call the callback */ 72238b464eb8Smec ret = page_capture_take_action(pp, flags, datap); 72248b464eb8Smec 72258b464eb8Smec if (ret == 0) { 72268b464eb8Smec return (0); 72278b464eb8Smec } 72288b464eb8Smec 72298b464eb8Smec /* 72308b464eb8Smec * Note that in the failure cases from page_capture_take_action, the 72318b464eb8Smec * EXCL lock will have already been dropped. 72328b464eb8Smec */ 72338b464eb8Smec if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) { 72348b464eb8Smec page_capture_add_hash(pp, szc, flags, datap); 72358b464eb8Smec } 72368b464eb8Smec return (EAGAIN); 72378b464eb8Smec } 72388b464eb8Smec 72398b464eb8Smec int 72408b464eb8Smec page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap) 72418b464eb8Smec { 72428b464eb8Smec int ret; 72438b464eb8Smec 72448b464eb8Smec curthread->t_flag |= T_CAPTURING; 72458b464eb8Smec ret = page_itrycapture(pp, szc, flags, datap); 72468b464eb8Smec curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */ 72478b464eb8Smec return (ret); 72488b464eb8Smec } 72498b464eb8Smec 72508b464eb8Smec /* 72518b464eb8Smec * When unlocking a page which has the PR_CAPTURE bit set, this routine 72528b464eb8Smec * gets called to try and capture the page. 72538b464eb8Smec */ 72548b464eb8Smec void 72558b464eb8Smec page_unlock_capture(page_t *pp) 72568b464eb8Smec { 72578b464eb8Smec page_capture_hash_bucket_t *bp; 72588b464eb8Smec int index; 72598b464eb8Smec int i; 72608b464eb8Smec uint_t szc; 72618b464eb8Smec uint_t flags = 0; 72628b464eb8Smec void *datap; 72638b464eb8Smec kmutex_t *mp; 72648b464eb8Smec extern vnode_t retired_pages; 72658b464eb8Smec 72668b464eb8Smec /* 72678b464eb8Smec * We need to protect against a possible deadlock here where we own 72688b464eb8Smec * the vnode page hash mutex and want to acquire it again as there 72698b464eb8Smec * are locations in the code, where we unlock a page while holding 72708b464eb8Smec * the mutex which can lead to the page being captured and eventually 72718b464eb8Smec * end up here. As we may be hashing out the old page and hashing into 72728b464eb8Smec * the retire vnode, we need to make sure we don't own them. 72738b464eb8Smec * Other callbacks who do hash operations also need to make sure that 72748b464eb8Smec * before they hashin to a vnode that they do not currently own the 72758b464eb8Smec * vphm mutex otherwise there will be a panic. 72768b464eb8Smec */ 72778b464eb8Smec if (mutex_owned(page_vnode_mutex(&retired_pages))) { 727822addef7Smec page_unlock_nocapture(pp); 72798b464eb8Smec return; 72808b464eb8Smec } 72818b464eb8Smec if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) { 728222addef7Smec page_unlock_nocapture(pp); 72838b464eb8Smec return; 72848b464eb8Smec } 72858b464eb8Smec 72868b464eb8Smec index = PAGE_CAPTURE_HASH(pp); 72878b464eb8Smec 72888b464eb8Smec mp = &page_capture_hash[index].pchh_mutex; 72898b464eb8Smec mutex_enter(mp); 72908b464eb8Smec for (i = 0; i < 2; i++) { 72918b464eb8Smec bp = page_capture_hash[index].lists[i].next; 72928b464eb8Smec while (bp != &page_capture_hash[index].lists[i]) { 72938b464eb8Smec if (bp->pp == pp) { 72948b464eb8Smec szc = bp->szc; 72958b464eb8Smec flags = bp->flags | CAPTURE_ASYNC; 72968b464eb8Smec datap = bp->datap; 72978b464eb8Smec mutex_exit(mp); 72988b464eb8Smec (void) page_trycapture(pp, szc, flags, datap); 72998b464eb8Smec return; 73008b464eb8Smec } 73018b464eb8Smec bp = bp->next; 73028b464eb8Smec } 73038b464eb8Smec } 73048b464eb8Smec 73058b464eb8Smec /* Failed to find page in hash so clear flags and unlock it. */ 73068b464eb8Smec page_clrtoxic(pp, PR_CAPTURE); 73078b464eb8Smec page_unlock(pp); 73088b464eb8Smec 73098b464eb8Smec mutex_exit(mp); 73108b464eb8Smec } 73118b464eb8Smec 73128b464eb8Smec void 73138b464eb8Smec page_capture_init() 73148b464eb8Smec { 73158b464eb8Smec int i; 73168b464eb8Smec for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 73178b464eb8Smec page_capture_hash[i].lists[0].next = 73188b464eb8Smec &page_capture_hash[i].lists[0]; 73198b464eb8Smec page_capture_hash[i].lists[0].prev = 73208b464eb8Smec &page_capture_hash[i].lists[0]; 73218b464eb8Smec page_capture_hash[i].lists[1].next = 73228b464eb8Smec &page_capture_hash[i].lists[1]; 73238b464eb8Smec page_capture_hash[i].lists[1].prev = 73248b464eb8Smec &page_capture_hash[i].lists[1]; 73258b464eb8Smec } 73268b464eb8Smec 73278b464eb8Smec pc_thread_shortwait = 23 * hz; 73288b464eb8Smec pc_thread_longwait = 1201 * hz; 7329*cee1d74bSjfrank pc_thread_ism_retry = 3; 73308b464eb8Smec mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL); 73318b464eb8Smec cv_init(&pc_cv, NULL, CV_DEFAULT, NULL); 73328b464eb8Smec pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0, 73338b464eb8Smec TS_RUN, minclsyspri); 73348b464eb8Smec } 73358b464eb8Smec 73368b464eb8Smec /* 73378b464eb8Smec * It is necessary to scrub any failing pages prior to reboot in order to 73388b464eb8Smec * prevent a latent error trap from occurring on the next boot. 73398b464eb8Smec */ 73408b464eb8Smec void 73418b464eb8Smec page_retire_mdboot() 73428b464eb8Smec { 73438b464eb8Smec page_t *pp; 73448b464eb8Smec int i, j; 73458b464eb8Smec page_capture_hash_bucket_t *bp; 73468b464eb8Smec 73478b464eb8Smec /* walk lists looking for pages to scrub */ 73488b464eb8Smec for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 73498b464eb8Smec if (page_capture_hash[i].num_pages == 0) 73508b464eb8Smec continue; 73518b464eb8Smec 73528b464eb8Smec mutex_enter(&page_capture_hash[i].pchh_mutex); 73538b464eb8Smec 73548b464eb8Smec for (j = 0; j < 2; j++) { 73558b464eb8Smec bp = page_capture_hash[i].lists[j].next; 73568b464eb8Smec while (bp != &page_capture_hash[i].lists[j]) { 73578b464eb8Smec pp = bp->pp; 7358ad23a2dbSjohansen if (!PP_ISKAS(pp) && PP_TOXIC(pp)) { 73598b464eb8Smec pp->p_selock = -1; /* pacify ASSERTs */ 73608b464eb8Smec PP_CLRFREE(pp); 73618b464eb8Smec pagescrub(pp, 0, PAGESIZE); 73628b464eb8Smec pp->p_selock = 0; 73638b464eb8Smec } 73648b464eb8Smec bp = bp->next; 73658b464eb8Smec } 73668b464eb8Smec } 73678b464eb8Smec mutex_exit(&page_capture_hash[i].pchh_mutex); 73688b464eb8Smec } 73698b464eb8Smec } 73708b464eb8Smec 73718b464eb8Smec /* 73728b464eb8Smec * Walk the page_capture_hash trying to capture pages and also cleanup old 73738b464eb8Smec * entries which have expired. 73748b464eb8Smec */ 73758b464eb8Smec void 73768b464eb8Smec page_capture_async() 73778b464eb8Smec { 73788b464eb8Smec page_t *pp; 73798b464eb8Smec int i; 73808b464eb8Smec int ret; 73818b464eb8Smec page_capture_hash_bucket_t *bp1, *bp2; 73828b464eb8Smec uint_t szc; 73838b464eb8Smec uint_t flags; 73848b464eb8Smec void *datap; 73858b464eb8Smec 73868b464eb8Smec /* If there are outstanding pages to be captured, get to work */ 73878b464eb8Smec for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) { 73888b464eb8Smec if (page_capture_hash[i].num_pages == 0) 73898b464eb8Smec continue; 73908b464eb8Smec /* Append list 1 to list 0 and then walk through list 0 */ 73918b464eb8Smec mutex_enter(&page_capture_hash[i].pchh_mutex); 73928b464eb8Smec bp1 = &page_capture_hash[i].lists[1]; 73938b464eb8Smec bp2 = bp1->next; 73948b464eb8Smec if (bp1 != bp2) { 73958b464eb8Smec bp1->prev->next = page_capture_hash[i].lists[0].next; 73968b464eb8Smec bp2->prev = &page_capture_hash[i].lists[0]; 73978b464eb8Smec page_capture_hash[i].lists[0].next->prev = bp1->prev; 73988b464eb8Smec page_capture_hash[i].lists[0].next = bp2; 73998b464eb8Smec bp1->next = bp1; 74008b464eb8Smec bp1->prev = bp1; 74018b464eb8Smec } 74028b464eb8Smec 74038b464eb8Smec /* list[1] will be empty now */ 74048b464eb8Smec 74058b464eb8Smec bp1 = page_capture_hash[i].lists[0].next; 74068b464eb8Smec while (bp1 != &page_capture_hash[i].lists[0]) { 74078b464eb8Smec /* Check expiration time */ 74088b464eb8Smec if ((lbolt > bp1->expires && bp1->expires != -1) || 74098b464eb8Smec page_deleted(bp1->pp)) { 74108b464eb8Smec page_capture_hash[i].lists[0].next = bp1->next; 74118b464eb8Smec bp1->next->prev = 74128b464eb8Smec &page_capture_hash[i].lists[0]; 74138b464eb8Smec page_capture_hash[i].num_pages--; 74148b464eb8Smec 74158b464eb8Smec /* 74168b464eb8Smec * We can safely remove the PR_CAPTURE bit 74178b464eb8Smec * without holding the EXCL lock on the page 74188b464eb8Smec * as the PR_CAPTURE bit requres that the 74198b464eb8Smec * page_capture_hash[].pchh_mutex be held 74208b464eb8Smec * to modify it. 74218b464eb8Smec */ 74228b464eb8Smec page_clrtoxic(bp1->pp, PR_CAPTURE); 74238b464eb8Smec mutex_exit(&page_capture_hash[i].pchh_mutex); 74248b464eb8Smec kmem_free(bp1, sizeof (*bp1)); 74258b464eb8Smec mutex_enter(&page_capture_hash[i].pchh_mutex); 74268b464eb8Smec bp1 = page_capture_hash[i].lists[0].next; 74278b464eb8Smec continue; 74288b464eb8Smec } 74298b464eb8Smec pp = bp1->pp; 74308b464eb8Smec szc = bp1->szc; 74318b464eb8Smec flags = bp1->flags; 74328b464eb8Smec datap = bp1->datap; 74338b464eb8Smec mutex_exit(&page_capture_hash[i].pchh_mutex); 74348b464eb8Smec if (page_trylock(pp, SE_EXCL)) { 74358b464eb8Smec ret = page_trycapture(pp, szc, 74368b464eb8Smec flags | CAPTURE_ASYNC, datap); 74378b464eb8Smec } else { 74388b464eb8Smec ret = 1; /* move to walked hash */ 74398b464eb8Smec } 74408b464eb8Smec 74418b464eb8Smec if (ret != 0) { 74428b464eb8Smec /* Move to walked hash */ 74438b464eb8Smec (void) page_capture_move_to_walked(pp); 74448b464eb8Smec } 74458b464eb8Smec mutex_enter(&page_capture_hash[i].pchh_mutex); 74468b464eb8Smec bp1 = page_capture_hash[i].lists[0].next; 74478b464eb8Smec } 74488b464eb8Smec 74498b464eb8Smec mutex_exit(&page_capture_hash[i].pchh_mutex); 74508b464eb8Smec } 74518b464eb8Smec } 74528b464eb8Smec 7453*cee1d74bSjfrank /* 7454*cee1d74bSjfrank * This function is called by the page_capture_thread, and is needed in 7455*cee1d74bSjfrank * in order to initiate aio cleanup, so that pages used in aio 7456*cee1d74bSjfrank * will be unlocked and subsequently retired by page_capture_thread. 7457*cee1d74bSjfrank */ 7458*cee1d74bSjfrank static int 7459*cee1d74bSjfrank do_aio_cleanup(void) 7460*cee1d74bSjfrank { 7461*cee1d74bSjfrank proc_t *procp; 7462*cee1d74bSjfrank int (*aio_cleanup_dr_delete_memory)(proc_t *); 7463*cee1d74bSjfrank int cleaned = 0; 7464*cee1d74bSjfrank 7465*cee1d74bSjfrank if (modload("sys", "kaio") == -1) { 7466*cee1d74bSjfrank cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio"); 7467*cee1d74bSjfrank return (0); 7468*cee1d74bSjfrank } 7469*cee1d74bSjfrank /* 7470*cee1d74bSjfrank * We use the aio_cleanup_dr_delete_memory function to 7471*cee1d74bSjfrank * initiate the actual clean up; this function will wake 7472*cee1d74bSjfrank * up the per-process aio_cleanup_thread. 7473*cee1d74bSjfrank */ 7474*cee1d74bSjfrank aio_cleanup_dr_delete_memory = (int (*)(proc_t *)) 7475*cee1d74bSjfrank modgetsymvalue("aio_cleanup_dr_delete_memory", 0); 7476*cee1d74bSjfrank if (aio_cleanup_dr_delete_memory == NULL) { 7477*cee1d74bSjfrank cmn_err(CE_WARN, 7478*cee1d74bSjfrank "aio_cleanup_dr_delete_memory not found in kaio"); 7479*cee1d74bSjfrank return (0); 7480*cee1d74bSjfrank } 7481*cee1d74bSjfrank mutex_enter(&pidlock); 7482*cee1d74bSjfrank for (procp = practive; (procp != NULL); procp = procp->p_next) { 7483*cee1d74bSjfrank mutex_enter(&procp->p_lock); 7484*cee1d74bSjfrank if (procp->p_aio != NULL) { 7485*cee1d74bSjfrank /* cleanup proc's outstanding kaio */ 7486*cee1d74bSjfrank cleaned += (*aio_cleanup_dr_delete_memory)(procp); 7487*cee1d74bSjfrank } 7488*cee1d74bSjfrank mutex_exit(&procp->p_lock); 7489*cee1d74bSjfrank } 7490*cee1d74bSjfrank mutex_exit(&pidlock); 7491*cee1d74bSjfrank return (cleaned); 7492*cee1d74bSjfrank } 7493*cee1d74bSjfrank 7494*cee1d74bSjfrank /* 7495*cee1d74bSjfrank * helper function for page_capture_thread 7496*cee1d74bSjfrank */ 7497*cee1d74bSjfrank static void 7498*cee1d74bSjfrank page_capture_handle_outstanding(void) 7499*cee1d74bSjfrank { 7500*cee1d74bSjfrank extern size_t spt_used; 7501*cee1d74bSjfrank int ntry; 7502*cee1d74bSjfrank 7503*cee1d74bSjfrank if (!page_retire_pend_count()) { 7504*cee1d74bSjfrank /* 7505*cee1d74bSjfrank * Do we really want to be this aggressive 7506*cee1d74bSjfrank * for things other than page_retire? 7507*cee1d74bSjfrank * Maybe have a counter for each callback 7508*cee1d74bSjfrank * type to guide how aggressive we should 7509*cee1d74bSjfrank * be here. Thus if there's at least one 7510*cee1d74bSjfrank * page for page_retire we go ahead and reap 7511*cee1d74bSjfrank * like this. 7512*cee1d74bSjfrank */ 7513*cee1d74bSjfrank kmem_reap(); 7514*cee1d74bSjfrank seg_preap(); 7515*cee1d74bSjfrank page_capture_async(); 7516*cee1d74bSjfrank } else { 7517*cee1d74bSjfrank /* 7518*cee1d74bSjfrank * There are pages pending retirement, so 7519*cee1d74bSjfrank * we reap prior to attempting to capture. 7520*cee1d74bSjfrank */ 7521*cee1d74bSjfrank kmem_reap(); 7522*cee1d74bSjfrank /* 7523*cee1d74bSjfrank * When ISM is in use, we need to disable and 7524*cee1d74bSjfrank * purge the seg_pcache, and initiate aio 7525*cee1d74bSjfrank * cleanup in order to release page locks and 7526*cee1d74bSjfrank * subsquently retire pages in need of 7527*cee1d74bSjfrank * retirement. 7528*cee1d74bSjfrank */ 7529*cee1d74bSjfrank if (spt_used) { 7530*cee1d74bSjfrank /* disable and purge seg_pcache */ 7531*cee1d74bSjfrank (void) seg_p_disable(); 7532*cee1d74bSjfrank for (ntry = 0; ntry < pc_thread_ism_retry; ntry++) { 7533*cee1d74bSjfrank if (!page_retire_pend_count()) 7534*cee1d74bSjfrank break; 7535*cee1d74bSjfrank if (do_aio_cleanup()) { 7536*cee1d74bSjfrank /* 7537*cee1d74bSjfrank * allow the apps cleanup threads 7538*cee1d74bSjfrank * to run 7539*cee1d74bSjfrank */ 7540*cee1d74bSjfrank delay(pc_thread_shortwait); 7541*cee1d74bSjfrank } 7542*cee1d74bSjfrank page_capture_async(); 7543*cee1d74bSjfrank } 7544*cee1d74bSjfrank /* reenable seg_pcache */ 7545*cee1d74bSjfrank seg_p_enable(); 7546*cee1d74bSjfrank } else { 7547*cee1d74bSjfrank seg_preap(); 7548*cee1d74bSjfrank page_capture_async(); 7549*cee1d74bSjfrank } 7550*cee1d74bSjfrank } 7551*cee1d74bSjfrank } 7552*cee1d74bSjfrank 75538b464eb8Smec /* 75548b464eb8Smec * The page_capture_thread loops forever, looking to see if there are 75558b464eb8Smec * pages still waiting to be captured. 75568b464eb8Smec */ 75578b464eb8Smec static void 75588b464eb8Smec page_capture_thread(void) 75598b464eb8Smec { 75608b464eb8Smec callb_cpr_t c; 75618b464eb8Smec int outstanding; 75628b464eb8Smec int i; 75638b464eb8Smec 75648b464eb8Smec CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture"); 75658b464eb8Smec 75668b464eb8Smec mutex_enter(&pc_thread_mutex); 75678b464eb8Smec for (;;) { 75688b464eb8Smec outstanding = 0; 75698b464eb8Smec for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) 75708b464eb8Smec outstanding += page_capture_hash[i].num_pages; 75718b464eb8Smec if (outstanding) { 7572*cee1d74bSjfrank page_capture_handle_outstanding(); 75738b464eb8Smec CALLB_CPR_SAFE_BEGIN(&c); 75748b464eb8Smec (void) cv_timedwait(&pc_cv, &pc_thread_mutex, 75758b464eb8Smec lbolt + pc_thread_shortwait); 75768b464eb8Smec CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 75778b464eb8Smec } else { 75788b464eb8Smec CALLB_CPR_SAFE_BEGIN(&c); 75798b464eb8Smec (void) cv_timedwait(&pc_cv, &pc_thread_mutex, 75808b464eb8Smec lbolt + pc_thread_longwait); 75818b464eb8Smec CALLB_CPR_SAFE_END(&c, &pc_thread_mutex); 75828b464eb8Smec } 75838b464eb8Smec } 75848b464eb8Smec /*NOTREACHED*/ 75858b464eb8Smec } 7586