xref: /illumos-gate/usr/src/uts/common/vm/vm_page.c (revision cee1d74b)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
53cff2f43Sstans  * Common Development and Distribution License (the "License").
63cff2f43Sstans  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22*cee1d74bSjfrank  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989  AT&T	*/
277c478bd9Sstevel@tonic-gate /*	  All Rights Reserved  	*/
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate  * University Copyright- Copyright (c) 1982, 1986, 1988
317c478bd9Sstevel@tonic-gate  * The Regents of the University of California
327c478bd9Sstevel@tonic-gate  * All Rights Reserved
337c478bd9Sstevel@tonic-gate  *
347c478bd9Sstevel@tonic-gate  * University Acknowledgment- Portions of this document are derived from
357c478bd9Sstevel@tonic-gate  * software developed by the University of California, Berkeley, and its
367c478bd9Sstevel@tonic-gate  * contributors.
377c478bd9Sstevel@tonic-gate  */
387c478bd9Sstevel@tonic-gate 
397c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate /*
427c478bd9Sstevel@tonic-gate  * VM - physical page management.
437c478bd9Sstevel@tonic-gate  */
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate #include <sys/types.h>
467c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
477c478bd9Sstevel@tonic-gate #include <sys/param.h>
487c478bd9Sstevel@tonic-gate #include <sys/systm.h>
497c478bd9Sstevel@tonic-gate #include <sys/errno.h>
507c478bd9Sstevel@tonic-gate #include <sys/time.h>
517c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
527c478bd9Sstevel@tonic-gate #include <sys/vm.h>
537c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
547c478bd9Sstevel@tonic-gate #include <sys/swap.h>
557c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
567c478bd9Sstevel@tonic-gate #include <sys/tuneable.h>
577c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
587c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
597c478bd9Sstevel@tonic-gate #include <sys/callb.h>
607c478bd9Sstevel@tonic-gate #include <sys/debug.h>
617c478bd9Sstevel@tonic-gate #include <sys/tnf_probe.h>
627c478bd9Sstevel@tonic-gate #include <sys/condvar_impl.h>
637c478bd9Sstevel@tonic-gate #include <sys/mem_config.h>
647c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h>
657c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
667c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
677c478bd9Sstevel@tonic-gate #include <sys/strlog.h>
687c478bd9Sstevel@tonic-gate #include <sys/mman.h>
697c478bd9Sstevel@tonic-gate #include <sys/ontrap.h>
707c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
717c478bd9Sstevel@tonic-gate #include <sys/vfs.h>
727c478bd9Sstevel@tonic-gate 
737c478bd9Sstevel@tonic-gate #include <vm/hat.h>
747c478bd9Sstevel@tonic-gate #include <vm/anon.h>
757c478bd9Sstevel@tonic-gate #include <vm/page.h>
767c478bd9Sstevel@tonic-gate #include <vm/seg.h>
777c478bd9Sstevel@tonic-gate #include <vm/pvn.h>
787c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
797c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h>
800209230bSgjelinek #include <sys/vm_usage.h>
817c478bd9Sstevel@tonic-gate #include <fs/fs_subr.h>
82*cee1d74bSjfrank #include <sys/ddi.h>
83*cee1d74bSjfrank #include <sys/modctl.h>
847c478bd9Sstevel@tonic-gate 
857c478bd9Sstevel@tonic-gate static int nopageage = 0;
867c478bd9Sstevel@tonic-gate 
877c478bd9Sstevel@tonic-gate static pgcnt_t max_page_get;	/* max page_get request size in pages */
887c478bd9Sstevel@tonic-gate pgcnt_t total_pages = 0;	/* total number of pages (used by /proc) */
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate /*
917c478bd9Sstevel@tonic-gate  * freemem_lock protects all freemem variables:
927c478bd9Sstevel@tonic-gate  * availrmem. Also this lock protects the globals which track the
937c478bd9Sstevel@tonic-gate  * availrmem changes for accurate kernel footprint calculation.
947c478bd9Sstevel@tonic-gate  * See below for an explanation of these
957c478bd9Sstevel@tonic-gate  * globals.
967c478bd9Sstevel@tonic-gate  */
977c478bd9Sstevel@tonic-gate kmutex_t freemem_lock;
987c478bd9Sstevel@tonic-gate pgcnt_t availrmem;
997c478bd9Sstevel@tonic-gate pgcnt_t availrmem_initial;
1007c478bd9Sstevel@tonic-gate 
1017c478bd9Sstevel@tonic-gate /*
1027c478bd9Sstevel@tonic-gate  * These globals track availrmem changes to get a more accurate
1037c478bd9Sstevel@tonic-gate  * estimate of tke kernel size. Historically pp_kernel is used for
1047c478bd9Sstevel@tonic-gate  * kernel size and is based on availrmem. But availrmem is adjusted for
1057c478bd9Sstevel@tonic-gate  * locked pages in the system not just for kernel locked pages.
1067c478bd9Sstevel@tonic-gate  * These new counters will track the pages locked through segvn and
1077c478bd9Sstevel@tonic-gate  * by explicit user locking.
1087c478bd9Sstevel@tonic-gate  *
1097c478bd9Sstevel@tonic-gate  * segvn_pages_locked : This keeps track on a global basis how many pages
1107c478bd9Sstevel@tonic-gate  * are currently locked because of I/O.
1117c478bd9Sstevel@tonic-gate  *
1127c478bd9Sstevel@tonic-gate  * pages_locked : How many pages are locked becuase of user specified
1137c478bd9Sstevel@tonic-gate  * locking through mlock or plock.
1147c478bd9Sstevel@tonic-gate  *
1157c478bd9Sstevel@tonic-gate  * pages_useclaim,pages_claimed : These two variables track the
1167c478bd9Sstevel@tonic-gate  * cliam adjustments because of the protection changes on a segvn segment.
1177c478bd9Sstevel@tonic-gate  *
1187c478bd9Sstevel@tonic-gate  * All these globals are protected by the same lock which protects availrmem.
1197c478bd9Sstevel@tonic-gate  */
1207c478bd9Sstevel@tonic-gate pgcnt_t segvn_pages_locked;
1217c478bd9Sstevel@tonic-gate pgcnt_t pages_locked;
1227c478bd9Sstevel@tonic-gate pgcnt_t pages_useclaim;
1237c478bd9Sstevel@tonic-gate pgcnt_t pages_claimed;
1247c478bd9Sstevel@tonic-gate 
1257c478bd9Sstevel@tonic-gate 
1267c478bd9Sstevel@tonic-gate /*
1277c478bd9Sstevel@tonic-gate  * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
1287c478bd9Sstevel@tonic-gate  */
1297c478bd9Sstevel@tonic-gate static kmutex_t	new_freemem_lock;
1307c478bd9Sstevel@tonic-gate static uint_t	freemem_wait;	/* someone waiting for freemem */
1317c478bd9Sstevel@tonic-gate static kcondvar_t freemem_cv;
1327c478bd9Sstevel@tonic-gate 
1337c478bd9Sstevel@tonic-gate /*
1347c478bd9Sstevel@tonic-gate  * The logical page free list is maintained as two lists, the 'free'
1357c478bd9Sstevel@tonic-gate  * and the 'cache' lists.
1367c478bd9Sstevel@tonic-gate  * The free list contains those pages that should be reused first.
1377c478bd9Sstevel@tonic-gate  *
1387c478bd9Sstevel@tonic-gate  * The implementation of the lists is machine dependent.
1397c478bd9Sstevel@tonic-gate  * page_get_freelist(), page_get_cachelist(),
1407c478bd9Sstevel@tonic-gate  * page_list_sub(), and page_list_add()
1417c478bd9Sstevel@tonic-gate  * form the interface to the machine dependent implementation.
1427c478bd9Sstevel@tonic-gate  *
1437c478bd9Sstevel@tonic-gate  * Pages with p_free set are on the cache list.
1447c478bd9Sstevel@tonic-gate  * Pages with p_free and p_age set are on the free list,
1457c478bd9Sstevel@tonic-gate  *
1467c478bd9Sstevel@tonic-gate  * A page may be locked while on either list.
1477c478bd9Sstevel@tonic-gate  */
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate /*
1507c478bd9Sstevel@tonic-gate  * free list accounting stuff.
1517c478bd9Sstevel@tonic-gate  *
1527c478bd9Sstevel@tonic-gate  *
1537c478bd9Sstevel@tonic-gate  * Spread out the value for the number of pages on the
1547c478bd9Sstevel@tonic-gate  * page free and page cache lists.  If there is just one
1557c478bd9Sstevel@tonic-gate  * value, then it must be under just one lock.
1567c478bd9Sstevel@tonic-gate  * The lock contention and cache traffic are a real bother.
1577c478bd9Sstevel@tonic-gate  *
1587c478bd9Sstevel@tonic-gate  * When we acquire and then drop a single pcf lock
1597c478bd9Sstevel@tonic-gate  * we can start in the middle of the array of pcf structures.
1607c478bd9Sstevel@tonic-gate  * If we acquire more than one pcf lock at a time, we need to
1617c478bd9Sstevel@tonic-gate  * start at the front to avoid deadlocking.
1627c478bd9Sstevel@tonic-gate  *
1637c478bd9Sstevel@tonic-gate  * pcf_count holds the number of pages in each pool.
1647c478bd9Sstevel@tonic-gate  *
1657c478bd9Sstevel@tonic-gate  * pcf_block is set when page_create_get_something() has asked the
1667c478bd9Sstevel@tonic-gate  * PSM page freelist and page cachelist routines without specifying
1677c478bd9Sstevel@tonic-gate  * a color and nothing came back.  This is used to block anything
1687c478bd9Sstevel@tonic-gate  * else from moving pages from one list to the other while the
1697c478bd9Sstevel@tonic-gate  * lists are searched again.  If a page is freeed while pcf_block is
1707c478bd9Sstevel@tonic-gate  * set, then pcf_reserve is incremented.  pcgs_unblock() takes care
1717c478bd9Sstevel@tonic-gate  * of clearning pcf_block, doing the wakeups, etc.
1727c478bd9Sstevel@tonic-gate  */
1737c478bd9Sstevel@tonic-gate 
1747c478bd9Sstevel@tonic-gate #if NCPU <= 4
175f2b37d75Sfr #define	PAD	2
1767c478bd9Sstevel@tonic-gate #define	PCF_FANOUT	4
1777c478bd9Sstevel@tonic-gate static	uint_t	pcf_mask = PCF_FANOUT - 1;
1787c478bd9Sstevel@tonic-gate #else
179f2b37d75Sfr #define	PAD	10
1807c478bd9Sstevel@tonic-gate #ifdef sun4v
1817c478bd9Sstevel@tonic-gate #define	PCF_FANOUT	32
1827c478bd9Sstevel@tonic-gate #else
1837c478bd9Sstevel@tonic-gate #define	PCF_FANOUT	128
1847c478bd9Sstevel@tonic-gate #endif
1857c478bd9Sstevel@tonic-gate static	uint_t	pcf_mask = PCF_FANOUT - 1;
1867c478bd9Sstevel@tonic-gate #endif
1877c478bd9Sstevel@tonic-gate 
1887c478bd9Sstevel@tonic-gate struct pcf {
1897c478bd9Sstevel@tonic-gate 	kmutex_t	pcf_lock;	/* protects the structure */
190f2b37d75Sfr 	uint_t		pcf_count;	/* page count */
1917c478bd9Sstevel@tonic-gate 	uint_t		pcf_wait;	/* number of waiters */
1927c478bd9Sstevel@tonic-gate 	uint_t		pcf_block; 	/* pcgs flag to page_free() */
1937c478bd9Sstevel@tonic-gate 	uint_t		pcf_reserve; 	/* pages freed after pcf_block set */
1947c478bd9Sstevel@tonic-gate 	uint_t		pcf_fill[PAD];	/* to line up on the caches */
1957c478bd9Sstevel@tonic-gate };
1967c478bd9Sstevel@tonic-gate 
1977c478bd9Sstevel@tonic-gate static struct	pcf	pcf[PCF_FANOUT];
1987c478bd9Sstevel@tonic-gate #define	PCF_INDEX()	((CPU->cpu_id) & (pcf_mask))
1997c478bd9Sstevel@tonic-gate 
2007c478bd9Sstevel@tonic-gate kmutex_t	pcgs_lock;		/* serializes page_create_get_ */
2017c478bd9Sstevel@tonic-gate kmutex_t	pcgs_cagelock;		/* serializes NOSLEEP cage allocs */
2027c478bd9Sstevel@tonic-gate kmutex_t	pcgs_wait_lock;		/* used for delay in pcgs */
2037c478bd9Sstevel@tonic-gate static kcondvar_t	pcgs_cv;	/* cv for delay in pcgs */
2047c478bd9Sstevel@tonic-gate 
2057c478bd9Sstevel@tonic-gate #ifdef VM_STATS
2067c478bd9Sstevel@tonic-gate 
2077c478bd9Sstevel@tonic-gate /*
2087c478bd9Sstevel@tonic-gate  * No locks, but so what, they are only statistics.
2097c478bd9Sstevel@tonic-gate  */
2107c478bd9Sstevel@tonic-gate 
2117c478bd9Sstevel@tonic-gate static struct page_tcnt {
2127c478bd9Sstevel@tonic-gate 	int	pc_free_cache;		/* free's into cache list */
2137c478bd9Sstevel@tonic-gate 	int	pc_free_dontneed;	/* free's with dontneed */
2147c478bd9Sstevel@tonic-gate 	int	pc_free_pageout;	/* free's from pageout */
2157c478bd9Sstevel@tonic-gate 	int	pc_free_free;		/* free's into free list */
2167c478bd9Sstevel@tonic-gate 	int	pc_free_pages;		/* free's into large page free list */
2177c478bd9Sstevel@tonic-gate 	int	pc_destroy_pages;	/* large page destroy's */
2187c478bd9Sstevel@tonic-gate 	int	pc_get_cache;		/* get's from cache list */
2197c478bd9Sstevel@tonic-gate 	int	pc_get_free;		/* get's from free list */
2207c478bd9Sstevel@tonic-gate 	int	pc_reclaim;		/* reclaim's */
2217c478bd9Sstevel@tonic-gate 	int	pc_abortfree;		/* abort's of free pages */
2227c478bd9Sstevel@tonic-gate 	int	pc_find_hit;		/* find's that find page */
2237c478bd9Sstevel@tonic-gate 	int	pc_find_miss;		/* find's that don't find page */
2247c478bd9Sstevel@tonic-gate 	int	pc_destroy_free;	/* # of free pages destroyed */
2257c478bd9Sstevel@tonic-gate #define	PC_HASH_CNT	(4*PAGE_HASHAVELEN)
2267c478bd9Sstevel@tonic-gate 	int	pc_find_hashlen[PC_HASH_CNT+1];
2277c478bd9Sstevel@tonic-gate 	int	pc_addclaim_pages;
2287c478bd9Sstevel@tonic-gate 	int	pc_subclaim_pages;
2297c478bd9Sstevel@tonic-gate 	int	pc_free_replacement_page[2];
2307c478bd9Sstevel@tonic-gate 	int	pc_try_demote_pages[6];
2317c478bd9Sstevel@tonic-gate 	int	pc_demote_pages[2];
2327c478bd9Sstevel@tonic-gate } pagecnt;
2337c478bd9Sstevel@tonic-gate 
2347c478bd9Sstevel@tonic-gate uint_t	hashin_count;
2357c478bd9Sstevel@tonic-gate uint_t	hashin_not_held;
2367c478bd9Sstevel@tonic-gate uint_t	hashin_already;
2377c478bd9Sstevel@tonic-gate 
2387c478bd9Sstevel@tonic-gate uint_t	hashout_count;
2397c478bd9Sstevel@tonic-gate uint_t	hashout_not_held;
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate uint_t	page_create_count;
2427c478bd9Sstevel@tonic-gate uint_t	page_create_not_enough;
2437c478bd9Sstevel@tonic-gate uint_t	page_create_not_enough_again;
2447c478bd9Sstevel@tonic-gate uint_t	page_create_zero;
2457c478bd9Sstevel@tonic-gate uint_t	page_create_hashout;
2467c478bd9Sstevel@tonic-gate uint_t	page_create_page_lock_failed;
2477c478bd9Sstevel@tonic-gate uint_t	page_create_trylock_failed;
2487c478bd9Sstevel@tonic-gate uint_t	page_create_found_one;
2497c478bd9Sstevel@tonic-gate uint_t	page_create_hashin_failed;
2507c478bd9Sstevel@tonic-gate uint_t	page_create_dropped_phm;
2517c478bd9Sstevel@tonic-gate 
2527c478bd9Sstevel@tonic-gate uint_t	page_create_new;
2537c478bd9Sstevel@tonic-gate uint_t	page_create_exists;
2547c478bd9Sstevel@tonic-gate uint_t	page_create_putbacks;
2557c478bd9Sstevel@tonic-gate uint_t	page_create_overshoot;
2567c478bd9Sstevel@tonic-gate 
2577c478bd9Sstevel@tonic-gate uint_t	page_reclaim_zero;
2587c478bd9Sstevel@tonic-gate uint_t	page_reclaim_zero_locked;
2597c478bd9Sstevel@tonic-gate 
2607c478bd9Sstevel@tonic-gate uint_t	page_rename_exists;
2617c478bd9Sstevel@tonic-gate uint_t	page_rename_count;
2627c478bd9Sstevel@tonic-gate 
2637c478bd9Sstevel@tonic-gate uint_t	page_lookup_cnt[20];
2647c478bd9Sstevel@tonic-gate uint_t	page_lookup_nowait_cnt[10];
2657c478bd9Sstevel@tonic-gate uint_t	page_find_cnt;
2667c478bd9Sstevel@tonic-gate uint_t	page_exists_cnt;
2677c478bd9Sstevel@tonic-gate uint_t	page_exists_forreal_cnt;
2687c478bd9Sstevel@tonic-gate uint_t	page_lookup_dev_cnt;
2697c478bd9Sstevel@tonic-gate uint_t	get_cachelist_cnt;
2707c478bd9Sstevel@tonic-gate uint_t	page_create_cnt[10];
2717c478bd9Sstevel@tonic-gate uint_t	alloc_pages[8];
2727c478bd9Sstevel@tonic-gate uint_t	page_exphcontg[19];
2737c478bd9Sstevel@tonic-gate uint_t  page_create_large_cnt[10];
2747c478bd9Sstevel@tonic-gate 
2757c478bd9Sstevel@tonic-gate /*
2767c478bd9Sstevel@tonic-gate  * Collects statistics.
2777c478bd9Sstevel@tonic-gate  */
2787c478bd9Sstevel@tonic-gate #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
2797c478bd9Sstevel@tonic-gate 	uint_t	mylen = 0; \
2807c478bd9Sstevel@tonic-gate 			\
2817c478bd9Sstevel@tonic-gate 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash, mylen++) { \
2827c478bd9Sstevel@tonic-gate 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
2837c478bd9Sstevel@tonic-gate 			break; \
2847c478bd9Sstevel@tonic-gate 	} \
2857c478bd9Sstevel@tonic-gate 	if ((pp) != NULL) \
2867c478bd9Sstevel@tonic-gate 		pagecnt.pc_find_hit++; \
2877c478bd9Sstevel@tonic-gate 	else \
2887c478bd9Sstevel@tonic-gate 		pagecnt.pc_find_miss++; \
2897c478bd9Sstevel@tonic-gate 	if (mylen > PC_HASH_CNT) \
2907c478bd9Sstevel@tonic-gate 		mylen = PC_HASH_CNT; \
2917c478bd9Sstevel@tonic-gate 	pagecnt.pc_find_hashlen[mylen]++; \
2927c478bd9Sstevel@tonic-gate }
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate #else	/* VM_STATS */
2957c478bd9Sstevel@tonic-gate 
2967c478bd9Sstevel@tonic-gate /*
2977c478bd9Sstevel@tonic-gate  * Don't collect statistics
2987c478bd9Sstevel@tonic-gate  */
2997c478bd9Sstevel@tonic-gate #define	PAGE_HASH_SEARCH(index, pp, vp, off) { \
3007c478bd9Sstevel@tonic-gate 	for ((pp) = page_hash[(index)]; (pp); (pp) = (pp)->p_hash) { \
3017c478bd9Sstevel@tonic-gate 		if ((pp)->p_vnode == (vp) && (pp)->p_offset == (off)) \
3027c478bd9Sstevel@tonic-gate 			break; \
3037c478bd9Sstevel@tonic-gate 	} \
3047c478bd9Sstevel@tonic-gate }
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate #endif	/* VM_STATS */
3077c478bd9Sstevel@tonic-gate 
3087c478bd9Sstevel@tonic-gate 
3097c478bd9Sstevel@tonic-gate 
3107c478bd9Sstevel@tonic-gate #ifdef DEBUG
3117c478bd9Sstevel@tonic-gate #define	MEMSEG_SEARCH_STATS
3127c478bd9Sstevel@tonic-gate #endif
3137c478bd9Sstevel@tonic-gate 
3147c478bd9Sstevel@tonic-gate #ifdef MEMSEG_SEARCH_STATS
3157c478bd9Sstevel@tonic-gate struct memseg_stats {
3167c478bd9Sstevel@tonic-gate     uint_t nsearch;
3177c478bd9Sstevel@tonic-gate     uint_t nlastwon;
3187c478bd9Sstevel@tonic-gate     uint_t nhashwon;
3197c478bd9Sstevel@tonic-gate     uint_t nnotfound;
3207c478bd9Sstevel@tonic-gate } memseg_stats;
3217c478bd9Sstevel@tonic-gate 
3227c478bd9Sstevel@tonic-gate #define	MEMSEG_STAT_INCR(v) \
3237c478bd9Sstevel@tonic-gate 	atomic_add_32(&memseg_stats.v, 1)
3247c478bd9Sstevel@tonic-gate #else
3257c478bd9Sstevel@tonic-gate #define	MEMSEG_STAT_INCR(x)
3267c478bd9Sstevel@tonic-gate #endif
3277c478bd9Sstevel@tonic-gate 
3287c478bd9Sstevel@tonic-gate struct memseg *memsegs;		/* list of memory segments */
3297c478bd9Sstevel@tonic-gate 
3307c478bd9Sstevel@tonic-gate 
3317c478bd9Sstevel@tonic-gate static void page_init_mem_config(void);
3327c478bd9Sstevel@tonic-gate static int page_do_hashin(page_t *, vnode_t *, u_offset_t);
3337c478bd9Sstevel@tonic-gate static void page_do_hashout(page_t *);
3348b464eb8Smec static void page_capture_init();
3358b464eb8Smec int page_capture_take_action(page_t *, uint_t, void *);
3367c478bd9Sstevel@tonic-gate 
3377c478bd9Sstevel@tonic-gate static void page_demote_vp_pages(page_t *);
3387c478bd9Sstevel@tonic-gate 
3397c478bd9Sstevel@tonic-gate /*
3407c478bd9Sstevel@tonic-gate  * vm subsystem related initialization
3417c478bd9Sstevel@tonic-gate  */
3427c478bd9Sstevel@tonic-gate void
3437c478bd9Sstevel@tonic-gate vm_init(void)
3447c478bd9Sstevel@tonic-gate {
3457c478bd9Sstevel@tonic-gate 	boolean_t callb_vm_cpr(void *, int);
3467c478bd9Sstevel@tonic-gate 
3477c478bd9Sstevel@tonic-gate 	(void) callb_add(callb_vm_cpr, 0, CB_CL_CPR_VM, "vm");
3487c478bd9Sstevel@tonic-gate 	page_init_mem_config();
349db874c57Selowe 	page_retire_init();
3500209230bSgjelinek 	vm_usage_init();
3518b464eb8Smec 	page_capture_init();
3527c478bd9Sstevel@tonic-gate }
3537c478bd9Sstevel@tonic-gate 
3547c478bd9Sstevel@tonic-gate /*
3557c478bd9Sstevel@tonic-gate  * This function is called at startup and when memory is added or deleted.
3567c478bd9Sstevel@tonic-gate  */
3577c478bd9Sstevel@tonic-gate void
3587c478bd9Sstevel@tonic-gate init_pages_pp_maximum()
3597c478bd9Sstevel@tonic-gate {
3607c478bd9Sstevel@tonic-gate 	static pgcnt_t p_min;
3617c478bd9Sstevel@tonic-gate 	static pgcnt_t pages_pp_maximum_startup;
3627c478bd9Sstevel@tonic-gate 	static pgcnt_t avrmem_delta;
3637c478bd9Sstevel@tonic-gate 	static int init_done;
3647c478bd9Sstevel@tonic-gate 	static int user_set;	/* true if set in /etc/system */
3657c478bd9Sstevel@tonic-gate 
3667c478bd9Sstevel@tonic-gate 	if (init_done == 0) {
3677c478bd9Sstevel@tonic-gate 
3687c478bd9Sstevel@tonic-gate 		/* If the user specified a value, save it */
3697c478bd9Sstevel@tonic-gate 		if (pages_pp_maximum != 0) {
3707c478bd9Sstevel@tonic-gate 			user_set = 1;
3717c478bd9Sstevel@tonic-gate 			pages_pp_maximum_startup = pages_pp_maximum;
3727c478bd9Sstevel@tonic-gate 		}
3737c478bd9Sstevel@tonic-gate 
3747c478bd9Sstevel@tonic-gate 		/*
3757c478bd9Sstevel@tonic-gate 		 * Setting of pages_pp_maximum is based first time
3767c478bd9Sstevel@tonic-gate 		 * on the value of availrmem just after the start-up
3777c478bd9Sstevel@tonic-gate 		 * allocations. To preserve this relationship at run
3787c478bd9Sstevel@tonic-gate 		 * time, use a delta from availrmem_initial.
3797c478bd9Sstevel@tonic-gate 		 */
3807c478bd9Sstevel@tonic-gate 		ASSERT(availrmem_initial >= availrmem);
3817c478bd9Sstevel@tonic-gate 		avrmem_delta = availrmem_initial - availrmem;
3827c478bd9Sstevel@tonic-gate 
3837c478bd9Sstevel@tonic-gate 		/* The allowable floor of pages_pp_maximum */
3847c478bd9Sstevel@tonic-gate 		p_min = tune.t_minarmem + 100;
3857c478bd9Sstevel@tonic-gate 
3867c478bd9Sstevel@tonic-gate 		/* Make sure we don't come through here again. */
3877c478bd9Sstevel@tonic-gate 		init_done = 1;
3887c478bd9Sstevel@tonic-gate 	}
3897c478bd9Sstevel@tonic-gate 	/*
3907c478bd9Sstevel@tonic-gate 	 * Determine pages_pp_maximum, the number of currently available
3917c478bd9Sstevel@tonic-gate 	 * pages (availrmem) that can't be `locked'. If not set by
3927c478bd9Sstevel@tonic-gate 	 * the user, we set it to 4% of the currently available memory
3937c478bd9Sstevel@tonic-gate 	 * plus 4MB.
3947c478bd9Sstevel@tonic-gate 	 * But we also insist that it be greater than tune.t_minarmem;
3957c478bd9Sstevel@tonic-gate 	 * otherwise a process could lock down a lot of memory, get swapped
3967c478bd9Sstevel@tonic-gate 	 * out, and never have enough to get swapped back in.
3977c478bd9Sstevel@tonic-gate 	 */
3987c478bd9Sstevel@tonic-gate 	if (user_set)
3997c478bd9Sstevel@tonic-gate 		pages_pp_maximum = pages_pp_maximum_startup;
4007c478bd9Sstevel@tonic-gate 	else
4017c478bd9Sstevel@tonic-gate 		pages_pp_maximum = ((availrmem_initial - avrmem_delta) / 25)
4027c478bd9Sstevel@tonic-gate 		    + btop(4 * 1024 * 1024);
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 	if (pages_pp_maximum <= p_min) {
4057c478bd9Sstevel@tonic-gate 		pages_pp_maximum = p_min;
4067c478bd9Sstevel@tonic-gate 	}
4077c478bd9Sstevel@tonic-gate }
4087c478bd9Sstevel@tonic-gate 
4097c478bd9Sstevel@tonic-gate void
4107c478bd9Sstevel@tonic-gate set_max_page_get(pgcnt_t target_total_pages)
4117c478bd9Sstevel@tonic-gate {
4127c478bd9Sstevel@tonic-gate 	max_page_get = target_total_pages / 2;
4137c478bd9Sstevel@tonic-gate }
4147c478bd9Sstevel@tonic-gate 
4157c478bd9Sstevel@tonic-gate static pgcnt_t pending_delete;
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4187c478bd9Sstevel@tonic-gate static void
4197c478bd9Sstevel@tonic-gate page_mem_config_post_add(
4207c478bd9Sstevel@tonic-gate 	void *arg,
4217c478bd9Sstevel@tonic-gate 	pgcnt_t delta_pages)
4227c478bd9Sstevel@tonic-gate {
4237c478bd9Sstevel@tonic-gate 	set_max_page_get(total_pages - pending_delete);
4247c478bd9Sstevel@tonic-gate 	init_pages_pp_maximum();
4257c478bd9Sstevel@tonic-gate }
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4287c478bd9Sstevel@tonic-gate static int
4297c478bd9Sstevel@tonic-gate page_mem_config_pre_del(
4307c478bd9Sstevel@tonic-gate 	void *arg,
4317c478bd9Sstevel@tonic-gate 	pgcnt_t delta_pages)
4327c478bd9Sstevel@tonic-gate {
4337c478bd9Sstevel@tonic-gate 	pgcnt_t nv;
4347c478bd9Sstevel@tonic-gate 
4357c478bd9Sstevel@tonic-gate 	nv = atomic_add_long_nv(&pending_delete, (spgcnt_t)delta_pages);
4367c478bd9Sstevel@tonic-gate 	set_max_page_get(total_pages - nv);
4377c478bd9Sstevel@tonic-gate 	return (0);
4387c478bd9Sstevel@tonic-gate }
4397c478bd9Sstevel@tonic-gate 
4407c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4417c478bd9Sstevel@tonic-gate static void
4427c478bd9Sstevel@tonic-gate page_mem_config_post_del(
4437c478bd9Sstevel@tonic-gate 	void *arg,
4447c478bd9Sstevel@tonic-gate 	pgcnt_t delta_pages,
4457c478bd9Sstevel@tonic-gate 	int cancelled)
4467c478bd9Sstevel@tonic-gate {
4477c478bd9Sstevel@tonic-gate 	pgcnt_t nv;
4487c478bd9Sstevel@tonic-gate 
4497c478bd9Sstevel@tonic-gate 	nv = atomic_add_long_nv(&pending_delete, -(spgcnt_t)delta_pages);
4507c478bd9Sstevel@tonic-gate 	set_max_page_get(total_pages - nv);
4517c478bd9Sstevel@tonic-gate 	if (!cancelled)
4527c478bd9Sstevel@tonic-gate 		init_pages_pp_maximum();
4537c478bd9Sstevel@tonic-gate }
4547c478bd9Sstevel@tonic-gate 
4557c478bd9Sstevel@tonic-gate static kphysm_setup_vector_t page_mem_config_vec = {
4567c478bd9Sstevel@tonic-gate 	KPHYSM_SETUP_VECTOR_VERSION,
4577c478bd9Sstevel@tonic-gate 	page_mem_config_post_add,
4587c478bd9Sstevel@tonic-gate 	page_mem_config_pre_del,
4597c478bd9Sstevel@tonic-gate 	page_mem_config_post_del,
4607c478bd9Sstevel@tonic-gate };
4617c478bd9Sstevel@tonic-gate 
4627c478bd9Sstevel@tonic-gate static void
4637c478bd9Sstevel@tonic-gate page_init_mem_config(void)
4647c478bd9Sstevel@tonic-gate {
4657c478bd9Sstevel@tonic-gate 	int ret;
4667c478bd9Sstevel@tonic-gate 
4677c478bd9Sstevel@tonic-gate 	ret = kphysm_setup_func_register(&page_mem_config_vec, (void *)NULL);
4687c478bd9Sstevel@tonic-gate 	ASSERT(ret == 0);
4697c478bd9Sstevel@tonic-gate }
4707c478bd9Sstevel@tonic-gate 
4717c478bd9Sstevel@tonic-gate /*
4727c478bd9Sstevel@tonic-gate  * Evenly spread out the PCF counters for large free pages
4737c478bd9Sstevel@tonic-gate  */
4747c478bd9Sstevel@tonic-gate static void
4757c478bd9Sstevel@tonic-gate page_free_large_ctr(pgcnt_t npages)
4767c478bd9Sstevel@tonic-gate {
4777c478bd9Sstevel@tonic-gate 	static struct pcf	*p = pcf;
4787c478bd9Sstevel@tonic-gate 	pgcnt_t			lump;
4797c478bd9Sstevel@tonic-gate 
4807c478bd9Sstevel@tonic-gate 	freemem += npages;
4817c478bd9Sstevel@tonic-gate 
4827c478bd9Sstevel@tonic-gate 	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;
4837c478bd9Sstevel@tonic-gate 
4847c478bd9Sstevel@tonic-gate 	while (npages > 0) {
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate 		ASSERT(!p->pcf_block);
4877c478bd9Sstevel@tonic-gate 
4887c478bd9Sstevel@tonic-gate 		if (lump < npages) {
4897c478bd9Sstevel@tonic-gate 			p->pcf_count += (uint_t)lump;
4907c478bd9Sstevel@tonic-gate 			npages -= lump;
4917c478bd9Sstevel@tonic-gate 		} else {
4927c478bd9Sstevel@tonic-gate 			p->pcf_count += (uint_t)npages;
4937c478bd9Sstevel@tonic-gate 			npages = 0;
4947c478bd9Sstevel@tonic-gate 		}
4957c478bd9Sstevel@tonic-gate 
4967c478bd9Sstevel@tonic-gate 		ASSERT(!p->pcf_wait);
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate 		if (++p > &pcf[PCF_FANOUT - 1])
4997c478bd9Sstevel@tonic-gate 			p = pcf;
5007c478bd9Sstevel@tonic-gate 	}
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate 	ASSERT(npages == 0);
5037c478bd9Sstevel@tonic-gate }
5047c478bd9Sstevel@tonic-gate 
5057c478bd9Sstevel@tonic-gate /*
5067c478bd9Sstevel@tonic-gate  * Add a physical chunk of memory to the system freee lists during startup.
5077c478bd9Sstevel@tonic-gate  * Platform specific startup() allocates the memory for the page structs.
5087c478bd9Sstevel@tonic-gate  *
5097c478bd9Sstevel@tonic-gate  * num	- number of page structures
5107c478bd9Sstevel@tonic-gate  * base - page number (pfn) to be associated with the first page.
5117c478bd9Sstevel@tonic-gate  *
5127c478bd9Sstevel@tonic-gate  * Since we are doing this during startup (ie. single threaded), we will
5137c478bd9Sstevel@tonic-gate  * use shortcut routines to avoid any locking overhead while putting all
5147c478bd9Sstevel@tonic-gate  * these pages on the freelists.
5157c478bd9Sstevel@tonic-gate  *
5167c478bd9Sstevel@tonic-gate  * NOTE: Any changes performed to page_free(), must also be performed to
5177c478bd9Sstevel@tonic-gate  *	 add_physmem() since this is how we initialize all page_t's at
5187c478bd9Sstevel@tonic-gate  *	 boot time.
5197c478bd9Sstevel@tonic-gate  */
5207c478bd9Sstevel@tonic-gate void
5217c478bd9Sstevel@tonic-gate add_physmem(
5227c478bd9Sstevel@tonic-gate 	page_t	*pp,
5237c478bd9Sstevel@tonic-gate 	pgcnt_t	num,
5247c478bd9Sstevel@tonic-gate 	pfn_t	pnum)
5257c478bd9Sstevel@tonic-gate {
5267c478bd9Sstevel@tonic-gate 	page_t	*root = NULL;
5277c478bd9Sstevel@tonic-gate 	uint_t	szc = page_num_pagesizes() - 1;
5287c478bd9Sstevel@tonic-gate 	pgcnt_t	large = page_get_pagecnt(szc);
5297c478bd9Sstevel@tonic-gate 	pgcnt_t	cnt = 0;
5307c478bd9Sstevel@tonic-gate 
5317c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_VM, TR_PAGE_INIT,
5327c478bd9Sstevel@tonic-gate 		"add_physmem:pp %p num %lu", pp, num);
5337c478bd9Sstevel@tonic-gate 
5347c478bd9Sstevel@tonic-gate 	/*
5357c478bd9Sstevel@tonic-gate 	 * Arbitrarily limit the max page_get request
5367c478bd9Sstevel@tonic-gate 	 * to 1/2 of the page structs we have.
5377c478bd9Sstevel@tonic-gate 	 */
5387c478bd9Sstevel@tonic-gate 	total_pages += num;
5397c478bd9Sstevel@tonic-gate 	set_max_page_get(total_pages);
5407c478bd9Sstevel@tonic-gate 
541e21bae1bSkchow 	PLCNT_MODIFY_MAX(pnum, (long)num);
542e21bae1bSkchow 
5437c478bd9Sstevel@tonic-gate 	/*
5447c478bd9Sstevel@tonic-gate 	 * The physical space for the pages array
5457c478bd9Sstevel@tonic-gate 	 * representing ram pages has already been
5467c478bd9Sstevel@tonic-gate 	 * allocated.  Here we initialize each lock
5477c478bd9Sstevel@tonic-gate 	 * in the page structure, and put each on
5487c478bd9Sstevel@tonic-gate 	 * the free list
5497c478bd9Sstevel@tonic-gate 	 */
550affbd3ccSkchow 	for (; num; pp++, pnum++, num--) {
5517c478bd9Sstevel@tonic-gate 
5527c478bd9Sstevel@tonic-gate 		/*
5537c478bd9Sstevel@tonic-gate 		 * this needs to fill in the page number
5547c478bd9Sstevel@tonic-gate 		 * and do any other arch specific initialization
5557c478bd9Sstevel@tonic-gate 		 */
5567c478bd9Sstevel@tonic-gate 		add_physmem_cb(pp, pnum);
5577c478bd9Sstevel@tonic-gate 
55807b65a64Saguzovsk 		pp->p_lckcnt = 0;
55907b65a64Saguzovsk 		pp->p_cowcnt = 0;
56007b65a64Saguzovsk 		pp->p_slckcnt = 0;
56107b65a64Saguzovsk 
5627c478bd9Sstevel@tonic-gate 		/*
5637c478bd9Sstevel@tonic-gate 		 * Initialize the page lock as unlocked, since nobody
5647c478bd9Sstevel@tonic-gate 		 * can see or access this page yet.
5657c478bd9Sstevel@tonic-gate 		 */
5667c478bd9Sstevel@tonic-gate 		pp->p_selock = 0;
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate 		/*
5697c478bd9Sstevel@tonic-gate 		 * Initialize IO lock
5707c478bd9Sstevel@tonic-gate 		 */
5717c478bd9Sstevel@tonic-gate 		page_iolock_init(pp);
5727c478bd9Sstevel@tonic-gate 
5737c478bd9Sstevel@tonic-gate 		/*
5747c478bd9Sstevel@tonic-gate 		 * initialize other fields in the page_t
5757c478bd9Sstevel@tonic-gate 		 */
5767c478bd9Sstevel@tonic-gate 		PP_SETFREE(pp);
5777c478bd9Sstevel@tonic-gate 		page_clr_all_props(pp);
5787c478bd9Sstevel@tonic-gate 		PP_SETAGED(pp);
5797c478bd9Sstevel@tonic-gate 		pp->p_offset = (u_offset_t)-1;
5807c478bd9Sstevel@tonic-gate 		pp->p_next = pp;
5817c478bd9Sstevel@tonic-gate 		pp->p_prev = pp;
5827c478bd9Sstevel@tonic-gate 
5837c478bd9Sstevel@tonic-gate 		/*
5847c478bd9Sstevel@tonic-gate 		 * Simple case: System doesn't support large pages.
5857c478bd9Sstevel@tonic-gate 		 */
5867c478bd9Sstevel@tonic-gate 		if (szc == 0) {
5877c478bd9Sstevel@tonic-gate 			pp->p_szc = 0;
5887c478bd9Sstevel@tonic-gate 			page_free_at_startup(pp);
5897c478bd9Sstevel@tonic-gate 			continue;
5907c478bd9Sstevel@tonic-gate 		}
5917c478bd9Sstevel@tonic-gate 
5927c478bd9Sstevel@tonic-gate 		/*
5937c478bd9Sstevel@tonic-gate 		 * Handle unaligned pages, we collect them up onto
5947c478bd9Sstevel@tonic-gate 		 * the root page until we have a full large page.
5957c478bd9Sstevel@tonic-gate 		 */
5967c478bd9Sstevel@tonic-gate 		if (!IS_P2ALIGNED(pnum, large)) {
5977c478bd9Sstevel@tonic-gate 
5987c478bd9Sstevel@tonic-gate 			/*
5997c478bd9Sstevel@tonic-gate 			 * If not in a large page,
6007c478bd9Sstevel@tonic-gate 			 * just free as small page.
6017c478bd9Sstevel@tonic-gate 			 */
6027c478bd9Sstevel@tonic-gate 			if (root == NULL) {
6037c478bd9Sstevel@tonic-gate 				pp->p_szc = 0;
6047c478bd9Sstevel@tonic-gate 				page_free_at_startup(pp);
6057c478bd9Sstevel@tonic-gate 				continue;
6067c478bd9Sstevel@tonic-gate 			}
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate 			/*
6097c478bd9Sstevel@tonic-gate 			 * Link a constituent page into the large page.
6107c478bd9Sstevel@tonic-gate 			 */
6117c478bd9Sstevel@tonic-gate 			pp->p_szc = szc;
6127c478bd9Sstevel@tonic-gate 			page_list_concat(&root, &pp);
6137c478bd9Sstevel@tonic-gate 
6147c478bd9Sstevel@tonic-gate 			/*
6157c478bd9Sstevel@tonic-gate 			 * When large page is fully formed, free it.
6167c478bd9Sstevel@tonic-gate 			 */
6177c478bd9Sstevel@tonic-gate 			if (++cnt == large) {
6187c478bd9Sstevel@tonic-gate 				page_free_large_ctr(cnt);
6197c478bd9Sstevel@tonic-gate 				page_list_add_pages(root, PG_LIST_ISINIT);
6207c478bd9Sstevel@tonic-gate 				root = NULL;
6217c478bd9Sstevel@tonic-gate 				cnt = 0;
6227c478bd9Sstevel@tonic-gate 			}
6237c478bd9Sstevel@tonic-gate 			continue;
6247c478bd9Sstevel@tonic-gate 		}
6257c478bd9Sstevel@tonic-gate 
6267c478bd9Sstevel@tonic-gate 		/*
6277c478bd9Sstevel@tonic-gate 		 * At this point we have a page number which
6287c478bd9Sstevel@tonic-gate 		 * is aligned. We assert that we aren't already
6297c478bd9Sstevel@tonic-gate 		 * in a different large page.
6307c478bd9Sstevel@tonic-gate 		 */
6317c478bd9Sstevel@tonic-gate 		ASSERT(IS_P2ALIGNED(pnum, large));
6327c478bd9Sstevel@tonic-gate 		ASSERT(root == NULL && cnt == 0);
6337c478bd9Sstevel@tonic-gate 
6347c478bd9Sstevel@tonic-gate 		/*
6357c478bd9Sstevel@tonic-gate 		 * If insufficient number of pages left to form
6367c478bd9Sstevel@tonic-gate 		 * a large page, just free the small page.
6377c478bd9Sstevel@tonic-gate 		 */
6387c478bd9Sstevel@tonic-gate 		if (num < large) {
6397c478bd9Sstevel@tonic-gate 			pp->p_szc = 0;
6407c478bd9Sstevel@tonic-gate 			page_free_at_startup(pp);
6417c478bd9Sstevel@tonic-gate 			continue;
6427c478bd9Sstevel@tonic-gate 		}
6437c478bd9Sstevel@tonic-gate 
6447c478bd9Sstevel@tonic-gate 		/*
6457c478bd9Sstevel@tonic-gate 		 * Otherwise start a new large page.
6467c478bd9Sstevel@tonic-gate 		 */
6477c478bd9Sstevel@tonic-gate 		pp->p_szc = szc;
6487c478bd9Sstevel@tonic-gate 		cnt++;
6497c478bd9Sstevel@tonic-gate 		root = pp;
6507c478bd9Sstevel@tonic-gate 	}
6517c478bd9Sstevel@tonic-gate 	ASSERT(root == NULL && cnt == 0);
6527c478bd9Sstevel@tonic-gate }
6537c478bd9Sstevel@tonic-gate 
6547c478bd9Sstevel@tonic-gate /*
6557c478bd9Sstevel@tonic-gate  * Find a page representing the specified [vp, offset].
6567c478bd9Sstevel@tonic-gate  * If we find the page but it is intransit coming in,
6577c478bd9Sstevel@tonic-gate  * it will have an "exclusive" lock and we wait for
6587c478bd9Sstevel@tonic-gate  * the i/o to complete.  A page found on the free list
6597c478bd9Sstevel@tonic-gate  * is always reclaimed and then locked.  On success, the page
6607c478bd9Sstevel@tonic-gate  * is locked, its data is valid and it isn't on the free
6617c478bd9Sstevel@tonic-gate  * list, while a NULL is returned if the page doesn't exist.
6627c478bd9Sstevel@tonic-gate  */
6637c478bd9Sstevel@tonic-gate page_t *
6647c478bd9Sstevel@tonic-gate page_lookup(vnode_t *vp, u_offset_t off, se_t se)
6657c478bd9Sstevel@tonic-gate {
6667c478bd9Sstevel@tonic-gate 	return (page_lookup_create(vp, off, se, NULL, NULL, 0));
6677c478bd9Sstevel@tonic-gate }
6687c478bd9Sstevel@tonic-gate 
6697c478bd9Sstevel@tonic-gate /*
6707c478bd9Sstevel@tonic-gate  * Find a page representing the specified [vp, offset].
6717c478bd9Sstevel@tonic-gate  * We either return the one we found or, if passed in,
6727c478bd9Sstevel@tonic-gate  * create one with identity of [vp, offset] of the
6737c478bd9Sstevel@tonic-gate  * pre-allocated page. If we find exsisting page but it is
6747c478bd9Sstevel@tonic-gate  * intransit coming in, it will have an "exclusive" lock
6757c478bd9Sstevel@tonic-gate  * and we wait for the i/o to complete.  A page found on
6767c478bd9Sstevel@tonic-gate  * the free list is always reclaimed and then locked.
6777c478bd9Sstevel@tonic-gate  * On success, the page is locked, its data is valid and
6787c478bd9Sstevel@tonic-gate  * it isn't on the free list, while a NULL is returned
6797c478bd9Sstevel@tonic-gate  * if the page doesn't exist and newpp is NULL;
6807c478bd9Sstevel@tonic-gate  */
6817c478bd9Sstevel@tonic-gate page_t *
6827c478bd9Sstevel@tonic-gate page_lookup_create(
6837c478bd9Sstevel@tonic-gate 	vnode_t *vp,
6847c478bd9Sstevel@tonic-gate 	u_offset_t off,
6857c478bd9Sstevel@tonic-gate 	se_t se,
6867c478bd9Sstevel@tonic-gate 	page_t *newpp,
6877c478bd9Sstevel@tonic-gate 	spgcnt_t *nrelocp,
6887c478bd9Sstevel@tonic-gate 	int flags)
6897c478bd9Sstevel@tonic-gate {
6907c478bd9Sstevel@tonic-gate 	page_t		*pp;
6917c478bd9Sstevel@tonic-gate 	kmutex_t	*phm;
6927c478bd9Sstevel@tonic-gate 	ulong_t		index;
6937c478bd9Sstevel@tonic-gate 	uint_t		hash_locked;
6947c478bd9Sstevel@tonic-gate 	uint_t		es;
6957c478bd9Sstevel@tonic-gate 
6967c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
6977c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_lookup_cnt[0]);
6987c478bd9Sstevel@tonic-gate 	ASSERT(newpp ? PAGE_EXCL(newpp) : 1);
6997c478bd9Sstevel@tonic-gate 
7007c478bd9Sstevel@tonic-gate 	/*
7017c478bd9Sstevel@tonic-gate 	 * Acquire the appropriate page hash lock since
7027c478bd9Sstevel@tonic-gate 	 * we have to search the hash list.  Pages that
7037c478bd9Sstevel@tonic-gate 	 * hash to this list can't change identity while
7047c478bd9Sstevel@tonic-gate 	 * this lock is held.
7057c478bd9Sstevel@tonic-gate 	 */
7067c478bd9Sstevel@tonic-gate 	hash_locked = 0;
7077c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
7087c478bd9Sstevel@tonic-gate 	phm = NULL;
7097c478bd9Sstevel@tonic-gate top:
7107c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
7117c478bd9Sstevel@tonic-gate 	if (pp != NULL) {
7127c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_cnt[1]);
7137c478bd9Sstevel@tonic-gate 		es = (newpp != NULL) ? 1 : 0;
7147c478bd9Sstevel@tonic-gate 		es |= flags;
7157c478bd9Sstevel@tonic-gate 		if (!hash_locked) {
7167c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lookup_cnt[2]);
7177c478bd9Sstevel@tonic-gate 			if (!page_try_reclaim_lock(pp, se, es)) {
7187c478bd9Sstevel@tonic-gate 				/*
7197c478bd9Sstevel@tonic-gate 				 * On a miss, acquire the phm.  Then
7207c478bd9Sstevel@tonic-gate 				 * next time, page_lock() will be called,
7217c478bd9Sstevel@tonic-gate 				 * causing a wait if the page is busy.
7227c478bd9Sstevel@tonic-gate 				 * just looping with page_trylock() would
7237c478bd9Sstevel@tonic-gate 				 * get pretty boring.
7247c478bd9Sstevel@tonic-gate 				 */
7257c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_cnt[3]);
7267c478bd9Sstevel@tonic-gate 				phm = PAGE_HASH_MUTEX(index);
7277c478bd9Sstevel@tonic-gate 				mutex_enter(phm);
7287c478bd9Sstevel@tonic-gate 				hash_locked = 1;
7297c478bd9Sstevel@tonic-gate 				goto top;
7307c478bd9Sstevel@tonic-gate 			}
7317c478bd9Sstevel@tonic-gate 		} else {
7327c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lookup_cnt[4]);
7337c478bd9Sstevel@tonic-gate 			if (!page_lock_es(pp, se, phm, P_RECLAIM, es)) {
7347c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_cnt[5]);
7357c478bd9Sstevel@tonic-gate 				goto top;
7367c478bd9Sstevel@tonic-gate 			}
7377c478bd9Sstevel@tonic-gate 		}
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate 		/*
7407c478bd9Sstevel@tonic-gate 		 * Since `pp' is locked it can not change identity now.
7417c478bd9Sstevel@tonic-gate 		 * Reconfirm we locked the correct page.
7427c478bd9Sstevel@tonic-gate 		 *
7437c478bd9Sstevel@tonic-gate 		 * Both the p_vnode and p_offset *must* be cast volatile
7447c478bd9Sstevel@tonic-gate 		 * to force a reload of their values: The PAGE_HASH_SEARCH
7457c478bd9Sstevel@tonic-gate 		 * macro will have stuffed p_vnode and p_offset into
7467c478bd9Sstevel@tonic-gate 		 * registers before calling page_trylock(); another thread,
7477c478bd9Sstevel@tonic-gate 		 * actually holding the hash lock, could have changed the
7487c478bd9Sstevel@tonic-gate 		 * page's identity in memory, but our registers would not
7497c478bd9Sstevel@tonic-gate 		 * be changed, fooling the reconfirmation.  If the hash
7507c478bd9Sstevel@tonic-gate 		 * lock was held during the search, the casting would
7517c478bd9Sstevel@tonic-gate 		 * not be needed.
7527c478bd9Sstevel@tonic-gate 		 */
7537c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_cnt[6]);
7547c478bd9Sstevel@tonic-gate 		if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
7557c478bd9Sstevel@tonic-gate 		    ((volatile u_offset_t)(pp->p_offset) != off)) {
7567c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lookup_cnt[7]);
7577c478bd9Sstevel@tonic-gate 			if (hash_locked) {
7587c478bd9Sstevel@tonic-gate 				panic("page_lookup_create: lost page %p",
7597c478bd9Sstevel@tonic-gate 				    (void *)pp);
7607c478bd9Sstevel@tonic-gate 				/*NOTREACHED*/
7617c478bd9Sstevel@tonic-gate 			}
7627c478bd9Sstevel@tonic-gate 			page_unlock(pp);
7637c478bd9Sstevel@tonic-gate 			phm = PAGE_HASH_MUTEX(index);
7647c478bd9Sstevel@tonic-gate 			mutex_enter(phm);
7657c478bd9Sstevel@tonic-gate 			hash_locked = 1;
7667c478bd9Sstevel@tonic-gate 			goto top;
7677c478bd9Sstevel@tonic-gate 		}
7687c478bd9Sstevel@tonic-gate 
7697c478bd9Sstevel@tonic-gate 		/*
7707c478bd9Sstevel@tonic-gate 		 * If page_trylock() was called, then pp may still be on
7717c478bd9Sstevel@tonic-gate 		 * the cachelist (can't be on the free list, it would not
7727c478bd9Sstevel@tonic-gate 		 * have been found in the search).  If it is on the
7737c478bd9Sstevel@tonic-gate 		 * cachelist it must be pulled now. To pull the page from
7747c478bd9Sstevel@tonic-gate 		 * the cachelist, it must be exclusively locked.
7757c478bd9Sstevel@tonic-gate 		 *
7767c478bd9Sstevel@tonic-gate 		 * The other big difference between page_trylock() and
7777c478bd9Sstevel@tonic-gate 		 * page_lock(), is that page_lock() will pull the
7787c478bd9Sstevel@tonic-gate 		 * page from whatever free list (the cache list in this
7797c478bd9Sstevel@tonic-gate 		 * case) the page is on.  If page_trylock() was used
7807c478bd9Sstevel@tonic-gate 		 * above, then we have to do the reclaim ourselves.
7817c478bd9Sstevel@tonic-gate 		 */
7827c478bd9Sstevel@tonic-gate 		if ((!hash_locked) && (PP_ISFREE(pp))) {
7837c478bd9Sstevel@tonic-gate 			ASSERT(PP_ISAGED(pp) == 0);
7847c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lookup_cnt[8]);
7857c478bd9Sstevel@tonic-gate 
7867c478bd9Sstevel@tonic-gate 			/*
7877c478bd9Sstevel@tonic-gate 			 * page_relcaim will insure that we
7887c478bd9Sstevel@tonic-gate 			 * have this page exclusively
7897c478bd9Sstevel@tonic-gate 			 */
7907c478bd9Sstevel@tonic-gate 
7917c478bd9Sstevel@tonic-gate 			if (!page_reclaim(pp, NULL)) {
7927c478bd9Sstevel@tonic-gate 				/*
7937c478bd9Sstevel@tonic-gate 				 * Page_reclaim dropped whatever lock
7947c478bd9Sstevel@tonic-gate 				 * we held.
7957c478bd9Sstevel@tonic-gate 				 */
7967c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_cnt[9]);
7977c478bd9Sstevel@tonic-gate 				phm = PAGE_HASH_MUTEX(index);
7987c478bd9Sstevel@tonic-gate 				mutex_enter(phm);
7997c478bd9Sstevel@tonic-gate 				hash_locked = 1;
8007c478bd9Sstevel@tonic-gate 				goto top;
8017c478bd9Sstevel@tonic-gate 			} else if (se == SE_SHARED && newpp == NULL) {
8027c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_cnt[10]);
8037c478bd9Sstevel@tonic-gate 				page_downgrade(pp);
8047c478bd9Sstevel@tonic-gate 			}
8057c478bd9Sstevel@tonic-gate 		}
8067c478bd9Sstevel@tonic-gate 
8077c478bd9Sstevel@tonic-gate 		if (hash_locked) {
8087c478bd9Sstevel@tonic-gate 			mutex_exit(phm);
8097c478bd9Sstevel@tonic-gate 		}
8107c478bd9Sstevel@tonic-gate 
8117c478bd9Sstevel@tonic-gate 		if (newpp != NULL && pp->p_szc < newpp->p_szc &&
8127c478bd9Sstevel@tonic-gate 		    PAGE_EXCL(pp) && nrelocp != NULL) {
8137c478bd9Sstevel@tonic-gate 			ASSERT(nrelocp != NULL);
8147c478bd9Sstevel@tonic-gate 			(void) page_relocate(&pp, &newpp, 1, 1, nrelocp,
8157c478bd9Sstevel@tonic-gate 			    NULL);
8167c478bd9Sstevel@tonic-gate 			if (*nrelocp > 0) {
8177c478bd9Sstevel@tonic-gate 				VM_STAT_COND_ADD(*nrelocp == 1,
8187c478bd9Sstevel@tonic-gate 				    page_lookup_cnt[11]);
8197c478bd9Sstevel@tonic-gate 				VM_STAT_COND_ADD(*nrelocp > 1,
8207c478bd9Sstevel@tonic-gate 				    page_lookup_cnt[12]);
8217c478bd9Sstevel@tonic-gate 				pp = newpp;
8227c478bd9Sstevel@tonic-gate 				se = SE_EXCL;
8237c478bd9Sstevel@tonic-gate 			} else {
8247c478bd9Sstevel@tonic-gate 				if (se == SE_SHARED) {
8257c478bd9Sstevel@tonic-gate 					page_downgrade(pp);
8267c478bd9Sstevel@tonic-gate 				}
8277c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_cnt[13]);
8287c478bd9Sstevel@tonic-gate 			}
8297c478bd9Sstevel@tonic-gate 		} else if (newpp != NULL && nrelocp != NULL) {
8307c478bd9Sstevel@tonic-gate 			if (PAGE_EXCL(pp) && se == SE_SHARED) {
8317c478bd9Sstevel@tonic-gate 				page_downgrade(pp);
8327c478bd9Sstevel@tonic-gate 			}
8337c478bd9Sstevel@tonic-gate 			VM_STAT_COND_ADD(pp->p_szc < newpp->p_szc,
8347c478bd9Sstevel@tonic-gate 			    page_lookup_cnt[14]);
8357c478bd9Sstevel@tonic-gate 			VM_STAT_COND_ADD(pp->p_szc == newpp->p_szc,
8367c478bd9Sstevel@tonic-gate 			    page_lookup_cnt[15]);
8377c478bd9Sstevel@tonic-gate 			VM_STAT_COND_ADD(pp->p_szc > newpp->p_szc,
8387c478bd9Sstevel@tonic-gate 			    page_lookup_cnt[16]);
8397c478bd9Sstevel@tonic-gate 		} else if (newpp != NULL && PAGE_EXCL(pp)) {
8407c478bd9Sstevel@tonic-gate 			se = SE_EXCL;
8417c478bd9Sstevel@tonic-gate 		}
8427c478bd9Sstevel@tonic-gate 	} else if (!hash_locked) {
8437c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_cnt[17]);
8447c478bd9Sstevel@tonic-gate 		phm = PAGE_HASH_MUTEX(index);
8457c478bd9Sstevel@tonic-gate 		mutex_enter(phm);
8467c478bd9Sstevel@tonic-gate 		hash_locked = 1;
8477c478bd9Sstevel@tonic-gate 		goto top;
8487c478bd9Sstevel@tonic-gate 	} else if (newpp != NULL) {
8497c478bd9Sstevel@tonic-gate 		/*
8507c478bd9Sstevel@tonic-gate 		 * If we have a preallocated page then
8517c478bd9Sstevel@tonic-gate 		 * insert it now and basically behave like
8527c478bd9Sstevel@tonic-gate 		 * page_create.
8537c478bd9Sstevel@tonic-gate 		 */
8547c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_cnt[18]);
8557c478bd9Sstevel@tonic-gate 		/*
8567c478bd9Sstevel@tonic-gate 		 * Since we hold the page hash mutex and
8577c478bd9Sstevel@tonic-gate 		 * just searched for this page, page_hashin
8587c478bd9Sstevel@tonic-gate 		 * had better not fail.  If it does, that
8597c478bd9Sstevel@tonic-gate 		 * means some thread did not follow the
8607c478bd9Sstevel@tonic-gate 		 * page hash mutex rules.  Panic now and
8617c478bd9Sstevel@tonic-gate 		 * get it over with.  As usual, go down
8627c478bd9Sstevel@tonic-gate 		 * holding all the locks.
8637c478bd9Sstevel@tonic-gate 		 */
8647c478bd9Sstevel@tonic-gate 		ASSERT(MUTEX_HELD(phm));
8657c478bd9Sstevel@tonic-gate 		if (!page_hashin(newpp, vp, off, phm)) {
8667c478bd9Sstevel@tonic-gate 			ASSERT(MUTEX_HELD(phm));
8677c478bd9Sstevel@tonic-gate 			panic("page_lookup_create: hashin failed %p %p %llx %p",
8687c478bd9Sstevel@tonic-gate 			    (void *)newpp, (void *)vp, off, (void *)phm);
8697c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
8707c478bd9Sstevel@tonic-gate 		}
8717c478bd9Sstevel@tonic-gate 		ASSERT(MUTEX_HELD(phm));
8727c478bd9Sstevel@tonic-gate 		mutex_exit(phm);
8737c478bd9Sstevel@tonic-gate 		phm = NULL;
8747c478bd9Sstevel@tonic-gate 		page_set_props(newpp, P_REF);
8757c478bd9Sstevel@tonic-gate 		page_io_lock(newpp);
8767c478bd9Sstevel@tonic-gate 		pp = newpp;
8777c478bd9Sstevel@tonic-gate 		se = SE_EXCL;
8787c478bd9Sstevel@tonic-gate 	} else {
8797c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_cnt[19]);
8807c478bd9Sstevel@tonic-gate 		mutex_exit(phm);
8817c478bd9Sstevel@tonic-gate 	}
8827c478bd9Sstevel@tonic-gate 
8837c478bd9Sstevel@tonic-gate 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
8847c478bd9Sstevel@tonic-gate 
8857c478bd9Sstevel@tonic-gate 	ASSERT(pp ? ((PP_ISFREE(pp) == 0) && (PP_ISAGED(pp) == 0)) : 1);
8867c478bd9Sstevel@tonic-gate 
8877c478bd9Sstevel@tonic-gate 	return (pp);
8887c478bd9Sstevel@tonic-gate }
8897c478bd9Sstevel@tonic-gate 
8907c478bd9Sstevel@tonic-gate /*
8917c478bd9Sstevel@tonic-gate  * Search the hash list for the page representing the
8927c478bd9Sstevel@tonic-gate  * specified [vp, offset] and return it locked.  Skip
8937c478bd9Sstevel@tonic-gate  * free pages and pages that cannot be locked as requested.
8947c478bd9Sstevel@tonic-gate  * Used while attempting to kluster pages.
8957c478bd9Sstevel@tonic-gate  */
8967c478bd9Sstevel@tonic-gate page_t *
8977c478bd9Sstevel@tonic-gate page_lookup_nowait(vnode_t *vp, u_offset_t off, se_t se)
8987c478bd9Sstevel@tonic-gate {
8997c478bd9Sstevel@tonic-gate 	page_t		*pp;
9007c478bd9Sstevel@tonic-gate 	kmutex_t	*phm;
9017c478bd9Sstevel@tonic-gate 	ulong_t		index;
9027c478bd9Sstevel@tonic-gate 	uint_t		locked;
9037c478bd9Sstevel@tonic-gate 
9047c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
9057c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_lookup_nowait_cnt[0]);
9067c478bd9Sstevel@tonic-gate 
9077c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
9087c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
9097c478bd9Sstevel@tonic-gate 	locked = 0;
9107c478bd9Sstevel@tonic-gate 	if (pp == NULL) {
9117c478bd9Sstevel@tonic-gate top:
9127c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_nowait_cnt[1]);
9137c478bd9Sstevel@tonic-gate 		locked = 1;
9147c478bd9Sstevel@tonic-gate 		phm = PAGE_HASH_MUTEX(index);
9157c478bd9Sstevel@tonic-gate 		mutex_enter(phm);
9167c478bd9Sstevel@tonic-gate 		PAGE_HASH_SEARCH(index, pp, vp, off);
9177c478bd9Sstevel@tonic-gate 	}
9187c478bd9Sstevel@tonic-gate 
9197c478bd9Sstevel@tonic-gate 	if (pp == NULL || PP_ISFREE(pp)) {
9207c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_nowait_cnt[2]);
9217c478bd9Sstevel@tonic-gate 		pp = NULL;
9227c478bd9Sstevel@tonic-gate 	} else {
9237c478bd9Sstevel@tonic-gate 		if (!page_trylock(pp, se)) {
9247c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lookup_nowait_cnt[3]);
9257c478bd9Sstevel@tonic-gate 			pp = NULL;
9267c478bd9Sstevel@tonic-gate 		} else {
9277c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lookup_nowait_cnt[4]);
9287c478bd9Sstevel@tonic-gate 			/*
9297c478bd9Sstevel@tonic-gate 			 * See the comment in page_lookup()
9307c478bd9Sstevel@tonic-gate 			 */
9317c478bd9Sstevel@tonic-gate 			if (((volatile struct vnode *)(pp->p_vnode) != vp) ||
9327c478bd9Sstevel@tonic-gate 			    ((u_offset_t)(pp->p_offset) != off)) {
9337c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_nowait_cnt[5]);
9347c478bd9Sstevel@tonic-gate 				if (locked) {
9357c478bd9Sstevel@tonic-gate 					panic("page_lookup_nowait %p",
9367c478bd9Sstevel@tonic-gate 					    (void *)pp);
9377c478bd9Sstevel@tonic-gate 					/*NOTREACHED*/
9387c478bd9Sstevel@tonic-gate 				}
9397c478bd9Sstevel@tonic-gate 				page_unlock(pp);
9407c478bd9Sstevel@tonic-gate 				goto top;
9417c478bd9Sstevel@tonic-gate 			}
9427c478bd9Sstevel@tonic-gate 			if (PP_ISFREE(pp)) {
9437c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lookup_nowait_cnt[6]);
9447c478bd9Sstevel@tonic-gate 				page_unlock(pp);
9457c478bd9Sstevel@tonic-gate 				pp = NULL;
9467c478bd9Sstevel@tonic-gate 			}
9477c478bd9Sstevel@tonic-gate 		}
9487c478bd9Sstevel@tonic-gate 	}
9497c478bd9Sstevel@tonic-gate 	if (locked) {
9507c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lookup_nowait_cnt[7]);
9517c478bd9Sstevel@tonic-gate 		mutex_exit(phm);
9527c478bd9Sstevel@tonic-gate 	}
9537c478bd9Sstevel@tonic-gate 
9547c478bd9Sstevel@tonic-gate 	ASSERT(pp ? PAGE_LOCKED_SE(pp, se) : 1);
9557c478bd9Sstevel@tonic-gate 
9567c478bd9Sstevel@tonic-gate 	return (pp);
9577c478bd9Sstevel@tonic-gate }
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate /*
9607c478bd9Sstevel@tonic-gate  * Search the hash list for a page with the specified [vp, off]
9617c478bd9Sstevel@tonic-gate  * that is known to exist and is already locked.  This routine
9627c478bd9Sstevel@tonic-gate  * is typically used by segment SOFTUNLOCK routines.
9637c478bd9Sstevel@tonic-gate  */
9647c478bd9Sstevel@tonic-gate page_t *
9657c478bd9Sstevel@tonic-gate page_find(vnode_t *vp, u_offset_t off)
9667c478bd9Sstevel@tonic-gate {
9677c478bd9Sstevel@tonic-gate 	page_t		*pp;
9687c478bd9Sstevel@tonic-gate 	kmutex_t	*phm;
9697c478bd9Sstevel@tonic-gate 	ulong_t		index;
9707c478bd9Sstevel@tonic-gate 
9717c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
9727c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_find_cnt);
9737c478bd9Sstevel@tonic-gate 
9747c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
9757c478bd9Sstevel@tonic-gate 	phm = PAGE_HASH_MUTEX(index);
9767c478bd9Sstevel@tonic-gate 
9777c478bd9Sstevel@tonic-gate 	mutex_enter(phm);
9787c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
9797c478bd9Sstevel@tonic-gate 	mutex_exit(phm);
9807c478bd9Sstevel@tonic-gate 
9814fc2445aSelowe 	ASSERT(pp == NULL || PAGE_LOCKED(pp) || panicstr);
9827c478bd9Sstevel@tonic-gate 	return (pp);
9837c478bd9Sstevel@tonic-gate }
9847c478bd9Sstevel@tonic-gate 
9857c478bd9Sstevel@tonic-gate /*
9867c478bd9Sstevel@tonic-gate  * Determine whether a page with the specified [vp, off]
9877c478bd9Sstevel@tonic-gate  * currently exists in the system.  Obviously this should
9887c478bd9Sstevel@tonic-gate  * only be considered as a hint since nothing prevents the
9897c478bd9Sstevel@tonic-gate  * page from disappearing or appearing immediately after
9907c478bd9Sstevel@tonic-gate  * the return from this routine. Subsequently, we don't
9917c478bd9Sstevel@tonic-gate  * even bother to lock the list.
9927c478bd9Sstevel@tonic-gate  */
9937c478bd9Sstevel@tonic-gate page_t *
9947c478bd9Sstevel@tonic-gate page_exists(vnode_t *vp, u_offset_t off)
9957c478bd9Sstevel@tonic-gate {
9967c478bd9Sstevel@tonic-gate 	page_t	*pp;
9977c478bd9Sstevel@tonic-gate 	ulong_t		index;
9987c478bd9Sstevel@tonic-gate 
9997c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
10007c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_exists_cnt);
10017c478bd9Sstevel@tonic-gate 
10027c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
10037c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
10047c478bd9Sstevel@tonic-gate 
10057c478bd9Sstevel@tonic-gate 	return (pp);
10067c478bd9Sstevel@tonic-gate }
10077c478bd9Sstevel@tonic-gate 
10087c478bd9Sstevel@tonic-gate /*
10097c478bd9Sstevel@tonic-gate  * Determine if physically contiguous pages exist for [vp, off] - [vp, off +
10107c478bd9Sstevel@tonic-gate  * page_size(szc)) range.  if they exist and ppa is not NULL fill ppa array
10117c478bd9Sstevel@tonic-gate  * with these pages locked SHARED. If necessary reclaim pages from
10127c478bd9Sstevel@tonic-gate  * freelist. Return 1 if contiguous pages exist and 0 otherwise.
10137c478bd9Sstevel@tonic-gate  *
10147c478bd9Sstevel@tonic-gate  * If we fail to lock pages still return 1 if pages exist and contiguous.
10157c478bd9Sstevel@tonic-gate  * But in this case return value is just a hint. ppa array won't be filled.
10167c478bd9Sstevel@tonic-gate  * Caller should initialize ppa[0] as NULL to distinguish return value.
10177c478bd9Sstevel@tonic-gate  *
10187c478bd9Sstevel@tonic-gate  * Returns 0 if pages don't exist or not physically contiguous.
10197c478bd9Sstevel@tonic-gate  *
10207c478bd9Sstevel@tonic-gate  * This routine doesn't work for anonymous(swapfs) pages.
10217c478bd9Sstevel@tonic-gate  */
10227c478bd9Sstevel@tonic-gate int
10237c478bd9Sstevel@tonic-gate page_exists_physcontig(vnode_t *vp, u_offset_t off, uint_t szc, page_t *ppa[])
10247c478bd9Sstevel@tonic-gate {
10257c478bd9Sstevel@tonic-gate 	pgcnt_t pages;
10267c478bd9Sstevel@tonic-gate 	pfn_t pfn;
10277c478bd9Sstevel@tonic-gate 	page_t *rootpp;
10287c478bd9Sstevel@tonic-gate 	pgcnt_t i;
10297c478bd9Sstevel@tonic-gate 	pgcnt_t j;
10307c478bd9Sstevel@tonic-gate 	u_offset_t save_off = off;
10317c478bd9Sstevel@tonic-gate 	ulong_t index;
10327c478bd9Sstevel@tonic-gate 	kmutex_t *phm;
10337c478bd9Sstevel@tonic-gate 	page_t *pp;
10347c478bd9Sstevel@tonic-gate 	uint_t pszc;
10357c478bd9Sstevel@tonic-gate 	int loopcnt = 0;
10367c478bd9Sstevel@tonic-gate 
10377c478bd9Sstevel@tonic-gate 	ASSERT(szc != 0);
10387c478bd9Sstevel@tonic-gate 	ASSERT(vp != NULL);
10397c478bd9Sstevel@tonic-gate 	ASSERT(!IS_SWAPFSVP(vp));
1040ad23a2dbSjohansen 	ASSERT(!VN_ISKAS(vp));
10417c478bd9Sstevel@tonic-gate 
10427c478bd9Sstevel@tonic-gate again:
10437c478bd9Sstevel@tonic-gate 	if (++loopcnt > 3) {
10447c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[0]);
10457c478bd9Sstevel@tonic-gate 		return (0);
10467c478bd9Sstevel@tonic-gate 	}
10477c478bd9Sstevel@tonic-gate 
10487c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
10497c478bd9Sstevel@tonic-gate 	phm = PAGE_HASH_MUTEX(index);
10507c478bd9Sstevel@tonic-gate 
10517c478bd9Sstevel@tonic-gate 	mutex_enter(phm);
10527c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
10537c478bd9Sstevel@tonic-gate 	mutex_exit(phm);
10547c478bd9Sstevel@tonic-gate 
10557c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_exphcontg[1]);
10567c478bd9Sstevel@tonic-gate 
10577c478bd9Sstevel@tonic-gate 	if (pp == NULL) {
10587c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[2]);
10597c478bd9Sstevel@tonic-gate 		return (0);
10607c478bd9Sstevel@tonic-gate 	}
10617c478bd9Sstevel@tonic-gate 
10627c478bd9Sstevel@tonic-gate 	pages = page_get_pagecnt(szc);
10637c478bd9Sstevel@tonic-gate 	rootpp = pp;
10647c478bd9Sstevel@tonic-gate 	pfn = rootpp->p_pagenum;
10657c478bd9Sstevel@tonic-gate 
10667c478bd9Sstevel@tonic-gate 	if ((pszc = pp->p_szc) >= szc && ppa != NULL) {
10677c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[3]);
10687c478bd9Sstevel@tonic-gate 		if (!page_trylock(pp, SE_SHARED)) {
10697c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_exphcontg[4]);
10707c478bd9Sstevel@tonic-gate 			return (1);
10717c478bd9Sstevel@tonic-gate 		}
10727c478bd9Sstevel@tonic-gate 		if (pp->p_szc != pszc || pp->p_vnode != vp ||
10737c478bd9Sstevel@tonic-gate 		    pp->p_offset != off) {
10747c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_exphcontg[5]);
10757c478bd9Sstevel@tonic-gate 			page_unlock(pp);
10767c478bd9Sstevel@tonic-gate 			off = save_off;
10777c478bd9Sstevel@tonic-gate 			goto again;
10787c478bd9Sstevel@tonic-gate 		}
10797c478bd9Sstevel@tonic-gate 		/*
10807c478bd9Sstevel@tonic-gate 		 * szc was non zero and vnode and offset matched after we
10817c478bd9Sstevel@tonic-gate 		 * locked the page it means it can't become free on us.
10827c478bd9Sstevel@tonic-gate 		 */
10837c478bd9Sstevel@tonic-gate 		ASSERT(!PP_ISFREE(pp));
10847c478bd9Sstevel@tonic-gate 		if (!IS_P2ALIGNED(pfn, pages)) {
10857c478bd9Sstevel@tonic-gate 			page_unlock(pp);
10867c478bd9Sstevel@tonic-gate 			return (0);
10877c478bd9Sstevel@tonic-gate 		}
10887c478bd9Sstevel@tonic-gate 		ppa[0] = pp;
10897c478bd9Sstevel@tonic-gate 		pp++;
10907c478bd9Sstevel@tonic-gate 		off += PAGESIZE;
10917c478bd9Sstevel@tonic-gate 		pfn++;
10927c478bd9Sstevel@tonic-gate 		for (i = 1; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
10937c478bd9Sstevel@tonic-gate 			if (!page_trylock(pp, SE_SHARED)) {
10947c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_exphcontg[6]);
10957c478bd9Sstevel@tonic-gate 				pp--;
10967c478bd9Sstevel@tonic-gate 				while (i-- > 0) {
10977c478bd9Sstevel@tonic-gate 					page_unlock(pp);
10987c478bd9Sstevel@tonic-gate 					pp--;
10997c478bd9Sstevel@tonic-gate 				}
11007c478bd9Sstevel@tonic-gate 				ppa[0] = NULL;
11017c478bd9Sstevel@tonic-gate 				return (1);
11027c478bd9Sstevel@tonic-gate 			}
11037c478bd9Sstevel@tonic-gate 			if (pp->p_szc != pszc) {
11047c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_exphcontg[7]);
11057c478bd9Sstevel@tonic-gate 				page_unlock(pp);
11067c478bd9Sstevel@tonic-gate 				pp--;
11077c478bd9Sstevel@tonic-gate 				while (i-- > 0) {
11087c478bd9Sstevel@tonic-gate 					page_unlock(pp);
11097c478bd9Sstevel@tonic-gate 					pp--;
11107c478bd9Sstevel@tonic-gate 				}
11117c478bd9Sstevel@tonic-gate 				ppa[0] = NULL;
11127c478bd9Sstevel@tonic-gate 				off = save_off;
11137c478bd9Sstevel@tonic-gate 				goto again;
11147c478bd9Sstevel@tonic-gate 			}
11157c478bd9Sstevel@tonic-gate 			/*
11167c478bd9Sstevel@tonic-gate 			 * szc the same as for previous already locked pages
11177c478bd9Sstevel@tonic-gate 			 * with right identity. Since this page had correct
11187c478bd9Sstevel@tonic-gate 			 * szc after we locked it can't get freed or destroyed
11197c478bd9Sstevel@tonic-gate 			 * and therefore must have the expected identity.
11207c478bd9Sstevel@tonic-gate 			 */
11217c478bd9Sstevel@tonic-gate 			ASSERT(!PP_ISFREE(pp));
11227c478bd9Sstevel@tonic-gate 			if (pp->p_vnode != vp ||
11237c478bd9Sstevel@tonic-gate 			    pp->p_offset != off) {
11247c478bd9Sstevel@tonic-gate 				panic("page_exists_physcontig: "
11257c478bd9Sstevel@tonic-gate 				    "large page identity doesn't match");
11267c478bd9Sstevel@tonic-gate 			}
11277c478bd9Sstevel@tonic-gate 			ppa[i] = pp;
11287c478bd9Sstevel@tonic-gate 			ASSERT(pp->p_pagenum == pfn);
11297c478bd9Sstevel@tonic-gate 		}
11307c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[8]);
11317c478bd9Sstevel@tonic-gate 		ppa[pages] = NULL;
11327c478bd9Sstevel@tonic-gate 		return (1);
11337c478bd9Sstevel@tonic-gate 	} else if (pszc >= szc) {
11347c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[9]);
11357c478bd9Sstevel@tonic-gate 		if (!IS_P2ALIGNED(pfn, pages)) {
11367c478bd9Sstevel@tonic-gate 			return (0);
11377c478bd9Sstevel@tonic-gate 		}
11387c478bd9Sstevel@tonic-gate 		return (1);
11397c478bd9Sstevel@tonic-gate 	}
11407c478bd9Sstevel@tonic-gate 
11417c478bd9Sstevel@tonic-gate 	if (!IS_P2ALIGNED(pfn, pages)) {
11427c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[10]);
11437c478bd9Sstevel@tonic-gate 		return (0);
11447c478bd9Sstevel@tonic-gate 	}
11457c478bd9Sstevel@tonic-gate 
11467c478bd9Sstevel@tonic-gate 	if (page_numtomemseg_nolock(pfn) !=
11477c478bd9Sstevel@tonic-gate 	    page_numtomemseg_nolock(pfn + pages - 1)) {
11487c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[11]);
11497c478bd9Sstevel@tonic-gate 		return (0);
11507c478bd9Sstevel@tonic-gate 	}
11517c478bd9Sstevel@tonic-gate 
11527c478bd9Sstevel@tonic-gate 	/*
11537c478bd9Sstevel@tonic-gate 	 * We loop up 4 times across pages to promote page size.
11547c478bd9Sstevel@tonic-gate 	 * We're extra cautious to promote page size atomically with respect
11557c478bd9Sstevel@tonic-gate 	 * to everybody else.  But we can probably optimize into 1 loop if
11567c478bd9Sstevel@tonic-gate 	 * this becomes an issue.
11577c478bd9Sstevel@tonic-gate 	 */
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE, pfn++) {
11607c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_pagenum == pfn);
11617c478bd9Sstevel@tonic-gate 		if (!page_trylock(pp, SE_EXCL)) {
11627c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_exphcontg[12]);
11637c478bd9Sstevel@tonic-gate 			break;
11647c478bd9Sstevel@tonic-gate 		}
11657c478bd9Sstevel@tonic-gate 		if (pp->p_vnode != vp ||
11667c478bd9Sstevel@tonic-gate 		    pp->p_offset != off) {
11677c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_exphcontg[13]);
11687c478bd9Sstevel@tonic-gate 			page_unlock(pp);
11697c478bd9Sstevel@tonic-gate 			break;
11707c478bd9Sstevel@tonic-gate 		}
11717c478bd9Sstevel@tonic-gate 		if (pp->p_szc >= szc) {
11727c478bd9Sstevel@tonic-gate 			ASSERT(i == 0);
11737c478bd9Sstevel@tonic-gate 			page_unlock(pp);
11747c478bd9Sstevel@tonic-gate 			off = save_off;
11757c478bd9Sstevel@tonic-gate 			goto again;
11767c478bd9Sstevel@tonic-gate 		}
11777c478bd9Sstevel@tonic-gate 	}
11787c478bd9Sstevel@tonic-gate 
11797c478bd9Sstevel@tonic-gate 	if (i != pages) {
11807c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[14]);
11817c478bd9Sstevel@tonic-gate 		--pp;
11827c478bd9Sstevel@tonic-gate 		while (i-- > 0) {
11837c478bd9Sstevel@tonic-gate 			page_unlock(pp);
11847c478bd9Sstevel@tonic-gate 			--pp;
11857c478bd9Sstevel@tonic-gate 		}
11867c478bd9Sstevel@tonic-gate 		return (0);
11877c478bd9Sstevel@tonic-gate 	}
11887c478bd9Sstevel@tonic-gate 
11897c478bd9Sstevel@tonic-gate 	pp = rootpp;
11907c478bd9Sstevel@tonic-gate 	for (i = 0; i < pages; i++, pp++) {
11917c478bd9Sstevel@tonic-gate 		if (PP_ISFREE(pp)) {
11927c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_exphcontg[15]);
11937c478bd9Sstevel@tonic-gate 			ASSERT(!PP_ISAGED(pp));
11947c478bd9Sstevel@tonic-gate 			ASSERT(pp->p_szc == 0);
11957c478bd9Sstevel@tonic-gate 			if (!page_reclaim(pp, NULL)) {
11967c478bd9Sstevel@tonic-gate 				break;
11977c478bd9Sstevel@tonic-gate 			}
11987c478bd9Sstevel@tonic-gate 		} else {
11997c478bd9Sstevel@tonic-gate 			ASSERT(pp->p_szc < szc);
12007c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_exphcontg[16]);
12017c478bd9Sstevel@tonic-gate 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
12027c478bd9Sstevel@tonic-gate 		}
12037c478bd9Sstevel@tonic-gate 	}
12047c478bd9Sstevel@tonic-gate 	if (i < pages) {
12057c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_exphcontg[17]);
12067c478bd9Sstevel@tonic-gate 		/*
12077c478bd9Sstevel@tonic-gate 		 * page_reclaim failed because we were out of memory.
12087c478bd9Sstevel@tonic-gate 		 * drop the rest of the locks and return because this page
12097c478bd9Sstevel@tonic-gate 		 * must be already reallocated anyway.
12107c478bd9Sstevel@tonic-gate 		 */
12117c478bd9Sstevel@tonic-gate 		pp = rootpp;
12127c478bd9Sstevel@tonic-gate 		for (j = 0; j < pages; j++, pp++) {
12137c478bd9Sstevel@tonic-gate 			if (j != i) {
12147c478bd9Sstevel@tonic-gate 				page_unlock(pp);
12157c478bd9Sstevel@tonic-gate 			}
12167c478bd9Sstevel@tonic-gate 		}
12177c478bd9Sstevel@tonic-gate 		return (0);
12187c478bd9Sstevel@tonic-gate 	}
12197c478bd9Sstevel@tonic-gate 
12207c478bd9Sstevel@tonic-gate 	off = save_off;
12217c478bd9Sstevel@tonic-gate 	pp = rootpp;
12227c478bd9Sstevel@tonic-gate 	for (i = 0; i < pages; i++, pp++, off += PAGESIZE) {
12237c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_EXCL(pp));
12247c478bd9Sstevel@tonic-gate 		ASSERT(!PP_ISFREE(pp));
12257c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_is_mapped(pp));
12267c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_vnode == vp);
12277c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_offset == off);
12287c478bd9Sstevel@tonic-gate 		pp->p_szc = szc;
12297c478bd9Sstevel@tonic-gate 	}
12307c478bd9Sstevel@tonic-gate 	pp = rootpp;
12317c478bd9Sstevel@tonic-gate 	for (i = 0; i < pages; i++, pp++) {
12327c478bd9Sstevel@tonic-gate 		if (ppa == NULL) {
12337c478bd9Sstevel@tonic-gate 			page_unlock(pp);
12347c478bd9Sstevel@tonic-gate 		} else {
12357c478bd9Sstevel@tonic-gate 			ppa[i] = pp;
12367c478bd9Sstevel@tonic-gate 			page_downgrade(ppa[i]);
12377c478bd9Sstevel@tonic-gate 		}
12387c478bd9Sstevel@tonic-gate 	}
12397c478bd9Sstevel@tonic-gate 	if (ppa != NULL) {
12407c478bd9Sstevel@tonic-gate 		ppa[pages] = NULL;
12417c478bd9Sstevel@tonic-gate 	}
12427c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_exphcontg[18]);
12437c478bd9Sstevel@tonic-gate 	ASSERT(vp->v_pages != NULL);
12447c478bd9Sstevel@tonic-gate 	return (1);
12457c478bd9Sstevel@tonic-gate }
12467c478bd9Sstevel@tonic-gate 
12477c478bd9Sstevel@tonic-gate /*
12487c478bd9Sstevel@tonic-gate  * Determine whether a page with the specified [vp, off]
12497c478bd9Sstevel@tonic-gate  * currently exists in the system and if so return its
12507c478bd9Sstevel@tonic-gate  * size code. Obviously this should only be considered as
12517c478bd9Sstevel@tonic-gate  * a hint since nothing prevents the page from disappearing
12527c478bd9Sstevel@tonic-gate  * or appearing immediately after the return from this routine.
12537c478bd9Sstevel@tonic-gate  */
12547c478bd9Sstevel@tonic-gate int
12557c478bd9Sstevel@tonic-gate page_exists_forreal(vnode_t *vp, u_offset_t off, uint_t *szc)
12567c478bd9Sstevel@tonic-gate {
12577c478bd9Sstevel@tonic-gate 	page_t		*pp;
12587c478bd9Sstevel@tonic-gate 	kmutex_t	*phm;
12597c478bd9Sstevel@tonic-gate 	ulong_t		index;
12607c478bd9Sstevel@tonic-gate 	int		rc = 0;
12617c478bd9Sstevel@tonic-gate 
12627c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
12637c478bd9Sstevel@tonic-gate 	ASSERT(szc != NULL);
12647c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_exists_forreal_cnt);
12657c478bd9Sstevel@tonic-gate 
12667c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
12677c478bd9Sstevel@tonic-gate 	phm = PAGE_HASH_MUTEX(index);
12687c478bd9Sstevel@tonic-gate 
12697c478bd9Sstevel@tonic-gate 	mutex_enter(phm);
12707c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
12717c478bd9Sstevel@tonic-gate 	if (pp != NULL) {
12727c478bd9Sstevel@tonic-gate 		*szc = pp->p_szc;
12737c478bd9Sstevel@tonic-gate 		rc = 1;
12747c478bd9Sstevel@tonic-gate 	}
12757c478bd9Sstevel@tonic-gate 	mutex_exit(phm);
12767c478bd9Sstevel@tonic-gate 	return (rc);
12777c478bd9Sstevel@tonic-gate }
12787c478bd9Sstevel@tonic-gate 
12797c478bd9Sstevel@tonic-gate /* wakeup threads waiting for pages in page_create_get_something() */
12807c478bd9Sstevel@tonic-gate void
12817c478bd9Sstevel@tonic-gate wakeup_pcgs(void)
12827c478bd9Sstevel@tonic-gate {
12837c478bd9Sstevel@tonic-gate 	if (!CV_HAS_WAITERS(&pcgs_cv))
12847c478bd9Sstevel@tonic-gate 		return;
12857c478bd9Sstevel@tonic-gate 	cv_broadcast(&pcgs_cv);
12867c478bd9Sstevel@tonic-gate }
12877c478bd9Sstevel@tonic-gate 
12887c478bd9Sstevel@tonic-gate /*
12897c478bd9Sstevel@tonic-gate  * 'freemem' is used all over the kernel as an indication of how many
12907c478bd9Sstevel@tonic-gate  * pages are free (either on the cache list or on the free page list)
12917c478bd9Sstevel@tonic-gate  * in the system.  In very few places is a really accurate 'freemem'
12927c478bd9Sstevel@tonic-gate  * needed.  To avoid contention of the lock protecting a the
12937c478bd9Sstevel@tonic-gate  * single freemem, it was spread out into NCPU buckets.  Set_freemem
12947c478bd9Sstevel@tonic-gate  * sets freemem to the total of all NCPU buckets.  It is called from
12957c478bd9Sstevel@tonic-gate  * clock() on each TICK.
12967c478bd9Sstevel@tonic-gate  */
12977c478bd9Sstevel@tonic-gate void
12987c478bd9Sstevel@tonic-gate set_freemem()
12997c478bd9Sstevel@tonic-gate {
13007c478bd9Sstevel@tonic-gate 	struct pcf	*p;
13017c478bd9Sstevel@tonic-gate 	ulong_t		t;
13027c478bd9Sstevel@tonic-gate 	uint_t		i;
13037c478bd9Sstevel@tonic-gate 
13047c478bd9Sstevel@tonic-gate 	t = 0;
13057c478bd9Sstevel@tonic-gate 	p = pcf;
13067c478bd9Sstevel@tonic-gate 	for (i = 0;  i < PCF_FANOUT; i++) {
13077c478bd9Sstevel@tonic-gate 		t += p->pcf_count;
13087c478bd9Sstevel@tonic-gate 		p++;
13097c478bd9Sstevel@tonic-gate 	}
13107c478bd9Sstevel@tonic-gate 	freemem = t;
13117c478bd9Sstevel@tonic-gate 
13127c478bd9Sstevel@tonic-gate 	/*
13137c478bd9Sstevel@tonic-gate 	 * Don't worry about grabbing mutex.  It's not that
13147c478bd9Sstevel@tonic-gate 	 * critical if we miss a tick or two.  This is
13157c478bd9Sstevel@tonic-gate 	 * where we wakeup possible delayers in
13167c478bd9Sstevel@tonic-gate 	 * page_create_get_something().
13177c478bd9Sstevel@tonic-gate 	 */
13187c478bd9Sstevel@tonic-gate 	wakeup_pcgs();
13197c478bd9Sstevel@tonic-gate }
13207c478bd9Sstevel@tonic-gate 
13217c478bd9Sstevel@tonic-gate ulong_t
13227c478bd9Sstevel@tonic-gate get_freemem()
13237c478bd9Sstevel@tonic-gate {
13247c478bd9Sstevel@tonic-gate 	struct pcf	*p;
13257c478bd9Sstevel@tonic-gate 	ulong_t		t;
13267c478bd9Sstevel@tonic-gate 	uint_t		i;
13277c478bd9Sstevel@tonic-gate 
13287c478bd9Sstevel@tonic-gate 	t = 0;
13297c478bd9Sstevel@tonic-gate 	p = pcf;
13307c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
13317c478bd9Sstevel@tonic-gate 		t += p->pcf_count;
13327c478bd9Sstevel@tonic-gate 		p++;
13337c478bd9Sstevel@tonic-gate 	}
13347c478bd9Sstevel@tonic-gate 	/*
13357c478bd9Sstevel@tonic-gate 	 * We just calculated it, might as well set it.
13367c478bd9Sstevel@tonic-gate 	 */
13377c478bd9Sstevel@tonic-gate 	freemem = t;
13387c478bd9Sstevel@tonic-gate 	return (t);
13397c478bd9Sstevel@tonic-gate }
13407c478bd9Sstevel@tonic-gate 
13417c478bd9Sstevel@tonic-gate /*
13427c478bd9Sstevel@tonic-gate  * Acquire all of the page cache & free (pcf) locks.
13437c478bd9Sstevel@tonic-gate  */
13447c478bd9Sstevel@tonic-gate void
13457c478bd9Sstevel@tonic-gate pcf_acquire_all()
13467c478bd9Sstevel@tonic-gate {
13477c478bd9Sstevel@tonic-gate 	struct pcf	*p;
13487c478bd9Sstevel@tonic-gate 	uint_t		i;
13497c478bd9Sstevel@tonic-gate 
13507c478bd9Sstevel@tonic-gate 	p = pcf;
13517c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
13527c478bd9Sstevel@tonic-gate 		mutex_enter(&p->pcf_lock);
13537c478bd9Sstevel@tonic-gate 		p++;
13547c478bd9Sstevel@tonic-gate 	}
13557c478bd9Sstevel@tonic-gate }
13567c478bd9Sstevel@tonic-gate 
13577c478bd9Sstevel@tonic-gate /*
13587c478bd9Sstevel@tonic-gate  * Release all the pcf_locks.
13597c478bd9Sstevel@tonic-gate  */
13607c478bd9Sstevel@tonic-gate void
13617c478bd9Sstevel@tonic-gate pcf_release_all()
13627c478bd9Sstevel@tonic-gate {
13637c478bd9Sstevel@tonic-gate 	struct pcf	*p;
13647c478bd9Sstevel@tonic-gate 	uint_t		i;
13657c478bd9Sstevel@tonic-gate 
13667c478bd9Sstevel@tonic-gate 	p = pcf;
13677c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
13687c478bd9Sstevel@tonic-gate 		mutex_exit(&p->pcf_lock);
13697c478bd9Sstevel@tonic-gate 		p++;
13707c478bd9Sstevel@tonic-gate 	}
13717c478bd9Sstevel@tonic-gate }
13727c478bd9Sstevel@tonic-gate 
13737c478bd9Sstevel@tonic-gate /*
13747c478bd9Sstevel@tonic-gate  * Inform the VM system that we need some pages freed up.
13757c478bd9Sstevel@tonic-gate  * Calls must be symmetric, e.g.:
13767c478bd9Sstevel@tonic-gate  *
13777c478bd9Sstevel@tonic-gate  *	page_needfree(100);
13787c478bd9Sstevel@tonic-gate  *	wait a bit;
13797c478bd9Sstevel@tonic-gate  *	page_needfree(-100);
13807c478bd9Sstevel@tonic-gate  */
13817c478bd9Sstevel@tonic-gate void
13827c478bd9Sstevel@tonic-gate page_needfree(spgcnt_t npages)
13837c478bd9Sstevel@tonic-gate {
13847c478bd9Sstevel@tonic-gate 	mutex_enter(&new_freemem_lock);
13857c478bd9Sstevel@tonic-gate 	needfree += npages;
13867c478bd9Sstevel@tonic-gate 	mutex_exit(&new_freemem_lock);
13877c478bd9Sstevel@tonic-gate }
13887c478bd9Sstevel@tonic-gate 
13897c478bd9Sstevel@tonic-gate /*
13907c478bd9Sstevel@tonic-gate  * Throttle for page_create(): try to prevent freemem from dropping
13917c478bd9Sstevel@tonic-gate  * below throttlefree.  We can't provide a 100% guarantee because
13927c478bd9Sstevel@tonic-gate  * KM_NOSLEEP allocations, page_reclaim(), and various other things
13937c478bd9Sstevel@tonic-gate  * nibble away at the freelist.  However, we can block all PG_WAIT
13947c478bd9Sstevel@tonic-gate  * allocations until memory becomes available.  The motivation is
13957c478bd9Sstevel@tonic-gate  * that several things can fall apart when there's no free memory:
13967c478bd9Sstevel@tonic-gate  *
13977c478bd9Sstevel@tonic-gate  * (1) If pageout() needs memory to push a page, the system deadlocks.
13987c478bd9Sstevel@tonic-gate  *
13997c478bd9Sstevel@tonic-gate  * (2) By (broken) specification, timeout(9F) can neither fail nor
14007c478bd9Sstevel@tonic-gate  *     block, so it has no choice but to panic the system if it
14017c478bd9Sstevel@tonic-gate  *     cannot allocate a callout structure.
14027c478bd9Sstevel@tonic-gate  *
14037c478bd9Sstevel@tonic-gate  * (3) Like timeout(), ddi_set_callback() cannot fail and cannot block;
14047c478bd9Sstevel@tonic-gate  *     it panics if it cannot allocate a callback structure.
14057c478bd9Sstevel@tonic-gate  *
14067c478bd9Sstevel@tonic-gate  * (4) Untold numbers of third-party drivers have not yet been hardened
14077c478bd9Sstevel@tonic-gate  *     against KM_NOSLEEP and/or allocb() failures; they simply assume
14087c478bd9Sstevel@tonic-gate  *     success and panic the system with a data fault on failure.
14097c478bd9Sstevel@tonic-gate  *     (The long-term solution to this particular problem is to ship
14107c478bd9Sstevel@tonic-gate  *     hostile fault-injecting DEBUG kernels with the DDK.)
14117c478bd9Sstevel@tonic-gate  *
14127c478bd9Sstevel@tonic-gate  * It is theoretically impossible to guarantee success of non-blocking
14137c478bd9Sstevel@tonic-gate  * allocations, but in practice, this throttle is very hard to break.
14147c478bd9Sstevel@tonic-gate  */
14157c478bd9Sstevel@tonic-gate static int
14167c478bd9Sstevel@tonic-gate page_create_throttle(pgcnt_t npages, int flags)
14177c478bd9Sstevel@tonic-gate {
14187c478bd9Sstevel@tonic-gate 	ulong_t	fm;
14197c478bd9Sstevel@tonic-gate 	uint_t	i;
14207c478bd9Sstevel@tonic-gate 	pgcnt_t tf;	/* effective value of throttlefree */
14217c478bd9Sstevel@tonic-gate 
14227c478bd9Sstevel@tonic-gate 	/*
14237c478bd9Sstevel@tonic-gate 	 * Never deny pages when:
14247c478bd9Sstevel@tonic-gate 	 * - it's a thread that cannot block [NOMEMWAIT()]
14257c478bd9Sstevel@tonic-gate 	 * - the allocation cannot block and must not fail
14267c478bd9Sstevel@tonic-gate 	 * - the allocation cannot block and is pageout dispensated
14277c478bd9Sstevel@tonic-gate 	 */
14287c478bd9Sstevel@tonic-gate 	if (NOMEMWAIT() ||
14297c478bd9Sstevel@tonic-gate 	    ((flags & (PG_WAIT | PG_PANIC)) == PG_PANIC) ||
14307c478bd9Sstevel@tonic-gate 	    ((flags & (PG_WAIT | PG_PUSHPAGE)) == PG_PUSHPAGE))
14317c478bd9Sstevel@tonic-gate 		return (1);
14327c478bd9Sstevel@tonic-gate 
14337c478bd9Sstevel@tonic-gate 	/*
14347c478bd9Sstevel@tonic-gate 	 * If the allocation can't block, we look favorably upon it
14357c478bd9Sstevel@tonic-gate 	 * unless we're below pageout_reserve.  In that case we fail
14367c478bd9Sstevel@tonic-gate 	 * the allocation because we want to make sure there are a few
14377c478bd9Sstevel@tonic-gate 	 * pages available for pageout.
14387c478bd9Sstevel@tonic-gate 	 */
14397c478bd9Sstevel@tonic-gate 	if ((flags & PG_WAIT) == 0)
14407c478bd9Sstevel@tonic-gate 		return (freemem >= npages + pageout_reserve);
14417c478bd9Sstevel@tonic-gate 
14427c478bd9Sstevel@tonic-gate 	/* Calculate the effective throttlefree value */
14437c478bd9Sstevel@tonic-gate 	tf = throttlefree -
14447c478bd9Sstevel@tonic-gate 	    ((flags & PG_PUSHPAGE) ? pageout_reserve : 0);
14457c478bd9Sstevel@tonic-gate 
14467c478bd9Sstevel@tonic-gate 	cv_signal(&proc_pageout->p_cv);
14477c478bd9Sstevel@tonic-gate 
14487c478bd9Sstevel@tonic-gate 	while (freemem < npages + tf) {
14497c478bd9Sstevel@tonic-gate 		pcf_acquire_all();
14507c478bd9Sstevel@tonic-gate 		mutex_enter(&new_freemem_lock);
14517c478bd9Sstevel@tonic-gate 		fm = 0;
14527c478bd9Sstevel@tonic-gate 		for (i = 0; i < PCF_FANOUT; i++) {
14537c478bd9Sstevel@tonic-gate 			fm += pcf[i].pcf_count;
14547c478bd9Sstevel@tonic-gate 			pcf[i].pcf_wait++;
14557c478bd9Sstevel@tonic-gate 			mutex_exit(&pcf[i].pcf_lock);
14567c478bd9Sstevel@tonic-gate 		}
14577c478bd9Sstevel@tonic-gate 		freemem = fm;
14587c478bd9Sstevel@tonic-gate 		needfree += npages;
14597c478bd9Sstevel@tonic-gate 		freemem_wait++;
14607c478bd9Sstevel@tonic-gate 		cv_wait(&freemem_cv, &new_freemem_lock);
14617c478bd9Sstevel@tonic-gate 		freemem_wait--;
14627c478bd9Sstevel@tonic-gate 		needfree -= npages;
14637c478bd9Sstevel@tonic-gate 		mutex_exit(&new_freemem_lock);
14647c478bd9Sstevel@tonic-gate 	}
14657c478bd9Sstevel@tonic-gate 	return (1);
14667c478bd9Sstevel@tonic-gate }
14677c478bd9Sstevel@tonic-gate 
14687c478bd9Sstevel@tonic-gate /*
14697c478bd9Sstevel@tonic-gate  * page_create_wait() is called to either coalecse pages from the
14707c478bd9Sstevel@tonic-gate  * different pcf buckets or to wait because there simply are not
14717c478bd9Sstevel@tonic-gate  * enough pages to satisfy the caller's request.
14727c478bd9Sstevel@tonic-gate  *
14737c478bd9Sstevel@tonic-gate  * Sadly, this is called from platform/vm/vm_machdep.c
14747c478bd9Sstevel@tonic-gate  */
14757c478bd9Sstevel@tonic-gate int
14767c478bd9Sstevel@tonic-gate page_create_wait(size_t npages, uint_t flags)
14777c478bd9Sstevel@tonic-gate {
14787c478bd9Sstevel@tonic-gate 	pgcnt_t		total;
14797c478bd9Sstevel@tonic-gate 	uint_t		i;
14807c478bd9Sstevel@tonic-gate 	struct pcf	*p;
14817c478bd9Sstevel@tonic-gate 
14827c478bd9Sstevel@tonic-gate 	/*
14837c478bd9Sstevel@tonic-gate 	 * Wait until there are enough free pages to satisfy our
14847c478bd9Sstevel@tonic-gate 	 * entire request.
14857c478bd9Sstevel@tonic-gate 	 * We set needfree += npages before prodding pageout, to make sure
14867c478bd9Sstevel@tonic-gate 	 * it does real work when npages > lotsfree > freemem.
14877c478bd9Sstevel@tonic-gate 	 */
14887c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_create_not_enough);
14897c478bd9Sstevel@tonic-gate 
14907c478bd9Sstevel@tonic-gate 	ASSERT(!kcage_on ? !(flags & PG_NORELOC) : 1);
14917c478bd9Sstevel@tonic-gate checkagain:
14927c478bd9Sstevel@tonic-gate 	if ((flags & PG_NORELOC) &&
14937c478bd9Sstevel@tonic-gate 	    kcage_freemem < kcage_throttlefree + npages)
14947c478bd9Sstevel@tonic-gate 		(void) kcage_create_throttle(npages, flags);
14957c478bd9Sstevel@tonic-gate 
14967c478bd9Sstevel@tonic-gate 	if (freemem < npages + throttlefree)
14977c478bd9Sstevel@tonic-gate 		if (!page_create_throttle(npages, flags))
14987c478bd9Sstevel@tonic-gate 			return (0);
14997c478bd9Sstevel@tonic-gate 
15007c478bd9Sstevel@tonic-gate 	/*
15017c478bd9Sstevel@tonic-gate 	 * Since page_create_va() looked at every
15027c478bd9Sstevel@tonic-gate 	 * bucket, assume we are going to have to wait.
15037c478bd9Sstevel@tonic-gate 	 * Get all of the pcf locks.
15047c478bd9Sstevel@tonic-gate 	 */
15057c478bd9Sstevel@tonic-gate 	total = 0;
15067c478bd9Sstevel@tonic-gate 	p = pcf;
15077c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
15087c478bd9Sstevel@tonic-gate 		mutex_enter(&p->pcf_lock);
15097c478bd9Sstevel@tonic-gate 		total += p->pcf_count;
15107c478bd9Sstevel@tonic-gate 		if (total >= npages) {
15117c478bd9Sstevel@tonic-gate 			/*
15127c478bd9Sstevel@tonic-gate 			 * Wow!  There are enough pages laying around
15137c478bd9Sstevel@tonic-gate 			 * to satisfy the request.  Do the accounting,
15147c478bd9Sstevel@tonic-gate 			 * drop the locks we acquired, and go back.
15157c478bd9Sstevel@tonic-gate 			 *
15167c478bd9Sstevel@tonic-gate 			 * freemem is not protected by any lock. So,
15177c478bd9Sstevel@tonic-gate 			 * we cannot have any assertion containing
15187c478bd9Sstevel@tonic-gate 			 * freemem.
15197c478bd9Sstevel@tonic-gate 			 */
15207c478bd9Sstevel@tonic-gate 			freemem -= npages;
15217c478bd9Sstevel@tonic-gate 
15227c478bd9Sstevel@tonic-gate 			while (p >= pcf) {
15237c478bd9Sstevel@tonic-gate 				if (p->pcf_count <= npages) {
15247c478bd9Sstevel@tonic-gate 					npages -= p->pcf_count;
15257c478bd9Sstevel@tonic-gate 					p->pcf_count = 0;
15267c478bd9Sstevel@tonic-gate 				} else {
15277c478bd9Sstevel@tonic-gate 					p->pcf_count -= (uint_t)npages;
15287c478bd9Sstevel@tonic-gate 					npages = 0;
15297c478bd9Sstevel@tonic-gate 				}
15307c478bd9Sstevel@tonic-gate 				mutex_exit(&p->pcf_lock);
15317c478bd9Sstevel@tonic-gate 				p--;
15327c478bd9Sstevel@tonic-gate 			}
15337c478bd9Sstevel@tonic-gate 			ASSERT(npages == 0);
15347c478bd9Sstevel@tonic-gate 			return (1);
15357c478bd9Sstevel@tonic-gate 		}
15367c478bd9Sstevel@tonic-gate 		p++;
15377c478bd9Sstevel@tonic-gate 	}
15387c478bd9Sstevel@tonic-gate 
15397c478bd9Sstevel@tonic-gate 	/*
15407c478bd9Sstevel@tonic-gate 	 * All of the pcf locks are held, there are not enough pages
15417c478bd9Sstevel@tonic-gate 	 * to satisfy the request (npages < total).
15427c478bd9Sstevel@tonic-gate 	 * Be sure to acquire the new_freemem_lock before dropping
15437c478bd9Sstevel@tonic-gate 	 * the pcf locks.  This prevents dropping wakeups in page_free().
15447c478bd9Sstevel@tonic-gate 	 * The order is always pcf_lock then new_freemem_lock.
15457c478bd9Sstevel@tonic-gate 	 *
15467c478bd9Sstevel@tonic-gate 	 * Since we hold all the pcf locks, it is a good time to set freemem.
15477c478bd9Sstevel@tonic-gate 	 *
15487c478bd9Sstevel@tonic-gate 	 * If the caller does not want to wait, return now.
15497c478bd9Sstevel@tonic-gate 	 * Else turn the pageout daemon loose to find something
15507c478bd9Sstevel@tonic-gate 	 * and wait till it does.
15517c478bd9Sstevel@tonic-gate 	 *
15527c478bd9Sstevel@tonic-gate 	 */
15537c478bd9Sstevel@tonic-gate 	freemem = total;
15547c478bd9Sstevel@tonic-gate 
15557c478bd9Sstevel@tonic-gate 	if ((flags & PG_WAIT) == 0) {
15567c478bd9Sstevel@tonic-gate 		pcf_release_all();
15577c478bd9Sstevel@tonic-gate 
15587c478bd9Sstevel@tonic-gate 		TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_NOMEM,
15597c478bd9Sstevel@tonic-gate 		"page_create_nomem:npages %ld freemem %ld", npages, freemem);
15607c478bd9Sstevel@tonic-gate 		return (0);
15617c478bd9Sstevel@tonic-gate 	}
15627c478bd9Sstevel@tonic-gate 
15637c478bd9Sstevel@tonic-gate 	ASSERT(proc_pageout != NULL);
15647c478bd9Sstevel@tonic-gate 	cv_signal(&proc_pageout->p_cv);
15657c478bd9Sstevel@tonic-gate 
15667c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_START,
15677c478bd9Sstevel@tonic-gate 	    "page_create_sleep_start: freemem %ld needfree %ld",
15687c478bd9Sstevel@tonic-gate 	    freemem, needfree);
15697c478bd9Sstevel@tonic-gate 
15707c478bd9Sstevel@tonic-gate 	/*
15717c478bd9Sstevel@tonic-gate 	 * We are going to wait.
15727c478bd9Sstevel@tonic-gate 	 * We currently hold all of the pcf_locks,
15737c478bd9Sstevel@tonic-gate 	 * get the new_freemem_lock (it protects freemem_wait),
15747c478bd9Sstevel@tonic-gate 	 * before dropping the pcf_locks.
15757c478bd9Sstevel@tonic-gate 	 */
15767c478bd9Sstevel@tonic-gate 	mutex_enter(&new_freemem_lock);
15777c478bd9Sstevel@tonic-gate 
15787c478bd9Sstevel@tonic-gate 	p = pcf;
15797c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
15807c478bd9Sstevel@tonic-gate 		p->pcf_wait++;
15817c478bd9Sstevel@tonic-gate 		mutex_exit(&p->pcf_lock);
15827c478bd9Sstevel@tonic-gate 		p++;
15837c478bd9Sstevel@tonic-gate 	}
15847c478bd9Sstevel@tonic-gate 
15857c478bd9Sstevel@tonic-gate 	needfree += npages;
15867c478bd9Sstevel@tonic-gate 	freemem_wait++;
15877c478bd9Sstevel@tonic-gate 
15887c478bd9Sstevel@tonic-gate 	cv_wait(&freemem_cv, &new_freemem_lock);
15897c478bd9Sstevel@tonic-gate 
15907c478bd9Sstevel@tonic-gate 	freemem_wait--;
15917c478bd9Sstevel@tonic-gate 	needfree -= npages;
15927c478bd9Sstevel@tonic-gate 
15937c478bd9Sstevel@tonic-gate 	mutex_exit(&new_freemem_lock);
15947c478bd9Sstevel@tonic-gate 
15957c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SLEEP_END,
15967c478bd9Sstevel@tonic-gate 	    "page_create_sleep_end: freemem %ld needfree %ld",
15977c478bd9Sstevel@tonic-gate 	    freemem, needfree);
15987c478bd9Sstevel@tonic-gate 
15997c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_create_not_enough_again);
16007c478bd9Sstevel@tonic-gate 	goto checkagain;
16017c478bd9Sstevel@tonic-gate }
16027c478bd9Sstevel@tonic-gate 
16037c478bd9Sstevel@tonic-gate /*
16047c478bd9Sstevel@tonic-gate  * A routine to do the opposite of page_create_wait().
16057c478bd9Sstevel@tonic-gate  */
16067c478bd9Sstevel@tonic-gate void
16077c478bd9Sstevel@tonic-gate page_create_putback(spgcnt_t npages)
16087c478bd9Sstevel@tonic-gate {
16097c478bd9Sstevel@tonic-gate 	struct pcf	*p;
16107c478bd9Sstevel@tonic-gate 	pgcnt_t		lump;
16117c478bd9Sstevel@tonic-gate 	uint_t		*which;
16127c478bd9Sstevel@tonic-gate 
16137c478bd9Sstevel@tonic-gate 	/*
16147c478bd9Sstevel@tonic-gate 	 * When a contiguous lump is broken up, we have to
16157c478bd9Sstevel@tonic-gate 	 * deal with lots of pages (min 64) so lets spread
16167c478bd9Sstevel@tonic-gate 	 * the wealth around.
16177c478bd9Sstevel@tonic-gate 	 */
16187c478bd9Sstevel@tonic-gate 	lump = roundup(npages, PCF_FANOUT) / PCF_FANOUT;
16197c478bd9Sstevel@tonic-gate 	freemem += npages;
16207c478bd9Sstevel@tonic-gate 
16217c478bd9Sstevel@tonic-gate 	for (p = pcf; (npages > 0) && (p < &pcf[PCF_FANOUT]); p++) {
16227c478bd9Sstevel@tonic-gate 		which = &p->pcf_count;
16237c478bd9Sstevel@tonic-gate 
16247c478bd9Sstevel@tonic-gate 		mutex_enter(&p->pcf_lock);
16257c478bd9Sstevel@tonic-gate 
16267c478bd9Sstevel@tonic-gate 		if (p->pcf_block) {
16277c478bd9Sstevel@tonic-gate 			which = &p->pcf_reserve;
16287c478bd9Sstevel@tonic-gate 		}
16297c478bd9Sstevel@tonic-gate 
16307c478bd9Sstevel@tonic-gate 		if (lump < npages) {
16317c478bd9Sstevel@tonic-gate 			*which += (uint_t)lump;
16327c478bd9Sstevel@tonic-gate 			npages -= lump;
16337c478bd9Sstevel@tonic-gate 		} else {
16347c478bd9Sstevel@tonic-gate 			*which += (uint_t)npages;
16357c478bd9Sstevel@tonic-gate 			npages = 0;
16367c478bd9Sstevel@tonic-gate 		}
16377c478bd9Sstevel@tonic-gate 
16387c478bd9Sstevel@tonic-gate 		if (p->pcf_wait) {
16397c478bd9Sstevel@tonic-gate 			mutex_enter(&new_freemem_lock);
16407c478bd9Sstevel@tonic-gate 			/*
16417c478bd9Sstevel@tonic-gate 			 * Check to see if some other thread
16427c478bd9Sstevel@tonic-gate 			 * is actually waiting.  Another bucket
16437c478bd9Sstevel@tonic-gate 			 * may have woken it up by now.  If there
16447c478bd9Sstevel@tonic-gate 			 * are no waiters, then set our pcf_wait
16457c478bd9Sstevel@tonic-gate 			 * count to zero to avoid coming in here
16467c478bd9Sstevel@tonic-gate 			 * next time.
16477c478bd9Sstevel@tonic-gate 			 */
16487c478bd9Sstevel@tonic-gate 			if (freemem_wait) {
16497c478bd9Sstevel@tonic-gate 				if (npages > 1) {
16507c478bd9Sstevel@tonic-gate 					cv_broadcast(&freemem_cv);
16517c478bd9Sstevel@tonic-gate 				} else {
16527c478bd9Sstevel@tonic-gate 					cv_signal(&freemem_cv);
16537c478bd9Sstevel@tonic-gate 				}
16547c478bd9Sstevel@tonic-gate 				p->pcf_wait--;
16557c478bd9Sstevel@tonic-gate 			} else {
16567c478bd9Sstevel@tonic-gate 				p->pcf_wait = 0;
16577c478bd9Sstevel@tonic-gate 			}
16587c478bd9Sstevel@tonic-gate 			mutex_exit(&new_freemem_lock);
16597c478bd9Sstevel@tonic-gate 		}
16607c478bd9Sstevel@tonic-gate 		mutex_exit(&p->pcf_lock);
16617c478bd9Sstevel@tonic-gate 	}
16627c478bd9Sstevel@tonic-gate 	ASSERT(npages == 0);
16637c478bd9Sstevel@tonic-gate }
16647c478bd9Sstevel@tonic-gate 
16657c478bd9Sstevel@tonic-gate /*
16667c478bd9Sstevel@tonic-gate  * A helper routine for page_create_get_something.
16677c478bd9Sstevel@tonic-gate  * The indenting got to deep down there.
16687c478bd9Sstevel@tonic-gate  * Unblock the pcf counters.  Any pages freed after
16697c478bd9Sstevel@tonic-gate  * pcf_block got set are moved to pcf_count and
16707c478bd9Sstevel@tonic-gate  * wakeups (cv_broadcast() or cv_signal()) are done as needed.
16717c478bd9Sstevel@tonic-gate  */
16727c478bd9Sstevel@tonic-gate static void
16737c478bd9Sstevel@tonic-gate pcgs_unblock(void)
16747c478bd9Sstevel@tonic-gate {
16757c478bd9Sstevel@tonic-gate 	int		i;
16767c478bd9Sstevel@tonic-gate 	struct pcf	*p;
16777c478bd9Sstevel@tonic-gate 
16787c478bd9Sstevel@tonic-gate 	/* Update freemem while we're here. */
16797c478bd9Sstevel@tonic-gate 	freemem = 0;
16807c478bd9Sstevel@tonic-gate 	p = pcf;
16817c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
16827c478bd9Sstevel@tonic-gate 		mutex_enter(&p->pcf_lock);
16837c478bd9Sstevel@tonic-gate 		ASSERT(p->pcf_count == 0);
16847c478bd9Sstevel@tonic-gate 		p->pcf_count = p->pcf_reserve;
16857c478bd9Sstevel@tonic-gate 		p->pcf_block = 0;
16867c478bd9Sstevel@tonic-gate 		freemem += p->pcf_count;
16877c478bd9Sstevel@tonic-gate 		if (p->pcf_wait) {
16887c478bd9Sstevel@tonic-gate 			mutex_enter(&new_freemem_lock);
16897c478bd9Sstevel@tonic-gate 			if (freemem_wait) {
16907c478bd9Sstevel@tonic-gate 				if (p->pcf_reserve > 1) {
16917c478bd9Sstevel@tonic-gate 					cv_broadcast(&freemem_cv);
16927c478bd9Sstevel@tonic-gate 					p->pcf_wait = 0;
16937c478bd9Sstevel@tonic-gate 				} else {
16947c478bd9Sstevel@tonic-gate 					cv_signal(&freemem_cv);
16957c478bd9Sstevel@tonic-gate 					p->pcf_wait--;
16967c478bd9Sstevel@tonic-gate 				}
16977c478bd9Sstevel@tonic-gate 			} else {
16987c478bd9Sstevel@tonic-gate 				p->pcf_wait = 0;
16997c478bd9Sstevel@tonic-gate 			}
17007c478bd9Sstevel@tonic-gate 			mutex_exit(&new_freemem_lock);
17017c478bd9Sstevel@tonic-gate 		}
17027c478bd9Sstevel@tonic-gate 		p->pcf_reserve = 0;
17037c478bd9Sstevel@tonic-gate 		mutex_exit(&p->pcf_lock);
17047c478bd9Sstevel@tonic-gate 		p++;
17057c478bd9Sstevel@tonic-gate 	}
17067c478bd9Sstevel@tonic-gate }
17077c478bd9Sstevel@tonic-gate 
17087c478bd9Sstevel@tonic-gate /*
17097c478bd9Sstevel@tonic-gate  * Called from page_create_va() when both the cache and free lists
17107c478bd9Sstevel@tonic-gate  * have been checked once.
17117c478bd9Sstevel@tonic-gate  *
17127c478bd9Sstevel@tonic-gate  * Either returns a page or panics since the accounting was done
17137c478bd9Sstevel@tonic-gate  * way before we got here.
17147c478bd9Sstevel@tonic-gate  *
17157c478bd9Sstevel@tonic-gate  * We don't come here often, so leave the accounting on permanently.
17167c478bd9Sstevel@tonic-gate  */
17177c478bd9Sstevel@tonic-gate 
17187c478bd9Sstevel@tonic-gate #define	MAX_PCGS	100
17197c478bd9Sstevel@tonic-gate 
17207c478bd9Sstevel@tonic-gate #ifdef	DEBUG
17217c478bd9Sstevel@tonic-gate #define	PCGS_TRIES	100
17227c478bd9Sstevel@tonic-gate #else	/* DEBUG */
17237c478bd9Sstevel@tonic-gate #define	PCGS_TRIES	10
17247c478bd9Sstevel@tonic-gate #endif	/* DEBUG */
17257c478bd9Sstevel@tonic-gate 
17267c478bd9Sstevel@tonic-gate #ifdef	VM_STATS
17277c478bd9Sstevel@tonic-gate uint_t	pcgs_counts[PCGS_TRIES];
17287c478bd9Sstevel@tonic-gate uint_t	pcgs_too_many;
17297c478bd9Sstevel@tonic-gate uint_t	pcgs_entered;
17307c478bd9Sstevel@tonic-gate uint_t	pcgs_entered_noreloc;
17317c478bd9Sstevel@tonic-gate uint_t	pcgs_locked;
17327c478bd9Sstevel@tonic-gate uint_t	pcgs_cagelocked;
17337c478bd9Sstevel@tonic-gate #endif	/* VM_STATS */
17347c478bd9Sstevel@tonic-gate 
17357c478bd9Sstevel@tonic-gate static page_t *
17367c478bd9Sstevel@tonic-gate page_create_get_something(vnode_t *vp, u_offset_t off, struct seg *seg,
17377c478bd9Sstevel@tonic-gate     caddr_t vaddr, uint_t flags)
17387c478bd9Sstevel@tonic-gate {
17397c478bd9Sstevel@tonic-gate 	uint_t		count;
17407c478bd9Sstevel@tonic-gate 	page_t		*pp;
17417c478bd9Sstevel@tonic-gate 	uint_t		locked, i;
17427c478bd9Sstevel@tonic-gate 	struct	pcf	*p;
17437c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
17447c478bd9Sstevel@tonic-gate 	int		cagelocked = 0;
17457c478bd9Sstevel@tonic-gate 
17467c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pcgs_entered);
17477c478bd9Sstevel@tonic-gate 
17487c478bd9Sstevel@tonic-gate 	/*
17497c478bd9Sstevel@tonic-gate 	 * Tap any reserve freelists: if we fail now, we'll die
17507c478bd9Sstevel@tonic-gate 	 * since the page(s) we're looking for have already been
17517c478bd9Sstevel@tonic-gate 	 * accounted for.
17527c478bd9Sstevel@tonic-gate 	 */
17537c478bd9Sstevel@tonic-gate 	flags |= PG_PANIC;
17547c478bd9Sstevel@tonic-gate 
17557c478bd9Sstevel@tonic-gate 	if ((flags & PG_NORELOC) != 0) {
17567c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pcgs_entered_noreloc);
17577c478bd9Sstevel@tonic-gate 		/*
17587c478bd9Sstevel@tonic-gate 		 * Requests for free pages from critical threads
17597c478bd9Sstevel@tonic-gate 		 * such as pageout still won't throttle here, but
17607c478bd9Sstevel@tonic-gate 		 * we must try again, to give the cageout thread
17617c478bd9Sstevel@tonic-gate 		 * another chance to catch up. Since we already
17627c478bd9Sstevel@tonic-gate 		 * accounted for the pages, we had better get them
17637c478bd9Sstevel@tonic-gate 		 * this time.
17647c478bd9Sstevel@tonic-gate 		 *
17657c478bd9Sstevel@tonic-gate 		 * N.B. All non-critical threads acquire the pcgs_cagelock
17667c478bd9Sstevel@tonic-gate 		 * to serialize access to the freelists. This implements a
17677c478bd9Sstevel@tonic-gate 		 * turnstile-type synchornization to avoid starvation of
17687c478bd9Sstevel@tonic-gate 		 * critical requests for PG_NORELOC memory by non-critical
17697c478bd9Sstevel@tonic-gate 		 * threads: all non-critical threads must acquire a 'ticket'
17707c478bd9Sstevel@tonic-gate 		 * before passing through, which entails making sure
17717c478bd9Sstevel@tonic-gate 		 * kcage_freemem won't fall below minfree prior to grabbing
17727c478bd9Sstevel@tonic-gate 		 * pages from the freelists.
17737c478bd9Sstevel@tonic-gate 		 */
17747c478bd9Sstevel@tonic-gate 		if (kcage_create_throttle(1, flags) == KCT_NONCRIT) {
17757c478bd9Sstevel@tonic-gate 			mutex_enter(&pcgs_cagelock);
17767c478bd9Sstevel@tonic-gate 			cagelocked = 1;
17777c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(pcgs_cagelocked);
17787c478bd9Sstevel@tonic-gate 		}
17797c478bd9Sstevel@tonic-gate 	}
17807c478bd9Sstevel@tonic-gate 
17817c478bd9Sstevel@tonic-gate 	/*
17827c478bd9Sstevel@tonic-gate 	 * Time to get serious.
17837c478bd9Sstevel@tonic-gate 	 * We failed to get a `correctly colored' page from both the
17847c478bd9Sstevel@tonic-gate 	 * free and cache lists.
17857c478bd9Sstevel@tonic-gate 	 * We escalate in stage.
17867c478bd9Sstevel@tonic-gate 	 *
17877c478bd9Sstevel@tonic-gate 	 * First try both lists without worring about color.
17887c478bd9Sstevel@tonic-gate 	 *
17897c478bd9Sstevel@tonic-gate 	 * Then, grab all page accounting locks (ie. pcf[]) and
17907c478bd9Sstevel@tonic-gate 	 * steal any pages that they have and set the pcf_block flag to
17917c478bd9Sstevel@tonic-gate 	 * stop deletions from the lists.  This will help because
17927c478bd9Sstevel@tonic-gate 	 * a page can get added to the free list while we are looking
17937c478bd9Sstevel@tonic-gate 	 * at the cache list, then another page could be added to the cache
17947c478bd9Sstevel@tonic-gate 	 * list allowing the page on the free list to be removed as we
17957c478bd9Sstevel@tonic-gate 	 * move from looking at the cache list to the free list. This
17967c478bd9Sstevel@tonic-gate 	 * could happen over and over. We would never find the page
17977c478bd9Sstevel@tonic-gate 	 * we have accounted for.
17987c478bd9Sstevel@tonic-gate 	 *
17997c478bd9Sstevel@tonic-gate 	 * Noreloc pages are a subset of the global (relocatable) page pool.
18007c478bd9Sstevel@tonic-gate 	 * They are not tracked separately in the pcf bins, so it is
18017c478bd9Sstevel@tonic-gate 	 * impossible to know when doing pcf accounting if the available
18027c478bd9Sstevel@tonic-gate 	 * page(s) are noreloc pages or not. When looking for a noreloc page
18037c478bd9Sstevel@tonic-gate 	 * it is quite easy to end up here even if the global (relocatable)
18047c478bd9Sstevel@tonic-gate 	 * page pool has plenty of free pages but the noreloc pool is empty.
18057c478bd9Sstevel@tonic-gate 	 *
18067c478bd9Sstevel@tonic-gate 	 * When the noreloc pool is empty (or low), additional noreloc pages
18077c478bd9Sstevel@tonic-gate 	 * are created by converting pages from the global page pool. This
18087c478bd9Sstevel@tonic-gate 	 * process will stall during pcf accounting if the pcf bins are
18097c478bd9Sstevel@tonic-gate 	 * already locked. Such is the case when a noreloc allocation is
18107c478bd9Sstevel@tonic-gate 	 * looping here in page_create_get_something waiting for more noreloc
18117c478bd9Sstevel@tonic-gate 	 * pages to appear.
18127c478bd9Sstevel@tonic-gate 	 *
18137c478bd9Sstevel@tonic-gate 	 * Short of adding a new field to the pcf bins to accurately track
18147c478bd9Sstevel@tonic-gate 	 * the number of free noreloc pages, we instead do not grab the
18157c478bd9Sstevel@tonic-gate 	 * pcgs_lock, do not set the pcf blocks and do not timeout when
18167c478bd9Sstevel@tonic-gate 	 * allocating a noreloc page. This allows noreloc allocations to
18177c478bd9Sstevel@tonic-gate 	 * loop without blocking global page pool allocations.
18187c478bd9Sstevel@tonic-gate 	 *
18197c478bd9Sstevel@tonic-gate 	 * NOTE: the behaviour of page_create_get_something has not changed
18207c478bd9Sstevel@tonic-gate 	 * for the case of global page pool allocations.
18217c478bd9Sstevel@tonic-gate 	 */
18227c478bd9Sstevel@tonic-gate 
18237c478bd9Sstevel@tonic-gate 	flags &= ~PG_MATCH_COLOR;
18247c478bd9Sstevel@tonic-gate 	locked = 0;
182507ad560dSkchow #if defined(__i386) || defined(__amd64)
18267c478bd9Sstevel@tonic-gate 	/*
18277c478bd9Sstevel@tonic-gate 	 * page_create_get_something may be called because 4g memory may be
18287c478bd9Sstevel@tonic-gate 	 * depleted. Set flags to allow for relocation of base page below
18297c478bd9Sstevel@tonic-gate 	 * 4g if necessary.
18307c478bd9Sstevel@tonic-gate 	 */
18317c478bd9Sstevel@tonic-gate 	if (physmax4g)
18327c478bd9Sstevel@tonic-gate 		flags |= (PGI_PGCPSZC0 | PGI_PGCPHIPRI);
18337c478bd9Sstevel@tonic-gate #endif
18347c478bd9Sstevel@tonic-gate 
18357c478bd9Sstevel@tonic-gate 	lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
18367c478bd9Sstevel@tonic-gate 
18377c478bd9Sstevel@tonic-gate 	for (count = 0; kcage_on || count < MAX_PCGS; count++) {
18387c478bd9Sstevel@tonic-gate 		pp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
18397c478bd9Sstevel@tonic-gate 		    flags, lgrp);
18407c478bd9Sstevel@tonic-gate 		if (pp == NULL) {
18417c478bd9Sstevel@tonic-gate 			pp = page_get_cachelist(vp, off, seg, vaddr,
18427c478bd9Sstevel@tonic-gate 				flags, lgrp);
18437c478bd9Sstevel@tonic-gate 		}
18447c478bd9Sstevel@tonic-gate 		if (pp == NULL) {
18457c478bd9Sstevel@tonic-gate 			/*
18467c478bd9Sstevel@tonic-gate 			 * Serialize.  Don't fight with other pcgs().
18477c478bd9Sstevel@tonic-gate 			 */
18487c478bd9Sstevel@tonic-gate 			if (!locked && (!kcage_on || !(flags & PG_NORELOC))) {
18497c478bd9Sstevel@tonic-gate 				mutex_enter(&pcgs_lock);
18507c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(pcgs_locked);
18517c478bd9Sstevel@tonic-gate 				locked = 1;
18527c478bd9Sstevel@tonic-gate 				p = pcf;
18537c478bd9Sstevel@tonic-gate 				for (i = 0; i < PCF_FANOUT; i++) {
18547c478bd9Sstevel@tonic-gate 					mutex_enter(&p->pcf_lock);
18557c478bd9Sstevel@tonic-gate 					ASSERT(p->pcf_block == 0);
18567c478bd9Sstevel@tonic-gate 					p->pcf_block = 1;
18577c478bd9Sstevel@tonic-gate 					p->pcf_reserve = p->pcf_count;
18587c478bd9Sstevel@tonic-gate 					p->pcf_count = 0;
18597c478bd9Sstevel@tonic-gate 					mutex_exit(&p->pcf_lock);
18607c478bd9Sstevel@tonic-gate 					p++;
18617c478bd9Sstevel@tonic-gate 				}
18627c478bd9Sstevel@tonic-gate 				freemem = 0;
18637c478bd9Sstevel@tonic-gate 			}
18647c478bd9Sstevel@tonic-gate 
18657c478bd9Sstevel@tonic-gate 			if (count) {
18667c478bd9Sstevel@tonic-gate 				/*
18677c478bd9Sstevel@tonic-gate 				 * Since page_free() puts pages on
18687c478bd9Sstevel@tonic-gate 				 * a list then accounts for it, we
18697c478bd9Sstevel@tonic-gate 				 * just have to wait for page_free()
18707c478bd9Sstevel@tonic-gate 				 * to unlock any page it was working
18717c478bd9Sstevel@tonic-gate 				 * with. The page_lock()-page_reclaim()
18727c478bd9Sstevel@tonic-gate 				 * path falls in the same boat.
18737c478bd9Sstevel@tonic-gate 				 *
18747c478bd9Sstevel@tonic-gate 				 * We don't need to check on the
18757c478bd9Sstevel@tonic-gate 				 * PG_WAIT flag, we have already
18767c478bd9Sstevel@tonic-gate 				 * accounted for the page we are
18777c478bd9Sstevel@tonic-gate 				 * looking for in page_create_va().
18787c478bd9Sstevel@tonic-gate 				 *
18797c478bd9Sstevel@tonic-gate 				 * We just wait a moment to let any
18807c478bd9Sstevel@tonic-gate 				 * locked pages on the lists free up,
18817c478bd9Sstevel@tonic-gate 				 * then continue around and try again.
18827c478bd9Sstevel@tonic-gate 				 *
18837c478bd9Sstevel@tonic-gate 				 * Will be awakened by set_freemem().
18847c478bd9Sstevel@tonic-gate 				 */
18857c478bd9Sstevel@tonic-gate 				mutex_enter(&pcgs_wait_lock);
18867c478bd9Sstevel@tonic-gate 				cv_wait(&pcgs_cv, &pcgs_wait_lock);
18877c478bd9Sstevel@tonic-gate 				mutex_exit(&pcgs_wait_lock);
18887c478bd9Sstevel@tonic-gate 			}
18897c478bd9Sstevel@tonic-gate 		} else {
18907c478bd9Sstevel@tonic-gate #ifdef VM_STATS
18917c478bd9Sstevel@tonic-gate 			if (count >= PCGS_TRIES) {
18927c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(pcgs_too_many);
18937c478bd9Sstevel@tonic-gate 			} else {
18947c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(pcgs_counts[count]);
18957c478bd9Sstevel@tonic-gate 			}
18967c478bd9Sstevel@tonic-gate #endif
18977c478bd9Sstevel@tonic-gate 			if (locked) {
18987c478bd9Sstevel@tonic-gate 				pcgs_unblock();
18997c478bd9Sstevel@tonic-gate 				mutex_exit(&pcgs_lock);
19007c478bd9Sstevel@tonic-gate 			}
19017c478bd9Sstevel@tonic-gate 			if (cagelocked)
19027c478bd9Sstevel@tonic-gate 				mutex_exit(&pcgs_cagelock);
19037c478bd9Sstevel@tonic-gate 			return (pp);
19047c478bd9Sstevel@tonic-gate 		}
19057c478bd9Sstevel@tonic-gate 	}
19067c478bd9Sstevel@tonic-gate 	/*
19077c478bd9Sstevel@tonic-gate 	 * we go down holding the pcf locks.
19087c478bd9Sstevel@tonic-gate 	 */
19097c478bd9Sstevel@tonic-gate 	panic("no %spage found %d",
19107c478bd9Sstevel@tonic-gate 	    ((flags & PG_NORELOC) ? "non-reloc " : ""), count);
19117c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
19127c478bd9Sstevel@tonic-gate }
19137c478bd9Sstevel@tonic-gate 
19147c478bd9Sstevel@tonic-gate /*
19157c478bd9Sstevel@tonic-gate  * Create enough pages for "bytes" worth of data starting at
19167c478bd9Sstevel@tonic-gate  * "off" in "vp".
19177c478bd9Sstevel@tonic-gate  *
19187c478bd9Sstevel@tonic-gate  *	Where flag must be one of:
19197c478bd9Sstevel@tonic-gate  *
19207c478bd9Sstevel@tonic-gate  *		PG_EXCL:	Exclusive create (fail if any page already
19217c478bd9Sstevel@tonic-gate  *				exists in the page cache) which does not
19227c478bd9Sstevel@tonic-gate  *				wait for memory to become available.
19237c478bd9Sstevel@tonic-gate  *
19247c478bd9Sstevel@tonic-gate  *		PG_WAIT:	Non-exclusive create which can wait for
19257c478bd9Sstevel@tonic-gate  *				memory to become available.
19267c478bd9Sstevel@tonic-gate  *
19277c478bd9Sstevel@tonic-gate  *		PG_PHYSCONTIG:	Allocate physically contiguous pages.
19287c478bd9Sstevel@tonic-gate  *				(Not Supported)
19297c478bd9Sstevel@tonic-gate  *
19307c478bd9Sstevel@tonic-gate  * A doubly linked list of pages is returned to the caller.  Each page
19317c478bd9Sstevel@tonic-gate  * on the list has the "exclusive" (p_selock) lock and "iolock" (p_iolock)
19327c478bd9Sstevel@tonic-gate  * lock.
19337c478bd9Sstevel@tonic-gate  *
19347c478bd9Sstevel@tonic-gate  * Unable to change the parameters to page_create() in a minor release,
19357c478bd9Sstevel@tonic-gate  * we renamed page_create() to page_create_va(), changed all known calls
19367c478bd9Sstevel@tonic-gate  * from page_create() to page_create_va(), and created this wrapper.
19377c478bd9Sstevel@tonic-gate  *
19387c478bd9Sstevel@tonic-gate  * Upon a major release, we should break compatibility by deleting this
19397c478bd9Sstevel@tonic-gate  * wrapper, and replacing all the strings "page_create_va", with "page_create".
19407c478bd9Sstevel@tonic-gate  *
19417c478bd9Sstevel@tonic-gate  * NOTE: There is a copy of this interface as page_create_io() in
19427c478bd9Sstevel@tonic-gate  *	 i86/vm/vm_machdep.c. Any bugs fixed here should be applied
19437c478bd9Sstevel@tonic-gate  *	 there.
19447c478bd9Sstevel@tonic-gate  */
19457c478bd9Sstevel@tonic-gate page_t *
19467c478bd9Sstevel@tonic-gate page_create(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags)
19477c478bd9Sstevel@tonic-gate {
19487c478bd9Sstevel@tonic-gate 	caddr_t random_vaddr;
19497c478bd9Sstevel@tonic-gate 	struct seg kseg;
19507c478bd9Sstevel@tonic-gate 
19517c478bd9Sstevel@tonic-gate #ifdef DEBUG
19527c478bd9Sstevel@tonic-gate 	cmn_err(CE_WARN, "Using deprecated interface page_create: caller %p",
19537c478bd9Sstevel@tonic-gate 	    (void *)caller());
19547c478bd9Sstevel@tonic-gate #endif
19557c478bd9Sstevel@tonic-gate 
19567c478bd9Sstevel@tonic-gate 	random_vaddr = (caddr_t)(((uintptr_t)vp >> 7) ^
19577c478bd9Sstevel@tonic-gate 	    (uintptr_t)(off >> PAGESHIFT));
19587c478bd9Sstevel@tonic-gate 	kseg.s_as = &kas;
19597c478bd9Sstevel@tonic-gate 
19607c478bd9Sstevel@tonic-gate 	return (page_create_va(vp, off, bytes, flags, &kseg, random_vaddr));
19617c478bd9Sstevel@tonic-gate }
19627c478bd9Sstevel@tonic-gate 
19637c478bd9Sstevel@tonic-gate #ifdef DEBUG
19647c478bd9Sstevel@tonic-gate uint32_t pg_alloc_pgs_mtbf = 0;
19657c478bd9Sstevel@tonic-gate #endif
19667c478bd9Sstevel@tonic-gate 
19677c478bd9Sstevel@tonic-gate /*
19687c478bd9Sstevel@tonic-gate  * Used for large page support. It will attempt to allocate
19697c478bd9Sstevel@tonic-gate  * a large page(s) off the freelist.
19707c478bd9Sstevel@tonic-gate  *
19717c478bd9Sstevel@tonic-gate  * Returns non zero on failure.
19727c478bd9Sstevel@tonic-gate  */
19737c478bd9Sstevel@tonic-gate int
1974e44bd21cSsusans page_alloc_pages(struct vnode *vp, struct seg *seg, caddr_t addr,
1975e44bd21cSsusans     page_t **basepp, page_t *ppa[], uint_t szc, int anypgsz)
19767c478bd9Sstevel@tonic-gate {
19777c478bd9Sstevel@tonic-gate 	pgcnt_t		npgs, curnpgs, totpgs;
19787c478bd9Sstevel@tonic-gate 	size_t		pgsz;
19797c478bd9Sstevel@tonic-gate 	page_t		*pplist = NULL, *pp;
19807c478bd9Sstevel@tonic-gate 	int		err = 0;
19817c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
19827c478bd9Sstevel@tonic-gate 
19837c478bd9Sstevel@tonic-gate 	ASSERT(szc != 0 && szc <= (page_num_pagesizes() - 1));
19847c478bd9Sstevel@tonic-gate 
19857c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(alloc_pages[0]);
19867c478bd9Sstevel@tonic-gate 
19877c478bd9Sstevel@tonic-gate #ifdef DEBUG
19887c478bd9Sstevel@tonic-gate 	if (pg_alloc_pgs_mtbf && !(gethrtime() % pg_alloc_pgs_mtbf)) {
19897c478bd9Sstevel@tonic-gate 		return (ENOMEM);
19907c478bd9Sstevel@tonic-gate 	}
19917c478bd9Sstevel@tonic-gate #endif
19927c478bd9Sstevel@tonic-gate 
19937c478bd9Sstevel@tonic-gate 	pgsz = page_get_pagesize(szc);
19947c478bd9Sstevel@tonic-gate 	totpgs = curnpgs = npgs = pgsz >> PAGESHIFT;
19957c478bd9Sstevel@tonic-gate 
19967c478bd9Sstevel@tonic-gate 	ASSERT(((uintptr_t)addr & (pgsz - 1)) == 0);
19977c478bd9Sstevel@tonic-gate 	/*
19987c478bd9Sstevel@tonic-gate 	 * One must be NULL but not both.
19997c478bd9Sstevel@tonic-gate 	 * And one must be non NULL but not both.
20007c478bd9Sstevel@tonic-gate 	 */
20017c478bd9Sstevel@tonic-gate 	ASSERT(basepp != NULL || ppa != NULL);
20027c478bd9Sstevel@tonic-gate 	ASSERT(basepp == NULL || ppa == NULL);
20037c478bd9Sstevel@tonic-gate 
20047c478bd9Sstevel@tonic-gate 	(void) page_create_wait(npgs, PG_WAIT);
20057c478bd9Sstevel@tonic-gate 
20067c478bd9Sstevel@tonic-gate 	while (npgs && szc) {
20077c478bd9Sstevel@tonic-gate 		lgrp = lgrp_mem_choose(seg, addr, pgsz);
2008e44bd21cSsusans 		pp = page_get_freelist(vp, 0, seg, addr, pgsz, 0, lgrp);
20097c478bd9Sstevel@tonic-gate 		if (pp != NULL) {
20107c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(alloc_pages[1]);
20117c478bd9Sstevel@tonic-gate 			page_list_concat(&pplist, &pp);
20127c478bd9Sstevel@tonic-gate 			ASSERT(npgs >= curnpgs);
20137c478bd9Sstevel@tonic-gate 			npgs -= curnpgs;
20147c478bd9Sstevel@tonic-gate 		} else if (anypgsz) {
20157c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(alloc_pages[2]);
20167c478bd9Sstevel@tonic-gate 			szc--;
20177c478bd9Sstevel@tonic-gate 			pgsz = page_get_pagesize(szc);
20187c478bd9Sstevel@tonic-gate 			curnpgs = pgsz >> PAGESHIFT;
20197c478bd9Sstevel@tonic-gate 		} else {
20207c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(alloc_pages[3]);
20217c478bd9Sstevel@tonic-gate 			ASSERT(npgs == totpgs);
20227c478bd9Sstevel@tonic-gate 			page_create_putback(npgs);
20237c478bd9Sstevel@tonic-gate 			return (ENOMEM);
20247c478bd9Sstevel@tonic-gate 		}
20257c478bd9Sstevel@tonic-gate 	}
20267c478bd9Sstevel@tonic-gate 	if (szc == 0) {
20277c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(alloc_pages[4]);
20287c478bd9Sstevel@tonic-gate 		ASSERT(npgs != 0);
20297c478bd9Sstevel@tonic-gate 		page_create_putback(npgs);
20307c478bd9Sstevel@tonic-gate 		err = ENOMEM;
20317c478bd9Sstevel@tonic-gate 	} else if (basepp != NULL) {
20327c478bd9Sstevel@tonic-gate 		ASSERT(npgs == 0);
20337c478bd9Sstevel@tonic-gate 		ASSERT(ppa == NULL);
20347c478bd9Sstevel@tonic-gate 		*basepp = pplist;
20357c478bd9Sstevel@tonic-gate 	}
20367c478bd9Sstevel@tonic-gate 
20377c478bd9Sstevel@tonic-gate 	npgs = totpgs - npgs;
20387c478bd9Sstevel@tonic-gate 	pp = pplist;
20397c478bd9Sstevel@tonic-gate 
20407c478bd9Sstevel@tonic-gate 	/*
20417c478bd9Sstevel@tonic-gate 	 * Clear the free and age bits. Also if we were passed in a ppa then
20427c478bd9Sstevel@tonic-gate 	 * fill it in with all the constituent pages from the large page. But
20437c478bd9Sstevel@tonic-gate 	 * if we failed to allocate all the pages just free what we got.
20447c478bd9Sstevel@tonic-gate 	 */
20457c478bd9Sstevel@tonic-gate 	while (npgs != 0) {
20467c478bd9Sstevel@tonic-gate 		ASSERT(PP_ISFREE(pp));
20477c478bd9Sstevel@tonic-gate 		ASSERT(PP_ISAGED(pp));
20487c478bd9Sstevel@tonic-gate 		if (ppa != NULL || err != 0) {
20497c478bd9Sstevel@tonic-gate 			if (err == 0) {
20507c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(alloc_pages[5]);
20517c478bd9Sstevel@tonic-gate 				PP_CLRFREE(pp);
20527c478bd9Sstevel@tonic-gate 				PP_CLRAGED(pp);
20537c478bd9Sstevel@tonic-gate 				page_sub(&pplist, pp);
20547c478bd9Sstevel@tonic-gate 				*ppa++ = pp;
20557c478bd9Sstevel@tonic-gate 				npgs--;
20567c478bd9Sstevel@tonic-gate 			} else {
20577c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(alloc_pages[6]);
20587c478bd9Sstevel@tonic-gate 				ASSERT(pp->p_szc != 0);
20597c478bd9Sstevel@tonic-gate 				curnpgs = page_get_pagecnt(pp->p_szc);
20607c478bd9Sstevel@tonic-gate 				page_list_break(&pp, &pplist, curnpgs);
20617c478bd9Sstevel@tonic-gate 				page_list_add_pages(pp, 0);
20627c478bd9Sstevel@tonic-gate 				page_create_putback(curnpgs);
20637c478bd9Sstevel@tonic-gate 				ASSERT(npgs >= curnpgs);
20647c478bd9Sstevel@tonic-gate 				npgs -= curnpgs;
20657c478bd9Sstevel@tonic-gate 			}
20667c478bd9Sstevel@tonic-gate 			pp = pplist;
20677c478bd9Sstevel@tonic-gate 		} else {
20687c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(alloc_pages[7]);
20697c478bd9Sstevel@tonic-gate 			PP_CLRFREE(pp);
20707c478bd9Sstevel@tonic-gate 			PP_CLRAGED(pp);
20717c478bd9Sstevel@tonic-gate 			pp = pp->p_next;
20727c478bd9Sstevel@tonic-gate 			npgs--;
20737c478bd9Sstevel@tonic-gate 		}
20747c478bd9Sstevel@tonic-gate 	}
20757c478bd9Sstevel@tonic-gate 	return (err);
20767c478bd9Sstevel@tonic-gate }
20777c478bd9Sstevel@tonic-gate 
20787c478bd9Sstevel@tonic-gate /*
20797c478bd9Sstevel@tonic-gate  * Get a single large page off of the freelists, and set it up for use.
20807c478bd9Sstevel@tonic-gate  * Number of bytes requested must be a supported page size.
20817c478bd9Sstevel@tonic-gate  *
20827c478bd9Sstevel@tonic-gate  * Note that this call may fail even if there is sufficient
20837c478bd9Sstevel@tonic-gate  * memory available or PG_WAIT is set, so the caller must
20847c478bd9Sstevel@tonic-gate  * be willing to fallback on page_create_va(), block and retry,
20857c478bd9Sstevel@tonic-gate  * or fail the requester.
20867c478bd9Sstevel@tonic-gate  */
20877c478bd9Sstevel@tonic-gate page_t *
20887c478bd9Sstevel@tonic-gate page_create_va_large(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
20897c478bd9Sstevel@tonic-gate     struct seg *seg, caddr_t vaddr, void *arg)
20907c478bd9Sstevel@tonic-gate {
20917c478bd9Sstevel@tonic-gate 	pgcnt_t		npages, pcftotal;
20927c478bd9Sstevel@tonic-gate 	page_t		*pp;
20937c478bd9Sstevel@tonic-gate 	page_t		*rootpp;
20947c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
20957c478bd9Sstevel@tonic-gate 	uint_t		enough;
20967c478bd9Sstevel@tonic-gate 	uint_t		pcf_index;
20977c478bd9Sstevel@tonic-gate 	uint_t		i;
20987c478bd9Sstevel@tonic-gate 	struct pcf	*p;
20997c478bd9Sstevel@tonic-gate 	struct pcf	*q;
21007c478bd9Sstevel@tonic-gate 	lgrp_id_t	*lgrpid = (lgrp_id_t *)arg;
21017c478bd9Sstevel@tonic-gate 
21027c478bd9Sstevel@tonic-gate 	ASSERT(vp != NULL);
21037c478bd9Sstevel@tonic-gate 
21047c478bd9Sstevel@tonic-gate 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
21057c478bd9Sstevel@tonic-gate 		    PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0);
21067c478bd9Sstevel@tonic-gate 	/* but no others */
21077c478bd9Sstevel@tonic-gate 
21087c478bd9Sstevel@tonic-gate 	ASSERT((flags & PG_EXCL) == PG_EXCL);
21097c478bd9Sstevel@tonic-gate 
21107c478bd9Sstevel@tonic-gate 	npages = btop(bytes);
21117c478bd9Sstevel@tonic-gate 
21127c478bd9Sstevel@tonic-gate 	if (!kcage_on || panicstr) {
21137c478bd9Sstevel@tonic-gate 		/*
21147c478bd9Sstevel@tonic-gate 		 * Cage is OFF, or we are single threaded in
21157c478bd9Sstevel@tonic-gate 		 * panic, so make everything a RELOC request.
21167c478bd9Sstevel@tonic-gate 		 */
21177c478bd9Sstevel@tonic-gate 		flags &= ~PG_NORELOC;
21187c478bd9Sstevel@tonic-gate 	}
21197c478bd9Sstevel@tonic-gate 
21207c478bd9Sstevel@tonic-gate 	/*
21217c478bd9Sstevel@tonic-gate 	 * Make sure there's adequate physical memory available.
21227c478bd9Sstevel@tonic-gate 	 * Note: PG_WAIT is ignored here.
21237c478bd9Sstevel@tonic-gate 	 */
21247c478bd9Sstevel@tonic-gate 	if (freemem <= throttlefree + npages) {
21257c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_create_large_cnt[1]);
21267c478bd9Sstevel@tonic-gate 		return (NULL);
21277c478bd9Sstevel@tonic-gate 	}
21287c478bd9Sstevel@tonic-gate 
21297c478bd9Sstevel@tonic-gate 	/*
21307c478bd9Sstevel@tonic-gate 	 * If cage is on, dampen draw from cage when available
21317c478bd9Sstevel@tonic-gate 	 * cage space is low.
21327c478bd9Sstevel@tonic-gate 	 */
21337c478bd9Sstevel@tonic-gate 	if ((flags & (PG_NORELOC | PG_WAIT)) ==  (PG_NORELOC | PG_WAIT) &&
21347c478bd9Sstevel@tonic-gate 	    kcage_freemem < kcage_throttlefree + npages) {
21357c478bd9Sstevel@tonic-gate 
21367c478bd9Sstevel@tonic-gate 		/*
21377c478bd9Sstevel@tonic-gate 		 * The cage is on, the caller wants PG_NORELOC
21387c478bd9Sstevel@tonic-gate 		 * pages and available cage memory is very low.
21397c478bd9Sstevel@tonic-gate 		 * Call kcage_create_throttle() to attempt to
21407c478bd9Sstevel@tonic-gate 		 * control demand on the cage.
21417c478bd9Sstevel@tonic-gate 		 */
21427c478bd9Sstevel@tonic-gate 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE) {
21437c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_large_cnt[2]);
21447c478bd9Sstevel@tonic-gate 			return (NULL);
21457c478bd9Sstevel@tonic-gate 		}
21467c478bd9Sstevel@tonic-gate 	}
21477c478bd9Sstevel@tonic-gate 
21487c478bd9Sstevel@tonic-gate 	enough = 0;
21497c478bd9Sstevel@tonic-gate 	pcf_index = PCF_INDEX();
21507c478bd9Sstevel@tonic-gate 	p = &pcf[pcf_index];
21517c478bd9Sstevel@tonic-gate 	q = &pcf[PCF_FANOUT];
21527c478bd9Sstevel@tonic-gate 	for (pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
21537c478bd9Sstevel@tonic-gate 		if (p->pcf_count > npages) {
21547c478bd9Sstevel@tonic-gate 			/*
21557c478bd9Sstevel@tonic-gate 			 * a good one to try.
21567c478bd9Sstevel@tonic-gate 			 */
21577c478bd9Sstevel@tonic-gate 			mutex_enter(&p->pcf_lock);
21587c478bd9Sstevel@tonic-gate 			if (p->pcf_count > npages) {
21597c478bd9Sstevel@tonic-gate 				p->pcf_count -= (uint_t)npages;
21607c478bd9Sstevel@tonic-gate 				/*
21617c478bd9Sstevel@tonic-gate 				 * freemem is not protected by any lock.
21627c478bd9Sstevel@tonic-gate 				 * Thus, we cannot have any assertion
21637c478bd9Sstevel@tonic-gate 				 * containing freemem here.
21647c478bd9Sstevel@tonic-gate 				 */
21657c478bd9Sstevel@tonic-gate 				freemem -= npages;
21667c478bd9Sstevel@tonic-gate 				enough = 1;
21677c478bd9Sstevel@tonic-gate 				mutex_exit(&p->pcf_lock);
21687c478bd9Sstevel@tonic-gate 				break;
21697c478bd9Sstevel@tonic-gate 			}
21707c478bd9Sstevel@tonic-gate 			mutex_exit(&p->pcf_lock);
21717c478bd9Sstevel@tonic-gate 		}
21727c478bd9Sstevel@tonic-gate 		pcftotal += p->pcf_count;
21737c478bd9Sstevel@tonic-gate 		p++;
21747c478bd9Sstevel@tonic-gate 		if (p >= q) {
21757c478bd9Sstevel@tonic-gate 			p = pcf;
21767c478bd9Sstevel@tonic-gate 		}
21777c478bd9Sstevel@tonic-gate 	}
21787c478bd9Sstevel@tonic-gate 
21797c478bd9Sstevel@tonic-gate 	if (!enough) {
21807c478bd9Sstevel@tonic-gate 		/* If there isn't enough memory available, give up. */
21817c478bd9Sstevel@tonic-gate 		if (pcftotal < npages) {
21827c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_large_cnt[3]);
21837c478bd9Sstevel@tonic-gate 			return (NULL);
21847c478bd9Sstevel@tonic-gate 		}
21857c478bd9Sstevel@tonic-gate 
21867c478bd9Sstevel@tonic-gate 		/* try to collect pages from several pcf bins */
21877c478bd9Sstevel@tonic-gate 		for (p = pcf, pcftotal = 0, i = 0; i < PCF_FANOUT; i++) {
21887c478bd9Sstevel@tonic-gate 			mutex_enter(&p->pcf_lock);
21897c478bd9Sstevel@tonic-gate 			pcftotal += p->pcf_count;
21907c478bd9Sstevel@tonic-gate 			if (pcftotal >= npages) {
21917c478bd9Sstevel@tonic-gate 				/*
21927c478bd9Sstevel@tonic-gate 				 * Wow!  There are enough pages laying around
21937c478bd9Sstevel@tonic-gate 				 * to satisfy the request.  Do the accounting,
21947c478bd9Sstevel@tonic-gate 				 * drop the locks we acquired, and go back.
21957c478bd9Sstevel@tonic-gate 				 *
21967c478bd9Sstevel@tonic-gate 				 * freemem is not protected by any lock. So,
21977c478bd9Sstevel@tonic-gate 				 * we cannot have any assertion containing
21987c478bd9Sstevel@tonic-gate 				 * freemem.
21997c478bd9Sstevel@tonic-gate 				 */
22007c478bd9Sstevel@tonic-gate 				pgcnt_t	tpages = npages;
22017c478bd9Sstevel@tonic-gate 				freemem -= npages;
22027c478bd9Sstevel@tonic-gate 				while (p >= pcf) {
22037c478bd9Sstevel@tonic-gate 					if (p->pcf_count <= tpages) {
22047c478bd9Sstevel@tonic-gate 						tpages -= p->pcf_count;
22057c478bd9Sstevel@tonic-gate 						p->pcf_count = 0;
22067c478bd9Sstevel@tonic-gate 					} else {
22077c478bd9Sstevel@tonic-gate 						p->pcf_count -= (uint_t)tpages;
22087c478bd9Sstevel@tonic-gate 						tpages = 0;
22097c478bd9Sstevel@tonic-gate 					}
22107c478bd9Sstevel@tonic-gate 					mutex_exit(&p->pcf_lock);
22117c478bd9Sstevel@tonic-gate 					p--;
22127c478bd9Sstevel@tonic-gate 				}
22137c478bd9Sstevel@tonic-gate 				ASSERT(tpages == 0);
22147c478bd9Sstevel@tonic-gate 				break;
22157c478bd9Sstevel@tonic-gate 			}
22167c478bd9Sstevel@tonic-gate 			p++;
22177c478bd9Sstevel@tonic-gate 		}
22187c478bd9Sstevel@tonic-gate 		if (i == PCF_FANOUT) {
22197c478bd9Sstevel@tonic-gate 			/* failed to collect pages - release the locks */
22207c478bd9Sstevel@tonic-gate 			while (--p >= pcf) {
22217c478bd9Sstevel@tonic-gate 				mutex_exit(&p->pcf_lock);
22227c478bd9Sstevel@tonic-gate 			}
22237c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_large_cnt[4]);
22247c478bd9Sstevel@tonic-gate 			return (NULL);
22257c478bd9Sstevel@tonic-gate 		}
22267c478bd9Sstevel@tonic-gate 	}
22277c478bd9Sstevel@tonic-gate 
22287c478bd9Sstevel@tonic-gate 	/*
22297c478bd9Sstevel@tonic-gate 	 * This is where this function behaves fundamentally differently
22307c478bd9Sstevel@tonic-gate 	 * than page_create_va(); since we're intending to map the page
22317c478bd9Sstevel@tonic-gate 	 * with a single TTE, we have to get it as a physically contiguous
22327c478bd9Sstevel@tonic-gate 	 * hardware pagesize chunk.  If we can't, we fail.
22337c478bd9Sstevel@tonic-gate 	 */
22347c478bd9Sstevel@tonic-gate 	if (lgrpid != NULL && *lgrpid >= 0 && *lgrpid <= lgrp_alloc_max &&
22357c478bd9Sstevel@tonic-gate 		LGRP_EXISTS(lgrp_table[*lgrpid]))
22367c478bd9Sstevel@tonic-gate 		lgrp = lgrp_table[*lgrpid];
22377c478bd9Sstevel@tonic-gate 	else
22387c478bd9Sstevel@tonic-gate 		lgrp = lgrp_mem_choose(seg, vaddr, bytes);
22397c478bd9Sstevel@tonic-gate 
22407c478bd9Sstevel@tonic-gate 	if ((rootpp = page_get_freelist(&kvp, off, seg, vaddr,
22417c478bd9Sstevel@tonic-gate 	    bytes, flags & ~PG_MATCH_COLOR, lgrp)) == NULL) {
22427c478bd9Sstevel@tonic-gate 		page_create_putback(npages);
22437c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_create_large_cnt[5]);
22447c478bd9Sstevel@tonic-gate 		return (NULL);
22457c478bd9Sstevel@tonic-gate 	}
22467c478bd9Sstevel@tonic-gate 
22477c478bd9Sstevel@tonic-gate 	/*
22487c478bd9Sstevel@tonic-gate 	 * if we got the page with the wrong mtype give it back this is a
22497c478bd9Sstevel@tonic-gate 	 * workaround for CR 6249718. When CR 6249718 is fixed we never get
22507c478bd9Sstevel@tonic-gate 	 * inside "if" and the workaround becomes just a nop
22517c478bd9Sstevel@tonic-gate 	 */
22527c478bd9Sstevel@tonic-gate 	if (kcage_on && (flags & PG_NORELOC) && !PP_ISNORELOC(rootpp)) {
22537c478bd9Sstevel@tonic-gate 		page_list_add_pages(rootpp, 0);
22547c478bd9Sstevel@tonic-gate 		page_create_putback(npages);
22557c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_create_large_cnt[6]);
22567c478bd9Sstevel@tonic-gate 		return (NULL);
22577c478bd9Sstevel@tonic-gate 	}
22587c478bd9Sstevel@tonic-gate 
22597c478bd9Sstevel@tonic-gate 	/*
22607c478bd9Sstevel@tonic-gate 	 * If satisfying this request has left us with too little
22617c478bd9Sstevel@tonic-gate 	 * memory, start the wheels turning to get some back.  The
22627c478bd9Sstevel@tonic-gate 	 * first clause of the test prevents waking up the pageout
22637c478bd9Sstevel@tonic-gate 	 * daemon in situations where it would decide that there's
22647c478bd9Sstevel@tonic-gate 	 * nothing to do.
22657c478bd9Sstevel@tonic-gate 	 */
22667c478bd9Sstevel@tonic-gate 	if (nscan < desscan && freemem < minfree) {
22677c478bd9Sstevel@tonic-gate 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
22687c478bd9Sstevel@tonic-gate 		    "pageout_cv_signal:freemem %ld", freemem);
22697c478bd9Sstevel@tonic-gate 		cv_signal(&proc_pageout->p_cv);
22707c478bd9Sstevel@tonic-gate 	}
22717c478bd9Sstevel@tonic-gate 
22727c478bd9Sstevel@tonic-gate 	pp = rootpp;
22737c478bd9Sstevel@tonic-gate 	while (npages--) {
22747c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_EXCL(pp));
22757c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_vnode == NULL);
22767c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_is_mapped(pp));
22777c478bd9Sstevel@tonic-gate 		PP_CLRFREE(pp);
22787c478bd9Sstevel@tonic-gate 		PP_CLRAGED(pp);
22797c478bd9Sstevel@tonic-gate 		if (!page_hashin(pp, vp, off, NULL))
22807c478bd9Sstevel@tonic-gate 			panic("page_create_large: hashin failed: page %p",
22817c478bd9Sstevel@tonic-gate 			    (void *)pp);
22827c478bd9Sstevel@tonic-gate 		page_io_lock(pp);
22837c478bd9Sstevel@tonic-gate 		off += PAGESIZE;
22847c478bd9Sstevel@tonic-gate 		pp = pp->p_next;
22857c478bd9Sstevel@tonic-gate 	}
22867c478bd9Sstevel@tonic-gate 
22877c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_create_large_cnt[0]);
22887c478bd9Sstevel@tonic-gate 	return (rootpp);
22897c478bd9Sstevel@tonic-gate }
22907c478bd9Sstevel@tonic-gate 
22917c478bd9Sstevel@tonic-gate page_t *
22927c478bd9Sstevel@tonic-gate page_create_va(vnode_t *vp, u_offset_t off, size_t bytes, uint_t flags,
22937c478bd9Sstevel@tonic-gate     struct seg *seg, caddr_t vaddr)
22947c478bd9Sstevel@tonic-gate {
22957c478bd9Sstevel@tonic-gate 	page_t		*plist = NULL;
22967c478bd9Sstevel@tonic-gate 	pgcnt_t		npages;
22977c478bd9Sstevel@tonic-gate 	pgcnt_t		found_on_free = 0;
22987c478bd9Sstevel@tonic-gate 	pgcnt_t		pages_req;
22997c478bd9Sstevel@tonic-gate 	page_t		*npp = NULL;
23007c478bd9Sstevel@tonic-gate 	uint_t		enough;
23017c478bd9Sstevel@tonic-gate 	uint_t		i;
23027c478bd9Sstevel@tonic-gate 	uint_t		pcf_index;
23037c478bd9Sstevel@tonic-gate 	struct pcf	*p;
23047c478bd9Sstevel@tonic-gate 	struct pcf	*q;
23057c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
23067c478bd9Sstevel@tonic-gate 
23077c478bd9Sstevel@tonic-gate 	TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_START,
23087c478bd9Sstevel@tonic-gate 		"page_create_start:vp %p off %llx bytes %lu flags %x",
23097c478bd9Sstevel@tonic-gate 		vp, off, bytes, flags);
23107c478bd9Sstevel@tonic-gate 
23117c478bd9Sstevel@tonic-gate 	ASSERT(bytes != 0 && vp != NULL);
23127c478bd9Sstevel@tonic-gate 
23137c478bd9Sstevel@tonic-gate 	if ((flags & PG_EXCL) == 0 && (flags & PG_WAIT) == 0) {
23147c478bd9Sstevel@tonic-gate 		panic("page_create: invalid flags");
23157c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
23167c478bd9Sstevel@tonic-gate 	}
23177c478bd9Sstevel@tonic-gate 	ASSERT((flags & ~(PG_EXCL | PG_WAIT |
23187c478bd9Sstevel@tonic-gate 	    PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == 0);
23197c478bd9Sstevel@tonic-gate 	    /* but no others */
23207c478bd9Sstevel@tonic-gate 
23217c478bd9Sstevel@tonic-gate 	pages_req = npages = btopr(bytes);
23227c478bd9Sstevel@tonic-gate 	/*
23237c478bd9Sstevel@tonic-gate 	 * Try to see whether request is too large to *ever* be
23247c478bd9Sstevel@tonic-gate 	 * satisfied, in order to prevent deadlock.  We arbitrarily
23257c478bd9Sstevel@tonic-gate 	 * decide to limit maximum size requests to max_page_get.
23267c478bd9Sstevel@tonic-gate 	 */
23277c478bd9Sstevel@tonic-gate 	if (npages >= max_page_get) {
23287c478bd9Sstevel@tonic-gate 		if ((flags & PG_WAIT) == 0) {
23297c478bd9Sstevel@tonic-gate 			TRACE_4(TR_FAC_VM, TR_PAGE_CREATE_TOOBIG,
23307c478bd9Sstevel@tonic-gate 			    "page_create_toobig:vp %p off %llx npages "
23317c478bd9Sstevel@tonic-gate 			    "%lu max_page_get %lu",
23327c478bd9Sstevel@tonic-gate 			    vp, off, npages, max_page_get);
23337c478bd9Sstevel@tonic-gate 			return (NULL);
23347c478bd9Sstevel@tonic-gate 		} else {
23357c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN,
23367c478bd9Sstevel@tonic-gate 			    "Request for too much kernel memory "
23377c478bd9Sstevel@tonic-gate 			    "(%lu bytes), will hang forever", bytes);
23387c478bd9Sstevel@tonic-gate 			for (;;)
23397c478bd9Sstevel@tonic-gate 				delay(1000000000);
23407c478bd9Sstevel@tonic-gate 		}
23417c478bd9Sstevel@tonic-gate 	}
23427c478bd9Sstevel@tonic-gate 
23437c478bd9Sstevel@tonic-gate 	if (!kcage_on || panicstr) {
23447c478bd9Sstevel@tonic-gate 		/*
23457c478bd9Sstevel@tonic-gate 		 * Cage is OFF, or we are single threaded in
23467c478bd9Sstevel@tonic-gate 		 * panic, so make everything a RELOC request.
23477c478bd9Sstevel@tonic-gate 		 */
23487c478bd9Sstevel@tonic-gate 		flags &= ~PG_NORELOC;
23497c478bd9Sstevel@tonic-gate 	}
23507c478bd9Sstevel@tonic-gate 
23517c478bd9Sstevel@tonic-gate 	if (freemem <= throttlefree + npages)
23527c478bd9Sstevel@tonic-gate 		if (!page_create_throttle(npages, flags))
23537c478bd9Sstevel@tonic-gate 			return (NULL);
23547c478bd9Sstevel@tonic-gate 
23557c478bd9Sstevel@tonic-gate 	/*
23567c478bd9Sstevel@tonic-gate 	 * If cage is on, dampen draw from cage when available
23577c478bd9Sstevel@tonic-gate 	 * cage space is low.
23587c478bd9Sstevel@tonic-gate 	 */
23597c478bd9Sstevel@tonic-gate 	if ((flags & PG_NORELOC) &&
23607c478bd9Sstevel@tonic-gate 		kcage_freemem < kcage_throttlefree + npages) {
23617c478bd9Sstevel@tonic-gate 
23627c478bd9Sstevel@tonic-gate 		/*
23637c478bd9Sstevel@tonic-gate 		 * The cage is on, the caller wants PG_NORELOC
23647c478bd9Sstevel@tonic-gate 		 * pages and available cage memory is very low.
23657c478bd9Sstevel@tonic-gate 		 * Call kcage_create_throttle() to attempt to
23667c478bd9Sstevel@tonic-gate 		 * control demand on the cage.
23677c478bd9Sstevel@tonic-gate 		 */
23687c478bd9Sstevel@tonic-gate 		if (kcage_create_throttle(npages, flags) == KCT_FAILURE)
23697c478bd9Sstevel@tonic-gate 			return (NULL);
23707c478bd9Sstevel@tonic-gate 	}
23717c478bd9Sstevel@tonic-gate 
23727c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_create_cnt[0]);
23737c478bd9Sstevel@tonic-gate 
23747c478bd9Sstevel@tonic-gate 	enough = 0;
23757c478bd9Sstevel@tonic-gate 	pcf_index = PCF_INDEX();
23767c478bd9Sstevel@tonic-gate 
23777c478bd9Sstevel@tonic-gate 	p = &pcf[pcf_index];
23787c478bd9Sstevel@tonic-gate 	q = &pcf[PCF_FANOUT];
23797c478bd9Sstevel@tonic-gate 	for (i = 0; i < PCF_FANOUT; i++) {
23807c478bd9Sstevel@tonic-gate 		if (p->pcf_count > npages) {
23817c478bd9Sstevel@tonic-gate 			/*
23827c478bd9Sstevel@tonic-gate 			 * a good one to try.
23837c478bd9Sstevel@tonic-gate 			 */
23847c478bd9Sstevel@tonic-gate 			mutex_enter(&p->pcf_lock);
23857c478bd9Sstevel@tonic-gate 			if (p->pcf_count > npages) {
23867c478bd9Sstevel@tonic-gate 				p->pcf_count -= (uint_t)npages;
23877c478bd9Sstevel@tonic-gate 				/*
23887c478bd9Sstevel@tonic-gate 				 * freemem is not protected by any lock.
23897c478bd9Sstevel@tonic-gate 				 * Thus, we cannot have any assertion
23907c478bd9Sstevel@tonic-gate 				 * containing freemem here.
23917c478bd9Sstevel@tonic-gate 				 */
23927c478bd9Sstevel@tonic-gate 				freemem -= npages;
23937c478bd9Sstevel@tonic-gate 				enough = 1;
23947c478bd9Sstevel@tonic-gate 				mutex_exit(&p->pcf_lock);
23957c478bd9Sstevel@tonic-gate 				break;
23967c478bd9Sstevel@tonic-gate 			}
23977c478bd9Sstevel@tonic-gate 			mutex_exit(&p->pcf_lock);
23987c478bd9Sstevel@tonic-gate 		}
23997c478bd9Sstevel@tonic-gate 		p++;
24007c478bd9Sstevel@tonic-gate 		if (p >= q) {
24017c478bd9Sstevel@tonic-gate 			p = pcf;
24027c478bd9Sstevel@tonic-gate 		}
24037c478bd9Sstevel@tonic-gate 	}
24047c478bd9Sstevel@tonic-gate 
24057c478bd9Sstevel@tonic-gate 	if (!enough) {
24067c478bd9Sstevel@tonic-gate 		/*
24077c478bd9Sstevel@tonic-gate 		 * Have to look harder.  If npages is greater than
24087c478bd9Sstevel@tonic-gate 		 * one, then we might have to coalecse the counters.
24097c478bd9Sstevel@tonic-gate 		 *
24107c478bd9Sstevel@tonic-gate 		 * Go wait.  We come back having accounted
24117c478bd9Sstevel@tonic-gate 		 * for the memory.
24127c478bd9Sstevel@tonic-gate 		 */
24137c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_create_cnt[1]);
24147c478bd9Sstevel@tonic-gate 		if (!page_create_wait(npages, flags)) {
24157c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_cnt[2]);
24167c478bd9Sstevel@tonic-gate 			return (NULL);
24177c478bd9Sstevel@tonic-gate 		}
24187c478bd9Sstevel@tonic-gate 	}
24197c478bd9Sstevel@tonic-gate 
24207c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_VM, TR_PAGE_CREATE_SUCCESS,
24217c478bd9Sstevel@tonic-gate 		"page_create_success:vp %p off %llx", vp, off);
24227c478bd9Sstevel@tonic-gate 
24237c478bd9Sstevel@tonic-gate 	/*
24247c478bd9Sstevel@tonic-gate 	 * If satisfying this request has left us with too little
24257c478bd9Sstevel@tonic-gate 	 * memory, start the wheels turning to get some back.  The
24267c478bd9Sstevel@tonic-gate 	 * first clause of the test prevents waking up the pageout
24277c478bd9Sstevel@tonic-gate 	 * daemon in situations where it would decide that there's
24287c478bd9Sstevel@tonic-gate 	 * nothing to do.
24297c478bd9Sstevel@tonic-gate 	 */
24307c478bd9Sstevel@tonic-gate 	if (nscan < desscan && freemem < minfree) {
24317c478bd9Sstevel@tonic-gate 		TRACE_1(TR_FAC_VM, TR_PAGEOUT_CV_SIGNAL,
24327c478bd9Sstevel@tonic-gate 			"pageout_cv_signal:freemem %ld", freemem);
24337c478bd9Sstevel@tonic-gate 		cv_signal(&proc_pageout->p_cv);
24347c478bd9Sstevel@tonic-gate 	}
24357c478bd9Sstevel@tonic-gate 
24367c478bd9Sstevel@tonic-gate 	/*
24377c478bd9Sstevel@tonic-gate 	 * Loop around collecting the requested number of pages.
24387c478bd9Sstevel@tonic-gate 	 * Most of the time, we have to `create' a new page. With
24397c478bd9Sstevel@tonic-gate 	 * this in mind, pull the page off the free list before
24407c478bd9Sstevel@tonic-gate 	 * getting the hash lock.  This will minimize the hash
24417c478bd9Sstevel@tonic-gate 	 * lock hold time, nesting, and the like.  If it turns
24427c478bd9Sstevel@tonic-gate 	 * out we don't need the page, we put it back at the end.
24437c478bd9Sstevel@tonic-gate 	 */
24447c478bd9Sstevel@tonic-gate 	while (npages--) {
24457c478bd9Sstevel@tonic-gate 		page_t		*pp;
24467c478bd9Sstevel@tonic-gate 		kmutex_t	*phm = NULL;
24477c478bd9Sstevel@tonic-gate 		ulong_t		index;
24487c478bd9Sstevel@tonic-gate 
24497c478bd9Sstevel@tonic-gate 		index = PAGE_HASH_FUNC(vp, off);
24507c478bd9Sstevel@tonic-gate top:
24517c478bd9Sstevel@tonic-gate 		ASSERT(phm == NULL);
24527c478bd9Sstevel@tonic-gate 		ASSERT(index == PAGE_HASH_FUNC(vp, off));
24537c478bd9Sstevel@tonic-gate 		ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
24547c478bd9Sstevel@tonic-gate 
24557c478bd9Sstevel@tonic-gate 		if (npp == NULL) {
24567c478bd9Sstevel@tonic-gate 			/*
24577c478bd9Sstevel@tonic-gate 			 * Try to get a page from the freelist (ie,
24587c478bd9Sstevel@tonic-gate 			 * a page with no [vp, off] tag).  If that
24597c478bd9Sstevel@tonic-gate 			 * fails, use the cachelist.
24607c478bd9Sstevel@tonic-gate 			 *
24617c478bd9Sstevel@tonic-gate 			 * During the first attempt at both the free
24627c478bd9Sstevel@tonic-gate 			 * and cache lists we try for the correct color.
24637c478bd9Sstevel@tonic-gate 			 */
24647c478bd9Sstevel@tonic-gate 			/*
24657c478bd9Sstevel@tonic-gate 			 * XXXX-how do we deal with virtual indexed
24667c478bd9Sstevel@tonic-gate 			 * caches and and colors?
24677c478bd9Sstevel@tonic-gate 			 */
24687c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_cnt[4]);
24697c478bd9Sstevel@tonic-gate 			/*
24707c478bd9Sstevel@tonic-gate 			 * Get lgroup to allocate next page of shared memory
24717c478bd9Sstevel@tonic-gate 			 * from and use it to specify where to allocate
24727c478bd9Sstevel@tonic-gate 			 * the physical memory
24737c478bd9Sstevel@tonic-gate 			 */
24747c478bd9Sstevel@tonic-gate 			lgrp = lgrp_mem_choose(seg, vaddr, PAGESIZE);
24757c478bd9Sstevel@tonic-gate 			npp = page_get_freelist(vp, off, seg, vaddr, PAGESIZE,
24767c478bd9Sstevel@tonic-gate 			    flags | PG_MATCH_COLOR, lgrp);
24777c478bd9Sstevel@tonic-gate 			if (npp == NULL) {
24787c478bd9Sstevel@tonic-gate 				npp = page_get_cachelist(vp, off, seg,
24797c478bd9Sstevel@tonic-gate 				    vaddr, flags | PG_MATCH_COLOR, lgrp);
24807c478bd9Sstevel@tonic-gate 				if (npp == NULL) {
24817c478bd9Sstevel@tonic-gate 					npp = page_create_get_something(vp,
24827c478bd9Sstevel@tonic-gate 					    off, seg, vaddr,
24837c478bd9Sstevel@tonic-gate 					    flags & ~PG_MATCH_COLOR);
24847c478bd9Sstevel@tonic-gate 				}
24857c478bd9Sstevel@tonic-gate 
24867c478bd9Sstevel@tonic-gate 				if (PP_ISAGED(npp) == 0) {
24877c478bd9Sstevel@tonic-gate 					/*
24887c478bd9Sstevel@tonic-gate 					 * Since this page came from the
24897c478bd9Sstevel@tonic-gate 					 * cachelist, we must destroy the
24907c478bd9Sstevel@tonic-gate 					 * old vnode association.
24917c478bd9Sstevel@tonic-gate 					 */
24927c478bd9Sstevel@tonic-gate 					page_hashout(npp, NULL);
24937c478bd9Sstevel@tonic-gate 				}
24947c478bd9Sstevel@tonic-gate 			}
24957c478bd9Sstevel@tonic-gate 		}
24967c478bd9Sstevel@tonic-gate 
24977c478bd9Sstevel@tonic-gate 		/*
24987c478bd9Sstevel@tonic-gate 		 * We own this page!
24997c478bd9Sstevel@tonic-gate 		 */
25007c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_EXCL(npp));
25017c478bd9Sstevel@tonic-gate 		ASSERT(npp->p_vnode == NULL);
25027c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_is_mapped(npp));
25037c478bd9Sstevel@tonic-gate 		PP_CLRFREE(npp);
25047c478bd9Sstevel@tonic-gate 		PP_CLRAGED(npp);
25057c478bd9Sstevel@tonic-gate 
25067c478bd9Sstevel@tonic-gate 		/*
25077c478bd9Sstevel@tonic-gate 		 * Here we have a page in our hot little mits and are
25087c478bd9Sstevel@tonic-gate 		 * just waiting to stuff it on the appropriate lists.
25097c478bd9Sstevel@tonic-gate 		 * Get the mutex and check to see if it really does
25107c478bd9Sstevel@tonic-gate 		 * not exist.
25117c478bd9Sstevel@tonic-gate 		 */
25127c478bd9Sstevel@tonic-gate 		phm = PAGE_HASH_MUTEX(index);
25137c478bd9Sstevel@tonic-gate 		mutex_enter(phm);
25147c478bd9Sstevel@tonic-gate 		PAGE_HASH_SEARCH(index, pp, vp, off);
25157c478bd9Sstevel@tonic-gate 		if (pp == NULL) {
25167c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_new);
25177c478bd9Sstevel@tonic-gate 			pp = npp;
25187c478bd9Sstevel@tonic-gate 			npp = NULL;
25197c478bd9Sstevel@tonic-gate 			if (!page_hashin(pp, vp, off, phm)) {
25207c478bd9Sstevel@tonic-gate 				/*
25217c478bd9Sstevel@tonic-gate 				 * Since we hold the page hash mutex and
25227c478bd9Sstevel@tonic-gate 				 * just searched for this page, page_hashin
25237c478bd9Sstevel@tonic-gate 				 * had better not fail.  If it does, that
25247c478bd9Sstevel@tonic-gate 				 * means somethread did not follow the
25257c478bd9Sstevel@tonic-gate 				 * page hash mutex rules.  Panic now and
25267c478bd9Sstevel@tonic-gate 				 * get it over with.  As usual, go down
25277c478bd9Sstevel@tonic-gate 				 * holding all the locks.
25287c478bd9Sstevel@tonic-gate 				 */
25297c478bd9Sstevel@tonic-gate 				ASSERT(MUTEX_HELD(phm));
25307c478bd9Sstevel@tonic-gate 				panic("page_create: "
25317c478bd9Sstevel@tonic-gate 				    "hashin failed %p %p %llx %p",
25327c478bd9Sstevel@tonic-gate 				    (void *)pp, (void *)vp, off, (void *)phm);
25337c478bd9Sstevel@tonic-gate 				/*NOTREACHED*/
25347c478bd9Sstevel@tonic-gate 			}
25357c478bd9Sstevel@tonic-gate 			ASSERT(MUTEX_HELD(phm));
25367c478bd9Sstevel@tonic-gate 			mutex_exit(phm);
25377c478bd9Sstevel@tonic-gate 			phm = NULL;
25387c478bd9Sstevel@tonic-gate 
25397c478bd9Sstevel@tonic-gate 			/*
25407c478bd9Sstevel@tonic-gate 			 * Hat layer locking need not be done to set
25417c478bd9Sstevel@tonic-gate 			 * the following bits since the page is not hashed
25427c478bd9Sstevel@tonic-gate 			 * and was on the free list (i.e., had no mappings).
25437c478bd9Sstevel@tonic-gate 			 *
25447c478bd9Sstevel@tonic-gate 			 * Set the reference bit to protect
25457c478bd9Sstevel@tonic-gate 			 * against immediate pageout
25467c478bd9Sstevel@tonic-gate 			 *
25477c478bd9Sstevel@tonic-gate 			 * XXXmh modify freelist code to set reference
25487c478bd9Sstevel@tonic-gate 			 * bit so we don't have to do it here.
25497c478bd9Sstevel@tonic-gate 			 */
25507c478bd9Sstevel@tonic-gate 			page_set_props(pp, P_REF);
25517c478bd9Sstevel@tonic-gate 			found_on_free++;
25527c478bd9Sstevel@tonic-gate 		} else {
25537c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_exists);
25547c478bd9Sstevel@tonic-gate 			if (flags & PG_EXCL) {
25557c478bd9Sstevel@tonic-gate 				/*
25567c478bd9Sstevel@tonic-gate 				 * Found an existing page, and the caller
25577c478bd9Sstevel@tonic-gate 				 * wanted all new pages.  Undo all of the work
25587c478bd9Sstevel@tonic-gate 				 * we have done.
25597c478bd9Sstevel@tonic-gate 				 */
25607c478bd9Sstevel@tonic-gate 				mutex_exit(phm);
25617c478bd9Sstevel@tonic-gate 				phm = NULL;
25627c478bd9Sstevel@tonic-gate 				while (plist != NULL) {
25637c478bd9Sstevel@tonic-gate 					pp = plist;
25647c478bd9Sstevel@tonic-gate 					page_sub(&plist, pp);
25657c478bd9Sstevel@tonic-gate 					page_io_unlock(pp);
25667c478bd9Sstevel@tonic-gate 					/* large pages should not end up here */
25677c478bd9Sstevel@tonic-gate 					ASSERT(pp->p_szc == 0);
25687c478bd9Sstevel@tonic-gate 					/*LINTED: constant in conditional ctx*/
25697c478bd9Sstevel@tonic-gate 					VN_DISPOSE(pp, B_INVAL, 0, kcred);
25707c478bd9Sstevel@tonic-gate 				}
25717c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_create_found_one);
25727c478bd9Sstevel@tonic-gate 				goto fail;
25737c478bd9Sstevel@tonic-gate 			}
25747c478bd9Sstevel@tonic-gate 			ASSERT(flags & PG_WAIT);
25757c478bd9Sstevel@tonic-gate 			if (!page_lock(pp, SE_EXCL, phm, P_NO_RECLAIM)) {
25767c478bd9Sstevel@tonic-gate 				/*
25777c478bd9Sstevel@tonic-gate 				 * Start all over again if we blocked trying
25787c478bd9Sstevel@tonic-gate 				 * to lock the page.
25797c478bd9Sstevel@tonic-gate 				 */
25807c478bd9Sstevel@tonic-gate 				mutex_exit(phm);
25817c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_create_page_lock_failed);
25827c478bd9Sstevel@tonic-gate 				phm = NULL;
25837c478bd9Sstevel@tonic-gate 				goto top;
25847c478bd9Sstevel@tonic-gate 			}
25857c478bd9Sstevel@tonic-gate 			mutex_exit(phm);
25867c478bd9Sstevel@tonic-gate 			phm = NULL;
25877c478bd9Sstevel@tonic-gate 
25887c478bd9Sstevel@tonic-gate 			if (PP_ISFREE(pp)) {
25897c478bd9Sstevel@tonic-gate 				ASSERT(PP_ISAGED(pp) == 0);
25907c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(pagecnt.pc_get_cache);
25917c478bd9Sstevel@tonic-gate 				page_list_sub(pp, PG_CACHE_LIST);
25927c478bd9Sstevel@tonic-gate 				PP_CLRFREE(pp);
25937c478bd9Sstevel@tonic-gate 				found_on_free++;
25947c478bd9Sstevel@tonic-gate 			}
25957c478bd9Sstevel@tonic-gate 		}
25967c478bd9Sstevel@tonic-gate 
25977c478bd9Sstevel@tonic-gate 		/*
25987c478bd9Sstevel@tonic-gate 		 * Got a page!  It is locked.  Acquire the i/o
25997c478bd9Sstevel@tonic-gate 		 * lock since we are going to use the p_next and
26007c478bd9Sstevel@tonic-gate 		 * p_prev fields to link the requested pages together.
26017c478bd9Sstevel@tonic-gate 		 */
26027c478bd9Sstevel@tonic-gate 		page_io_lock(pp);
26037c478bd9Sstevel@tonic-gate 		page_add(&plist, pp);
26047c478bd9Sstevel@tonic-gate 		plist = plist->p_next;
26057c478bd9Sstevel@tonic-gate 		off += PAGESIZE;
26067c478bd9Sstevel@tonic-gate 		vaddr += PAGESIZE;
26077c478bd9Sstevel@tonic-gate 	}
26087c478bd9Sstevel@tonic-gate 
26097c478bd9Sstevel@tonic-gate 	ASSERT((flags & PG_EXCL) ? (found_on_free == pages_req) : 1);
26107c478bd9Sstevel@tonic-gate fail:
26117c478bd9Sstevel@tonic-gate 	if (npp != NULL) {
26127c478bd9Sstevel@tonic-gate 		/*
26137c478bd9Sstevel@tonic-gate 		 * Did not need this page after all.
26147c478bd9Sstevel@tonic-gate 		 * Put it back on the free list.
26157c478bd9Sstevel@tonic-gate 		 */
26167c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_create_putbacks);
26177c478bd9Sstevel@tonic-gate 		PP_SETFREE(npp);
26187c478bd9Sstevel@tonic-gate 		PP_SETAGED(npp);
26197c478bd9Sstevel@tonic-gate 		npp->p_offset = (u_offset_t)-1;
26207c478bd9Sstevel@tonic-gate 		page_list_add(npp, PG_FREE_LIST | PG_LIST_TAIL);
26217c478bd9Sstevel@tonic-gate 		page_unlock(npp);
26227c478bd9Sstevel@tonic-gate 
26237c478bd9Sstevel@tonic-gate 	}
26247c478bd9Sstevel@tonic-gate 
26257c478bd9Sstevel@tonic-gate 	ASSERT(pages_req >= found_on_free);
26267c478bd9Sstevel@tonic-gate 
26277c478bd9Sstevel@tonic-gate 	{
26287c478bd9Sstevel@tonic-gate 		uint_t overshoot = (uint_t)(pages_req - found_on_free);
26297c478bd9Sstevel@tonic-gate 
26307c478bd9Sstevel@tonic-gate 		if (overshoot) {
26317c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_create_overshoot);
26327c478bd9Sstevel@tonic-gate 			p = &pcf[pcf_index];
26337c478bd9Sstevel@tonic-gate 			mutex_enter(&p->pcf_lock);
26347c478bd9Sstevel@tonic-gate 			if (p->pcf_block) {
26357c478bd9Sstevel@tonic-gate 				p->pcf_reserve += overshoot;
26367c478bd9Sstevel@tonic-gate 			} else {
26377c478bd9Sstevel@tonic-gate 				p->pcf_count += overshoot;
26387c478bd9Sstevel@tonic-gate 				if (p->pcf_wait) {
26397c478bd9Sstevel@tonic-gate 					mutex_enter(&new_freemem_lock);
26407c478bd9Sstevel@tonic-gate 					if (freemem_wait) {
26417c478bd9Sstevel@tonic-gate 						cv_signal(&freemem_cv);
26427c478bd9Sstevel@tonic-gate 						p->pcf_wait--;
26437c478bd9Sstevel@tonic-gate 					} else {
26447c478bd9Sstevel@tonic-gate 						p->pcf_wait = 0;
26457c478bd9Sstevel@tonic-gate 					}
26467c478bd9Sstevel@tonic-gate 					mutex_exit(&new_freemem_lock);
26477c478bd9Sstevel@tonic-gate 				}
26487c478bd9Sstevel@tonic-gate 			}
26497c478bd9Sstevel@tonic-gate 			mutex_exit(&p->pcf_lock);
26507c478bd9Sstevel@tonic-gate 			/* freemem is approximate, so this test OK */
26517c478bd9Sstevel@tonic-gate 			if (!p->pcf_block)
26527c478bd9Sstevel@tonic-gate 				freemem += overshoot;
26537c478bd9Sstevel@tonic-gate 		}
26547c478bd9Sstevel@tonic-gate 	}
26557c478bd9Sstevel@tonic-gate 
26567c478bd9Sstevel@tonic-gate 	return (plist);
26577c478bd9Sstevel@tonic-gate }
26587c478bd9Sstevel@tonic-gate 
26597c478bd9Sstevel@tonic-gate /*
26607c478bd9Sstevel@tonic-gate  * One or more constituent pages of this large page has been marked
26617c478bd9Sstevel@tonic-gate  * toxic. Simply demote the large page to PAGESIZE pages and let
26627c478bd9Sstevel@tonic-gate  * page_free() handle it. This routine should only be called by
26637c478bd9Sstevel@tonic-gate  * large page free routines (page_free_pages() and page_destroy_pages().
26647c478bd9Sstevel@tonic-gate  * All pages are locked SE_EXCL and have already been marked free.
26657c478bd9Sstevel@tonic-gate  */
26667c478bd9Sstevel@tonic-gate static void
26677c478bd9Sstevel@tonic-gate page_free_toxic_pages(page_t *rootpp)
26687c478bd9Sstevel@tonic-gate {
26697c478bd9Sstevel@tonic-gate 	page_t	*tpp;
26707c478bd9Sstevel@tonic-gate 	pgcnt_t	i, pgcnt = page_get_pagecnt(rootpp->p_szc);
26717c478bd9Sstevel@tonic-gate 	uint_t	szc = rootpp->p_szc;
26727c478bd9Sstevel@tonic-gate 
26737c478bd9Sstevel@tonic-gate 	for (i = 0, tpp = rootpp; i < pgcnt; i++, tpp = tpp->p_next) {
26747c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_szc == szc);
26757c478bd9Sstevel@tonic-gate 		ASSERT((PAGE_EXCL(tpp) &&
26767c478bd9Sstevel@tonic-gate 		    !page_iolock_assert(tpp)) || panicstr);
26777c478bd9Sstevel@tonic-gate 		tpp->p_szc = 0;
26787c478bd9Sstevel@tonic-gate 	}
26797c478bd9Sstevel@tonic-gate 
26807c478bd9Sstevel@tonic-gate 	while (rootpp != NULL) {
26817c478bd9Sstevel@tonic-gate 		tpp = rootpp;
26827c478bd9Sstevel@tonic-gate 		page_sub(&rootpp, tpp);
26837c478bd9Sstevel@tonic-gate 		ASSERT(PP_ISFREE(tpp));
26847c478bd9Sstevel@tonic-gate 		PP_CLRFREE(tpp);
26857c478bd9Sstevel@tonic-gate 		page_free(tpp, 1);
26867c478bd9Sstevel@tonic-gate 	}
26877c478bd9Sstevel@tonic-gate }
26887c478bd9Sstevel@tonic-gate 
26897c478bd9Sstevel@tonic-gate /*
26907c478bd9Sstevel@tonic-gate  * Put page on the "free" list.
26917c478bd9Sstevel@tonic-gate  * The free list is really two lists maintained by
26927c478bd9Sstevel@tonic-gate  * the PSM of whatever machine we happen to be on.
26937c478bd9Sstevel@tonic-gate  */
26947c478bd9Sstevel@tonic-gate void
26957c478bd9Sstevel@tonic-gate page_free(page_t *pp, int dontneed)
26967c478bd9Sstevel@tonic-gate {
26977c478bd9Sstevel@tonic-gate 	struct pcf	*p;
26987c478bd9Sstevel@tonic-gate 	uint_t		pcf_index;
26997c478bd9Sstevel@tonic-gate 
27007c478bd9Sstevel@tonic-gate 	ASSERT((PAGE_EXCL(pp) &&
27017c478bd9Sstevel@tonic-gate 	    !page_iolock_assert(pp)) || panicstr);
27027c478bd9Sstevel@tonic-gate 
27037c478bd9Sstevel@tonic-gate 	if (PP_ISFREE(pp)) {
27047c478bd9Sstevel@tonic-gate 		panic("page_free: page %p is free", (void *)pp);
27057c478bd9Sstevel@tonic-gate 	}
27067c478bd9Sstevel@tonic-gate 
27077c478bd9Sstevel@tonic-gate 	if (pp->p_szc != 0) {
27087c478bd9Sstevel@tonic-gate 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
2709ad23a2dbSjohansen 		    PP_ISKAS(pp)) {
27107c478bd9Sstevel@tonic-gate 			panic("page_free: anon or kernel "
27117c478bd9Sstevel@tonic-gate 			    "or no vnode large page %p", (void *)pp);
27127c478bd9Sstevel@tonic-gate 		}
27137c478bd9Sstevel@tonic-gate 		page_demote_vp_pages(pp);
27147c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_szc == 0);
27157c478bd9Sstevel@tonic-gate 	}
27167c478bd9Sstevel@tonic-gate 
27177c478bd9Sstevel@tonic-gate 	/*
27187c478bd9Sstevel@tonic-gate 	 * The page_struct_lock need not be acquired to examine these
27197c478bd9Sstevel@tonic-gate 	 * fields since the page has an "exclusive" lock.
27207c478bd9Sstevel@tonic-gate 	 */
272107b65a64Saguzovsk 	if (hat_page_is_mapped(pp) || pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
272207b65a64Saguzovsk 	    pp->p_slckcnt != 0) {
272307b65a64Saguzovsk 		panic("page_free pp=%p, pfn=%lx, lckcnt=%d, cowcnt=%d "
272407b65a64Saguzovsk 		    "slckcnt = %d", pp, page_pptonum(pp), pp->p_lckcnt,
272507b65a64Saguzovsk 		    pp->p_cowcnt, pp->p_slckcnt);
27267c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
27277c478bd9Sstevel@tonic-gate 	}
27287c478bd9Sstevel@tonic-gate 
27297c478bd9Sstevel@tonic-gate 	ASSERT(!hat_page_getshare(pp));
27307c478bd9Sstevel@tonic-gate 
27317c478bd9Sstevel@tonic-gate 	PP_SETFREE(pp);
27327c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode == NULL || !IS_VMODSORT(pp->p_vnode) ||
27337c478bd9Sstevel@tonic-gate 	    !hat_ismod(pp));
27347c478bd9Sstevel@tonic-gate 	page_clr_all_props(pp);
27357c478bd9Sstevel@tonic-gate 	ASSERT(!hat_page_getshare(pp));
27367c478bd9Sstevel@tonic-gate 
27377c478bd9Sstevel@tonic-gate 	/*
27387c478bd9Sstevel@tonic-gate 	 * Now we add the page to the head of the free list.
27397c478bd9Sstevel@tonic-gate 	 * But if this page is associated with a paged vnode
27407c478bd9Sstevel@tonic-gate 	 * then we adjust the head forward so that the page is
27417c478bd9Sstevel@tonic-gate 	 * effectively at the end of the list.
27427c478bd9Sstevel@tonic-gate 	 */
27437c478bd9Sstevel@tonic-gate 	if (pp->p_vnode == NULL) {
27447c478bd9Sstevel@tonic-gate 		/*
27457c478bd9Sstevel@tonic-gate 		 * Page has no identity, put it on the free list.
27467c478bd9Sstevel@tonic-gate 		 */
27477c478bd9Sstevel@tonic-gate 		PP_SETAGED(pp);
27487c478bd9Sstevel@tonic-gate 		pp->p_offset = (u_offset_t)-1;
27497c478bd9Sstevel@tonic-gate 		page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
27507c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pagecnt.pc_free_free);
27517c478bd9Sstevel@tonic-gate 		TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
27527c478bd9Sstevel@tonic-gate 		    "page_free_free:pp %p", pp);
27537c478bd9Sstevel@tonic-gate 	} else {
27547c478bd9Sstevel@tonic-gate 		PP_CLRAGED(pp);
27557c478bd9Sstevel@tonic-gate 
27567c478bd9Sstevel@tonic-gate 		if (!dontneed || nopageage) {
27577c478bd9Sstevel@tonic-gate 			/* move it to the tail of the list */
27587c478bd9Sstevel@tonic-gate 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_TAIL);
27597c478bd9Sstevel@tonic-gate 
27607c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(pagecnt.pc_free_cache);
27617c478bd9Sstevel@tonic-gate 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_TAIL,
27627c478bd9Sstevel@tonic-gate 			    "page_free_cache_tail:pp %p", pp);
27637c478bd9Sstevel@tonic-gate 		} else {
27647c478bd9Sstevel@tonic-gate 			page_list_add(pp, PG_CACHE_LIST | PG_LIST_HEAD);
27657c478bd9Sstevel@tonic-gate 
27667c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(pagecnt.pc_free_dontneed);
27677c478bd9Sstevel@tonic-gate 			TRACE_1(TR_FAC_VM, TR_PAGE_FREE_CACHE_HEAD,
27687c478bd9Sstevel@tonic-gate 			    "page_free_cache_head:pp %p", pp);
27697c478bd9Sstevel@tonic-gate 		}
27707c478bd9Sstevel@tonic-gate 	}
27717c478bd9Sstevel@tonic-gate 	page_unlock(pp);
27727c478bd9Sstevel@tonic-gate 
27737c478bd9Sstevel@tonic-gate 	/*
27747c478bd9Sstevel@tonic-gate 	 * Now do the `freemem' accounting.
27757c478bd9Sstevel@tonic-gate 	 */
27767c478bd9Sstevel@tonic-gate 	pcf_index = PCF_INDEX();
27777c478bd9Sstevel@tonic-gate 	p = &pcf[pcf_index];
27787c478bd9Sstevel@tonic-gate 
27797c478bd9Sstevel@tonic-gate 	mutex_enter(&p->pcf_lock);
27807c478bd9Sstevel@tonic-gate 	if (p->pcf_block) {
27817c478bd9Sstevel@tonic-gate 		p->pcf_reserve += 1;
27827c478bd9Sstevel@tonic-gate 	} else {
27837c478bd9Sstevel@tonic-gate 		p->pcf_count += 1;
27847c478bd9Sstevel@tonic-gate 		if (p->pcf_wait) {
27857c478bd9Sstevel@tonic-gate 			mutex_enter(&new_freemem_lock);
27867c478bd9Sstevel@tonic-gate 			/*
27877c478bd9Sstevel@tonic-gate 			 * Check to see if some other thread
27887c478bd9Sstevel@tonic-gate 			 * is actually waiting.  Another bucket
27897c478bd9Sstevel@tonic-gate 			 * may have woken it up by now.  If there
27907c478bd9Sstevel@tonic-gate 			 * are no waiters, then set our pcf_wait
27917c478bd9Sstevel@tonic-gate 			 * count to zero to avoid coming in here
27927c478bd9Sstevel@tonic-gate 			 * next time.  Also, since only one page
27937c478bd9Sstevel@tonic-gate 			 * was put on the free list, just wake
27947c478bd9Sstevel@tonic-gate 			 * up one waiter.
27957c478bd9Sstevel@tonic-gate 			 */
27967c478bd9Sstevel@tonic-gate 			if (freemem_wait) {
27977c478bd9Sstevel@tonic-gate 				cv_signal(&freemem_cv);
27987c478bd9Sstevel@tonic-gate 				p->pcf_wait--;
27997c478bd9Sstevel@tonic-gate 			} else {
28007c478bd9Sstevel@tonic-gate 				p->pcf_wait = 0;
28017c478bd9Sstevel@tonic-gate 			}
28027c478bd9Sstevel@tonic-gate 			mutex_exit(&new_freemem_lock);
28037c478bd9Sstevel@tonic-gate 		}
28047c478bd9Sstevel@tonic-gate 	}
28057c478bd9Sstevel@tonic-gate 	mutex_exit(&p->pcf_lock);
28067c478bd9Sstevel@tonic-gate 
28077c478bd9Sstevel@tonic-gate 	/* freemem is approximate, so this test OK */
28087c478bd9Sstevel@tonic-gate 	if (!p->pcf_block)
28097c478bd9Sstevel@tonic-gate 		freemem += 1;
28107c478bd9Sstevel@tonic-gate }
28117c478bd9Sstevel@tonic-gate 
28127c478bd9Sstevel@tonic-gate /*
28137c478bd9Sstevel@tonic-gate  * Put page on the "free" list during intial startup.
28147c478bd9Sstevel@tonic-gate  * This happens during initial single threaded execution.
28157c478bd9Sstevel@tonic-gate  */
28167c478bd9Sstevel@tonic-gate void
28177c478bd9Sstevel@tonic-gate page_free_at_startup(page_t *pp)
28187c478bd9Sstevel@tonic-gate {
28197c478bd9Sstevel@tonic-gate 	struct pcf	*p;
28207c478bd9Sstevel@tonic-gate 	uint_t		pcf_index;
28217c478bd9Sstevel@tonic-gate 
28227c478bd9Sstevel@tonic-gate 	page_list_add(pp, PG_FREE_LIST | PG_LIST_HEAD | PG_LIST_ISINIT);
28237c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_free_free);
28247c478bd9Sstevel@tonic-gate 
28257c478bd9Sstevel@tonic-gate 	/*
28267c478bd9Sstevel@tonic-gate 	 * Now do the `freemem' accounting.
28277c478bd9Sstevel@tonic-gate 	 */
28287c478bd9Sstevel@tonic-gate 	pcf_index = PCF_INDEX();
28297c478bd9Sstevel@tonic-gate 	p = &pcf[pcf_index];
28307c478bd9Sstevel@tonic-gate 
28317c478bd9Sstevel@tonic-gate 	ASSERT(p->pcf_block == 0);
28327c478bd9Sstevel@tonic-gate 	ASSERT(p->pcf_wait == 0);
28337c478bd9Sstevel@tonic-gate 	p->pcf_count += 1;
28347c478bd9Sstevel@tonic-gate 
28357c478bd9Sstevel@tonic-gate 	/* freemem is approximate, so this is OK */
28367c478bd9Sstevel@tonic-gate 	freemem += 1;
28377c478bd9Sstevel@tonic-gate }
28387c478bd9Sstevel@tonic-gate 
28397c478bd9Sstevel@tonic-gate void
28407c478bd9Sstevel@tonic-gate page_free_pages(page_t *pp)
28417c478bd9Sstevel@tonic-gate {
28427c478bd9Sstevel@tonic-gate 	page_t	*tpp, *rootpp = NULL;
28437c478bd9Sstevel@tonic-gate 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
28447c478bd9Sstevel@tonic-gate 	pgcnt_t	i;
28457c478bd9Sstevel@tonic-gate 	uint_t	szc = pp->p_szc;
28467c478bd9Sstevel@tonic-gate 
28477c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_free_pages);
28487c478bd9Sstevel@tonic-gate 	TRACE_1(TR_FAC_VM, TR_PAGE_FREE_FREE,
28497c478bd9Sstevel@tonic-gate 	    "page_free_free:pp %p", pp);
28507c478bd9Sstevel@tonic-gate 
28517c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
28527c478bd9Sstevel@tonic-gate 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
28537c478bd9Sstevel@tonic-gate 		panic("page_free_pages: not root page %p", (void *)pp);
28547c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
28557c478bd9Sstevel@tonic-gate 	}
28567c478bd9Sstevel@tonic-gate 
2857affbd3ccSkchow 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
28587c478bd9Sstevel@tonic-gate 		ASSERT((PAGE_EXCL(tpp) &&
28597c478bd9Sstevel@tonic-gate 		    !page_iolock_assert(tpp)) || panicstr);
28607c478bd9Sstevel@tonic-gate 		if (PP_ISFREE(tpp)) {
28617c478bd9Sstevel@tonic-gate 			panic("page_free_pages: page %p is free", (void *)tpp);
28627c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
28637c478bd9Sstevel@tonic-gate 		}
28647c478bd9Sstevel@tonic-gate 		if (hat_page_is_mapped(tpp) || tpp->p_lckcnt != 0 ||
286507b65a64Saguzovsk 		    tpp->p_cowcnt != 0 || tpp->p_slckcnt != 0) {
28667c478bd9Sstevel@tonic-gate 			panic("page_free_pages %p", (void *)tpp);
28677c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
28687c478bd9Sstevel@tonic-gate 		}
28697c478bd9Sstevel@tonic-gate 
28707c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_getshare(tpp));
28717c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_vnode == NULL);
28727c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_szc == szc);
28737c478bd9Sstevel@tonic-gate 
28747c478bd9Sstevel@tonic-gate 		PP_SETFREE(tpp);
28757c478bd9Sstevel@tonic-gate 		page_clr_all_props(tpp);
28767c478bd9Sstevel@tonic-gate 		PP_SETAGED(tpp);
28777c478bd9Sstevel@tonic-gate 		tpp->p_offset = (u_offset_t)-1;
28787c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_next == tpp);
28797c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_prev == tpp);
28807c478bd9Sstevel@tonic-gate 		page_list_concat(&rootpp, &tpp);
28817c478bd9Sstevel@tonic-gate 	}
28827c478bd9Sstevel@tonic-gate 	ASSERT(rootpp == pp);
28837c478bd9Sstevel@tonic-gate 
28847c478bd9Sstevel@tonic-gate 	page_list_add_pages(rootpp, 0);
28857c478bd9Sstevel@tonic-gate 	page_create_putback(pgcnt);
28867c478bd9Sstevel@tonic-gate }
28877c478bd9Sstevel@tonic-gate 
28887c478bd9Sstevel@tonic-gate int free_pages = 1;
28897c478bd9Sstevel@tonic-gate 
28907c478bd9Sstevel@tonic-gate /*
28917c478bd9Sstevel@tonic-gate  * This routine attempts to return pages to the cachelist via page_release().
28927c478bd9Sstevel@tonic-gate  * It does not *have* to be successful in all cases, since the pageout scanner
28937c478bd9Sstevel@tonic-gate  * will catch any pages it misses.  It does need to be fast and not introduce
28947c478bd9Sstevel@tonic-gate  * too much overhead.
28957c478bd9Sstevel@tonic-gate  *
28967c478bd9Sstevel@tonic-gate  * If a page isn't found on the unlocked sweep of the page_hash bucket, we
28977c478bd9Sstevel@tonic-gate  * don't lock and retry.  This is ok, since the page scanner will eventually
28987c478bd9Sstevel@tonic-gate  * find any page we miss in free_vp_pages().
28997c478bd9Sstevel@tonic-gate  */
29007c478bd9Sstevel@tonic-gate void
29017c478bd9Sstevel@tonic-gate free_vp_pages(vnode_t *vp, u_offset_t off, size_t len)
29027c478bd9Sstevel@tonic-gate {
29037c478bd9Sstevel@tonic-gate 	page_t *pp;
29047c478bd9Sstevel@tonic-gate 	u_offset_t eoff;
29057c478bd9Sstevel@tonic-gate 	extern int swap_in_range(vnode_t *, u_offset_t, size_t);
29067c478bd9Sstevel@tonic-gate 
29077c478bd9Sstevel@tonic-gate 	eoff = off + len;
29087c478bd9Sstevel@tonic-gate 
29097c478bd9Sstevel@tonic-gate 	if (free_pages == 0)
29107c478bd9Sstevel@tonic-gate 		return;
29117c478bd9Sstevel@tonic-gate 	if (swap_in_range(vp, off, len))
29127c478bd9Sstevel@tonic-gate 		return;
29137c478bd9Sstevel@tonic-gate 
29147c478bd9Sstevel@tonic-gate 	for (; off < eoff; off += PAGESIZE) {
29157c478bd9Sstevel@tonic-gate 
29167c478bd9Sstevel@tonic-gate 		/*
29177c478bd9Sstevel@tonic-gate 		 * find the page using a fast, but inexact search. It'll be OK
29187c478bd9Sstevel@tonic-gate 		 * if a few pages slip through the cracks here.
29197c478bd9Sstevel@tonic-gate 		 */
29207c478bd9Sstevel@tonic-gate 		pp = page_exists(vp, off);
29217c478bd9Sstevel@tonic-gate 
29227c478bd9Sstevel@tonic-gate 		/*
29237c478bd9Sstevel@tonic-gate 		 * If we didn't find the page (it may not exist), the page
29247c478bd9Sstevel@tonic-gate 		 * is free, looks still in use (shared), or we can't lock it,
29257c478bd9Sstevel@tonic-gate 		 * just give up.
29267c478bd9Sstevel@tonic-gate 		 */
29277c478bd9Sstevel@tonic-gate 		if (pp == NULL ||
29287c478bd9Sstevel@tonic-gate 		    PP_ISFREE(pp) ||
29297c478bd9Sstevel@tonic-gate 		    page_share_cnt(pp) > 0 ||
29307c478bd9Sstevel@tonic-gate 		    !page_trylock(pp, SE_EXCL))
29317c478bd9Sstevel@tonic-gate 			continue;
29327c478bd9Sstevel@tonic-gate 
29337c478bd9Sstevel@tonic-gate 		/*
29347c478bd9Sstevel@tonic-gate 		 * Once we have locked pp, verify that it's still the
29357c478bd9Sstevel@tonic-gate 		 * correct page and not already free
29367c478bd9Sstevel@tonic-gate 		 */
29377c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_LOCKED_SE(pp, SE_EXCL));
29387c478bd9Sstevel@tonic-gate 		if (pp->p_vnode != vp || pp->p_offset != off || PP_ISFREE(pp)) {
29397c478bd9Sstevel@tonic-gate 			page_unlock(pp);
29407c478bd9Sstevel@tonic-gate 			continue;
29417c478bd9Sstevel@tonic-gate 		}
29427c478bd9Sstevel@tonic-gate 
29437c478bd9Sstevel@tonic-gate 		/*
29447c478bd9Sstevel@tonic-gate 		 * try to release the page...
29457c478bd9Sstevel@tonic-gate 		 */
29467c478bd9Sstevel@tonic-gate 		(void) page_release(pp, 1);
29477c478bd9Sstevel@tonic-gate 	}
29487c478bd9Sstevel@tonic-gate }
29497c478bd9Sstevel@tonic-gate 
29507c478bd9Sstevel@tonic-gate /*
29517c478bd9Sstevel@tonic-gate  * Reclaim the given page from the free list.
29527c478bd9Sstevel@tonic-gate  * Returns 1 on success or 0 on failure.
29537c478bd9Sstevel@tonic-gate  *
29547c478bd9Sstevel@tonic-gate  * The page is unlocked if it can't be reclaimed (when freemem == 0).
29557c478bd9Sstevel@tonic-gate  * If `lock' is non-null, it will be dropped and re-acquired if
29567c478bd9Sstevel@tonic-gate  * the routine must wait while freemem is 0.
29577c478bd9Sstevel@tonic-gate  *
29587c478bd9Sstevel@tonic-gate  * As it turns out, boot_getpages() does this.  It picks a page,
29597c478bd9Sstevel@tonic-gate  * based on where OBP mapped in some address, gets its pfn, searches
29607c478bd9Sstevel@tonic-gate  * the memsegs, locks the page, then pulls it off the free list!
29617c478bd9Sstevel@tonic-gate  */
29627c478bd9Sstevel@tonic-gate int
29637c478bd9Sstevel@tonic-gate page_reclaim(page_t *pp, kmutex_t *lock)
29647c478bd9Sstevel@tonic-gate {
29657c478bd9Sstevel@tonic-gate 	struct pcf	*p;
29667c478bd9Sstevel@tonic-gate 	uint_t		pcf_index;
29677c478bd9Sstevel@tonic-gate 	struct cpu	*cpup;
29687c478bd9Sstevel@tonic-gate 	uint_t		i;
2969de62167aSmec 	pgcnt_t		npgs, need;
2970de62167aSmec 	pgcnt_t		collected = 0;
29717c478bd9Sstevel@tonic-gate 
29727c478bd9Sstevel@tonic-gate 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
29737c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
2974db874c57Selowe 
2975db874c57Selowe 	npgs = page_get_pagecnt(pp->p_szc);
29767c478bd9Sstevel@tonic-gate 
29777c478bd9Sstevel@tonic-gate 	/*
29787c478bd9Sstevel@tonic-gate 	 * If `freemem' is 0, we cannot reclaim this page from the
29797c478bd9Sstevel@tonic-gate 	 * freelist, so release every lock we might hold: the page,
29807c478bd9Sstevel@tonic-gate 	 * and the `lock' before blocking.
29817c478bd9Sstevel@tonic-gate 	 *
29827c478bd9Sstevel@tonic-gate 	 * The only way `freemem' can become 0 while there are pages
29837c478bd9Sstevel@tonic-gate 	 * marked free (have their p->p_free bit set) is when the
29847c478bd9Sstevel@tonic-gate 	 * system is low on memory and doing a page_create().  In
29857c478bd9Sstevel@tonic-gate 	 * order to guarantee that once page_create() starts acquiring
29867c478bd9Sstevel@tonic-gate 	 * pages it will be able to get all that it needs since `freemem'
29877c478bd9Sstevel@tonic-gate 	 * was decreased by the requested amount.  So, we need to release
29887c478bd9Sstevel@tonic-gate 	 * this page, and let page_create() have it.
29897c478bd9Sstevel@tonic-gate 	 *
29907c478bd9Sstevel@tonic-gate 	 * Since `freemem' being zero is not supposed to happen, just
29917c478bd9Sstevel@tonic-gate 	 * use the usual hash stuff as a starting point.  If that bucket
29927c478bd9Sstevel@tonic-gate 	 * is empty, then assume the worst, and start at the beginning
29937c478bd9Sstevel@tonic-gate 	 * of the pcf array.  If we always start at the beginning
29947c478bd9Sstevel@tonic-gate 	 * when acquiring more than one pcf lock, there won't be any
29957c478bd9Sstevel@tonic-gate 	 * deadlock problems.
29967c478bd9Sstevel@tonic-gate 	 */
29977c478bd9Sstevel@tonic-gate 
29987c478bd9Sstevel@tonic-gate 	/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
29997c478bd9Sstevel@tonic-gate 
300024e9c58bSelowe 	if (freemem <= throttlefree && !page_create_throttle(npgs, 0)) {
30017c478bd9Sstevel@tonic-gate 		pcf_acquire_all();
30027c478bd9Sstevel@tonic-gate 		goto page_reclaim_nomem;
30037c478bd9Sstevel@tonic-gate 	}
30047c478bd9Sstevel@tonic-gate 
30057c478bd9Sstevel@tonic-gate 	pcf_index = PCF_INDEX();
30067c478bd9Sstevel@tonic-gate 	p = &pcf[pcf_index];
30077c478bd9Sstevel@tonic-gate 	mutex_enter(&p->pcf_lock);
3008db874c57Selowe 	if (p->pcf_count >= npgs) {
3009db874c57Selowe 		collected = npgs;
3010db874c57Selowe 		p->pcf_count -= npgs;
30117c478bd9Sstevel@tonic-gate 	}
30127c478bd9Sstevel@tonic-gate 	mutex_exit(&p->pcf_lock);
3013db874c57Selowe 	need = npgs - collected;
30147c478bd9Sstevel@tonic-gate 
3015db874c57Selowe 	if (need > 0) {
30167c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_reclaim_zero);
30177c478bd9Sstevel@tonic-gate 		/*
30187c478bd9Sstevel@tonic-gate 		 * Check again. Its possible that some other thread
30197c478bd9Sstevel@tonic-gate 		 * could have been right behind us, and added one
30207c478bd9Sstevel@tonic-gate 		 * to a list somewhere.  Acquire each of the pcf locks
30217c478bd9Sstevel@tonic-gate 		 * until we find a page.
30227c478bd9Sstevel@tonic-gate 		 */
30237c478bd9Sstevel@tonic-gate 		p = pcf;
30247c478bd9Sstevel@tonic-gate 		for (i = 0; i < PCF_FANOUT; i++) {
30257c478bd9Sstevel@tonic-gate 			mutex_enter(&p->pcf_lock);
3026db874c57Selowe 			if (p->pcf_count) {
3027db874c57Selowe 				if (p->pcf_count >= need) {
3028db874c57Selowe 					p->pcf_count -= need;
3029db874c57Selowe 					collected += need;
3030db874c57Selowe 					need = 0;
3031db874c57Selowe 					break;
3032db874c57Selowe 				} else if (p->pcf_count) {
3033db874c57Selowe 					collected += p->pcf_count;
3034db874c57Selowe 					need -= p->pcf_count;
3035db874c57Selowe 					p->pcf_count = 0;
3036db874c57Selowe 				}
30377c478bd9Sstevel@tonic-gate 			}
30387c478bd9Sstevel@tonic-gate 			p++;
30397c478bd9Sstevel@tonic-gate 		}
30407c478bd9Sstevel@tonic-gate 
3041db874c57Selowe 		if (need > 0) {
30427c478bd9Sstevel@tonic-gate page_reclaim_nomem:
30437c478bd9Sstevel@tonic-gate 			/*
30447c478bd9Sstevel@tonic-gate 			 * We really can't have page `pp'.
30457c478bd9Sstevel@tonic-gate 			 * Time for the no-memory dance with
30467c478bd9Sstevel@tonic-gate 			 * page_free().  This is just like
30477c478bd9Sstevel@tonic-gate 			 * page_create_wait().  Plus the added
30487c478bd9Sstevel@tonic-gate 			 * attraction of releasing whatever mutex
30497c478bd9Sstevel@tonic-gate 			 * we held when we were called with in `lock'.
30507c478bd9Sstevel@tonic-gate 			 * Page_unlock() will wakeup any thread
30517c478bd9Sstevel@tonic-gate 			 * waiting around for this page.
30527c478bd9Sstevel@tonic-gate 			 */
30537c478bd9Sstevel@tonic-gate 			if (lock) {
30547c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_reclaim_zero_locked);
30557c478bd9Sstevel@tonic-gate 				mutex_exit(lock);
30567c478bd9Sstevel@tonic-gate 			}
30577c478bd9Sstevel@tonic-gate 			page_unlock(pp);
30587c478bd9Sstevel@tonic-gate 
30597c478bd9Sstevel@tonic-gate 			/*
30607c478bd9Sstevel@tonic-gate 			 * get this before we drop all the pcf locks.
30617c478bd9Sstevel@tonic-gate 			 */
30627c478bd9Sstevel@tonic-gate 			mutex_enter(&new_freemem_lock);
30637c478bd9Sstevel@tonic-gate 
30647c478bd9Sstevel@tonic-gate 			p = pcf;
3065db874c57Selowe 			p->pcf_count += collected;
30667c478bd9Sstevel@tonic-gate 			for (i = 0; i < PCF_FANOUT; i++) {
30677c478bd9Sstevel@tonic-gate 				p->pcf_wait++;
30687c478bd9Sstevel@tonic-gate 				mutex_exit(&p->pcf_lock);
30697c478bd9Sstevel@tonic-gate 				p++;
30707c478bd9Sstevel@tonic-gate 			}
30717c478bd9Sstevel@tonic-gate 
30727c478bd9Sstevel@tonic-gate 			freemem_wait++;
30737c478bd9Sstevel@tonic-gate 			cv_wait(&freemem_cv, &new_freemem_lock);
30747c478bd9Sstevel@tonic-gate 			freemem_wait--;
30757c478bd9Sstevel@tonic-gate 
30767c478bd9Sstevel@tonic-gate 			mutex_exit(&new_freemem_lock);
30777c478bd9Sstevel@tonic-gate 
30787c478bd9Sstevel@tonic-gate 			if (lock) {
30797c478bd9Sstevel@tonic-gate 				mutex_enter(lock);
30807c478bd9Sstevel@tonic-gate 			}
30817c478bd9Sstevel@tonic-gate 			return (0);
30827c478bd9Sstevel@tonic-gate 		}
30837c478bd9Sstevel@tonic-gate 
30847c478bd9Sstevel@tonic-gate 		/*
3085db874c57Selowe 		 * We beat the PCF bins over the head until
3086db874c57Selowe 		 * we got the memory that we wanted.
30877c478bd9Sstevel@tonic-gate 		 * The pcf accounting has been done,
30887c478bd9Sstevel@tonic-gate 		 * though none of the pcf_wait flags have been set,
30897c478bd9Sstevel@tonic-gate 		 * drop the locks and continue on.
30907c478bd9Sstevel@tonic-gate 		 */
3091db874c57Selowe 		ASSERT(collected == npgs);
30927c478bd9Sstevel@tonic-gate 		while (p >= pcf) {
30937c478bd9Sstevel@tonic-gate 			mutex_exit(&p->pcf_lock);
30947c478bd9Sstevel@tonic-gate 			p--;
30957c478bd9Sstevel@tonic-gate 		}
30967c478bd9Sstevel@tonic-gate 	}
30977c478bd9Sstevel@tonic-gate 
30987c478bd9Sstevel@tonic-gate 	/*
30997c478bd9Sstevel@tonic-gate 	 * freemem is not protected by any lock. Thus, we cannot
31007c478bd9Sstevel@tonic-gate 	 * have any assertion containing freemem here.
31017c478bd9Sstevel@tonic-gate 	 */
3102db874c57Selowe 	freemem -= npgs;
31037c478bd9Sstevel@tonic-gate 
31047c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_reclaim);
31057c478bd9Sstevel@tonic-gate 	if (PP_ISAGED(pp)) {
3106db874c57Selowe 		if (npgs > 1) {
3107db874c57Selowe 			page_list_sub_pages(pp, pp->p_szc);
3108db874c57Selowe 		} else {
3109db874c57Selowe 			page_list_sub(pp, PG_FREE_LIST);
3110db874c57Selowe 		}
31117c478bd9Sstevel@tonic-gate 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
31127c478bd9Sstevel@tonic-gate 		    "page_reclaim_free:pp %p", pp);
31137c478bd9Sstevel@tonic-gate 	} else {
3114db874c57Selowe 		ASSERT(npgs == 1);
31157c478bd9Sstevel@tonic-gate 		page_list_sub(pp, PG_CACHE_LIST);
31167c478bd9Sstevel@tonic-gate 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
31177c478bd9Sstevel@tonic-gate 		    "page_reclaim_cache:pp %p", pp);
31187c478bd9Sstevel@tonic-gate 	}
31197c478bd9Sstevel@tonic-gate 
31207c478bd9Sstevel@tonic-gate 	/*
31217c478bd9Sstevel@tonic-gate 	 * clear the p_free & p_age bits since this page is no longer
31227c478bd9Sstevel@tonic-gate 	 * on the free list.  Notice that there was a brief time where
31237c478bd9Sstevel@tonic-gate 	 * a page is marked as free, but is not on the list.
31247c478bd9Sstevel@tonic-gate 	 *
31257c478bd9Sstevel@tonic-gate 	 * Set the reference bit to protect against immediate pageout.
31267c478bd9Sstevel@tonic-gate 	 */
312724e9c58bSelowe 	for (i = 0; i < npgs; i++, pp++) {
3128db874c57Selowe 		PP_CLRFREE(pp);
3129db874c57Selowe 		PP_CLRAGED(pp);
3130db874c57Selowe 		page_set_props(pp, P_REF);
3131db874c57Selowe 	}
31327c478bd9Sstevel@tonic-gate 
31337c478bd9Sstevel@tonic-gate 	CPU_STATS_ENTER_K();
31347c478bd9Sstevel@tonic-gate 	cpup = CPU;	/* get cpup now that CPU cannot change */
31357c478bd9Sstevel@tonic-gate 	CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
31367c478bd9Sstevel@tonic-gate 	CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
31377c478bd9Sstevel@tonic-gate 	CPU_STATS_EXIT_K();
31387c478bd9Sstevel@tonic-gate 
31397c478bd9Sstevel@tonic-gate 	return (1);
31407c478bd9Sstevel@tonic-gate }
31417c478bd9Sstevel@tonic-gate 
31427c478bd9Sstevel@tonic-gate 
31437c478bd9Sstevel@tonic-gate 
31447c478bd9Sstevel@tonic-gate /*
31457c478bd9Sstevel@tonic-gate  * Destroy identity of the page and put it back on
31467c478bd9Sstevel@tonic-gate  * the page free list.  Assumes that the caller has
31477c478bd9Sstevel@tonic-gate  * acquired the "exclusive" lock on the page.
31487c478bd9Sstevel@tonic-gate  */
31497c478bd9Sstevel@tonic-gate void
31507c478bd9Sstevel@tonic-gate page_destroy(page_t *pp, int dontfree)
31517c478bd9Sstevel@tonic-gate {
31527c478bd9Sstevel@tonic-gate 	ASSERT((PAGE_EXCL(pp) &&
31537c478bd9Sstevel@tonic-gate 	    !page_iolock_assert(pp)) || panicstr);
315407b65a64Saguzovsk 	ASSERT(pp->p_slckcnt == 0 || panicstr);
31557c478bd9Sstevel@tonic-gate 
31567c478bd9Sstevel@tonic-gate 	if (pp->p_szc != 0) {
31577c478bd9Sstevel@tonic-gate 		if (pp->p_vnode == NULL || IS_SWAPFSVP(pp->p_vnode) ||
3158ad23a2dbSjohansen 		    PP_ISKAS(pp)) {
31597c478bd9Sstevel@tonic-gate 			panic("page_destroy: anon or kernel or no vnode "
31607c478bd9Sstevel@tonic-gate 			    "large page %p", (void *)pp);
31617c478bd9Sstevel@tonic-gate 		}
31627c478bd9Sstevel@tonic-gate 		page_demote_vp_pages(pp);
31637c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_szc == 0);
31647c478bd9Sstevel@tonic-gate 	}
31657c478bd9Sstevel@tonic-gate 
31667c478bd9Sstevel@tonic-gate 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy:pp %p", pp);
31677c478bd9Sstevel@tonic-gate 
31687c478bd9Sstevel@tonic-gate 	/*
31697c478bd9Sstevel@tonic-gate 	 * Unload translations, if any, then hash out the
31707c478bd9Sstevel@tonic-gate 	 * page to erase its identity.
31717c478bd9Sstevel@tonic-gate 	 */
31727c478bd9Sstevel@tonic-gate 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
31737c478bd9Sstevel@tonic-gate 	page_hashout(pp, NULL);
31747c478bd9Sstevel@tonic-gate 
31757c478bd9Sstevel@tonic-gate 	if (!dontfree) {
31767c478bd9Sstevel@tonic-gate 		/*
31777c478bd9Sstevel@tonic-gate 		 * Acquire the "freemem_lock" for availrmem.
31787c478bd9Sstevel@tonic-gate 		 * The page_struct_lock need not be acquired for lckcnt
31797c478bd9Sstevel@tonic-gate 		 * and cowcnt since the page has an "exclusive" lock.
31807c478bd9Sstevel@tonic-gate 		 */
31817c478bd9Sstevel@tonic-gate 		if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
31827c478bd9Sstevel@tonic-gate 			mutex_enter(&freemem_lock);
31837c478bd9Sstevel@tonic-gate 			if (pp->p_lckcnt != 0) {
31847c478bd9Sstevel@tonic-gate 				availrmem++;
31857c478bd9Sstevel@tonic-gate 				pp->p_lckcnt = 0;
31867c478bd9Sstevel@tonic-gate 			}
31877c478bd9Sstevel@tonic-gate 			if (pp->p_cowcnt != 0) {
31887c478bd9Sstevel@tonic-gate 				availrmem += pp->p_cowcnt;
31897c478bd9Sstevel@tonic-gate 				pp->p_cowcnt = 0;
31907c478bd9Sstevel@tonic-gate 			}
31917c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
31927c478bd9Sstevel@tonic-gate 		}
31937c478bd9Sstevel@tonic-gate 		/*
31947c478bd9Sstevel@tonic-gate 		 * Put the page on the "free" list.
31957c478bd9Sstevel@tonic-gate 		 */
31967c478bd9Sstevel@tonic-gate 		page_free(pp, 0);
31977c478bd9Sstevel@tonic-gate 	}
31987c478bd9Sstevel@tonic-gate }
31997c478bd9Sstevel@tonic-gate 
32007c478bd9Sstevel@tonic-gate void
32017c478bd9Sstevel@tonic-gate page_destroy_pages(page_t *pp)
32027c478bd9Sstevel@tonic-gate {
32037c478bd9Sstevel@tonic-gate 
32047c478bd9Sstevel@tonic-gate 	page_t	*tpp, *rootpp = NULL;
32057c478bd9Sstevel@tonic-gate 	pgcnt_t	pgcnt = page_get_pagecnt(pp->p_szc);
32067c478bd9Sstevel@tonic-gate 	pgcnt_t	i, pglcks = 0;
32077c478bd9Sstevel@tonic-gate 	uint_t	szc = pp->p_szc;
32087c478bd9Sstevel@tonic-gate 
32097c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_szc != 0 && pp->p_szc < page_num_pagesizes());
32107c478bd9Sstevel@tonic-gate 
32117c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_destroy_pages);
32127c478bd9Sstevel@tonic-gate 
32137c478bd9Sstevel@tonic-gate 	TRACE_1(TR_FAC_VM, TR_PAGE_DESTROY, "page_destroy_pages:pp %p", pp);
32147c478bd9Sstevel@tonic-gate 
32157c478bd9Sstevel@tonic-gate 	if ((page_pptonum(pp) & (pgcnt - 1)) != 0) {
32167c478bd9Sstevel@tonic-gate 		panic("page_destroy_pages: not root page %p", (void *)pp);
32177c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
32187c478bd9Sstevel@tonic-gate 	}
32197c478bd9Sstevel@tonic-gate 
3220affbd3ccSkchow 	for (i = 0, tpp = pp; i < pgcnt; i++, tpp++) {
32217c478bd9Sstevel@tonic-gate 		ASSERT((PAGE_EXCL(tpp) &&
32227c478bd9Sstevel@tonic-gate 		    !page_iolock_assert(tpp)) || panicstr);
322307b65a64Saguzovsk 		ASSERT(tpp->p_slckcnt == 0 || panicstr);
32247c478bd9Sstevel@tonic-gate 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
32257c478bd9Sstevel@tonic-gate 		page_hashout(tpp, NULL);
32267c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_offset == (u_offset_t)-1);
32277c478bd9Sstevel@tonic-gate 		if (tpp->p_lckcnt != 0) {
32287c478bd9Sstevel@tonic-gate 			pglcks++;
32297c478bd9Sstevel@tonic-gate 			tpp->p_lckcnt = 0;
32307c478bd9Sstevel@tonic-gate 		} else if (tpp->p_cowcnt != 0) {
32317c478bd9Sstevel@tonic-gate 			pglcks += tpp->p_cowcnt;
32327c478bd9Sstevel@tonic-gate 			tpp->p_cowcnt = 0;
32337c478bd9Sstevel@tonic-gate 		}
32347c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_getshare(tpp));
32357c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_vnode == NULL);
32367c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_szc == szc);
32377c478bd9Sstevel@tonic-gate 
32387c478bd9Sstevel@tonic-gate 		PP_SETFREE(tpp);
32397c478bd9Sstevel@tonic-gate 		page_clr_all_props(tpp);
32407c478bd9Sstevel@tonic-gate 		PP_SETAGED(tpp);
32417c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_next == tpp);
32427c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_prev == tpp);
32437c478bd9Sstevel@tonic-gate 		page_list_concat(&rootpp, &tpp);
32447c478bd9Sstevel@tonic-gate 	}
32457c478bd9Sstevel@tonic-gate 
32467c478bd9Sstevel@tonic-gate 	ASSERT(rootpp == pp);
32477c478bd9Sstevel@tonic-gate 	if (pglcks != 0) {
32487c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
32497c478bd9Sstevel@tonic-gate 		availrmem += pglcks;
32507c478bd9Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
32517c478bd9Sstevel@tonic-gate 	}
32527c478bd9Sstevel@tonic-gate 
32537c478bd9Sstevel@tonic-gate 	page_list_add_pages(rootpp, 0);
32547c478bd9Sstevel@tonic-gate 	page_create_putback(pgcnt);
32557c478bd9Sstevel@tonic-gate }
32567c478bd9Sstevel@tonic-gate 
32577c478bd9Sstevel@tonic-gate /*
32587c478bd9Sstevel@tonic-gate  * Similar to page_destroy(), but destroys pages which are
32597c478bd9Sstevel@tonic-gate  * locked and known to be on the page free list.  Since
32607c478bd9Sstevel@tonic-gate  * the page is known to be free and locked, no one can access
32617c478bd9Sstevel@tonic-gate  * it.
32627c478bd9Sstevel@tonic-gate  *
32637c478bd9Sstevel@tonic-gate  * Also, the number of free pages does not change.
32647c478bd9Sstevel@tonic-gate  */
32657c478bd9Sstevel@tonic-gate void
32667c478bd9Sstevel@tonic-gate page_destroy_free(page_t *pp)
32677c478bd9Sstevel@tonic-gate {
32687c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp));
32697c478bd9Sstevel@tonic-gate 	ASSERT(PP_ISFREE(pp));
32707c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode);
32717c478bd9Sstevel@tonic-gate 	ASSERT(hat_page_getattr(pp, P_MOD | P_REF | P_RO) == 0);
32727c478bd9Sstevel@tonic-gate 	ASSERT(!hat_page_is_mapped(pp));
32737c478bd9Sstevel@tonic-gate 	ASSERT(PP_ISAGED(pp) == 0);
32747c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_szc == 0);
32757c478bd9Sstevel@tonic-gate 
32767c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_destroy_free);
32777c478bd9Sstevel@tonic-gate 	page_list_sub(pp, PG_CACHE_LIST);
32787c478bd9Sstevel@tonic-gate 
32797c478bd9Sstevel@tonic-gate 	page_hashout(pp, NULL);
32807c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode == NULL);
32817c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_offset == (u_offset_t)-1);
32827c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_hash == NULL);
32837c478bd9Sstevel@tonic-gate 
32847c478bd9Sstevel@tonic-gate 	PP_SETAGED(pp);
32857c478bd9Sstevel@tonic-gate 	page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
32867c478bd9Sstevel@tonic-gate 	page_unlock(pp);
32877c478bd9Sstevel@tonic-gate 
32887c478bd9Sstevel@tonic-gate 	mutex_enter(&new_freemem_lock);
32897c478bd9Sstevel@tonic-gate 	if (freemem_wait) {
32907c478bd9Sstevel@tonic-gate 		cv_signal(&freemem_cv);
32917c478bd9Sstevel@tonic-gate 	}
32927c478bd9Sstevel@tonic-gate 	mutex_exit(&new_freemem_lock);
32937c478bd9Sstevel@tonic-gate }
32947c478bd9Sstevel@tonic-gate 
32957c478bd9Sstevel@tonic-gate /*
32967c478bd9Sstevel@tonic-gate  * Rename the page "opp" to have an identity specified
32977c478bd9Sstevel@tonic-gate  * by [vp, off].  If a page already exists with this name
32987c478bd9Sstevel@tonic-gate  * it is locked and destroyed.  Note that the page's
32997c478bd9Sstevel@tonic-gate  * translations are not unloaded during the rename.
33007c478bd9Sstevel@tonic-gate  *
33017c478bd9Sstevel@tonic-gate  * This routine is used by the anon layer to "steal" the
33027c478bd9Sstevel@tonic-gate  * original page and is not unlike destroying a page and
33037c478bd9Sstevel@tonic-gate  * creating a new page using the same page frame.
33047c478bd9Sstevel@tonic-gate  *
33057c478bd9Sstevel@tonic-gate  * XXX -- Could deadlock if caller 1 tries to rename A to B while
33067c478bd9Sstevel@tonic-gate  * caller 2 tries to rename B to A.
33077c478bd9Sstevel@tonic-gate  */
33087c478bd9Sstevel@tonic-gate void
33097c478bd9Sstevel@tonic-gate page_rename(page_t *opp, vnode_t *vp, u_offset_t off)
33107c478bd9Sstevel@tonic-gate {
33117c478bd9Sstevel@tonic-gate 	page_t		*pp;
33127c478bd9Sstevel@tonic-gate 	int		olckcnt = 0;
33137c478bd9Sstevel@tonic-gate 	int		ocowcnt = 0;
33147c478bd9Sstevel@tonic-gate 	kmutex_t	*phm;
33157c478bd9Sstevel@tonic-gate 	ulong_t		index;
33167c478bd9Sstevel@tonic-gate 
33177c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(opp) && !page_iolock_assert(opp));
33187c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
33197c478bd9Sstevel@tonic-gate 	ASSERT(PP_ISFREE(opp) == 0);
33207c478bd9Sstevel@tonic-gate 
33217c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_rename_count);
33227c478bd9Sstevel@tonic-gate 
33237c478bd9Sstevel@tonic-gate 	TRACE_3(TR_FAC_VM, TR_PAGE_RENAME,
33247c478bd9Sstevel@tonic-gate 		"page rename:pp %p vp %p off %llx", opp, vp, off);
33257c478bd9Sstevel@tonic-gate 
332637fbc076Saguzovsk 	/*
332737fbc076Saguzovsk 	 * CacheFS may call page_rename for a large NFS page
332837fbc076Saguzovsk 	 * when both CacheFS and NFS mount points are used
332937fbc076Saguzovsk 	 * by applications. Demote this large page before
333037fbc076Saguzovsk 	 * renaming it, to ensure that there are no "partial"
333137fbc076Saguzovsk 	 * large pages left lying around.
333237fbc076Saguzovsk 	 */
333337fbc076Saguzovsk 	if (opp->p_szc != 0) {
333437fbc076Saguzovsk 		vnode_t *ovp = opp->p_vnode;
333537fbc076Saguzovsk 		ASSERT(ovp != NULL);
333637fbc076Saguzovsk 		ASSERT(!IS_SWAPFSVP(ovp));
3337ad23a2dbSjohansen 		ASSERT(!VN_ISKAS(ovp));
333837fbc076Saguzovsk 		page_demote_vp_pages(opp);
333937fbc076Saguzovsk 		ASSERT(opp->p_szc == 0);
334037fbc076Saguzovsk 	}
334137fbc076Saguzovsk 
33427c478bd9Sstevel@tonic-gate 	page_hashout(opp, NULL);
33437c478bd9Sstevel@tonic-gate 	PP_CLRAGED(opp);
33447c478bd9Sstevel@tonic-gate 
33457c478bd9Sstevel@tonic-gate 	/*
33467c478bd9Sstevel@tonic-gate 	 * Acquire the appropriate page hash lock, since
33477c478bd9Sstevel@tonic-gate 	 * we're going to rename the page.
33487c478bd9Sstevel@tonic-gate 	 */
33497c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, off);
33507c478bd9Sstevel@tonic-gate 	phm = PAGE_HASH_MUTEX(index);
33517c478bd9Sstevel@tonic-gate 	mutex_enter(phm);
33527c478bd9Sstevel@tonic-gate top:
33537c478bd9Sstevel@tonic-gate 	/*
33547c478bd9Sstevel@tonic-gate 	 * Look for an existing page with this name and destroy it if found.
33557c478bd9Sstevel@tonic-gate 	 * By holding the page hash lock all the way to the page_hashin()
33567c478bd9Sstevel@tonic-gate 	 * call, we are assured that no page can be created with this
33577c478bd9Sstevel@tonic-gate 	 * identity.  In the case when the phm lock is dropped to undo any
33587c478bd9Sstevel@tonic-gate 	 * hat layer mappings, the existing page is held with an "exclusive"
33597c478bd9Sstevel@tonic-gate 	 * lock, again preventing another page from being created with
33607c478bd9Sstevel@tonic-gate 	 * this identity.
33617c478bd9Sstevel@tonic-gate 	 */
33627c478bd9Sstevel@tonic-gate 	PAGE_HASH_SEARCH(index, pp, vp, off);
33637c478bd9Sstevel@tonic-gate 	if (pp != NULL) {
33647c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_rename_exists);
33657c478bd9Sstevel@tonic-gate 
33667c478bd9Sstevel@tonic-gate 		/*
33677c478bd9Sstevel@tonic-gate 		 * As it turns out, this is one of only two places where
33687c478bd9Sstevel@tonic-gate 		 * page_lock() needs to hold the passed in lock in the
33697c478bd9Sstevel@tonic-gate 		 * successful case.  In all of the others, the lock could
33707c478bd9Sstevel@tonic-gate 		 * be dropped as soon as the attempt is made to lock
33717c478bd9Sstevel@tonic-gate 		 * the page.  It is tempting to add yet another arguement,
33727c478bd9Sstevel@tonic-gate 		 * PL_KEEP or PL_DROP, to let page_lock know what to do.
33737c478bd9Sstevel@tonic-gate 		 */
33747c478bd9Sstevel@tonic-gate 		if (!page_lock(pp, SE_EXCL, phm, P_RECLAIM)) {
33757c478bd9Sstevel@tonic-gate 			/*
33767c478bd9Sstevel@tonic-gate 			 * Went to sleep because the page could not
33777c478bd9Sstevel@tonic-gate 			 * be locked.  We were woken up when the page
33787c478bd9Sstevel@tonic-gate 			 * was unlocked, or when the page was destroyed.
33797c478bd9Sstevel@tonic-gate 			 * In either case, `phm' was dropped while we
33807c478bd9Sstevel@tonic-gate 			 * slept.  Hence we should not just roar through
33817c478bd9Sstevel@tonic-gate 			 * this loop.
33827c478bd9Sstevel@tonic-gate 			 */
33837c478bd9Sstevel@tonic-gate 			goto top;
33847c478bd9Sstevel@tonic-gate 		}
33857c478bd9Sstevel@tonic-gate 
338637fbc076Saguzovsk 		/*
338737fbc076Saguzovsk 		 * If an existing page is a large page, then demote
338837fbc076Saguzovsk 		 * it to ensure that no "partial" large pages are
338937fbc076Saguzovsk 		 * "created" after page_rename. An existing page
339037fbc076Saguzovsk 		 * can be a CacheFS page, and can't belong to swapfs.
339137fbc076Saguzovsk 		 */
33927c478bd9Sstevel@tonic-gate 		if (hat_page_is_mapped(pp)) {
33937c478bd9Sstevel@tonic-gate 			/*
33947c478bd9Sstevel@tonic-gate 			 * Unload translations.  Since we hold the
33957c478bd9Sstevel@tonic-gate 			 * exclusive lock on this page, the page
33967c478bd9Sstevel@tonic-gate 			 * can not be changed while we drop phm.
33977c478bd9Sstevel@tonic-gate 			 * This is also not a lock protocol violation,
33987c478bd9Sstevel@tonic-gate 			 * but rather the proper way to do things.
33997c478bd9Sstevel@tonic-gate 			 */
34007c478bd9Sstevel@tonic-gate 			mutex_exit(phm);
34017c478bd9Sstevel@tonic-gate 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
340237fbc076Saguzovsk 			if (pp->p_szc != 0) {
340337fbc076Saguzovsk 				ASSERT(!IS_SWAPFSVP(vp));
3404ad23a2dbSjohansen 				ASSERT(!VN_ISKAS(vp));
340537fbc076Saguzovsk 				page_demote_vp_pages(pp);
340637fbc076Saguzovsk 				ASSERT(pp->p_szc == 0);
340737fbc076Saguzovsk 			}
340837fbc076Saguzovsk 			mutex_enter(phm);
340937fbc076Saguzovsk 		} else if (pp->p_szc != 0) {
341037fbc076Saguzovsk 			ASSERT(!IS_SWAPFSVP(vp));
3411ad23a2dbSjohansen 			ASSERT(!VN_ISKAS(vp));
341237fbc076Saguzovsk 			mutex_exit(phm);
341337fbc076Saguzovsk 			page_demote_vp_pages(pp);
341437fbc076Saguzovsk 			ASSERT(pp->p_szc == 0);
34157c478bd9Sstevel@tonic-gate 			mutex_enter(phm);
34167c478bd9Sstevel@tonic-gate 		}
34177c478bd9Sstevel@tonic-gate 		page_hashout(pp, phm);
34187c478bd9Sstevel@tonic-gate 	}
34197c478bd9Sstevel@tonic-gate 	/*
34207c478bd9Sstevel@tonic-gate 	 * Hash in the page with the new identity.
34217c478bd9Sstevel@tonic-gate 	 */
34227c478bd9Sstevel@tonic-gate 	if (!page_hashin(opp, vp, off, phm)) {
34237c478bd9Sstevel@tonic-gate 		/*
34247c478bd9Sstevel@tonic-gate 		 * We were holding phm while we searched for [vp, off]
34257c478bd9Sstevel@tonic-gate 		 * and only dropped phm if we found and locked a page.
34267c478bd9Sstevel@tonic-gate 		 * If we can't create this page now, then some thing
34277c478bd9Sstevel@tonic-gate 		 * is really broken.
34287c478bd9Sstevel@tonic-gate 		 */
34297c478bd9Sstevel@tonic-gate 		panic("page_rename: Can't hash in page: %p", (void *)pp);
34307c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
34317c478bd9Sstevel@tonic-gate 	}
34327c478bd9Sstevel@tonic-gate 
34337c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(phm));
34347c478bd9Sstevel@tonic-gate 	mutex_exit(phm);
34357c478bd9Sstevel@tonic-gate 
34367c478bd9Sstevel@tonic-gate 	/*
34377c478bd9Sstevel@tonic-gate 	 * Now that we have dropped phm, lets get around to finishing up
34387c478bd9Sstevel@tonic-gate 	 * with pp.
34397c478bd9Sstevel@tonic-gate 	 */
34407c478bd9Sstevel@tonic-gate 	if (pp != NULL) {
34417c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_is_mapped(pp));
34427c478bd9Sstevel@tonic-gate 		/* for now large pages should not end up here */
34437c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_szc == 0);
34447c478bd9Sstevel@tonic-gate 		/*
34457c478bd9Sstevel@tonic-gate 		 * Save the locks for transfer to the new page and then
34467c478bd9Sstevel@tonic-gate 		 * clear them so page_free doesn't think they're important.
34477c478bd9Sstevel@tonic-gate 		 * The page_struct_lock need not be acquired for lckcnt and
34487c478bd9Sstevel@tonic-gate 		 * cowcnt since the page has an "exclusive" lock.
34497c478bd9Sstevel@tonic-gate 		 */
34507c478bd9Sstevel@tonic-gate 		olckcnt = pp->p_lckcnt;
34517c478bd9Sstevel@tonic-gate 		ocowcnt = pp->p_cowcnt;
34527c478bd9Sstevel@tonic-gate 		pp->p_lckcnt = pp->p_cowcnt = 0;
34537c478bd9Sstevel@tonic-gate 
34547c478bd9Sstevel@tonic-gate 		/*
34557c478bd9Sstevel@tonic-gate 		 * Put the page on the "free" list after we drop
34567c478bd9Sstevel@tonic-gate 		 * the lock.  The less work under the lock the better.
34577c478bd9Sstevel@tonic-gate 		 */
34587c478bd9Sstevel@tonic-gate 		/*LINTED: constant in conditional context*/
34597c478bd9Sstevel@tonic-gate 		VN_DISPOSE(pp, B_FREE, 0, kcred);
34607c478bd9Sstevel@tonic-gate 	}
34617c478bd9Sstevel@tonic-gate 
34627c478bd9Sstevel@tonic-gate 	/*
34637c478bd9Sstevel@tonic-gate 	 * Transfer the lock count from the old page (if any).
34647c478bd9Sstevel@tonic-gate 	 * The page_struct_lock need not be acquired for lckcnt and
34657c478bd9Sstevel@tonic-gate 	 * cowcnt since the page has an "exclusive" lock.
34667c478bd9Sstevel@tonic-gate 	 */
34677c478bd9Sstevel@tonic-gate 	opp->p_lckcnt += olckcnt;
34687c478bd9Sstevel@tonic-gate 	opp->p_cowcnt += ocowcnt;
34697c478bd9Sstevel@tonic-gate }
34707c478bd9Sstevel@tonic-gate 
34717c478bd9Sstevel@tonic-gate /*
34727c478bd9Sstevel@tonic-gate  * low level routine to add page `pp' to the hash and vp chains for [vp, offset]
34737c478bd9Sstevel@tonic-gate  *
34747c478bd9Sstevel@tonic-gate  * Pages are normally inserted at the start of a vnode's v_pages list.
34757c478bd9Sstevel@tonic-gate  * If the vnode is VMODSORT and the page is modified, it goes at the end.
34767c478bd9Sstevel@tonic-gate  * This can happen when a modified page is relocated for DR.
34777c478bd9Sstevel@tonic-gate  *
34787c478bd9Sstevel@tonic-gate  * Returns 1 on success and 0 on failure.
34797c478bd9Sstevel@tonic-gate  */
34807c478bd9Sstevel@tonic-gate static int
34817c478bd9Sstevel@tonic-gate page_do_hashin(page_t *pp, vnode_t *vp, u_offset_t offset)
34827c478bd9Sstevel@tonic-gate {
34837c478bd9Sstevel@tonic-gate 	page_t		**listp;
34847c478bd9Sstevel@tonic-gate 	page_t		*tp;
34857c478bd9Sstevel@tonic-gate 	ulong_t		index;
34867c478bd9Sstevel@tonic-gate 
34877c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp));
34887c478bd9Sstevel@tonic-gate 	ASSERT(vp != NULL);
34897c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
34907c478bd9Sstevel@tonic-gate 
34917c478bd9Sstevel@tonic-gate 	/*
34927c478bd9Sstevel@tonic-gate 	 * Be sure to set these up before the page is inserted on the hash
34937c478bd9Sstevel@tonic-gate 	 * list.  As soon as the page is placed on the list some other
34947c478bd9Sstevel@tonic-gate 	 * thread might get confused and wonder how this page could
34957c478bd9Sstevel@tonic-gate 	 * possibly hash to this list.
34967c478bd9Sstevel@tonic-gate 	 */
34977c478bd9Sstevel@tonic-gate 	pp->p_vnode = vp;
34987c478bd9Sstevel@tonic-gate 	pp->p_offset = offset;
34997c478bd9Sstevel@tonic-gate 
35007c478bd9Sstevel@tonic-gate 	/*
35017c478bd9Sstevel@tonic-gate 	 * record if this page is on a swap vnode
35027c478bd9Sstevel@tonic-gate 	 */
35037c478bd9Sstevel@tonic-gate 	if ((vp->v_flag & VISSWAP) != 0)
35047c478bd9Sstevel@tonic-gate 		PP_SETSWAP(pp);
35057c478bd9Sstevel@tonic-gate 
35067c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, offset);
35077c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(index)));
35087c478bd9Sstevel@tonic-gate 	listp = &page_hash[index];
35097c478bd9Sstevel@tonic-gate 
35107c478bd9Sstevel@tonic-gate 	/*
35117c478bd9Sstevel@tonic-gate 	 * If this page is already hashed in, fail this attempt to add it.
35127c478bd9Sstevel@tonic-gate 	 */
35137c478bd9Sstevel@tonic-gate 	for (tp = *listp; tp != NULL; tp = tp->p_hash) {
35147c478bd9Sstevel@tonic-gate 		if (tp->p_vnode == vp && tp->p_offset == offset) {
35157c478bd9Sstevel@tonic-gate 			pp->p_vnode = NULL;
35167c478bd9Sstevel@tonic-gate 			pp->p_offset = (u_offset_t)(-1);
35177c478bd9Sstevel@tonic-gate 			return (0);
35187c478bd9Sstevel@tonic-gate 		}
35197c478bd9Sstevel@tonic-gate 	}
35207c478bd9Sstevel@tonic-gate 	pp->p_hash = *listp;
35217c478bd9Sstevel@tonic-gate 	*listp = pp;
35227c478bd9Sstevel@tonic-gate 
35237c478bd9Sstevel@tonic-gate 	/*
35247c478bd9Sstevel@tonic-gate 	 * Add the page to the vnode's list of pages
35257c478bd9Sstevel@tonic-gate 	 */
35267c478bd9Sstevel@tonic-gate 	if (vp->v_pages != NULL && IS_VMODSORT(vp) && hat_ismod(pp))
35277c478bd9Sstevel@tonic-gate 		listp = &vp->v_pages->p_vpprev->p_vpnext;
35287c478bd9Sstevel@tonic-gate 	else
35297c478bd9Sstevel@tonic-gate 		listp = &vp->v_pages;
35307c478bd9Sstevel@tonic-gate 
35317c478bd9Sstevel@tonic-gate 	page_vpadd(listp, pp);
35327c478bd9Sstevel@tonic-gate 
35337c478bd9Sstevel@tonic-gate 	return (1);
35347c478bd9Sstevel@tonic-gate }
35357c478bd9Sstevel@tonic-gate 
35367c478bd9Sstevel@tonic-gate /*
35377c478bd9Sstevel@tonic-gate  * Add page `pp' to both the hash and vp chains for [vp, offset].
35387c478bd9Sstevel@tonic-gate  *
35397c478bd9Sstevel@tonic-gate  * Returns 1 on success and 0 on failure.
35407c478bd9Sstevel@tonic-gate  * If hold is passed in, it is not dropped.
35417c478bd9Sstevel@tonic-gate  */
35427c478bd9Sstevel@tonic-gate int
35437c478bd9Sstevel@tonic-gate page_hashin(page_t *pp, vnode_t *vp, u_offset_t offset, kmutex_t *hold)
35447c478bd9Sstevel@tonic-gate {
35457c478bd9Sstevel@tonic-gate 	kmutex_t	*phm = NULL;
35467c478bd9Sstevel@tonic-gate 	kmutex_t	*vphm;
35477c478bd9Sstevel@tonic-gate 	int		rc;
35487c478bd9Sstevel@tonic-gate 
35497c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(vp)));
35507c478bd9Sstevel@tonic-gate 
35517c478bd9Sstevel@tonic-gate 	TRACE_3(TR_FAC_VM, TR_PAGE_HASHIN,
35527c478bd9Sstevel@tonic-gate 		"page_hashin:pp %p vp %p offset %llx",
35537c478bd9Sstevel@tonic-gate 		pp, vp, offset);
35547c478bd9Sstevel@tonic-gate 
35557c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(hashin_count);
35567c478bd9Sstevel@tonic-gate 
35577c478bd9Sstevel@tonic-gate 	if (hold != NULL)
35587c478bd9Sstevel@tonic-gate 		phm = hold;
35597c478bd9Sstevel@tonic-gate 	else {
35607c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(hashin_not_held);
35617c478bd9Sstevel@tonic-gate 		phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, offset));
35627c478bd9Sstevel@tonic-gate 		mutex_enter(phm);
35637c478bd9Sstevel@tonic-gate 	}
35647c478bd9Sstevel@tonic-gate 
35657c478bd9Sstevel@tonic-gate 	vphm = page_vnode_mutex(vp);
35667c478bd9Sstevel@tonic-gate 	mutex_enter(vphm);
35677c478bd9Sstevel@tonic-gate 	rc = page_do_hashin(pp, vp, offset);
35687c478bd9Sstevel@tonic-gate 	mutex_exit(vphm);
35697c478bd9Sstevel@tonic-gate 	if (hold == NULL)
35707c478bd9Sstevel@tonic-gate 		mutex_exit(phm);
35717c478bd9Sstevel@tonic-gate 	if (rc == 0)
35727c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(hashin_already);
35737c478bd9Sstevel@tonic-gate 	return (rc);
35747c478bd9Sstevel@tonic-gate }
35757c478bd9Sstevel@tonic-gate 
35767c478bd9Sstevel@tonic-gate /*
35777c478bd9Sstevel@tonic-gate  * Remove page ``pp'' from the hash and vp chains and remove vp association.
35787c478bd9Sstevel@tonic-gate  * All mutexes must be held
35797c478bd9Sstevel@tonic-gate  */
35807c478bd9Sstevel@tonic-gate static void
35817c478bd9Sstevel@tonic-gate page_do_hashout(page_t *pp)
35827c478bd9Sstevel@tonic-gate {
35837c478bd9Sstevel@tonic-gate 	page_t	**hpp;
35847c478bd9Sstevel@tonic-gate 	page_t	*hp;
35857c478bd9Sstevel@tonic-gate 	vnode_t	*vp = pp->p_vnode;
35867c478bd9Sstevel@tonic-gate 
35877c478bd9Sstevel@tonic-gate 	ASSERT(vp != NULL);
35887c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
35897c478bd9Sstevel@tonic-gate 
35907c478bd9Sstevel@tonic-gate 	/*
35917c478bd9Sstevel@tonic-gate 	 * First, take pp off of its hash chain.
35927c478bd9Sstevel@tonic-gate 	 */
35937c478bd9Sstevel@tonic-gate 	hpp = &page_hash[PAGE_HASH_FUNC(vp, pp->p_offset)];
35947c478bd9Sstevel@tonic-gate 
35957c478bd9Sstevel@tonic-gate 	for (;;) {
35967c478bd9Sstevel@tonic-gate 		hp = *hpp;
35977c478bd9Sstevel@tonic-gate 		if (hp == pp)
35987c478bd9Sstevel@tonic-gate 			break;
35997c478bd9Sstevel@tonic-gate 		if (hp == NULL) {
36007c478bd9Sstevel@tonic-gate 			panic("page_do_hashout");
36017c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
36027c478bd9Sstevel@tonic-gate 		}
36037c478bd9Sstevel@tonic-gate 		hpp = &hp->p_hash;
36047c478bd9Sstevel@tonic-gate 	}
36057c478bd9Sstevel@tonic-gate 	*hpp = pp->p_hash;
36067c478bd9Sstevel@tonic-gate 
36077c478bd9Sstevel@tonic-gate 	/*
36087c478bd9Sstevel@tonic-gate 	 * Now remove it from its associated vnode.
36097c478bd9Sstevel@tonic-gate 	 */
36107c478bd9Sstevel@tonic-gate 	if (vp->v_pages)
36117c478bd9Sstevel@tonic-gate 		page_vpsub(&vp->v_pages, pp);
36127c478bd9Sstevel@tonic-gate 
36137c478bd9Sstevel@tonic-gate 	pp->p_hash = NULL;
36147c478bd9Sstevel@tonic-gate 	page_clr_all_props(pp);
36157c478bd9Sstevel@tonic-gate 	PP_CLRSWAP(pp);
36167c478bd9Sstevel@tonic-gate 	pp->p_vnode = NULL;
36177c478bd9Sstevel@tonic-gate 	pp->p_offset = (u_offset_t)-1;
36187c478bd9Sstevel@tonic-gate }
36197c478bd9Sstevel@tonic-gate 
36207c478bd9Sstevel@tonic-gate /*
36217c478bd9Sstevel@tonic-gate  * Remove page ``pp'' from the hash and vp chains and remove vp association.
36227c478bd9Sstevel@tonic-gate  *
36237c478bd9Sstevel@tonic-gate  * When `phm' is non-NULL it contains the address of the mutex protecting the
36247c478bd9Sstevel@tonic-gate  * hash list pp is on.  It is not dropped.
36257c478bd9Sstevel@tonic-gate  */
36267c478bd9Sstevel@tonic-gate void
36277c478bd9Sstevel@tonic-gate page_hashout(page_t *pp, kmutex_t *phm)
36287c478bd9Sstevel@tonic-gate {
36297c478bd9Sstevel@tonic-gate 	vnode_t		*vp;
36307c478bd9Sstevel@tonic-gate 	ulong_t		index;
36317c478bd9Sstevel@tonic-gate 	kmutex_t	*nphm;
36327c478bd9Sstevel@tonic-gate 	kmutex_t	*vphm;
36337c478bd9Sstevel@tonic-gate 	kmutex_t	*sep;
36347c478bd9Sstevel@tonic-gate 
36357c478bd9Sstevel@tonic-gate 	ASSERT(phm != NULL ? MUTEX_HELD(phm) : 1);
36367c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode != NULL);
36377c478bd9Sstevel@tonic-gate 	ASSERT((PAGE_EXCL(pp) && !page_iolock_assert(pp)) || panicstr);
36387c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(page_vnode_mutex(pp->p_vnode)));
36397c478bd9Sstevel@tonic-gate 
36407c478bd9Sstevel@tonic-gate 	vp = pp->p_vnode;
36417c478bd9Sstevel@tonic-gate 
36427c478bd9Sstevel@tonic-gate 	TRACE_2(TR_FAC_VM, TR_PAGE_HASHOUT,
36437c478bd9Sstevel@tonic-gate 		"page_hashout:pp %p vp %p", pp, vp);
36447c478bd9Sstevel@tonic-gate 
36457c478bd9Sstevel@tonic-gate 	/* Kernel probe */
36467c478bd9Sstevel@tonic-gate 	TNF_PROBE_2(page_unmap, "vm pagefault", /* CSTYLED */,
36477c478bd9Sstevel@tonic-gate 	    tnf_opaque, vnode, vp,
36487c478bd9Sstevel@tonic-gate 	    tnf_offset, offset, pp->p_offset);
36497c478bd9Sstevel@tonic-gate 
36507c478bd9Sstevel@tonic-gate 	/*
36517c478bd9Sstevel@tonic-gate 	 *
36527c478bd9Sstevel@tonic-gate 	 */
36537c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(hashout_count);
36547c478bd9Sstevel@tonic-gate 	index = PAGE_HASH_FUNC(vp, pp->p_offset);
36557c478bd9Sstevel@tonic-gate 	if (phm == NULL) {
36567c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(hashout_not_held);
36577c478bd9Sstevel@tonic-gate 		nphm = PAGE_HASH_MUTEX(index);
36587c478bd9Sstevel@tonic-gate 		mutex_enter(nphm);
36597c478bd9Sstevel@tonic-gate 	}
36607c478bd9Sstevel@tonic-gate 	ASSERT(phm ? phm == PAGE_HASH_MUTEX(index) : 1);
36617c478bd9Sstevel@tonic-gate 
36627c478bd9Sstevel@tonic-gate 
36637c478bd9Sstevel@tonic-gate 	/*
36647c478bd9Sstevel@tonic-gate 	 * grab page vnode mutex and remove it...
36657c478bd9Sstevel@tonic-gate 	 */
36667c478bd9Sstevel@tonic-gate 	vphm = page_vnode_mutex(vp);
36677c478bd9Sstevel@tonic-gate 	mutex_enter(vphm);
36687c478bd9Sstevel@tonic-gate 
36697c478bd9Sstevel@tonic-gate 	page_do_hashout(pp);
36707c478bd9Sstevel@tonic-gate 
36717c478bd9Sstevel@tonic-gate 	mutex_exit(vphm);
36727c478bd9Sstevel@tonic-gate 	if (phm == NULL)
36737c478bd9Sstevel@tonic-gate 		mutex_exit(nphm);
36747c478bd9Sstevel@tonic-gate 
36757c478bd9Sstevel@tonic-gate 	/*
36767c478bd9Sstevel@tonic-gate 	 * Wake up processes waiting for this page.  The page's
36777c478bd9Sstevel@tonic-gate 	 * identity has been changed, and is probably not the
36787c478bd9Sstevel@tonic-gate 	 * desired page any longer.
36797c478bd9Sstevel@tonic-gate 	 */
36807c478bd9Sstevel@tonic-gate 	sep = page_se_mutex(pp);
36817c478bd9Sstevel@tonic-gate 	mutex_enter(sep);
368242787a71Sstans 	pp->p_selock &= ~SE_EWANTED;
36837c478bd9Sstevel@tonic-gate 	if (CV_HAS_WAITERS(&pp->p_cv))
36847c478bd9Sstevel@tonic-gate 		cv_broadcast(&pp->p_cv);
36857c478bd9Sstevel@tonic-gate 	mutex_exit(sep);
36867c478bd9Sstevel@tonic-gate }
36877c478bd9Sstevel@tonic-gate 
36887c478bd9Sstevel@tonic-gate /*
36897c478bd9Sstevel@tonic-gate  * Add the page to the front of a linked list of pages
36907c478bd9Sstevel@tonic-gate  * using the p_next & p_prev pointers for the list.
36917c478bd9Sstevel@tonic-gate  * The caller is responsible for protecting the list pointers.
36927c478bd9Sstevel@tonic-gate  */
36937c478bd9Sstevel@tonic-gate void
36947c478bd9Sstevel@tonic-gate page_add(page_t **ppp, page_t *pp)
36957c478bd9Sstevel@tonic-gate {
36967c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
36977c478bd9Sstevel@tonic-gate 
36987c478bd9Sstevel@tonic-gate 	page_add_common(ppp, pp);
36997c478bd9Sstevel@tonic-gate }
37007c478bd9Sstevel@tonic-gate 
37017c478bd9Sstevel@tonic-gate 
37027c478bd9Sstevel@tonic-gate 
37037c478bd9Sstevel@tonic-gate /*
37047c478bd9Sstevel@tonic-gate  *  Common code for page_add() and mach_page_add()
37057c478bd9Sstevel@tonic-gate  */
37067c478bd9Sstevel@tonic-gate void
37077c478bd9Sstevel@tonic-gate page_add_common(page_t **ppp, page_t *pp)
37087c478bd9Sstevel@tonic-gate {
37097c478bd9Sstevel@tonic-gate 	if (*ppp == NULL) {
37107c478bd9Sstevel@tonic-gate 		pp->p_next = pp->p_prev = pp;
37117c478bd9Sstevel@tonic-gate 	} else {
37127c478bd9Sstevel@tonic-gate 		pp->p_next = *ppp;
37137c478bd9Sstevel@tonic-gate 		pp->p_prev = (*ppp)->p_prev;
37147c478bd9Sstevel@tonic-gate 		(*ppp)->p_prev = pp;
37157c478bd9Sstevel@tonic-gate 		pp->p_prev->p_next = pp;
37167c478bd9Sstevel@tonic-gate 	}
37177c478bd9Sstevel@tonic-gate 	*ppp = pp;
37187c478bd9Sstevel@tonic-gate }
37197c478bd9Sstevel@tonic-gate 
37207c478bd9Sstevel@tonic-gate 
37217c478bd9Sstevel@tonic-gate /*
37227c478bd9Sstevel@tonic-gate  * Remove this page from a linked list of pages
37237c478bd9Sstevel@tonic-gate  * using the p_next & p_prev pointers for the list.
37247c478bd9Sstevel@tonic-gate  *
37257c478bd9Sstevel@tonic-gate  * The caller is responsible for protecting the list pointers.
37267c478bd9Sstevel@tonic-gate  */
37277c478bd9Sstevel@tonic-gate void
37287c478bd9Sstevel@tonic-gate page_sub(page_t **ppp, page_t *pp)
37297c478bd9Sstevel@tonic-gate {
37307c478bd9Sstevel@tonic-gate 	ASSERT((PP_ISFREE(pp)) ? 1 :
37317c478bd9Sstevel@tonic-gate 	    (PAGE_EXCL(pp)) || (PAGE_SHARED(pp) && page_iolock_assert(pp)));
37327c478bd9Sstevel@tonic-gate 
37337c478bd9Sstevel@tonic-gate 	if (*ppp == NULL || pp == NULL) {
37347c478bd9Sstevel@tonic-gate 		panic("page_sub: bad arg(s): pp %p, *ppp %p",
37357c478bd9Sstevel@tonic-gate 		    (void *)pp, (void *)(*ppp));
37367c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
37377c478bd9Sstevel@tonic-gate 	}
37387c478bd9Sstevel@tonic-gate 
37397c478bd9Sstevel@tonic-gate 	page_sub_common(ppp, pp);
37407c478bd9Sstevel@tonic-gate }
37417c478bd9Sstevel@tonic-gate 
37427c478bd9Sstevel@tonic-gate 
37437c478bd9Sstevel@tonic-gate /*
37447c478bd9Sstevel@tonic-gate  *  Common code for page_sub() and mach_page_sub()
37457c478bd9Sstevel@tonic-gate  */
37467c478bd9Sstevel@tonic-gate void
37477c478bd9Sstevel@tonic-gate page_sub_common(page_t **ppp, page_t *pp)
37487c478bd9Sstevel@tonic-gate {
37497c478bd9Sstevel@tonic-gate 	if (*ppp == pp)
37507c478bd9Sstevel@tonic-gate 		*ppp = pp->p_next;		/* go to next page */
37517c478bd9Sstevel@tonic-gate 
37527c478bd9Sstevel@tonic-gate 	if (*ppp == pp)
37537c478bd9Sstevel@tonic-gate 		*ppp = NULL;			/* page list is gone */
37547c478bd9Sstevel@tonic-gate 	else {
37557c478bd9Sstevel@tonic-gate 		pp->p_prev->p_next = pp->p_next;
37567c478bd9Sstevel@tonic-gate 		pp->p_next->p_prev = pp->p_prev;
37577c478bd9Sstevel@tonic-gate 	}
37587c478bd9Sstevel@tonic-gate 	pp->p_prev = pp->p_next = pp;		/* make pp a list of one */
37597c478bd9Sstevel@tonic-gate }
37607c478bd9Sstevel@tonic-gate 
37617c478bd9Sstevel@tonic-gate 
37627c478bd9Sstevel@tonic-gate /*
37637c478bd9Sstevel@tonic-gate  * Break page list cppp into two lists with npages in the first list.
37647c478bd9Sstevel@tonic-gate  * The tail is returned in nppp.
37657c478bd9Sstevel@tonic-gate  */
37667c478bd9Sstevel@tonic-gate void
37677c478bd9Sstevel@tonic-gate page_list_break(page_t **oppp, page_t **nppp, pgcnt_t npages)
37687c478bd9Sstevel@tonic-gate {
37697c478bd9Sstevel@tonic-gate 	page_t *s1pp = *oppp;
37707c478bd9Sstevel@tonic-gate 	page_t *s2pp;
37717c478bd9Sstevel@tonic-gate 	page_t *e1pp, *e2pp;
37727c478bd9Sstevel@tonic-gate 	long n = 0;
37737c478bd9Sstevel@tonic-gate 
37747c478bd9Sstevel@tonic-gate 	if (s1pp == NULL) {
37757c478bd9Sstevel@tonic-gate 		*nppp = NULL;
37767c478bd9Sstevel@tonic-gate 		return;
37777c478bd9Sstevel@tonic-gate 	}
37787c478bd9Sstevel@tonic-gate 	if (npages == 0) {
37797c478bd9Sstevel@tonic-gate 		*nppp = s1pp;
37807c478bd9Sstevel@tonic-gate 		*oppp = NULL;
37817c478bd9Sstevel@tonic-gate 		return;
37827c478bd9Sstevel@tonic-gate 	}
37837c478bd9Sstevel@tonic-gate 	for (n = 0, s2pp = *oppp; n < npages; n++) {
37847c478bd9Sstevel@tonic-gate 		s2pp = s2pp->p_next;
37857c478bd9Sstevel@tonic-gate 	}
37867c478bd9Sstevel@tonic-gate 	/* Fix head and tail of new lists */
37877c478bd9Sstevel@tonic-gate 	e1pp = s2pp->p_prev;
37887c478bd9Sstevel@tonic-gate 	e2pp = s1pp->p_prev;
37897c478bd9Sstevel@tonic-gate 	s1pp->p_prev = e1pp;
37907c478bd9Sstevel@tonic-gate 	e1pp->p_next = s1pp;
37917c478bd9Sstevel@tonic-gate 	s2pp->p_prev = e2pp;
37927c478bd9Sstevel@tonic-gate 	e2pp->p_next = s2pp;
37937c478bd9Sstevel@tonic-gate 
37947c478bd9Sstevel@tonic-gate 	/* second list empty */
37957c478bd9Sstevel@tonic-gate 	if (s2pp == s1pp) {
37967c478bd9Sstevel@tonic-gate 		*oppp = s1pp;
37977c478bd9Sstevel@tonic-gate 		*nppp = NULL;
37987c478bd9Sstevel@tonic-gate 	} else {
37997c478bd9Sstevel@tonic-gate 		*oppp = s1pp;
38007c478bd9Sstevel@tonic-gate 		*nppp = s2pp;
38017c478bd9Sstevel@tonic-gate 	}
38027c478bd9Sstevel@tonic-gate }
38037c478bd9Sstevel@tonic-gate 
38047c478bd9Sstevel@tonic-gate /*
38057c478bd9Sstevel@tonic-gate  * Concatenate page list nppp onto the end of list ppp.
38067c478bd9Sstevel@tonic-gate  */
38077c478bd9Sstevel@tonic-gate void
38087c478bd9Sstevel@tonic-gate page_list_concat(page_t **ppp, page_t **nppp)
38097c478bd9Sstevel@tonic-gate {
38107c478bd9Sstevel@tonic-gate 	page_t *s1pp, *s2pp, *e1pp, *e2pp;
38117c478bd9Sstevel@tonic-gate 
38127c478bd9Sstevel@tonic-gate 	if (*nppp == NULL) {
38137c478bd9Sstevel@tonic-gate 		return;
38147c478bd9Sstevel@tonic-gate 	}
38157c478bd9Sstevel@tonic-gate 	if (*ppp == NULL) {
38167c478bd9Sstevel@tonic-gate 		*ppp = *nppp;
38177c478bd9Sstevel@tonic-gate 		return;
38187c478bd9Sstevel@tonic-gate 	}
38197c478bd9Sstevel@tonic-gate 	s1pp = *ppp;
38207c478bd9Sstevel@tonic-gate 	e1pp =  s1pp->p_prev;
38217c478bd9Sstevel@tonic-gate 	s2pp = *nppp;
38227c478bd9Sstevel@tonic-gate 	e2pp = s2pp->p_prev;
38237c478bd9Sstevel@tonic-gate 	s1pp->p_prev = e2pp;
38247c478bd9Sstevel@tonic-gate 	e2pp->p_next = s1pp;
38257c478bd9Sstevel@tonic-gate 	e1pp->p_next = s2pp;
38267c478bd9Sstevel@tonic-gate 	s2pp->p_prev = e1pp;
38277c478bd9Sstevel@tonic-gate }
38287c478bd9Sstevel@tonic-gate 
38297c478bd9Sstevel@tonic-gate /*
38307c478bd9Sstevel@tonic-gate  * return the next page in the page list
38317c478bd9Sstevel@tonic-gate  */
38327c478bd9Sstevel@tonic-gate page_t *
38337c478bd9Sstevel@tonic-gate page_list_next(page_t *pp)
38347c478bd9Sstevel@tonic-gate {
38357c478bd9Sstevel@tonic-gate 	return (pp->p_next);
38367c478bd9Sstevel@tonic-gate }
38377c478bd9Sstevel@tonic-gate 
38387c478bd9Sstevel@tonic-gate 
38397c478bd9Sstevel@tonic-gate /*
38407c478bd9Sstevel@tonic-gate  * Add the page to the front of the linked list of pages
38417c478bd9Sstevel@tonic-gate  * using p_vpnext/p_vpprev pointers for the list.
38427c478bd9Sstevel@tonic-gate  *
38437c478bd9Sstevel@tonic-gate  * The caller is responsible for protecting the lists.
38447c478bd9Sstevel@tonic-gate  */
38457c478bd9Sstevel@tonic-gate void
38467c478bd9Sstevel@tonic-gate page_vpadd(page_t **ppp, page_t *pp)
38477c478bd9Sstevel@tonic-gate {
38487c478bd9Sstevel@tonic-gate 	if (*ppp == NULL) {
38497c478bd9Sstevel@tonic-gate 		pp->p_vpnext = pp->p_vpprev = pp;
38507c478bd9Sstevel@tonic-gate 	} else {
38517c478bd9Sstevel@tonic-gate 		pp->p_vpnext = *ppp;
38527c478bd9Sstevel@tonic-gate 		pp->p_vpprev = (*ppp)->p_vpprev;
38537c478bd9Sstevel@tonic-gate 		(*ppp)->p_vpprev = pp;
38547c478bd9Sstevel@tonic-gate 		pp->p_vpprev->p_vpnext = pp;
38557c478bd9Sstevel@tonic-gate 	}
38567c478bd9Sstevel@tonic-gate 	*ppp = pp;
38577c478bd9Sstevel@tonic-gate }
38587c478bd9Sstevel@tonic-gate 
38597c478bd9Sstevel@tonic-gate /*
38607c478bd9Sstevel@tonic-gate  * Remove this page from the linked list of pages
38617c478bd9Sstevel@tonic-gate  * using p_vpnext/p_vpprev pointers for the list.
38627c478bd9Sstevel@tonic-gate  *
38637c478bd9Sstevel@tonic-gate  * The caller is responsible for protecting the lists.
38647c478bd9Sstevel@tonic-gate  */
38657c478bd9Sstevel@tonic-gate void
38667c478bd9Sstevel@tonic-gate page_vpsub(page_t **ppp, page_t *pp)
38677c478bd9Sstevel@tonic-gate {
38687c478bd9Sstevel@tonic-gate 	if (*ppp == NULL || pp == NULL) {
38697c478bd9Sstevel@tonic-gate 		panic("page_vpsub: bad arg(s): pp %p, *ppp %p",
38707c478bd9Sstevel@tonic-gate 		    (void *)pp, (void *)(*ppp));
38717c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
38727c478bd9Sstevel@tonic-gate 	}
38737c478bd9Sstevel@tonic-gate 
38747c478bd9Sstevel@tonic-gate 	if (*ppp == pp)
38757c478bd9Sstevel@tonic-gate 		*ppp = pp->p_vpnext;		/* go to next page */
38767c478bd9Sstevel@tonic-gate 
38777c478bd9Sstevel@tonic-gate 	if (*ppp == pp)
38787c478bd9Sstevel@tonic-gate 		*ppp = NULL;			/* page list is gone */
38797c478bd9Sstevel@tonic-gate 	else {
38807c478bd9Sstevel@tonic-gate 		pp->p_vpprev->p_vpnext = pp->p_vpnext;
38817c478bd9Sstevel@tonic-gate 		pp->p_vpnext->p_vpprev = pp->p_vpprev;
38827c478bd9Sstevel@tonic-gate 	}
38837c478bd9Sstevel@tonic-gate 	pp->p_vpprev = pp->p_vpnext = pp;	/* make pp a list of one */
38847c478bd9Sstevel@tonic-gate }
38857c478bd9Sstevel@tonic-gate 
38867c478bd9Sstevel@tonic-gate /*
38877c478bd9Sstevel@tonic-gate  * Lock a physical page into memory "long term".  Used to support "lock
38887c478bd9Sstevel@tonic-gate  * in memory" functions.  Accepts the page to be locked, and a cow variable
38897c478bd9Sstevel@tonic-gate  * to indicate whether a the lock will travel to the new page during
38907c478bd9Sstevel@tonic-gate  * a potential copy-on-write.
38917c478bd9Sstevel@tonic-gate  */
38927c478bd9Sstevel@tonic-gate int
38937c478bd9Sstevel@tonic-gate page_pp_lock(
38947c478bd9Sstevel@tonic-gate 	page_t *pp,			/* page to be locked */
38957c478bd9Sstevel@tonic-gate 	int cow,			/* cow lock */
38967c478bd9Sstevel@tonic-gate 	int kernel)			/* must succeed -- ignore checking */
38977c478bd9Sstevel@tonic-gate {
38987c478bd9Sstevel@tonic-gate 	int r = 0;			/* result -- assume failure */
38997c478bd9Sstevel@tonic-gate 
39007c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
39017c478bd9Sstevel@tonic-gate 
39027c478bd9Sstevel@tonic-gate 	page_struct_lock(pp);
39037c478bd9Sstevel@tonic-gate 	/*
39047c478bd9Sstevel@tonic-gate 	 * Acquire the "freemem_lock" for availrmem.
39057c478bd9Sstevel@tonic-gate 	 */
39067c478bd9Sstevel@tonic-gate 	if (cow) {
39077c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
39087c478bd9Sstevel@tonic-gate 		if ((availrmem > pages_pp_maximum) &&
39097c478bd9Sstevel@tonic-gate 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
39107c478bd9Sstevel@tonic-gate 			availrmem--;
39117c478bd9Sstevel@tonic-gate 			pages_locked++;
39127c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
39137c478bd9Sstevel@tonic-gate 			r = 1;
39147c478bd9Sstevel@tonic-gate 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
39157c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
39167c478bd9Sstevel@tonic-gate 				    "COW lock limit reached on pfn 0x%lx",
39177c478bd9Sstevel@tonic-gate 				    page_pptonum(pp));
39187c478bd9Sstevel@tonic-gate 			}
39197c478bd9Sstevel@tonic-gate 		} else
39207c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
39217c478bd9Sstevel@tonic-gate 	} else {
39227c478bd9Sstevel@tonic-gate 		if (pp->p_lckcnt) {
39237c478bd9Sstevel@tonic-gate 			if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
39247c478bd9Sstevel@tonic-gate 				r = 1;
39257c478bd9Sstevel@tonic-gate 				if (++pp->p_lckcnt ==
39267c478bd9Sstevel@tonic-gate 				    (ushort_t)PAGE_LOCK_MAXIMUM) {
39277c478bd9Sstevel@tonic-gate 					cmn_err(CE_WARN, "Page lock limit "
39287c478bd9Sstevel@tonic-gate 					    "reached on pfn 0x%lx",
39297c478bd9Sstevel@tonic-gate 					    page_pptonum(pp));
39307c478bd9Sstevel@tonic-gate 				}
39317c478bd9Sstevel@tonic-gate 			}
39327c478bd9Sstevel@tonic-gate 		} else {
39337c478bd9Sstevel@tonic-gate 			if (kernel) {
39347c478bd9Sstevel@tonic-gate 				/* availrmem accounting done by caller */
39357c478bd9Sstevel@tonic-gate 				++pp->p_lckcnt;
39367c478bd9Sstevel@tonic-gate 				r = 1;
39377c478bd9Sstevel@tonic-gate 			} else {
39387c478bd9Sstevel@tonic-gate 				mutex_enter(&freemem_lock);
39397c478bd9Sstevel@tonic-gate 				if (availrmem > pages_pp_maximum) {
39407c478bd9Sstevel@tonic-gate 					availrmem--;
39417c478bd9Sstevel@tonic-gate 					pages_locked++;
39427c478bd9Sstevel@tonic-gate 					++pp->p_lckcnt;
39437c478bd9Sstevel@tonic-gate 					r = 1;
39447c478bd9Sstevel@tonic-gate 				}
39457c478bd9Sstevel@tonic-gate 				mutex_exit(&freemem_lock);
39467c478bd9Sstevel@tonic-gate 			}
39477c478bd9Sstevel@tonic-gate 		}
39487c478bd9Sstevel@tonic-gate 	}
39497c478bd9Sstevel@tonic-gate 	page_struct_unlock(pp);
39507c478bd9Sstevel@tonic-gate 	return (r);
39517c478bd9Sstevel@tonic-gate }
39527c478bd9Sstevel@tonic-gate 
39537c478bd9Sstevel@tonic-gate /*
39547c478bd9Sstevel@tonic-gate  * Decommit a lock on a physical page frame.  Account for cow locks if
39557c478bd9Sstevel@tonic-gate  * appropriate.
39567c478bd9Sstevel@tonic-gate  */
39577c478bd9Sstevel@tonic-gate void
39587c478bd9Sstevel@tonic-gate page_pp_unlock(
39597c478bd9Sstevel@tonic-gate 	page_t *pp,			/* page to be unlocked */
39607c478bd9Sstevel@tonic-gate 	int cow,			/* expect cow lock */
39617c478bd9Sstevel@tonic-gate 	int kernel)			/* this was a kernel lock */
39627c478bd9Sstevel@tonic-gate {
39637c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
39647c478bd9Sstevel@tonic-gate 
39657c478bd9Sstevel@tonic-gate 	page_struct_lock(pp);
39667c478bd9Sstevel@tonic-gate 	/*
39677c478bd9Sstevel@tonic-gate 	 * Acquire the "freemem_lock" for availrmem.
39687c478bd9Sstevel@tonic-gate 	 * If cowcnt or lcknt is already 0 do nothing; i.e., we
39697c478bd9Sstevel@tonic-gate 	 * could be called to unlock even if nothing is locked. This could
39707c478bd9Sstevel@tonic-gate 	 * happen if locked file pages were truncated (removing the lock)
39717c478bd9Sstevel@tonic-gate 	 * and the file was grown again and new pages faulted in; the new
39727c478bd9Sstevel@tonic-gate 	 * pages are unlocked but the segment still thinks they're locked.
39737c478bd9Sstevel@tonic-gate 	 */
39747c478bd9Sstevel@tonic-gate 	if (cow) {
39757c478bd9Sstevel@tonic-gate 		if (pp->p_cowcnt) {
39767c478bd9Sstevel@tonic-gate 			mutex_enter(&freemem_lock);
39777c478bd9Sstevel@tonic-gate 			pp->p_cowcnt--;
39787c478bd9Sstevel@tonic-gate 			availrmem++;
39797c478bd9Sstevel@tonic-gate 			pages_locked--;
39807c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
39817c478bd9Sstevel@tonic-gate 		}
39827c478bd9Sstevel@tonic-gate 	} else {
39837c478bd9Sstevel@tonic-gate 		if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
39847c478bd9Sstevel@tonic-gate 			if (!kernel) {
39857c478bd9Sstevel@tonic-gate 				mutex_enter(&freemem_lock);
39867c478bd9Sstevel@tonic-gate 				availrmem++;
39877c478bd9Sstevel@tonic-gate 				pages_locked--;
39887c478bd9Sstevel@tonic-gate 				mutex_exit(&freemem_lock);
39897c478bd9Sstevel@tonic-gate 			}
39907c478bd9Sstevel@tonic-gate 		}
39917c478bd9Sstevel@tonic-gate 	}
39927c478bd9Sstevel@tonic-gate 	page_struct_unlock(pp);
39937c478bd9Sstevel@tonic-gate }
39947c478bd9Sstevel@tonic-gate 
39957c478bd9Sstevel@tonic-gate /*
39967c478bd9Sstevel@tonic-gate  * This routine reserves availrmem for npages;
39977c478bd9Sstevel@tonic-gate  * 	flags: KM_NOSLEEP or KM_SLEEP
39987c478bd9Sstevel@tonic-gate  * 	returns 1 on success or 0 on failure
39997c478bd9Sstevel@tonic-gate  */
40007c478bd9Sstevel@tonic-gate int
40017c478bd9Sstevel@tonic-gate page_resv(pgcnt_t npages, uint_t flags)
40027c478bd9Sstevel@tonic-gate {
40037c478bd9Sstevel@tonic-gate 	mutex_enter(&freemem_lock);
40047c478bd9Sstevel@tonic-gate 	while (availrmem < tune.t_minarmem + npages) {
40057c478bd9Sstevel@tonic-gate 		if (flags & KM_NOSLEEP) {
40067c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
40077c478bd9Sstevel@tonic-gate 			return (0);
40087c478bd9Sstevel@tonic-gate 		}
40097c478bd9Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
40107c478bd9Sstevel@tonic-gate 		page_needfree(npages);
40117c478bd9Sstevel@tonic-gate 		kmem_reap();
40127c478bd9Sstevel@tonic-gate 		delay(hz >> 2);
40137c478bd9Sstevel@tonic-gate 		page_needfree(-(spgcnt_t)npages);
40147c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
40157c478bd9Sstevel@tonic-gate 	}
40167c478bd9Sstevel@tonic-gate 	availrmem -= npages;
40177c478bd9Sstevel@tonic-gate 	mutex_exit(&freemem_lock);
40187c478bd9Sstevel@tonic-gate 	return (1);
40197c478bd9Sstevel@tonic-gate }
40207c478bd9Sstevel@tonic-gate 
40217c478bd9Sstevel@tonic-gate /*
40227c478bd9Sstevel@tonic-gate  * This routine unreserves availrmem for npages;
40237c478bd9Sstevel@tonic-gate  */
40247c478bd9Sstevel@tonic-gate void
40257c478bd9Sstevel@tonic-gate page_unresv(pgcnt_t npages)
40267c478bd9Sstevel@tonic-gate {
40277c478bd9Sstevel@tonic-gate 	mutex_enter(&freemem_lock);
40287c478bd9Sstevel@tonic-gate 	availrmem += npages;
40297c478bd9Sstevel@tonic-gate 	mutex_exit(&freemem_lock);
40307c478bd9Sstevel@tonic-gate }
40317c478bd9Sstevel@tonic-gate 
40327c478bd9Sstevel@tonic-gate /*
40337c478bd9Sstevel@tonic-gate  * See Statement at the beginning of segvn_lockop() regarding
40347c478bd9Sstevel@tonic-gate  * the way we handle cowcnts and lckcnts.
40357c478bd9Sstevel@tonic-gate  *
40367c478bd9Sstevel@tonic-gate  * Transfer cowcnt on 'opp' to cowcnt on 'npp' if the vpage
40377c478bd9Sstevel@tonic-gate  * that breaks COW has PROT_WRITE.
40387c478bd9Sstevel@tonic-gate  *
40397c478bd9Sstevel@tonic-gate  * Note that, we may also break COW in case we are softlocking
40407c478bd9Sstevel@tonic-gate  * on read access during physio;
40417c478bd9Sstevel@tonic-gate  * in this softlock case, the vpage may not have PROT_WRITE.
40427c478bd9Sstevel@tonic-gate  * So, we need to transfer lckcnt on 'opp' to lckcnt on 'npp'
40437c478bd9Sstevel@tonic-gate  * if the vpage doesn't have PROT_WRITE.
40447c478bd9Sstevel@tonic-gate  *
40457c478bd9Sstevel@tonic-gate  * This routine is never called if we are stealing a page
40467c478bd9Sstevel@tonic-gate  * in anon_private.
40477c478bd9Sstevel@tonic-gate  *
40487c478bd9Sstevel@tonic-gate  * The caller subtracted from availrmem for read only mapping.
40497c478bd9Sstevel@tonic-gate  * if lckcnt is 1 increment availrmem.
40507c478bd9Sstevel@tonic-gate  */
40517c478bd9Sstevel@tonic-gate void
40527c478bd9Sstevel@tonic-gate page_pp_useclaim(
40537c478bd9Sstevel@tonic-gate 	page_t *opp,		/* original page frame losing lock */
40547c478bd9Sstevel@tonic-gate 	page_t *npp,		/* new page frame gaining lock */
40557c478bd9Sstevel@tonic-gate 	uint_t	write_perm) 	/* set if vpage has PROT_WRITE */
40567c478bd9Sstevel@tonic-gate {
40577c478bd9Sstevel@tonic-gate 	int payback = 0;
40587c478bd9Sstevel@tonic-gate 
40597c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(opp));
40607c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(npp));
40617c478bd9Sstevel@tonic-gate 
40627c478bd9Sstevel@tonic-gate 	page_struct_lock(opp);
40637c478bd9Sstevel@tonic-gate 
40647c478bd9Sstevel@tonic-gate 	ASSERT(npp->p_cowcnt == 0);
40657c478bd9Sstevel@tonic-gate 	ASSERT(npp->p_lckcnt == 0);
40667c478bd9Sstevel@tonic-gate 
40677c478bd9Sstevel@tonic-gate 	/* Don't use claim if nothing is locked (see page_pp_unlock above) */
40687c478bd9Sstevel@tonic-gate 	if ((write_perm && opp->p_cowcnt != 0) ||
40697c478bd9Sstevel@tonic-gate 	    (!write_perm && opp->p_lckcnt != 0)) {
40707c478bd9Sstevel@tonic-gate 
40717c478bd9Sstevel@tonic-gate 		if (write_perm) {
40727c478bd9Sstevel@tonic-gate 			npp->p_cowcnt++;
40737c478bd9Sstevel@tonic-gate 			ASSERT(opp->p_cowcnt != 0);
40747c478bd9Sstevel@tonic-gate 			opp->p_cowcnt--;
40757c478bd9Sstevel@tonic-gate 		} else {
40767c478bd9Sstevel@tonic-gate 
40777c478bd9Sstevel@tonic-gate 			ASSERT(opp->p_lckcnt != 0);
40787c478bd9Sstevel@tonic-gate 
40797c478bd9Sstevel@tonic-gate 			/*
40807c478bd9Sstevel@tonic-gate 			 * We didn't need availrmem decremented if p_lckcnt on
40817c478bd9Sstevel@tonic-gate 			 * original page is 1. Here, we are unlocking
40827c478bd9Sstevel@tonic-gate 			 * read-only copy belonging to original page and
40837c478bd9Sstevel@tonic-gate 			 * are locking a copy belonging to new page.
40847c478bd9Sstevel@tonic-gate 			 */
40857c478bd9Sstevel@tonic-gate 			if (opp->p_lckcnt == 1)
40867c478bd9Sstevel@tonic-gate 				payback = 1;
40877c478bd9Sstevel@tonic-gate 
40887c478bd9Sstevel@tonic-gate 			npp->p_lckcnt++;
40897c478bd9Sstevel@tonic-gate 			opp->p_lckcnt--;
40907c478bd9Sstevel@tonic-gate 		}
40917c478bd9Sstevel@tonic-gate 	}
40927c478bd9Sstevel@tonic-gate 	if (payback) {
40937c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
40947c478bd9Sstevel@tonic-gate 		availrmem++;
40957c478bd9Sstevel@tonic-gate 		pages_useclaim--;
40967c478bd9Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
40977c478bd9Sstevel@tonic-gate 	}
40987c478bd9Sstevel@tonic-gate 	page_struct_unlock(opp);
40997c478bd9Sstevel@tonic-gate }
41007c478bd9Sstevel@tonic-gate 
41017c478bd9Sstevel@tonic-gate /*
41027c478bd9Sstevel@tonic-gate  * Simple claim adjust functions -- used to support changes in
41037c478bd9Sstevel@tonic-gate  * claims due to changes in access permissions.  Used by segvn_setprot().
41047c478bd9Sstevel@tonic-gate  */
41057c478bd9Sstevel@tonic-gate int
41067c478bd9Sstevel@tonic-gate page_addclaim(page_t *pp)
41077c478bd9Sstevel@tonic-gate {
41087c478bd9Sstevel@tonic-gate 	int r = 0;			/* result */
41097c478bd9Sstevel@tonic-gate 
41107c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
41117c478bd9Sstevel@tonic-gate 
41127c478bd9Sstevel@tonic-gate 	page_struct_lock(pp);
41137c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_lckcnt != 0);
41147c478bd9Sstevel@tonic-gate 
41157c478bd9Sstevel@tonic-gate 	if (pp->p_lckcnt == 1) {
41167c478bd9Sstevel@tonic-gate 		if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
41177c478bd9Sstevel@tonic-gate 			--pp->p_lckcnt;
41187c478bd9Sstevel@tonic-gate 			r = 1;
41197c478bd9Sstevel@tonic-gate 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
41207c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
41217c478bd9Sstevel@tonic-gate 				    "COW lock limit reached on pfn 0x%lx",
41227c478bd9Sstevel@tonic-gate 				    page_pptonum(pp));
41237c478bd9Sstevel@tonic-gate 			}
41247c478bd9Sstevel@tonic-gate 		}
41257c478bd9Sstevel@tonic-gate 	} else {
41267c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
41277c478bd9Sstevel@tonic-gate 		if ((availrmem > pages_pp_maximum) &&
41287c478bd9Sstevel@tonic-gate 		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
41297c478bd9Sstevel@tonic-gate 			--availrmem;
41307c478bd9Sstevel@tonic-gate 			++pages_claimed;
41317c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
41327c478bd9Sstevel@tonic-gate 			--pp->p_lckcnt;
41337c478bd9Sstevel@tonic-gate 			r = 1;
41347c478bd9Sstevel@tonic-gate 			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
41357c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
41367c478bd9Sstevel@tonic-gate 				    "COW lock limit reached on pfn 0x%lx",
41377c478bd9Sstevel@tonic-gate 				    page_pptonum(pp));
41387c478bd9Sstevel@tonic-gate 			}
41397c478bd9Sstevel@tonic-gate 		} else
41407c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
41417c478bd9Sstevel@tonic-gate 	}
41427c478bd9Sstevel@tonic-gate 	page_struct_unlock(pp);
41437c478bd9Sstevel@tonic-gate 	return (r);
41447c478bd9Sstevel@tonic-gate }
41457c478bd9Sstevel@tonic-gate 
41467c478bd9Sstevel@tonic-gate int
41477c478bd9Sstevel@tonic-gate page_subclaim(page_t *pp)
41487c478bd9Sstevel@tonic-gate {
41497c478bd9Sstevel@tonic-gate 	int r = 0;
41507c478bd9Sstevel@tonic-gate 
41517c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
41527c478bd9Sstevel@tonic-gate 
41537c478bd9Sstevel@tonic-gate 	page_struct_lock(pp);
41547c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_cowcnt != 0);
41557c478bd9Sstevel@tonic-gate 
41567c478bd9Sstevel@tonic-gate 	if (pp->p_lckcnt) {
41577c478bd9Sstevel@tonic-gate 		if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
41587c478bd9Sstevel@tonic-gate 			r = 1;
41597c478bd9Sstevel@tonic-gate 			/*
41607c478bd9Sstevel@tonic-gate 			 * for availrmem
41617c478bd9Sstevel@tonic-gate 			 */
41627c478bd9Sstevel@tonic-gate 			mutex_enter(&freemem_lock);
41637c478bd9Sstevel@tonic-gate 			availrmem++;
41647c478bd9Sstevel@tonic-gate 			pages_claimed--;
41657c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
41667c478bd9Sstevel@tonic-gate 
41677c478bd9Sstevel@tonic-gate 			pp->p_cowcnt--;
41687c478bd9Sstevel@tonic-gate 
41697c478bd9Sstevel@tonic-gate 			if (++pp->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
41707c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
41717c478bd9Sstevel@tonic-gate 				    "Page lock limit reached on pfn 0x%lx",
41727c478bd9Sstevel@tonic-gate 				    page_pptonum(pp));
41737c478bd9Sstevel@tonic-gate 			}
41747c478bd9Sstevel@tonic-gate 		}
41757c478bd9Sstevel@tonic-gate 	} else {
41767c478bd9Sstevel@tonic-gate 		r = 1;
41777c478bd9Sstevel@tonic-gate 		pp->p_cowcnt--;
41787c478bd9Sstevel@tonic-gate 		pp->p_lckcnt++;
41797c478bd9Sstevel@tonic-gate 	}
41807c478bd9Sstevel@tonic-gate 	page_struct_unlock(pp);
41817c478bd9Sstevel@tonic-gate 	return (r);
41827c478bd9Sstevel@tonic-gate }
41837c478bd9Sstevel@tonic-gate 
41847c478bd9Sstevel@tonic-gate int
41857c478bd9Sstevel@tonic-gate page_addclaim_pages(page_t  **ppa)
41867c478bd9Sstevel@tonic-gate {
41877c478bd9Sstevel@tonic-gate 
41887c478bd9Sstevel@tonic-gate 	pgcnt_t	lckpgs = 0, pg_idx;
41897c478bd9Sstevel@tonic-gate 
41907c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_addclaim_pages);
41917c478bd9Sstevel@tonic-gate 
41927c478bd9Sstevel@tonic-gate 	mutex_enter(&page_llock);
41937c478bd9Sstevel@tonic-gate 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
41947c478bd9Sstevel@tonic-gate 
41957c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
41967c478bd9Sstevel@tonic-gate 		ASSERT(ppa[pg_idx]->p_lckcnt != 0);
41977c478bd9Sstevel@tonic-gate 		if (ppa[pg_idx]->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
41987c478bd9Sstevel@tonic-gate 			mutex_exit(&page_llock);
41997c478bd9Sstevel@tonic-gate 			return (0);
42007c478bd9Sstevel@tonic-gate 		}
42017c478bd9Sstevel@tonic-gate 		if (ppa[pg_idx]->p_lckcnt > 1)
42027c478bd9Sstevel@tonic-gate 			lckpgs++;
42037c478bd9Sstevel@tonic-gate 	}
42047c478bd9Sstevel@tonic-gate 
42057c478bd9Sstevel@tonic-gate 	if (lckpgs != 0) {
42067c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
42077c478bd9Sstevel@tonic-gate 		if (availrmem >= pages_pp_maximum + lckpgs) {
42087c478bd9Sstevel@tonic-gate 			availrmem -= lckpgs;
42097c478bd9Sstevel@tonic-gate 			pages_claimed += lckpgs;
42107c478bd9Sstevel@tonic-gate 		} else {
42117c478bd9Sstevel@tonic-gate 			mutex_exit(&freemem_lock);
42127c478bd9Sstevel@tonic-gate 			mutex_exit(&page_llock);
42137c478bd9Sstevel@tonic-gate 			return (0);
42147c478bd9Sstevel@tonic-gate 		}
42157c478bd9Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
42167c478bd9Sstevel@tonic-gate 	}
42177c478bd9Sstevel@tonic-gate 
42187c478bd9Sstevel@tonic-gate 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
42197c478bd9Sstevel@tonic-gate 		ppa[pg_idx]->p_lckcnt--;
42207c478bd9Sstevel@tonic-gate 		ppa[pg_idx]->p_cowcnt++;
42217c478bd9Sstevel@tonic-gate 	}
42227c478bd9Sstevel@tonic-gate 	mutex_exit(&page_llock);
42237c478bd9Sstevel@tonic-gate 	return (1);
42247c478bd9Sstevel@tonic-gate }
42257c478bd9Sstevel@tonic-gate 
42267c478bd9Sstevel@tonic-gate int
42277c478bd9Sstevel@tonic-gate page_subclaim_pages(page_t  **ppa)
42287c478bd9Sstevel@tonic-gate {
42297c478bd9Sstevel@tonic-gate 	pgcnt_t	ulckpgs = 0, pg_idx;
42307c478bd9Sstevel@tonic-gate 
42317c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_subclaim_pages);
42327c478bd9Sstevel@tonic-gate 
42337c478bd9Sstevel@tonic-gate 	mutex_enter(&page_llock);
42347c478bd9Sstevel@tonic-gate 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
42357c478bd9Sstevel@tonic-gate 
42367c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_LOCKED(ppa[pg_idx]));
42377c478bd9Sstevel@tonic-gate 		ASSERT(ppa[pg_idx]->p_cowcnt != 0);
42387c478bd9Sstevel@tonic-gate 		if (ppa[pg_idx]->p_lckcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
42397c478bd9Sstevel@tonic-gate 			mutex_exit(&page_llock);
42407c478bd9Sstevel@tonic-gate 			return (0);
42417c478bd9Sstevel@tonic-gate 		}
42427c478bd9Sstevel@tonic-gate 		if (ppa[pg_idx]->p_lckcnt != 0)
42437c478bd9Sstevel@tonic-gate 			ulckpgs++;
42447c478bd9Sstevel@tonic-gate 	}
42457c478bd9Sstevel@tonic-gate 
42467c478bd9Sstevel@tonic-gate 	if (ulckpgs != 0) {
42477c478bd9Sstevel@tonic-gate 		mutex_enter(&freemem_lock);
42487c478bd9Sstevel@tonic-gate 		availrmem += ulckpgs;
42497c478bd9Sstevel@tonic-gate 		pages_claimed -= ulckpgs;
42507c478bd9Sstevel@tonic-gate 		mutex_exit(&freemem_lock);
42517c478bd9Sstevel@tonic-gate 	}
42527c478bd9Sstevel@tonic-gate 
42537c478bd9Sstevel@tonic-gate 	for (pg_idx = 0; ppa[pg_idx] != NULL; pg_idx++) {
42547c478bd9Sstevel@tonic-gate 		ppa[pg_idx]->p_cowcnt--;
42557c478bd9Sstevel@tonic-gate 		ppa[pg_idx]->p_lckcnt++;
42567c478bd9Sstevel@tonic-gate 
42577c478bd9Sstevel@tonic-gate 	}
42587c478bd9Sstevel@tonic-gate 	mutex_exit(&page_llock);
42597c478bd9Sstevel@tonic-gate 	return (1);
42607c478bd9Sstevel@tonic-gate }
42617c478bd9Sstevel@tonic-gate 
42627c478bd9Sstevel@tonic-gate page_t *
42637c478bd9Sstevel@tonic-gate page_numtopp(pfn_t pfnum, se_t se)
42647c478bd9Sstevel@tonic-gate {
42657c478bd9Sstevel@tonic-gate 	page_t *pp;
42667c478bd9Sstevel@tonic-gate 
42677c478bd9Sstevel@tonic-gate retry:
42687c478bd9Sstevel@tonic-gate 	pp = page_numtopp_nolock(pfnum);
42697c478bd9Sstevel@tonic-gate 	if (pp == NULL) {
42707c478bd9Sstevel@tonic-gate 		return ((page_t *)NULL);
42717c478bd9Sstevel@tonic-gate 	}
42727c478bd9Sstevel@tonic-gate 
42737c478bd9Sstevel@tonic-gate 	/*
42747c478bd9Sstevel@tonic-gate 	 * Acquire the appropriate lock on the page.
42757c478bd9Sstevel@tonic-gate 	 */
42767c478bd9Sstevel@tonic-gate 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_RECLAIM)) {
42777c478bd9Sstevel@tonic-gate 		if (page_pptonum(pp) != pfnum)
42787c478bd9Sstevel@tonic-gate 			goto retry;
42797c478bd9Sstevel@tonic-gate 		continue;
42807c478bd9Sstevel@tonic-gate 	}
42817c478bd9Sstevel@tonic-gate 
42827c478bd9Sstevel@tonic-gate 	if (page_pptonum(pp) != pfnum) {
42837c478bd9Sstevel@tonic-gate 		page_unlock(pp);
42847c478bd9Sstevel@tonic-gate 		goto retry;
42857c478bd9Sstevel@tonic-gate 	}
42867c478bd9Sstevel@tonic-gate 
42877c478bd9Sstevel@tonic-gate 	return (pp);
42887c478bd9Sstevel@tonic-gate }
42897c478bd9Sstevel@tonic-gate 
42907c478bd9Sstevel@tonic-gate page_t *
42917c478bd9Sstevel@tonic-gate page_numtopp_noreclaim(pfn_t pfnum, se_t se)
42927c478bd9Sstevel@tonic-gate {
42937c478bd9Sstevel@tonic-gate 	page_t *pp;
42947c478bd9Sstevel@tonic-gate 
42957c478bd9Sstevel@tonic-gate retry:
42967c478bd9Sstevel@tonic-gate 	pp = page_numtopp_nolock(pfnum);
42977c478bd9Sstevel@tonic-gate 	if (pp == NULL) {
42987c478bd9Sstevel@tonic-gate 		return ((page_t *)NULL);
42997c478bd9Sstevel@tonic-gate 	}
43007c478bd9Sstevel@tonic-gate 
43017c478bd9Sstevel@tonic-gate 	/*
43027c478bd9Sstevel@tonic-gate 	 * Acquire the appropriate lock on the page.
43037c478bd9Sstevel@tonic-gate 	 */
43047c478bd9Sstevel@tonic-gate 	while (!page_lock(pp, se, (kmutex_t *)NULL, P_NO_RECLAIM)) {
43057c478bd9Sstevel@tonic-gate 		if (page_pptonum(pp) != pfnum)
43067c478bd9Sstevel@tonic-gate 			goto retry;
43077c478bd9Sstevel@tonic-gate 		continue;
43087c478bd9Sstevel@tonic-gate 	}
43097c478bd9Sstevel@tonic-gate 
43107c478bd9Sstevel@tonic-gate 	if (page_pptonum(pp) != pfnum) {
43117c478bd9Sstevel@tonic-gate 		page_unlock(pp);
43127c478bd9Sstevel@tonic-gate 		goto retry;
43137c478bd9Sstevel@tonic-gate 	}
43147c478bd9Sstevel@tonic-gate 
43157c478bd9Sstevel@tonic-gate 	return (pp);
43167c478bd9Sstevel@tonic-gate }
43177c478bd9Sstevel@tonic-gate 
43187c478bd9Sstevel@tonic-gate /*
43197c478bd9Sstevel@tonic-gate  * This routine is like page_numtopp, but will only return page structs
43207c478bd9Sstevel@tonic-gate  * for pages which are ok for loading into hardware using the page struct.
43217c478bd9Sstevel@tonic-gate  */
43227c478bd9Sstevel@tonic-gate page_t *
43237c478bd9Sstevel@tonic-gate page_numtopp_nowait(pfn_t pfnum, se_t se)
43247c478bd9Sstevel@tonic-gate {
43257c478bd9Sstevel@tonic-gate 	page_t *pp;
43267c478bd9Sstevel@tonic-gate 
43277c478bd9Sstevel@tonic-gate retry:
43287c478bd9Sstevel@tonic-gate 	pp = page_numtopp_nolock(pfnum);
43297c478bd9Sstevel@tonic-gate 	if (pp == NULL) {
43307c478bd9Sstevel@tonic-gate 		return ((page_t *)NULL);
43317c478bd9Sstevel@tonic-gate 	}
43327c478bd9Sstevel@tonic-gate 
43337c478bd9Sstevel@tonic-gate 	/*
43347c478bd9Sstevel@tonic-gate 	 * Try to acquire the appropriate lock on the page.
43357c478bd9Sstevel@tonic-gate 	 */
43367c478bd9Sstevel@tonic-gate 	if (PP_ISFREE(pp))
43377c478bd9Sstevel@tonic-gate 		pp = NULL;
43387c478bd9Sstevel@tonic-gate 	else {
43397c478bd9Sstevel@tonic-gate 		if (!page_trylock(pp, se))
43407c478bd9Sstevel@tonic-gate 			pp = NULL;
43417c478bd9Sstevel@tonic-gate 		else {
43427c478bd9Sstevel@tonic-gate 			if (page_pptonum(pp) != pfnum) {
43437c478bd9Sstevel@tonic-gate 				page_unlock(pp);
43447c478bd9Sstevel@tonic-gate 				goto retry;
43457c478bd9Sstevel@tonic-gate 			}
43467c478bd9Sstevel@tonic-gate 			if (PP_ISFREE(pp)) {
43477c478bd9Sstevel@tonic-gate 				page_unlock(pp);
43487c478bd9Sstevel@tonic-gate 				pp = NULL;
43497c478bd9Sstevel@tonic-gate 			}
43507c478bd9Sstevel@tonic-gate 		}
43517c478bd9Sstevel@tonic-gate 	}
43527c478bd9Sstevel@tonic-gate 	return (pp);
43537c478bd9Sstevel@tonic-gate }
43547c478bd9Sstevel@tonic-gate 
43557c478bd9Sstevel@tonic-gate /*
43567c478bd9Sstevel@tonic-gate  * Returns a count of dirty pages that are in the process
43577c478bd9Sstevel@tonic-gate  * of being written out.  If 'cleanit' is set, try to push the page.
43587c478bd9Sstevel@tonic-gate  */
43597c478bd9Sstevel@tonic-gate pgcnt_t
43607c478bd9Sstevel@tonic-gate page_busy(int cleanit)
43617c478bd9Sstevel@tonic-gate {
43627c478bd9Sstevel@tonic-gate 	page_t *page0 = page_first();
43637c478bd9Sstevel@tonic-gate 	page_t *pp = page0;
43647c478bd9Sstevel@tonic-gate 	pgcnt_t nppbusy = 0;
43657c478bd9Sstevel@tonic-gate 	u_offset_t off;
43667c478bd9Sstevel@tonic-gate 
43677c478bd9Sstevel@tonic-gate 	do {
43687c478bd9Sstevel@tonic-gate 		vnode_t *vp = pp->p_vnode;
43697c478bd9Sstevel@tonic-gate 
43707c478bd9Sstevel@tonic-gate 		/*
43717c478bd9Sstevel@tonic-gate 		 * A page is a candidate for syncing if it is:
43727c478bd9Sstevel@tonic-gate 		 *
43737c478bd9Sstevel@tonic-gate 		 * (a)	On neither the freelist nor the cachelist
43747c478bd9Sstevel@tonic-gate 		 * (b)	Hashed onto a vnode
43757c478bd9Sstevel@tonic-gate 		 * (c)	Not a kernel page
43767c478bd9Sstevel@tonic-gate 		 * (d)	Dirty
43777c478bd9Sstevel@tonic-gate 		 * (e)	Not part of a swapfile
43787c478bd9Sstevel@tonic-gate 		 * (f)	a page which belongs to a real vnode; eg has a non-null
43797c478bd9Sstevel@tonic-gate 		 *	v_vfsp pointer.
43807c478bd9Sstevel@tonic-gate 		 * (g)	Backed by a filesystem which doesn't have a
43817c478bd9Sstevel@tonic-gate 		 *	stubbed-out sync operation
43827c478bd9Sstevel@tonic-gate 		 */
4383ad23a2dbSjohansen 		if (!PP_ISFREE(pp) && vp != NULL && !VN_ISKAS(vp) &&
43847c478bd9Sstevel@tonic-gate 		    hat_ismod(pp) && !IS_SWAPVP(vp) && vp->v_vfsp != NULL &&
43857c478bd9Sstevel@tonic-gate 		    vfs_can_sync(vp->v_vfsp)) {
43867c478bd9Sstevel@tonic-gate 			nppbusy++;
43877c478bd9Sstevel@tonic-gate 			vfs_syncprogress();
43887c478bd9Sstevel@tonic-gate 
43897c478bd9Sstevel@tonic-gate 			if (!cleanit)
43907c478bd9Sstevel@tonic-gate 				continue;
43917c478bd9Sstevel@tonic-gate 			if (!page_trylock(pp, SE_EXCL))
43927c478bd9Sstevel@tonic-gate 				continue;
43937c478bd9Sstevel@tonic-gate 
43947c478bd9Sstevel@tonic-gate 			if (PP_ISFREE(pp) || vp == NULL || IS_SWAPVP(vp) ||
43957c478bd9Sstevel@tonic-gate 			    pp->p_lckcnt != 0 || pp->p_cowcnt != 0 ||
43967c478bd9Sstevel@tonic-gate 			    !(hat_pagesync(pp,
43977c478bd9Sstevel@tonic-gate 			    HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD) & P_MOD)) {
43987c478bd9Sstevel@tonic-gate 				page_unlock(pp);
43997c478bd9Sstevel@tonic-gate 				continue;
44007c478bd9Sstevel@tonic-gate 			}
44017c478bd9Sstevel@tonic-gate 			off = pp->p_offset;
44027c478bd9Sstevel@tonic-gate 			VN_HOLD(vp);
44037c478bd9Sstevel@tonic-gate 			page_unlock(pp);
44047c478bd9Sstevel@tonic-gate 			(void) VOP_PUTPAGE(vp, off, PAGESIZE,
44057c478bd9Sstevel@tonic-gate 			    B_ASYNC | B_FREE, kcred);
44067c478bd9Sstevel@tonic-gate 			VN_RELE(vp);
44077c478bd9Sstevel@tonic-gate 		}
44087c478bd9Sstevel@tonic-gate 	} while ((pp = page_next(pp)) != page0);
44097c478bd9Sstevel@tonic-gate 
44107c478bd9Sstevel@tonic-gate 	return (nppbusy);
44117c478bd9Sstevel@tonic-gate }
44127c478bd9Sstevel@tonic-gate 
44137c478bd9Sstevel@tonic-gate void page_invalidate_pages(void);
44147c478bd9Sstevel@tonic-gate 
44157c478bd9Sstevel@tonic-gate /*
44167c478bd9Sstevel@tonic-gate  * callback handler to vm sub-system
44177c478bd9Sstevel@tonic-gate  *
44187c478bd9Sstevel@tonic-gate  * callers make sure no recursive entries to this func.
44197c478bd9Sstevel@tonic-gate  */
44207c478bd9Sstevel@tonic-gate /*ARGSUSED*/
44217c478bd9Sstevel@tonic-gate boolean_t
44227c478bd9Sstevel@tonic-gate callb_vm_cpr(void *arg, int code)
44237c478bd9Sstevel@tonic-gate {
44247c478bd9Sstevel@tonic-gate 	if (code == CB_CODE_CPR_CHKPT)
44257c478bd9Sstevel@tonic-gate 		page_invalidate_pages();
44267c478bd9Sstevel@tonic-gate 	return (B_TRUE);
44277c478bd9Sstevel@tonic-gate }
44287c478bd9Sstevel@tonic-gate 
44297c478bd9Sstevel@tonic-gate /*
44307c478bd9Sstevel@tonic-gate  * Invalidate all pages of the system.
44317c478bd9Sstevel@tonic-gate  * It shouldn't be called until all user page activities are all stopped.
44327c478bd9Sstevel@tonic-gate  */
44337c478bd9Sstevel@tonic-gate void
44347c478bd9Sstevel@tonic-gate page_invalidate_pages()
44357c478bd9Sstevel@tonic-gate {
44367c478bd9Sstevel@tonic-gate 	page_t *pp;
44377c478bd9Sstevel@tonic-gate 	page_t *page0;
44387c478bd9Sstevel@tonic-gate 	pgcnt_t nbusypages;
44397c478bd9Sstevel@tonic-gate 	int retry = 0;
44407c478bd9Sstevel@tonic-gate 	const int MAXRETRIES = 4;
44417c478bd9Sstevel@tonic-gate #if defined(__sparc)
44427c478bd9Sstevel@tonic-gate 	extern struct vnode prom_ppages;
44437c478bd9Sstevel@tonic-gate #endif /* __sparc */
44447c478bd9Sstevel@tonic-gate 
44457c478bd9Sstevel@tonic-gate top:
44467c478bd9Sstevel@tonic-gate 	/*
44478b464eb8Smec 	 * Flush dirty pages and destroy the clean ones.
44487c478bd9Sstevel@tonic-gate 	 */
44497c478bd9Sstevel@tonic-gate 	nbusypages = 0;
44507c478bd9Sstevel@tonic-gate 
44517c478bd9Sstevel@tonic-gate 	pp = page0 = page_first();
44527c478bd9Sstevel@tonic-gate 	do {
44537c478bd9Sstevel@tonic-gate 		struct vnode	*vp;
44547c478bd9Sstevel@tonic-gate 		u_offset_t	offset;
44557c478bd9Sstevel@tonic-gate 		int		mod;
44567c478bd9Sstevel@tonic-gate 
44577c478bd9Sstevel@tonic-gate 		/*
44587c478bd9Sstevel@tonic-gate 		 * skip the page if it has no vnode or the page associated
44597c478bd9Sstevel@tonic-gate 		 * with the kernel vnode or prom allocated kernel mem.
44607c478bd9Sstevel@tonic-gate 		 */
44617c478bd9Sstevel@tonic-gate #if defined(__sparc)
4462ad23a2dbSjohansen 		if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp) ||
44637c478bd9Sstevel@tonic-gate 		    vp == &prom_ppages)
44647c478bd9Sstevel@tonic-gate #else /* x86 doesn't have prom or prom_ppage */
4465ad23a2dbSjohansen 		if ((vp = pp->p_vnode) == NULL || VN_ISKAS(vp))
44667c478bd9Sstevel@tonic-gate #endif /* __sparc */
44677c478bd9Sstevel@tonic-gate 			continue;
44687c478bd9Sstevel@tonic-gate 
44697c478bd9Sstevel@tonic-gate 		/*
44707c478bd9Sstevel@tonic-gate 		 * skip the page which is already free invalidated.
44717c478bd9Sstevel@tonic-gate 		 */
44727c478bd9Sstevel@tonic-gate 		if (PP_ISFREE(pp) && PP_ISAGED(pp))
44737c478bd9Sstevel@tonic-gate 			continue;
44747c478bd9Sstevel@tonic-gate 
44757c478bd9Sstevel@tonic-gate 		/*
44767c478bd9Sstevel@tonic-gate 		 * skip pages that are already locked or can't be "exclusively"
44777c478bd9Sstevel@tonic-gate 		 * locked or are already free.  After we lock the page, check
44787c478bd9Sstevel@tonic-gate 		 * the free and age bits again to be sure it's not destroied
44797c478bd9Sstevel@tonic-gate 		 * yet.
44807c478bd9Sstevel@tonic-gate 		 * To achieve max. parallelization, we use page_trylock instead
44817c478bd9Sstevel@tonic-gate 		 * of page_lock so that we don't get block on individual pages
44827c478bd9Sstevel@tonic-gate 		 * while we have thousands of other pages to process.
44837c478bd9Sstevel@tonic-gate 		 */
44847c478bd9Sstevel@tonic-gate 		if (!page_trylock(pp, SE_EXCL)) {
44857c478bd9Sstevel@tonic-gate 			nbusypages++;
44867c478bd9Sstevel@tonic-gate 			continue;
44877c478bd9Sstevel@tonic-gate 		} else if (PP_ISFREE(pp)) {
44887c478bd9Sstevel@tonic-gate 			if (!PP_ISAGED(pp)) {
44897c478bd9Sstevel@tonic-gate 				page_destroy_free(pp);
44907c478bd9Sstevel@tonic-gate 			} else {
44917c478bd9Sstevel@tonic-gate 				page_unlock(pp);
44927c478bd9Sstevel@tonic-gate 			}
44937c478bd9Sstevel@tonic-gate 			continue;
44947c478bd9Sstevel@tonic-gate 		}
44957c478bd9Sstevel@tonic-gate 		/*
44967c478bd9Sstevel@tonic-gate 		 * Is this page involved in some I/O? shared?
44977c478bd9Sstevel@tonic-gate 		 *
44987c478bd9Sstevel@tonic-gate 		 * The page_struct_lock need not be acquired to
44997c478bd9Sstevel@tonic-gate 		 * examine these fields since the page has an
45007c478bd9Sstevel@tonic-gate 		 * "exclusive" lock.
45017c478bd9Sstevel@tonic-gate 		 */
45027c478bd9Sstevel@tonic-gate 		if (pp->p_lckcnt != 0 || pp->p_cowcnt != 0) {
45037c478bd9Sstevel@tonic-gate 			page_unlock(pp);
45047c478bd9Sstevel@tonic-gate 			continue;
45057c478bd9Sstevel@tonic-gate 		}
45067c478bd9Sstevel@tonic-gate 
45077c478bd9Sstevel@tonic-gate 		if (vp->v_type == VCHR) {
45087c478bd9Sstevel@tonic-gate 			panic("vp->v_type == VCHR");
45097c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
45107c478bd9Sstevel@tonic-gate 		}
45117c478bd9Sstevel@tonic-gate 
45127c478bd9Sstevel@tonic-gate 		if (!page_try_demote_pages(pp)) {
45137c478bd9Sstevel@tonic-gate 			page_unlock(pp);
45147c478bd9Sstevel@tonic-gate 			continue;
45157c478bd9Sstevel@tonic-gate 		}
45167c478bd9Sstevel@tonic-gate 
45177c478bd9Sstevel@tonic-gate 		/*
45187c478bd9Sstevel@tonic-gate 		 * Check the modified bit. Leave the bits alone in hardware
45197c478bd9Sstevel@tonic-gate 		 * (they will be modified if we do the putpage).
45207c478bd9Sstevel@tonic-gate 		 */
45217c478bd9Sstevel@tonic-gate 		mod = (hat_pagesync(pp, HAT_SYNC_DONTZERO | HAT_SYNC_STOPON_MOD)
45227c478bd9Sstevel@tonic-gate 			& P_MOD);
45237c478bd9Sstevel@tonic-gate 		if (mod) {
45247c478bd9Sstevel@tonic-gate 			offset = pp->p_offset;
45257c478bd9Sstevel@tonic-gate 			/*
45267c478bd9Sstevel@tonic-gate 			 * Hold the vnode before releasing the page lock
45277c478bd9Sstevel@tonic-gate 			 * to prevent it from being freed and re-used by
45287c478bd9Sstevel@tonic-gate 			 * some other thread.
45297c478bd9Sstevel@tonic-gate 			 */
45307c478bd9Sstevel@tonic-gate 			VN_HOLD(vp);
45317c478bd9Sstevel@tonic-gate 			page_unlock(pp);
45327c478bd9Sstevel@tonic-gate 			/*
45337c478bd9Sstevel@tonic-gate 			 * No error return is checked here. Callers such as
45347c478bd9Sstevel@tonic-gate 			 * cpr deals with the dirty pages at the dump time
45357c478bd9Sstevel@tonic-gate 			 * if this putpage fails.
45367c478bd9Sstevel@tonic-gate 			 */
45377c478bd9Sstevel@tonic-gate 			(void) VOP_PUTPAGE(vp, offset, PAGESIZE, B_INVAL,
45387c478bd9Sstevel@tonic-gate 			    kcred);
45397c478bd9Sstevel@tonic-gate 			VN_RELE(vp);
45407c478bd9Sstevel@tonic-gate 		} else {
45417c478bd9Sstevel@tonic-gate 			page_destroy(pp, 0);
45427c478bd9Sstevel@tonic-gate 		}
45437c478bd9Sstevel@tonic-gate 	} while ((pp = page_next(pp)) != page0);
45447c478bd9Sstevel@tonic-gate 	if (nbusypages && retry++ < MAXRETRIES) {
45457c478bd9Sstevel@tonic-gate 		delay(1);
45467c478bd9Sstevel@tonic-gate 		goto top;
45477c478bd9Sstevel@tonic-gate 	}
45487c478bd9Sstevel@tonic-gate }
45497c478bd9Sstevel@tonic-gate 
45507c478bd9Sstevel@tonic-gate /*
45517c478bd9Sstevel@tonic-gate  * Replace the page "old" with the page "new" on the page hash and vnode lists
45527c478bd9Sstevel@tonic-gate  *
45537c478bd9Sstevel@tonic-gate  * the replacemnt must be done in place, ie the equivalent sequence:
45547c478bd9Sstevel@tonic-gate  *
45557c478bd9Sstevel@tonic-gate  *	vp = old->p_vnode;
45567c478bd9Sstevel@tonic-gate  *	off = old->p_offset;
45577c478bd9Sstevel@tonic-gate  *	page_do_hashout(old)
45587c478bd9Sstevel@tonic-gate  *	page_do_hashin(new, vp, off)
45597c478bd9Sstevel@tonic-gate  *
45607c478bd9Sstevel@tonic-gate  * doesn't work, since
45617c478bd9Sstevel@tonic-gate  *  1) if old is the only page on the vnode, the v_pages list has a window
45627c478bd9Sstevel@tonic-gate  *     where it looks empty. This will break file system assumptions.
45637c478bd9Sstevel@tonic-gate  * and
45647c478bd9Sstevel@tonic-gate  *  2) pvn_vplist_dirty() can't deal with pages moving on the v_pages list.
45657c478bd9Sstevel@tonic-gate  */
45667c478bd9Sstevel@tonic-gate static void
45677c478bd9Sstevel@tonic-gate page_do_relocate_hash(page_t *new, page_t *old)
45687c478bd9Sstevel@tonic-gate {
45697c478bd9Sstevel@tonic-gate 	page_t	**hash_list;
45707c478bd9Sstevel@tonic-gate 	vnode_t	*vp = old->p_vnode;
45717c478bd9Sstevel@tonic-gate 	kmutex_t *sep;
45727c478bd9Sstevel@tonic-gate 
45737c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(old));
45747c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(new));
45757c478bd9Sstevel@tonic-gate 	ASSERT(vp != NULL);
45767c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(page_vnode_mutex(vp)));
45777c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, old->p_offset))));
45787c478bd9Sstevel@tonic-gate 
45797c478bd9Sstevel@tonic-gate 	/*
45807c478bd9Sstevel@tonic-gate 	 * First find old page on the page hash list
45817c478bd9Sstevel@tonic-gate 	 */
45827c478bd9Sstevel@tonic-gate 	hash_list = &page_hash[PAGE_HASH_FUNC(vp, old->p_offset)];
45837c478bd9Sstevel@tonic-gate 
45847c478bd9Sstevel@tonic-gate 	for (;;) {
45857c478bd9Sstevel@tonic-gate 		if (*hash_list == old)
45867c478bd9Sstevel@tonic-gate 			break;
45877c478bd9Sstevel@tonic-gate 		if (*hash_list == NULL) {
45887c478bd9Sstevel@tonic-gate 			panic("page_do_hashout");
45897c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
45907c478bd9Sstevel@tonic-gate 		}
45917c478bd9Sstevel@tonic-gate 		hash_list = &(*hash_list)->p_hash;
45927c478bd9Sstevel@tonic-gate 	}
45937c478bd9Sstevel@tonic-gate 
45947c478bd9Sstevel@tonic-gate 	/*
45957c478bd9Sstevel@tonic-gate 	 * update new and replace old with new on the page hash list
45967c478bd9Sstevel@tonic-gate 	 */
45977c478bd9Sstevel@tonic-gate 	new->p_vnode = old->p_vnode;
45987c478bd9Sstevel@tonic-gate 	new->p_offset = old->p_offset;
45997c478bd9Sstevel@tonic-gate 	new->p_hash = old->p_hash;
46007c478bd9Sstevel@tonic-gate 	*hash_list = new;
46017c478bd9Sstevel@tonic-gate 
46027c478bd9Sstevel@tonic-gate 	if ((new->p_vnode->v_flag & VISSWAP) != 0)
46037c478bd9Sstevel@tonic-gate 		PP_SETSWAP(new);
46047c478bd9Sstevel@tonic-gate 
46057c478bd9Sstevel@tonic-gate 	/*
46067c478bd9Sstevel@tonic-gate 	 * replace old with new on the vnode's page list
46077c478bd9Sstevel@tonic-gate 	 */
46087c478bd9Sstevel@tonic-gate 	if (old->p_vpnext == old) {
46097c478bd9Sstevel@tonic-gate 		new->p_vpnext = new;
46107c478bd9Sstevel@tonic-gate 		new->p_vpprev = new;
46117c478bd9Sstevel@tonic-gate 	} else {
46127c478bd9Sstevel@tonic-gate 		new->p_vpnext = old->p_vpnext;
46137c478bd9Sstevel@tonic-gate 		new->p_vpprev = old->p_vpprev;
46147c478bd9Sstevel@tonic-gate 		new->p_vpnext->p_vpprev = new;
46157c478bd9Sstevel@tonic-gate 		new->p_vpprev->p_vpnext = new;
46167c478bd9Sstevel@tonic-gate 	}
46177c478bd9Sstevel@tonic-gate 	if (vp->v_pages == old)
46187c478bd9Sstevel@tonic-gate 		vp->v_pages = new;
46197c478bd9Sstevel@tonic-gate 
46207c478bd9Sstevel@tonic-gate 	/*
46217c478bd9Sstevel@tonic-gate 	 * clear out the old page
46227c478bd9Sstevel@tonic-gate 	 */
46237c478bd9Sstevel@tonic-gate 	old->p_hash = NULL;
46247c478bd9Sstevel@tonic-gate 	old->p_vpnext = NULL;
46257c478bd9Sstevel@tonic-gate 	old->p_vpprev = NULL;
46267c478bd9Sstevel@tonic-gate 	old->p_vnode = NULL;
46277c478bd9Sstevel@tonic-gate 	PP_CLRSWAP(old);
46287c478bd9Sstevel@tonic-gate 	old->p_offset = (u_offset_t)-1;
46297c478bd9Sstevel@tonic-gate 	page_clr_all_props(old);
46307c478bd9Sstevel@tonic-gate 
46317c478bd9Sstevel@tonic-gate 	/*
46327c478bd9Sstevel@tonic-gate 	 * Wake up processes waiting for this page.  The page's
46337c478bd9Sstevel@tonic-gate 	 * identity has been changed, and is probably not the
46347c478bd9Sstevel@tonic-gate 	 * desired page any longer.
46357c478bd9Sstevel@tonic-gate 	 */
46367c478bd9Sstevel@tonic-gate 	sep = page_se_mutex(old);
46377c478bd9Sstevel@tonic-gate 	mutex_enter(sep);
463842787a71Sstans 	old->p_selock &= ~SE_EWANTED;
46397c478bd9Sstevel@tonic-gate 	if (CV_HAS_WAITERS(&old->p_cv))
46407c478bd9Sstevel@tonic-gate 		cv_broadcast(&old->p_cv);
46417c478bd9Sstevel@tonic-gate 	mutex_exit(sep);
46427c478bd9Sstevel@tonic-gate }
46437c478bd9Sstevel@tonic-gate 
46447c478bd9Sstevel@tonic-gate /*
46457c478bd9Sstevel@tonic-gate  * This function moves the identity of page "pp_old" to page "pp_new".
46467c478bd9Sstevel@tonic-gate  * Both pages must be locked on entry.  "pp_new" is free, has no identity,
46477c478bd9Sstevel@tonic-gate  * and need not be hashed out from anywhere.
46487c478bd9Sstevel@tonic-gate  */
46497c478bd9Sstevel@tonic-gate void
46507c478bd9Sstevel@tonic-gate page_relocate_hash(page_t *pp_new, page_t *pp_old)
46517c478bd9Sstevel@tonic-gate {
46527c478bd9Sstevel@tonic-gate 	vnode_t *vp = pp_old->p_vnode;
46537c478bd9Sstevel@tonic-gate 	u_offset_t off = pp_old->p_offset;
46547c478bd9Sstevel@tonic-gate 	kmutex_t *phm, *vphm;
46557c478bd9Sstevel@tonic-gate 
46567c478bd9Sstevel@tonic-gate 	/*
46577c478bd9Sstevel@tonic-gate 	 * Rehash two pages
46587c478bd9Sstevel@tonic-gate 	 */
46597c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp_old));
46607c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp_new));
46617c478bd9Sstevel@tonic-gate 	ASSERT(vp != NULL);
46627c478bd9Sstevel@tonic-gate 	ASSERT(pp_new->p_vnode == NULL);
46637c478bd9Sstevel@tonic-gate 
46647c478bd9Sstevel@tonic-gate 	/*
46657c478bd9Sstevel@tonic-gate 	 * hashout then hashin while holding the mutexes
46667c478bd9Sstevel@tonic-gate 	 */
46677c478bd9Sstevel@tonic-gate 	phm = PAGE_HASH_MUTEX(PAGE_HASH_FUNC(vp, off));
46687c478bd9Sstevel@tonic-gate 	mutex_enter(phm);
46697c478bd9Sstevel@tonic-gate 	vphm = page_vnode_mutex(vp);
46707c478bd9Sstevel@tonic-gate 	mutex_enter(vphm);
46717c478bd9Sstevel@tonic-gate 
46727c478bd9Sstevel@tonic-gate 	page_do_relocate_hash(pp_new, pp_old);
46737c478bd9Sstevel@tonic-gate 
46747c478bd9Sstevel@tonic-gate 	mutex_exit(vphm);
46757c478bd9Sstevel@tonic-gate 	mutex_exit(phm);
46767c478bd9Sstevel@tonic-gate 
46777c478bd9Sstevel@tonic-gate 	/*
46787c478bd9Sstevel@tonic-gate 	 * The page_struct_lock need not be acquired for lckcnt and
46797c478bd9Sstevel@tonic-gate 	 * cowcnt since the page has an "exclusive" lock.
46807c478bd9Sstevel@tonic-gate 	 */
46817c478bd9Sstevel@tonic-gate 	ASSERT(pp_new->p_lckcnt == 0);
46827c478bd9Sstevel@tonic-gate 	ASSERT(pp_new->p_cowcnt == 0);
46837c478bd9Sstevel@tonic-gate 	pp_new->p_lckcnt = pp_old->p_lckcnt;
46847c478bd9Sstevel@tonic-gate 	pp_new->p_cowcnt = pp_old->p_cowcnt;
46857c478bd9Sstevel@tonic-gate 	pp_old->p_lckcnt = pp_old->p_cowcnt = 0;
46867c478bd9Sstevel@tonic-gate 
46877c478bd9Sstevel@tonic-gate 	/* The following comment preserved from page_flip(). */
46887c478bd9Sstevel@tonic-gate 	/* XXX - Do we need to protect fsdata? */
46897c478bd9Sstevel@tonic-gate 	pp_new->p_fsdata = pp_old->p_fsdata;
46907c478bd9Sstevel@tonic-gate }
46917c478bd9Sstevel@tonic-gate 
46927c478bd9Sstevel@tonic-gate /*
46937c478bd9Sstevel@tonic-gate  * Helper routine used to lock all remaining members of a
46947c478bd9Sstevel@tonic-gate  * large page. The caller is responsible for passing in a locked
46957c478bd9Sstevel@tonic-gate  * pp. If pp is a large page, then it succeeds in locking all the
46967c478bd9Sstevel@tonic-gate  * remaining constituent pages or it returns with only the
46977c478bd9Sstevel@tonic-gate  * original page locked.
46987c478bd9Sstevel@tonic-gate  *
46997c478bd9Sstevel@tonic-gate  * Returns 1 on success, 0 on failure.
47007c478bd9Sstevel@tonic-gate  *
47017c478bd9Sstevel@tonic-gate  * If success is returned this routine gurantees p_szc for all constituent
47027c478bd9Sstevel@tonic-gate  * pages of a large page pp belongs to can't change. To achieve this we
47037c478bd9Sstevel@tonic-gate  * recheck szc of pp after locking all constituent pages and retry if szc
47047c478bd9Sstevel@tonic-gate  * changed (it could only decrease). Since hat_page_demote() needs an EXCL
47057c478bd9Sstevel@tonic-gate  * lock on one of constituent pages it can't be running after all constituent
47067c478bd9Sstevel@tonic-gate  * pages are locked.  hat_page_demote() with a lock on a constituent page
47077c478bd9Sstevel@tonic-gate  * outside of this large page (i.e. pp belonged to a larger large page) is
47087c478bd9Sstevel@tonic-gate  * already done with all constituent pages of pp since the root's p_szc is
47097c478bd9Sstevel@tonic-gate  * changed last. Thefore no need to synchronize with hat_page_demote() that
47107c478bd9Sstevel@tonic-gate  * locked a constituent page outside of pp's current large page.
47117c478bd9Sstevel@tonic-gate  */
47127c478bd9Sstevel@tonic-gate #ifdef DEBUG
47137c478bd9Sstevel@tonic-gate uint32_t gpg_trylock_mtbf = 0;
47147c478bd9Sstevel@tonic-gate #endif
47157c478bd9Sstevel@tonic-gate 
47167c478bd9Sstevel@tonic-gate int
47177c478bd9Sstevel@tonic-gate group_page_trylock(page_t *pp, se_t se)
47187c478bd9Sstevel@tonic-gate {
47197c478bd9Sstevel@tonic-gate 	page_t  *tpp;
47207c478bd9Sstevel@tonic-gate 	pgcnt_t	npgs, i, j;
47217c478bd9Sstevel@tonic-gate 	uint_t pszc = pp->p_szc;
47227c478bd9Sstevel@tonic-gate 
47237c478bd9Sstevel@tonic-gate #ifdef DEBUG
47247c478bd9Sstevel@tonic-gate 	if (gpg_trylock_mtbf && !(gethrtime() % gpg_trylock_mtbf)) {
47257c478bd9Sstevel@tonic-gate 		return (0);
47267c478bd9Sstevel@tonic-gate 	}
47277c478bd9Sstevel@tonic-gate #endif
47287c478bd9Sstevel@tonic-gate 
47297c478bd9Sstevel@tonic-gate 	if (pp != PP_GROUPLEADER(pp, pszc)) {
47307c478bd9Sstevel@tonic-gate 		return (0);
47317c478bd9Sstevel@tonic-gate 	}
47327c478bd9Sstevel@tonic-gate 
47337c478bd9Sstevel@tonic-gate retry:
47347c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED_SE(pp, se));
47357c478bd9Sstevel@tonic-gate 	ASSERT(!PP_ISFREE(pp));
47367c478bd9Sstevel@tonic-gate 	if (pszc == 0) {
47377c478bd9Sstevel@tonic-gate 		return (1);
47387c478bd9Sstevel@tonic-gate 	}
47397c478bd9Sstevel@tonic-gate 	npgs = page_get_pagecnt(pszc);
47407c478bd9Sstevel@tonic-gate 	tpp = pp + 1;
47417c478bd9Sstevel@tonic-gate 	for (i = 1; i < npgs; i++, tpp++) {
47427c478bd9Sstevel@tonic-gate 		if (!page_trylock(tpp, se)) {
47437c478bd9Sstevel@tonic-gate 			tpp = pp + 1;
47447c478bd9Sstevel@tonic-gate 			for (j = 1; j < i; j++, tpp++) {
47457c478bd9Sstevel@tonic-gate 				page_unlock(tpp);
47467c478bd9Sstevel@tonic-gate 			}
47477c478bd9Sstevel@tonic-gate 			return (0);
47487c478bd9Sstevel@tonic-gate 		}
47497c478bd9Sstevel@tonic-gate 	}
47507c478bd9Sstevel@tonic-gate 	if (pp->p_szc != pszc) {
47517c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_szc < pszc);
4752ad23a2dbSjohansen 		ASSERT(pp->p_vnode != NULL && !PP_ISKAS(pp) &&
47537c478bd9Sstevel@tonic-gate 		    !IS_SWAPFSVP(pp->p_vnode));
47547c478bd9Sstevel@tonic-gate 		tpp = pp + 1;
47557c478bd9Sstevel@tonic-gate 		for (i = 1; i < npgs; i++, tpp++) {
47567c478bd9Sstevel@tonic-gate 			page_unlock(tpp);
47577c478bd9Sstevel@tonic-gate 		}
47587c478bd9Sstevel@tonic-gate 		pszc = pp->p_szc;
47597c478bd9Sstevel@tonic-gate 		goto retry;
47607c478bd9Sstevel@tonic-gate 	}
47617c478bd9Sstevel@tonic-gate 	return (1);
47627c478bd9Sstevel@tonic-gate }
47637c478bd9Sstevel@tonic-gate 
47647c478bd9Sstevel@tonic-gate void
47657c478bd9Sstevel@tonic-gate group_page_unlock(page_t *pp)
47667c478bd9Sstevel@tonic-gate {
47677c478bd9Sstevel@tonic-gate 	page_t *tpp;
47687c478bd9Sstevel@tonic-gate 	pgcnt_t	npgs, i;
47697c478bd9Sstevel@tonic-gate 
47707c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
47717c478bd9Sstevel@tonic-gate 	ASSERT(!PP_ISFREE(pp));
47727c478bd9Sstevel@tonic-gate 	ASSERT(pp == PP_PAGEROOT(pp));
47737c478bd9Sstevel@tonic-gate 	npgs = page_get_pagecnt(pp->p_szc);
47747c478bd9Sstevel@tonic-gate 	for (i = 1, tpp = pp + 1; i < npgs; i++, tpp++) {
47757c478bd9Sstevel@tonic-gate 		page_unlock(tpp);
47767c478bd9Sstevel@tonic-gate 	}
47777c478bd9Sstevel@tonic-gate }
47787c478bd9Sstevel@tonic-gate 
47797c478bd9Sstevel@tonic-gate /*
47807c478bd9Sstevel@tonic-gate  * returns
47817c478bd9Sstevel@tonic-gate  * 0 		: on success and *nrelocp is number of relocated PAGESIZE pages
47827c478bd9Sstevel@tonic-gate  * ERANGE	: this is not a base page
47837c478bd9Sstevel@tonic-gate  * EBUSY	: failure to get locks on the page/pages
47847c478bd9Sstevel@tonic-gate  * ENOMEM	: failure to obtain replacement pages
47857c478bd9Sstevel@tonic-gate  * EAGAIN	: OBP has not yet completed its boot-time handoff to the kernel
47868b464eb8Smec  * EIO		: An error occurred while trying to copy the page data
47877c478bd9Sstevel@tonic-gate  *
47887c478bd9Sstevel@tonic-gate  * Return with all constituent members of target and replacement
47897c478bd9Sstevel@tonic-gate  * SE_EXCL locked. It is the callers responsibility to drop the
47907c478bd9Sstevel@tonic-gate  * locks.
47917c478bd9Sstevel@tonic-gate  */
47927c478bd9Sstevel@tonic-gate int
47937c478bd9Sstevel@tonic-gate do_page_relocate(
47947c478bd9Sstevel@tonic-gate 	page_t **target,
47957c478bd9Sstevel@tonic-gate 	page_t **replacement,
47967c478bd9Sstevel@tonic-gate 	int grouplock,
47977c478bd9Sstevel@tonic-gate 	spgcnt_t *nrelocp,
47987c478bd9Sstevel@tonic-gate 	lgrp_t *lgrp)
47997c478bd9Sstevel@tonic-gate {
48007c478bd9Sstevel@tonic-gate 	page_t *first_repl;
48017c478bd9Sstevel@tonic-gate 	page_t *repl;
48027c478bd9Sstevel@tonic-gate 	page_t *targ;
48037c478bd9Sstevel@tonic-gate 	page_t *pl = NULL;
48047c478bd9Sstevel@tonic-gate 	uint_t ppattr;
48057c478bd9Sstevel@tonic-gate 	pfn_t   pfn, repl_pfn;
48067c478bd9Sstevel@tonic-gate 	uint_t	szc;
48077c478bd9Sstevel@tonic-gate 	spgcnt_t npgs, i;
48087c478bd9Sstevel@tonic-gate 	int repl_contig = 0;
48097c478bd9Sstevel@tonic-gate 	uint_t flags = 0;
48107c478bd9Sstevel@tonic-gate 	spgcnt_t dofree = 0;
48117c478bd9Sstevel@tonic-gate 
48127c478bd9Sstevel@tonic-gate 	*nrelocp = 0;
48137c478bd9Sstevel@tonic-gate 
48147c478bd9Sstevel@tonic-gate #if defined(__sparc)
48157c478bd9Sstevel@tonic-gate 	/*
48167c478bd9Sstevel@tonic-gate 	 * We need to wait till OBP has completed
48177c478bd9Sstevel@tonic-gate 	 * its boot-time handoff of its resources to the kernel
48187c478bd9Sstevel@tonic-gate 	 * before we allow page relocation
48197c478bd9Sstevel@tonic-gate 	 */
48207c478bd9Sstevel@tonic-gate 	if (page_relocate_ready == 0) {
48217c478bd9Sstevel@tonic-gate 		return (EAGAIN);
48227c478bd9Sstevel@tonic-gate 	}
48237c478bd9Sstevel@tonic-gate #endif
48247c478bd9Sstevel@tonic-gate 
48257c478bd9Sstevel@tonic-gate 	/*
48267c478bd9Sstevel@tonic-gate 	 * If this is not a base page,
48277c478bd9Sstevel@tonic-gate 	 * just return with 0x0 pages relocated.
48287c478bd9Sstevel@tonic-gate 	 */
48297c478bd9Sstevel@tonic-gate 	targ = *target;
48307c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(targ));
48317c478bd9Sstevel@tonic-gate 	ASSERT(!PP_ISFREE(targ));
48327c478bd9Sstevel@tonic-gate 	szc = targ->p_szc;
48337c478bd9Sstevel@tonic-gate 	ASSERT(szc < mmu_page_sizes);
48347c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
48357c478bd9Sstevel@tonic-gate 	pfn = targ->p_pagenum;
48367c478bd9Sstevel@tonic-gate 	if (pfn != PFN_BASE(pfn, szc)) {
48377c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(vmm_vmstats.ppr_relocnoroot[szc]);
48387c478bd9Sstevel@tonic-gate 		return (ERANGE);
48397c478bd9Sstevel@tonic-gate 	}
48407c478bd9Sstevel@tonic-gate 
48417c478bd9Sstevel@tonic-gate 	if ((repl = *replacement) != NULL && repl->p_szc >= szc) {
48427c478bd9Sstevel@tonic-gate 		repl_pfn = repl->p_pagenum;
48437c478bd9Sstevel@tonic-gate 		if (repl_pfn != PFN_BASE(repl_pfn, szc)) {
48447c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(vmm_vmstats.ppr_reloc_replnoroot[szc]);
48457c478bd9Sstevel@tonic-gate 			return (ERANGE);
48467c478bd9Sstevel@tonic-gate 		}
48477c478bd9Sstevel@tonic-gate 		repl_contig = 1;
48487c478bd9Sstevel@tonic-gate 	}
48497c478bd9Sstevel@tonic-gate 
48507c478bd9Sstevel@tonic-gate 	/*
48517c478bd9Sstevel@tonic-gate 	 * We must lock all members of this large page or we cannot
48527c478bd9Sstevel@tonic-gate 	 * relocate any part of it.
48537c478bd9Sstevel@tonic-gate 	 */
48547c478bd9Sstevel@tonic-gate 	if (grouplock != 0 && !group_page_trylock(targ, SE_EXCL)) {
48557c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(vmm_vmstats.ppr_relocnolock[targ->p_szc]);
48567c478bd9Sstevel@tonic-gate 		return (EBUSY);
48577c478bd9Sstevel@tonic-gate 	}
48587c478bd9Sstevel@tonic-gate 
48597c478bd9Sstevel@tonic-gate 	/*
48607c478bd9Sstevel@tonic-gate 	 * reread szc it could have been decreased before
48617c478bd9Sstevel@tonic-gate 	 * group_page_trylock() was done.
48627c478bd9Sstevel@tonic-gate 	 */
48637c478bd9Sstevel@tonic-gate 	szc = targ->p_szc;
48647c478bd9Sstevel@tonic-gate 	ASSERT(szc < mmu_page_sizes);
48657c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(vmm_vmstats.ppr_reloc[szc]);
48667c478bd9Sstevel@tonic-gate 	ASSERT(pfn == PFN_BASE(pfn, szc));
48677c478bd9Sstevel@tonic-gate 
48687c478bd9Sstevel@tonic-gate 	npgs = page_get_pagecnt(targ->p_szc);
48697c478bd9Sstevel@tonic-gate 
48707c478bd9Sstevel@tonic-gate 	if (repl == NULL) {
48717c478bd9Sstevel@tonic-gate 		dofree = npgs;		/* Size of target page in MMU pages */
48727c478bd9Sstevel@tonic-gate 		if (!page_create_wait(dofree, 0)) {
48737c478bd9Sstevel@tonic-gate 			if (grouplock != 0) {
48747c478bd9Sstevel@tonic-gate 				group_page_unlock(targ);
48757c478bd9Sstevel@tonic-gate 			}
48767c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
48777c478bd9Sstevel@tonic-gate 			return (ENOMEM);
48787c478bd9Sstevel@tonic-gate 		}
48797c478bd9Sstevel@tonic-gate 
48807c478bd9Sstevel@tonic-gate 		/*
48817c478bd9Sstevel@tonic-gate 		 * seg kmem pages require that the target and replacement
48827c478bd9Sstevel@tonic-gate 		 * page be the same pagesize.
48837c478bd9Sstevel@tonic-gate 		 */
4884ad23a2dbSjohansen 		flags = (VN_ISKAS(targ->p_vnode)) ? PGR_SAMESZC : 0;
48857c478bd9Sstevel@tonic-gate 		repl = page_get_replacement_page(targ, lgrp, flags);
48867c478bd9Sstevel@tonic-gate 		if (repl == NULL) {
48877c478bd9Sstevel@tonic-gate 			if (grouplock != 0) {
48887c478bd9Sstevel@tonic-gate 				group_page_unlock(targ);
48897c478bd9Sstevel@tonic-gate 			}
48907c478bd9Sstevel@tonic-gate 			page_create_putback(dofree);
48917c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(vmm_vmstats.ppr_relocnomem[szc]);
48927c478bd9Sstevel@tonic-gate 			return (ENOMEM);
48937c478bd9Sstevel@tonic-gate 		}
48947c478bd9Sstevel@tonic-gate 	}
48957c478bd9Sstevel@tonic-gate #ifdef DEBUG
48967c478bd9Sstevel@tonic-gate 	else {
48977c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_LOCKED(repl));
48987c478bd9Sstevel@tonic-gate 	}
48997c478bd9Sstevel@tonic-gate #endif /* DEBUG */
49007c478bd9Sstevel@tonic-gate 
49017c478bd9Sstevel@tonic-gate #if defined(__sparc)
49027c478bd9Sstevel@tonic-gate 	/*
49037c478bd9Sstevel@tonic-gate 	 * Let hat_page_relocate() complete the relocation if it's kernel page
49047c478bd9Sstevel@tonic-gate 	 */
4905ad23a2dbSjohansen 	if (VN_ISKAS(targ->p_vnode)) {
49067c478bd9Sstevel@tonic-gate 		*replacement = repl;
49077c478bd9Sstevel@tonic-gate 		if (hat_page_relocate(target, replacement, nrelocp) != 0) {
49087c478bd9Sstevel@tonic-gate 			if (grouplock != 0) {
49097c478bd9Sstevel@tonic-gate 				group_page_unlock(targ);
49107c478bd9Sstevel@tonic-gate 			}
49117c478bd9Sstevel@tonic-gate 			if (dofree) {
49127c478bd9Sstevel@tonic-gate 				*replacement = NULL;
49137c478bd9Sstevel@tonic-gate 				page_free_replacement_page(repl);
49147c478bd9Sstevel@tonic-gate 				page_create_putback(dofree);
49157c478bd9Sstevel@tonic-gate 			}
49167c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(vmm_vmstats.ppr_krelocfail[szc]);
49177c478bd9Sstevel@tonic-gate 			return (EAGAIN);
49187c478bd9Sstevel@tonic-gate 		}
49197c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
49207c478bd9Sstevel@tonic-gate 		return (0);
49217c478bd9Sstevel@tonic-gate 	}
49227c478bd9Sstevel@tonic-gate #else
49237c478bd9Sstevel@tonic-gate #if defined(lint)
49247c478bd9Sstevel@tonic-gate 	dofree = dofree;
49257c478bd9Sstevel@tonic-gate #endif
49267c478bd9Sstevel@tonic-gate #endif
49277c478bd9Sstevel@tonic-gate 
49287c478bd9Sstevel@tonic-gate 	first_repl = repl;
49297c478bd9Sstevel@tonic-gate 
49307c478bd9Sstevel@tonic-gate 	for (i = 0; i < npgs; i++) {
49317c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_EXCL(targ));
493207b65a64Saguzovsk 		ASSERT(targ->p_slckcnt == 0);
493307b65a64Saguzovsk 		ASSERT(repl->p_slckcnt == 0);
49347c478bd9Sstevel@tonic-gate 
49357c478bd9Sstevel@tonic-gate 		(void) hat_pageunload(targ, HAT_FORCE_PGUNLOAD);
49367c478bd9Sstevel@tonic-gate 
49377c478bd9Sstevel@tonic-gate 		ASSERT(hat_page_getshare(targ) == 0);
49387c478bd9Sstevel@tonic-gate 		ASSERT(!PP_ISFREE(targ));
49397c478bd9Sstevel@tonic-gate 		ASSERT(targ->p_pagenum == (pfn + i));
49407c478bd9Sstevel@tonic-gate 		ASSERT(repl_contig == 0 ||
49417c478bd9Sstevel@tonic-gate 		    repl->p_pagenum == (repl_pfn + i));
49427c478bd9Sstevel@tonic-gate 
49437c478bd9Sstevel@tonic-gate 		/*
49447c478bd9Sstevel@tonic-gate 		 * Copy the page contents and attributes then
49457c478bd9Sstevel@tonic-gate 		 * relocate the page in the page hash.
49467c478bd9Sstevel@tonic-gate 		 */
49478b464eb8Smec 		if (ppcopy(targ, repl) == 0) {
49488b464eb8Smec 			targ = *target;
49498b464eb8Smec 			repl = first_repl;
49508b464eb8Smec 			VM_STAT_ADD(vmm_vmstats.ppr_copyfail);
49518b464eb8Smec 			if (grouplock != 0) {
49528b464eb8Smec 				group_page_unlock(targ);
49538b464eb8Smec 			}
49548b464eb8Smec 			if (dofree) {
49558b464eb8Smec 				*replacement = NULL;
49568b464eb8Smec 				page_free_replacement_page(repl);
49578b464eb8Smec 				page_create_putback(dofree);
49588b464eb8Smec 			}
49598b464eb8Smec 			return (EIO);
49608b464eb8Smec 		}
49618b464eb8Smec 
49628b464eb8Smec 		targ++;
49638b464eb8Smec 		if (repl_contig != 0) {
49648b464eb8Smec 			repl++;
49658b464eb8Smec 		} else {
49668b464eb8Smec 			repl = repl->p_next;
49678b464eb8Smec 		}
49688b464eb8Smec 	}
49698b464eb8Smec 
49708b464eb8Smec 	repl = first_repl;
49718b464eb8Smec 	targ = *target;
49728b464eb8Smec 
49738b464eb8Smec 	for (i = 0; i < npgs; i++) {
49747c478bd9Sstevel@tonic-gate 		ppattr = hat_page_getattr(targ, (P_MOD | P_REF | P_RO));
49757c478bd9Sstevel@tonic-gate 		page_clr_all_props(repl);
49767c478bd9Sstevel@tonic-gate 		page_set_props(repl, ppattr);
49777c478bd9Sstevel@tonic-gate 		page_relocate_hash(repl, targ);
49787c478bd9Sstevel@tonic-gate 
49797c478bd9Sstevel@tonic-gate 		ASSERT(hat_page_getshare(targ) == 0);
49807c478bd9Sstevel@tonic-gate 		ASSERT(hat_page_getshare(repl) == 0);
49817c478bd9Sstevel@tonic-gate 		/*
49827c478bd9Sstevel@tonic-gate 		 * Now clear the props on targ, after the
49837c478bd9Sstevel@tonic-gate 		 * page_relocate_hash(), they no longer
49847c478bd9Sstevel@tonic-gate 		 * have any meaning.
49857c478bd9Sstevel@tonic-gate 		 */
49867c478bd9Sstevel@tonic-gate 		page_clr_all_props(targ);
49877c478bd9Sstevel@tonic-gate 		ASSERT(targ->p_next == targ);
49887c478bd9Sstevel@tonic-gate 		ASSERT(targ->p_prev == targ);
49897c478bd9Sstevel@tonic-gate 		page_list_concat(&pl, &targ);
49907c478bd9Sstevel@tonic-gate 
49917c478bd9Sstevel@tonic-gate 		targ++;
49927c478bd9Sstevel@tonic-gate 		if (repl_contig != 0) {
49937c478bd9Sstevel@tonic-gate 			repl++;
49947c478bd9Sstevel@tonic-gate 		} else {
49957c478bd9Sstevel@tonic-gate 			repl = repl->p_next;
49967c478bd9Sstevel@tonic-gate 		}
49977c478bd9Sstevel@tonic-gate 	}
49987c478bd9Sstevel@tonic-gate 	/* assert that we have come full circle with repl */
49997c478bd9Sstevel@tonic-gate 	ASSERT(repl_contig == 1 || first_repl == repl);
50007c478bd9Sstevel@tonic-gate 
50017c478bd9Sstevel@tonic-gate 	*target = pl;
50027c478bd9Sstevel@tonic-gate 	if (*replacement == NULL) {
50037c478bd9Sstevel@tonic-gate 		ASSERT(first_repl == repl);
50047c478bd9Sstevel@tonic-gate 		*replacement = repl;
50057c478bd9Sstevel@tonic-gate 	}
50067c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(vmm_vmstats.ppr_relocok[szc]);
50077c478bd9Sstevel@tonic-gate 	*nrelocp = npgs;
50087c478bd9Sstevel@tonic-gate 	return (0);
50097c478bd9Sstevel@tonic-gate }
50107c478bd9Sstevel@tonic-gate /*
50117c478bd9Sstevel@tonic-gate  * On success returns 0 and *nrelocp the number of PAGESIZE pages relocated.
50127c478bd9Sstevel@tonic-gate  */
50137c478bd9Sstevel@tonic-gate int
50147c478bd9Sstevel@tonic-gate page_relocate(
50157c478bd9Sstevel@tonic-gate 	page_t **target,
50167c478bd9Sstevel@tonic-gate 	page_t **replacement,
50177c478bd9Sstevel@tonic-gate 	int grouplock,
50187c478bd9Sstevel@tonic-gate 	int freetarget,
50197c478bd9Sstevel@tonic-gate 	spgcnt_t *nrelocp,
50207c478bd9Sstevel@tonic-gate 	lgrp_t *lgrp)
50217c478bd9Sstevel@tonic-gate {
50227c478bd9Sstevel@tonic-gate 	spgcnt_t ret;
50237c478bd9Sstevel@tonic-gate 
50247c478bd9Sstevel@tonic-gate 	/* do_page_relocate returns 0 on success or errno value */
50257c478bd9Sstevel@tonic-gate 	ret = do_page_relocate(target, replacement, grouplock, nrelocp, lgrp);
50267c478bd9Sstevel@tonic-gate 
50277c478bd9Sstevel@tonic-gate 	if (ret != 0 || freetarget == 0) {
50287c478bd9Sstevel@tonic-gate 		return (ret);
50297c478bd9Sstevel@tonic-gate 	}
50307c478bd9Sstevel@tonic-gate 	if (*nrelocp == 1) {
50317c478bd9Sstevel@tonic-gate 		ASSERT(*target != NULL);
50327c478bd9Sstevel@tonic-gate 		page_free(*target, 1);
50337c478bd9Sstevel@tonic-gate 	} else {
50347c478bd9Sstevel@tonic-gate 		page_t *tpp = *target;
50357c478bd9Sstevel@tonic-gate 		uint_t szc = tpp->p_szc;
50367c478bd9Sstevel@tonic-gate 		pgcnt_t npgs = page_get_pagecnt(szc);
50377c478bd9Sstevel@tonic-gate 		ASSERT(npgs > 1);
50387c478bd9Sstevel@tonic-gate 		ASSERT(szc != 0);
50397c478bd9Sstevel@tonic-gate 		do {
50407c478bd9Sstevel@tonic-gate 			ASSERT(PAGE_EXCL(tpp));
50417c478bd9Sstevel@tonic-gate 			ASSERT(!hat_page_is_mapped(tpp));
50427c478bd9Sstevel@tonic-gate 			ASSERT(tpp->p_szc == szc);
50437c478bd9Sstevel@tonic-gate 			PP_SETFREE(tpp);
50447c478bd9Sstevel@tonic-gate 			PP_SETAGED(tpp);
50457c478bd9Sstevel@tonic-gate 			npgs--;
50467c478bd9Sstevel@tonic-gate 		} while ((tpp = tpp->p_next) != *target);
50477c478bd9Sstevel@tonic-gate 		ASSERT(npgs == 0);
50487c478bd9Sstevel@tonic-gate 		page_list_add_pages(*target, 0);
50497c478bd9Sstevel@tonic-gate 		npgs = page_get_pagecnt(szc);
50507c478bd9Sstevel@tonic-gate 		page_create_putback(npgs);
50517c478bd9Sstevel@tonic-gate 	}
50527c478bd9Sstevel@tonic-gate 	return (ret);
50537c478bd9Sstevel@tonic-gate }
50547c478bd9Sstevel@tonic-gate 
50557c478bd9Sstevel@tonic-gate /*
50567c478bd9Sstevel@tonic-gate  * it is up to the caller to deal with pcf accounting.
50577c478bd9Sstevel@tonic-gate  */
50587c478bd9Sstevel@tonic-gate void
50597c478bd9Sstevel@tonic-gate page_free_replacement_page(page_t *pplist)
50607c478bd9Sstevel@tonic-gate {
50617c478bd9Sstevel@tonic-gate 	page_t *pp;
50627c478bd9Sstevel@tonic-gate 
50637c478bd9Sstevel@tonic-gate 	while (pplist != NULL) {
50647c478bd9Sstevel@tonic-gate 		/*
50657c478bd9Sstevel@tonic-gate 		 * pp_targ is a linked list.
50667c478bd9Sstevel@tonic-gate 		 */
50677c478bd9Sstevel@tonic-gate 		pp = pplist;
50687c478bd9Sstevel@tonic-gate 		if (pp->p_szc == 0) {
50697c478bd9Sstevel@tonic-gate 			page_sub(&pplist, pp);
50707c478bd9Sstevel@tonic-gate 			page_clr_all_props(pp);
50717c478bd9Sstevel@tonic-gate 			PP_SETFREE(pp);
50727c478bd9Sstevel@tonic-gate 			PP_SETAGED(pp);
50737c478bd9Sstevel@tonic-gate 			page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL);
50747c478bd9Sstevel@tonic-gate 			page_unlock(pp);
50757c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[0]);
50767c478bd9Sstevel@tonic-gate 		} else {
50777c478bd9Sstevel@tonic-gate 			spgcnt_t curnpgs = page_get_pagecnt(pp->p_szc);
50787c478bd9Sstevel@tonic-gate 			page_t *tpp;
50797c478bd9Sstevel@tonic-gate 			page_list_break(&pp, &pplist, curnpgs);
50807c478bd9Sstevel@tonic-gate 			tpp = pp;
50817c478bd9Sstevel@tonic-gate 			do {
50827c478bd9Sstevel@tonic-gate 				ASSERT(PAGE_EXCL(tpp));
50837c478bd9Sstevel@tonic-gate 				ASSERT(!hat_page_is_mapped(tpp));
50847c478bd9Sstevel@tonic-gate 				page_clr_all_props(pp);
50857c478bd9Sstevel@tonic-gate 				PP_SETFREE(tpp);
50867c478bd9Sstevel@tonic-gate 				PP_SETAGED(tpp);
50877c478bd9Sstevel@tonic-gate 			} while ((tpp = tpp->p_next) != pp);
50887c478bd9Sstevel@tonic-gate 			page_list_add_pages(pp, 0);
50897c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(pagecnt.pc_free_replacement_page[1]);
50907c478bd9Sstevel@tonic-gate 		}
50917c478bd9Sstevel@tonic-gate 	}
50927c478bd9Sstevel@tonic-gate }
50937c478bd9Sstevel@tonic-gate 
50947c478bd9Sstevel@tonic-gate /*
50957c478bd9Sstevel@tonic-gate  * Relocate target to non-relocatable replacement page.
50967c478bd9Sstevel@tonic-gate  */
50977c478bd9Sstevel@tonic-gate int
50987c478bd9Sstevel@tonic-gate page_relocate_cage(page_t **target, page_t **replacement)
50997c478bd9Sstevel@tonic-gate {
51007c478bd9Sstevel@tonic-gate 	page_t *tpp, *rpp;
51017c478bd9Sstevel@tonic-gate 	spgcnt_t pgcnt, npgs;
51027c478bd9Sstevel@tonic-gate 	int result;
51037c478bd9Sstevel@tonic-gate 
51047c478bd9Sstevel@tonic-gate 	tpp = *target;
51057c478bd9Sstevel@tonic-gate 
51067c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(tpp));
51077c478bd9Sstevel@tonic-gate 	ASSERT(tpp->p_szc == 0);
51087c478bd9Sstevel@tonic-gate 
51097c478bd9Sstevel@tonic-gate 	pgcnt = btop(page_get_pagesize(tpp->p_szc));
51107c478bd9Sstevel@tonic-gate 
51117c478bd9Sstevel@tonic-gate 	do {
51127c478bd9Sstevel@tonic-gate 		(void) page_create_wait(pgcnt, PG_WAIT | PG_NORELOC);
51137c478bd9Sstevel@tonic-gate 		rpp = page_get_replacement_page(tpp, NULL, PGR_NORELOC);
51147c478bd9Sstevel@tonic-gate 		if (rpp == NULL) {
51157c478bd9Sstevel@tonic-gate 			page_create_putback(pgcnt);
51167c478bd9Sstevel@tonic-gate 			kcage_cageout_wakeup();
51177c478bd9Sstevel@tonic-gate 		}
51187c478bd9Sstevel@tonic-gate 	} while (rpp == NULL);
51197c478bd9Sstevel@tonic-gate 
51207c478bd9Sstevel@tonic-gate 	ASSERT(PP_ISNORELOC(rpp));
51217c478bd9Sstevel@tonic-gate 
51227c478bd9Sstevel@tonic-gate 	result = page_relocate(&tpp, &rpp, 0, 1, &npgs, NULL);
51237c478bd9Sstevel@tonic-gate 
51247c478bd9Sstevel@tonic-gate 	if (result == 0) {
51257c478bd9Sstevel@tonic-gate 		*replacement = rpp;
51267c478bd9Sstevel@tonic-gate 		if (pgcnt != npgs)
51277c478bd9Sstevel@tonic-gate 			panic("page_relocate_cage: partial relocation");
51287c478bd9Sstevel@tonic-gate 	}
51297c478bd9Sstevel@tonic-gate 
51307c478bd9Sstevel@tonic-gate 	return (result);
51317c478bd9Sstevel@tonic-gate }
51327c478bd9Sstevel@tonic-gate 
51337c478bd9Sstevel@tonic-gate /*
51347c478bd9Sstevel@tonic-gate  * Release the page lock on a page, place on cachelist
51357c478bd9Sstevel@tonic-gate  * tail if no longer mapped. Caller can let us know if
51367c478bd9Sstevel@tonic-gate  * the page is known to be clean.
51377c478bd9Sstevel@tonic-gate  */
51387c478bd9Sstevel@tonic-gate int
51397c478bd9Sstevel@tonic-gate page_release(page_t *pp, int checkmod)
51407c478bd9Sstevel@tonic-gate {
51417c478bd9Sstevel@tonic-gate 	int status;
51427c478bd9Sstevel@tonic-gate 
51437c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp) && !PP_ISFREE(pp) &&
51447c478bd9Sstevel@tonic-gate 		(pp->p_vnode != NULL));
51457c478bd9Sstevel@tonic-gate 
51467c478bd9Sstevel@tonic-gate 	if (!hat_page_is_mapped(pp) && !IS_SWAPVP(pp->p_vnode) &&
51477c478bd9Sstevel@tonic-gate 	    ((PAGE_SHARED(pp) && page_tryupgrade(pp)) || PAGE_EXCL(pp)) &&
51487c478bd9Sstevel@tonic-gate 	    pp->p_lckcnt == 0 && pp->p_cowcnt == 0 &&
51497c478bd9Sstevel@tonic-gate 	    !hat_page_is_mapped(pp)) {
51507c478bd9Sstevel@tonic-gate 
51517c478bd9Sstevel@tonic-gate 		/*
51527c478bd9Sstevel@tonic-gate 		 * If page is modified, unlock it
51537c478bd9Sstevel@tonic-gate 		 *
51547c478bd9Sstevel@tonic-gate 		 * (p_nrm & P_MOD) bit has the latest stuff because:
51557c478bd9Sstevel@tonic-gate 		 * (1) We found that this page doesn't have any mappings
51567c478bd9Sstevel@tonic-gate 		 *	_after_ holding SE_EXCL and
51577c478bd9Sstevel@tonic-gate 		 * (2) We didn't drop SE_EXCL lock after the check in (1)
51587c478bd9Sstevel@tonic-gate 		 */
51597c478bd9Sstevel@tonic-gate 		if (checkmod && hat_ismod(pp)) {
51607c478bd9Sstevel@tonic-gate 			page_unlock(pp);
51617c478bd9Sstevel@tonic-gate 			status = PGREL_MOD;
51627c478bd9Sstevel@tonic-gate 		} else {
51637c478bd9Sstevel@tonic-gate 			/*LINTED: constant in conditional context*/
51647c478bd9Sstevel@tonic-gate 			VN_DISPOSE(pp, B_FREE, 0, kcred);
51657c478bd9Sstevel@tonic-gate 			status = PGREL_CLEAN;
51667c478bd9Sstevel@tonic-gate 		}
51677c478bd9Sstevel@tonic-gate 	} else {
51687c478bd9Sstevel@tonic-gate 		page_unlock(pp);
51697c478bd9Sstevel@tonic-gate 		status = PGREL_NOTREL;
51707c478bd9Sstevel@tonic-gate 	}
51717c478bd9Sstevel@tonic-gate 	return (status);
51727c478bd9Sstevel@tonic-gate }
51737c478bd9Sstevel@tonic-gate 
5174db874c57Selowe /*
5175db874c57Selowe  * Given a constituent page, try to demote the large page on the freelist.
5176db874c57Selowe  *
5177db874c57Selowe  * Returns nonzero if the page could be demoted successfully. Returns with
5178db874c57Selowe  * the constituent page still locked.
5179db874c57Selowe  */
5180db874c57Selowe int
5181db874c57Selowe page_try_demote_free_pages(page_t *pp)
5182db874c57Selowe {
5183db874c57Selowe 	page_t *rootpp = pp;
5184db874c57Selowe 	pfn_t	pfn = page_pptonum(pp);
5185db874c57Selowe 	spgcnt_t npgs;
5186db874c57Selowe 	uint_t	szc = pp->p_szc;
5187db874c57Selowe 
5188db874c57Selowe 	ASSERT(PP_ISFREE(pp));
5189db874c57Selowe 	ASSERT(PAGE_EXCL(pp));
5190db874c57Selowe 
5191db874c57Selowe 	/*
5192db874c57Selowe 	 * Adjust rootpp and lock it, if `pp' is not the base
5193db874c57Selowe 	 * constituent page.
5194db874c57Selowe 	 */
5195db874c57Selowe 	npgs = page_get_pagecnt(pp->p_szc);
5196db874c57Selowe 	if (npgs == 1) {
5197db874c57Selowe 		return (0);
5198db874c57Selowe 	}
5199db874c57Selowe 
5200db874c57Selowe 	if (!IS_P2ALIGNED(pfn, npgs)) {
5201db874c57Selowe 		pfn = P2ALIGN(pfn, npgs);
5202db874c57Selowe 		rootpp = page_numtopp_nolock(pfn);
5203db874c57Selowe 	}
5204db874c57Selowe 
5205db874c57Selowe 	if (pp != rootpp && !page_trylock(rootpp, SE_EXCL)) {
5206db874c57Selowe 		return (0);
5207db874c57Selowe 	}
5208db874c57Selowe 
5209db874c57Selowe 	if (rootpp->p_szc != szc) {
5210db874c57Selowe 		if (pp != rootpp)
5211db874c57Selowe 			page_unlock(rootpp);
5212db874c57Selowe 		return (0);
5213db874c57Selowe 	}
5214db874c57Selowe 
5215db874c57Selowe 	page_demote_free_pages(rootpp);
5216db874c57Selowe 
5217db874c57Selowe 	if (pp != rootpp)
5218db874c57Selowe 		page_unlock(rootpp);
5219db874c57Selowe 
5220db874c57Selowe 	ASSERT(PP_ISFREE(pp));
5221db874c57Selowe 	ASSERT(PAGE_EXCL(pp));
5222db874c57Selowe 	return (1);
5223db874c57Selowe }
5224db874c57Selowe 
5225db874c57Selowe /*
5226db874c57Selowe  * Given a constituent page, try to demote the large page.
5227db874c57Selowe  *
5228db874c57Selowe  * Returns nonzero if the page could be demoted successfully. Returns with
5229db874c57Selowe  * the constituent page still locked.
5230db874c57Selowe  */
52317c478bd9Sstevel@tonic-gate int
52327c478bd9Sstevel@tonic-gate page_try_demote_pages(page_t *pp)
52337c478bd9Sstevel@tonic-gate {
52347c478bd9Sstevel@tonic-gate 	page_t *tpp, *rootpp = pp;
52357c478bd9Sstevel@tonic-gate 	pfn_t	pfn = page_pptonum(pp);
52367c478bd9Sstevel@tonic-gate 	spgcnt_t i, npgs;
52377c478bd9Sstevel@tonic-gate 	uint_t	szc = pp->p_szc;
52387c478bd9Sstevel@tonic-gate 	vnode_t *vp = pp->p_vnode;
52397c478bd9Sstevel@tonic-gate 
5240db874c57Selowe 	ASSERT(PAGE_EXCL(pp));
52417c478bd9Sstevel@tonic-gate 
52427c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[0]);
52437c478bd9Sstevel@tonic-gate 
5244db874c57Selowe 	if (pp->p_szc == 0) {
52457c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[1]);
52467c478bd9Sstevel@tonic-gate 		return (1);
52477c478bd9Sstevel@tonic-gate 	}
52487c478bd9Sstevel@tonic-gate 
5249ad23a2dbSjohansen 	if (vp != NULL && !IS_SWAPFSVP(vp) && !VN_ISKAS(vp)) {
52507c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[2]);
5251db874c57Selowe 		page_demote_vp_pages(pp);
52527c478bd9Sstevel@tonic-gate 		ASSERT(pp->p_szc == 0);
52537c478bd9Sstevel@tonic-gate 		return (1);
52547c478bd9Sstevel@tonic-gate 	}
52557c478bd9Sstevel@tonic-gate 
52567c478bd9Sstevel@tonic-gate 	/*
5257db874c57Selowe 	 * Adjust rootpp if passed in is not the base
52587c478bd9Sstevel@tonic-gate 	 * constituent page.
52597c478bd9Sstevel@tonic-gate 	 */
5260db874c57Selowe 	npgs = page_get_pagecnt(pp->p_szc);
52617c478bd9Sstevel@tonic-gate 	ASSERT(npgs > 1);
52627c478bd9Sstevel@tonic-gate 	if (!IS_P2ALIGNED(pfn, npgs)) {
52637c478bd9Sstevel@tonic-gate 		pfn = P2ALIGN(pfn, npgs);
52647c478bd9Sstevel@tonic-gate 		rootpp = page_numtopp_nolock(pfn);
52657c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[3]);
52667c478bd9Sstevel@tonic-gate 		ASSERT(rootpp->p_vnode != NULL);
52677c478bd9Sstevel@tonic-gate 		ASSERT(rootpp->p_szc == szc);
52687c478bd9Sstevel@tonic-gate 	}
52697c478bd9Sstevel@tonic-gate 
52707c478bd9Sstevel@tonic-gate 	/*
52717c478bd9Sstevel@tonic-gate 	 * We can't demote kernel pages since we can't hat_unload()
52727c478bd9Sstevel@tonic-gate 	 * the mappings.
52737c478bd9Sstevel@tonic-gate 	 */
5274ad23a2dbSjohansen 	if (VN_ISKAS(rootpp->p_vnode))
52757c478bd9Sstevel@tonic-gate 		return (0);
52767c478bd9Sstevel@tonic-gate 
52777c478bd9Sstevel@tonic-gate 	/*
52787c478bd9Sstevel@tonic-gate 	 * Attempt to lock all constituent pages except the page passed
52797c478bd9Sstevel@tonic-gate 	 * in since it's already locked.
52807c478bd9Sstevel@tonic-gate 	 */
5281affbd3ccSkchow 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
52827c478bd9Sstevel@tonic-gate 		ASSERT(!PP_ISFREE(tpp));
52837c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_vnode != NULL);
52847c478bd9Sstevel@tonic-gate 
52857c478bd9Sstevel@tonic-gate 		if (tpp != pp && !page_trylock(tpp, SE_EXCL))
52867c478bd9Sstevel@tonic-gate 			break;
52877c478bd9Sstevel@tonic-gate 		ASSERT(tpp->p_szc == rootpp->p_szc);
52887c478bd9Sstevel@tonic-gate 		ASSERT(page_pptonum(tpp) == page_pptonum(rootpp) + i);
52897c478bd9Sstevel@tonic-gate 	}
52907c478bd9Sstevel@tonic-gate 
52917c478bd9Sstevel@tonic-gate 	/*
5292db874c57Selowe 	 * If we failed to lock them all then unlock what we have
5293db874c57Selowe 	 * locked so far and bail.
52947c478bd9Sstevel@tonic-gate 	 */
52957c478bd9Sstevel@tonic-gate 	if (i < npgs) {
52967c478bd9Sstevel@tonic-gate 		tpp = rootpp;
52977c478bd9Sstevel@tonic-gate 		while (i-- > 0) {
52987c478bd9Sstevel@tonic-gate 			if (tpp != pp)
52997c478bd9Sstevel@tonic-gate 				page_unlock(tpp);
5300affbd3ccSkchow 			tpp++;
53017c478bd9Sstevel@tonic-gate 		}
53027c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pagecnt.pc_try_demote_pages[4]);
53037c478bd9Sstevel@tonic-gate 		return (0);
53047c478bd9Sstevel@tonic-gate 	}
53057c478bd9Sstevel@tonic-gate 
5306affbd3ccSkchow 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
53077c478bd9Sstevel@tonic-gate 		ASSERT(PAGE_EXCL(tpp));
530807b65a64Saguzovsk 		ASSERT(tpp->p_slckcnt == 0);
5309db874c57Selowe 		(void) hat_pageunload(tpp, HAT_FORCE_PGUNLOAD);
53107c478bd9Sstevel@tonic-gate 		tpp->p_szc = 0;
53117c478bd9Sstevel@tonic-gate 	}
53127c478bd9Sstevel@tonic-gate 
53137c478bd9Sstevel@tonic-gate 	/*
53147c478bd9Sstevel@tonic-gate 	 * Unlock all pages except the page passed in.
53157c478bd9Sstevel@tonic-gate 	 */
5316affbd3ccSkchow 	for (tpp = rootpp, i = 0; i < npgs; i++, tpp++) {
53177c478bd9Sstevel@tonic-gate 		ASSERT(!hat_page_is_mapped(tpp));
53187c478bd9Sstevel@tonic-gate 		if (tpp != pp)
53197c478bd9Sstevel@tonic-gate 			page_unlock(tpp);
53207c478bd9Sstevel@tonic-gate 	}
5321db874c57Selowe 
53227c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_try_demote_pages[5]);
53237c478bd9Sstevel@tonic-gate 	return (1);
53247c478bd9Sstevel@tonic-gate }
53257c478bd9Sstevel@tonic-gate 
53267c478bd9Sstevel@tonic-gate /*
53277c478bd9Sstevel@tonic-gate  * Called by page_free() and page_destroy() to demote the page size code
53287c478bd9Sstevel@tonic-gate  * (p_szc) to 0 (since we can't just put a single PAGESIZE page with non zero
53297c478bd9Sstevel@tonic-gate  * p_szc on free list, neither can we just clear p_szc of a single page_t
53307c478bd9Sstevel@tonic-gate  * within a large page since it will break other code that relies on p_szc
53317c478bd9Sstevel@tonic-gate  * being the same for all page_t's of a large page). Anonymous pages should
53327c478bd9Sstevel@tonic-gate  * never end up here because anon_map_getpages() cannot deal with p_szc
53337c478bd9Sstevel@tonic-gate  * changes after a single constituent page is locked.  While anonymous or
53347c478bd9Sstevel@tonic-gate  * kernel large pages are demoted or freed the entire large page at a time
53357c478bd9Sstevel@tonic-gate  * with all constituent pages locked EXCL for the file system pages we
53367c478bd9Sstevel@tonic-gate  * have to be able to demote a large page (i.e. decrease all constituent pages
53377c478bd9Sstevel@tonic-gate  * p_szc) with only just an EXCL lock on one of constituent pages. The reason
53387c478bd9Sstevel@tonic-gate  * we can easily deal with anonymous page demotion the entire large page at a
53397c478bd9Sstevel@tonic-gate  * time is that those operation originate at address space level and concern
53407c478bd9Sstevel@tonic-gate  * the entire large page region with actual demotion only done when pages are
53417c478bd9Sstevel@tonic-gate  * not shared with any other processes (therefore we can always get EXCL lock
53427c478bd9Sstevel@tonic-gate  * on all anonymous constituent pages after clearing segment page
53437c478bd9Sstevel@tonic-gate  * cache). However file system pages can be truncated or invalidated at a
53447c478bd9Sstevel@tonic-gate  * PAGESIZE level from the file system side and end up in page_free() or
53457c478bd9Sstevel@tonic-gate  * page_destroy() (we also allow only part of the large page to be SOFTLOCKed
53467c478bd9Sstevel@tonic-gate  * and therfore pageout should be able to demote a large page by EXCL locking
53477c478bd9Sstevel@tonic-gate  * any constituent page that is not under SOFTLOCK). In those cases we cannot
53487c478bd9Sstevel@tonic-gate  * rely on being able to lock EXCL all constituent pages.
53497c478bd9Sstevel@tonic-gate  *
53507c478bd9Sstevel@tonic-gate  * To prevent szc changes on file system pages one has to lock all constituent
53517c478bd9Sstevel@tonic-gate  * pages at least SHARED (or call page_szc_lock()). The only subsystem that
53527c478bd9Sstevel@tonic-gate  * doesn't rely on locking all constituent pages (or using page_szc_lock()) to
53537c478bd9Sstevel@tonic-gate  * prevent szc changes is hat layer that uses its own page level mlist
53547c478bd9Sstevel@tonic-gate  * locks. hat assumes that szc doesn't change after mlist lock for a page is
53557c478bd9Sstevel@tonic-gate  * taken. Therefore we need to change szc under hat level locks if we only
53567c478bd9Sstevel@tonic-gate  * have an EXCL lock on a single constituent page and hat still references any
53577c478bd9Sstevel@tonic-gate  * of constituent pages.  (Note we can't "ignore" hat layer by simply
53587c478bd9Sstevel@tonic-gate  * hat_pageunload() all constituent pages without having EXCL locks on all of
53597c478bd9Sstevel@tonic-gate  * constituent pages). We use hat_page_demote() call to safely demote szc of
53607c478bd9Sstevel@tonic-gate  * all constituent pages under hat locks when we only have an EXCL lock on one
53617c478bd9Sstevel@tonic-gate  * of constituent pages.
53627c478bd9Sstevel@tonic-gate  *
53637c478bd9Sstevel@tonic-gate  * This routine calls page_szc_lock() before calling hat_page_demote() to
53647c478bd9Sstevel@tonic-gate  * allow segvn in one special case not to lock all constituent pages SHARED
53657c478bd9Sstevel@tonic-gate  * before calling hat_memload_array() that relies on p_szc not changeing even
53667c478bd9Sstevel@tonic-gate  * before hat level mlist lock is taken.  In that case segvn uses
53677c478bd9Sstevel@tonic-gate  * page_szc_lock() to prevent hat_page_demote() changeing p_szc values.
53687c478bd9Sstevel@tonic-gate  *
53697c478bd9Sstevel@tonic-gate  * Anonymous or kernel page demotion still has to lock all pages exclusively
53707c478bd9Sstevel@tonic-gate  * and do hat_pageunload() on all constituent pages before demoting the page
53717c478bd9Sstevel@tonic-gate  * therefore there's no need for anonymous or kernel page demotion to use
53727c478bd9Sstevel@tonic-gate  * hat_page_demote() mechanism.
53737c478bd9Sstevel@tonic-gate  *
53747c478bd9Sstevel@tonic-gate  * hat_page_demote() removes all large mappings that map pp and then decreases
53757c478bd9Sstevel@tonic-gate  * p_szc starting from the last constituent page of the large page. By working
53767c478bd9Sstevel@tonic-gate  * from the tail of a large page in pfn decreasing order allows one looking at
53777c478bd9Sstevel@tonic-gate  * the root page to know that hat_page_demote() is done for root's szc area.
53787c478bd9Sstevel@tonic-gate  * e.g. if a root page has szc 1 one knows it only has to lock all constituent
53797c478bd9Sstevel@tonic-gate  * pages within szc 1 area to prevent szc changes because hat_page_demote()
53807c478bd9Sstevel@tonic-gate  * that started on this page when it had szc > 1 is done for this szc 1 area.
53817c478bd9Sstevel@tonic-gate  *
53827c478bd9Sstevel@tonic-gate  * We are guranteed that all constituent pages of pp's large page belong to
53837c478bd9Sstevel@tonic-gate  * the same vnode with the consecutive offsets increasing in the direction of
53847c478bd9Sstevel@tonic-gate  * the pfn i.e. the identity of constituent pages can't change until their
53857c478bd9Sstevel@tonic-gate  * p_szc is decreased. Therefore it's safe for hat_page_demote() to remove
53867c478bd9Sstevel@tonic-gate  * large mappings to pp even though we don't lock any constituent page except
53877c478bd9Sstevel@tonic-gate  * pp (i.e. we won't unload e.g. kernel locked page).
53887c478bd9Sstevel@tonic-gate  */
53897c478bd9Sstevel@tonic-gate static void
53907c478bd9Sstevel@tonic-gate page_demote_vp_pages(page_t *pp)
53917c478bd9Sstevel@tonic-gate {
53927c478bd9Sstevel@tonic-gate 	kmutex_t *mtx;
53937c478bd9Sstevel@tonic-gate 
53947c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp));
53957c478bd9Sstevel@tonic-gate 	ASSERT(!PP_ISFREE(pp));
53967c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode != NULL);
53977c478bd9Sstevel@tonic-gate 	ASSERT(!IS_SWAPFSVP(pp->p_vnode));
5398ad23a2dbSjohansen 	ASSERT(!PP_ISKAS(pp));
53997c478bd9Sstevel@tonic-gate 
54007c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pagecnt.pc_demote_pages[0]);
54017c478bd9Sstevel@tonic-gate 
54027c478bd9Sstevel@tonic-gate 	mtx = page_szc_lock(pp);
54037c478bd9Sstevel@tonic-gate 	if (mtx != NULL) {
54047c478bd9Sstevel@tonic-gate 		hat_page_demote(pp);
54057c478bd9Sstevel@tonic-gate 		mutex_exit(mtx);
54067c478bd9Sstevel@tonic-gate 	}
54077c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_szc == 0);
54087c478bd9Sstevel@tonic-gate }
54097c478bd9Sstevel@tonic-gate 
54107c478bd9Sstevel@tonic-gate /*
54117c478bd9Sstevel@tonic-gate  * Mark any existing pages for migration in the given range
54127c478bd9Sstevel@tonic-gate  */
54137c478bd9Sstevel@tonic-gate void
54147c478bd9Sstevel@tonic-gate page_mark_migrate(struct seg *seg, caddr_t addr, size_t len,
54157c478bd9Sstevel@tonic-gate     struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
54167c478bd9Sstevel@tonic-gate     u_offset_t vnoff, int rflag)
54177c478bd9Sstevel@tonic-gate {
54187c478bd9Sstevel@tonic-gate 	struct anon	*ap;
54197c478bd9Sstevel@tonic-gate 	vnode_t		*curvp;
54207c478bd9Sstevel@tonic-gate 	lgrp_t		*from;
54217c478bd9Sstevel@tonic-gate 	pgcnt_t		i;
54227c478bd9Sstevel@tonic-gate 	pgcnt_t		nlocked;
54237c478bd9Sstevel@tonic-gate 	u_offset_t	off;
54247c478bd9Sstevel@tonic-gate 	pfn_t		pfn;
54257c478bd9Sstevel@tonic-gate 	size_t		pgsz;
54267c478bd9Sstevel@tonic-gate 	size_t		segpgsz;
54277c478bd9Sstevel@tonic-gate 	pgcnt_t		pages;
54287c478bd9Sstevel@tonic-gate 	uint_t		pszc;
54297c478bd9Sstevel@tonic-gate 	page_t		**ppa;
54307c478bd9Sstevel@tonic-gate 	pgcnt_t		ppa_nentries;
54317c478bd9Sstevel@tonic-gate 	page_t		*pp;
54327c478bd9Sstevel@tonic-gate 	caddr_t		va;
54337c478bd9Sstevel@tonic-gate 	ulong_t		an_idx;
54347c478bd9Sstevel@tonic-gate 	anon_sync_obj_t	cookie;
54357c478bd9Sstevel@tonic-gate 
54367c478bd9Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
54377c478bd9Sstevel@tonic-gate 
54387c478bd9Sstevel@tonic-gate 	/*
54397c478bd9Sstevel@tonic-gate 	 * Don't do anything if don't need to do lgroup optimizations
54407c478bd9Sstevel@tonic-gate 	 * on this system
54417c478bd9Sstevel@tonic-gate 	 */
54427c478bd9Sstevel@tonic-gate 	if (!lgrp_optimizations())
54437c478bd9Sstevel@tonic-gate 		return;
54447c478bd9Sstevel@tonic-gate 
54457c478bd9Sstevel@tonic-gate 	/*
54467c478bd9Sstevel@tonic-gate 	 * Align address and length to (potentially large) page boundary
54477c478bd9Sstevel@tonic-gate 	 */
54487c478bd9Sstevel@tonic-gate 	segpgsz = page_get_pagesize(seg->s_szc);
54497c478bd9Sstevel@tonic-gate 	addr = (caddr_t)P2ALIGN((uintptr_t)addr, segpgsz);
54507c478bd9Sstevel@tonic-gate 	if (rflag)
54517c478bd9Sstevel@tonic-gate 		len = P2ROUNDUP(len, segpgsz);
54527c478bd9Sstevel@tonic-gate 
54537c478bd9Sstevel@tonic-gate 	/*
54547c478bd9Sstevel@tonic-gate 	 * Allocate page array to accomodate largest page size
54557c478bd9Sstevel@tonic-gate 	 */
54567c478bd9Sstevel@tonic-gate 	pgsz = page_get_pagesize(page_num_pagesizes() - 1);
54577c478bd9Sstevel@tonic-gate 	ppa_nentries = btop(pgsz);
54587c478bd9Sstevel@tonic-gate 	ppa = kmem_zalloc(ppa_nentries * sizeof (page_t *), KM_SLEEP);
54597c478bd9Sstevel@tonic-gate 
54607c478bd9Sstevel@tonic-gate 	/*
54617c478bd9Sstevel@tonic-gate 	 * Do one (large) page at a time
54627c478bd9Sstevel@tonic-gate 	 */
54637c478bd9Sstevel@tonic-gate 	va = addr;
54647c478bd9Sstevel@tonic-gate 	while (va < addr + len) {
54657c478bd9Sstevel@tonic-gate 		/*
54667c478bd9Sstevel@tonic-gate 		 * Lookup (root) page for vnode and offset corresponding to
54677c478bd9Sstevel@tonic-gate 		 * this virtual address
54687c478bd9Sstevel@tonic-gate 		 * Try anonmap first since there may be copy-on-write
54697c478bd9Sstevel@tonic-gate 		 * pages, but initialize vnode pointer and offset using
54707c478bd9Sstevel@tonic-gate 		 * vnode arguments just in case there isn't an amp.
54717c478bd9Sstevel@tonic-gate 		 */
54727c478bd9Sstevel@tonic-gate 		curvp = vp;
54737c478bd9Sstevel@tonic-gate 		off = vnoff + va - seg->s_base;
54747c478bd9Sstevel@tonic-gate 		if (amp) {
54757c478bd9Sstevel@tonic-gate 			ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
54767c478bd9Sstevel@tonic-gate 			an_idx = anon_index + seg_page(seg, va);
54777c478bd9Sstevel@tonic-gate 			anon_array_enter(amp, an_idx, &cookie);
54787c478bd9Sstevel@tonic-gate 			ap = anon_get_ptr(amp->ahp, an_idx);
54797c478bd9Sstevel@tonic-gate 			if (ap)
54807c478bd9Sstevel@tonic-gate 				swap_xlate(ap, &curvp, &off);
54817c478bd9Sstevel@tonic-gate 			anon_array_exit(&cookie);
54827c478bd9Sstevel@tonic-gate 			ANON_LOCK_EXIT(&amp->a_rwlock);
54837c478bd9Sstevel@tonic-gate 		}
54847c478bd9Sstevel@tonic-gate 
54857c478bd9Sstevel@tonic-gate 		pp = NULL;
54867c478bd9Sstevel@tonic-gate 		if (curvp)
54877c478bd9Sstevel@tonic-gate 			pp = page_lookup(curvp, off, SE_SHARED);
54887c478bd9Sstevel@tonic-gate 
54897c478bd9Sstevel@tonic-gate 		/*
54907c478bd9Sstevel@tonic-gate 		 * If there isn't a page at this virtual address,
54917c478bd9Sstevel@tonic-gate 		 * skip to next page
54927c478bd9Sstevel@tonic-gate 		 */
54937c478bd9Sstevel@tonic-gate 		if (pp == NULL) {
54947c478bd9Sstevel@tonic-gate 			va += PAGESIZE;
54957c478bd9Sstevel@tonic-gate 			continue;
54967c478bd9Sstevel@tonic-gate 		}
54977c478bd9Sstevel@tonic-gate 
54987c478bd9Sstevel@tonic-gate 		/*
54997c478bd9Sstevel@tonic-gate 		 * Figure out which lgroup this page is in for kstats
55007c478bd9Sstevel@tonic-gate 		 */
55017c478bd9Sstevel@tonic-gate 		pfn = page_pptonum(pp);
55027c478bd9Sstevel@tonic-gate 		from = lgrp_pfn_to_lgrp(pfn);
55037c478bd9Sstevel@tonic-gate 
55047c478bd9Sstevel@tonic-gate 		/*
55057c478bd9Sstevel@tonic-gate 		 * Get page size, and round up and skip to next page boundary
55067c478bd9Sstevel@tonic-gate 		 * if unaligned address
55077c478bd9Sstevel@tonic-gate 		 */
55087c478bd9Sstevel@tonic-gate 		pszc = pp->p_szc;
55097c478bd9Sstevel@tonic-gate 		pgsz = page_get_pagesize(pszc);
55107c478bd9Sstevel@tonic-gate 		pages = btop(pgsz);
55117c478bd9Sstevel@tonic-gate 		if (!IS_P2ALIGNED(va, pgsz) ||
55127c478bd9Sstevel@tonic-gate 		    !IS_P2ALIGNED(pfn, pages) ||
55137c478bd9Sstevel@tonic-gate 		    pgsz > segpgsz) {
55147c478bd9Sstevel@tonic-gate 			pgsz = MIN(pgsz, segpgsz);
55157c478bd9Sstevel@tonic-gate 			page_unlock(pp);
55167c478bd9Sstevel@tonic-gate 			i = btop(P2END((uintptr_t)va, pgsz) -
55177c478bd9Sstevel@tonic-gate 			    (uintptr_t)va);
55187c478bd9Sstevel@tonic-gate 			va = (caddr_t)P2END((uintptr_t)va, pgsz);
55197c478bd9Sstevel@tonic-gate 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS, i);
55207c478bd9Sstevel@tonic-gate 			continue;
55217c478bd9Sstevel@tonic-gate 		}
55227c478bd9Sstevel@tonic-gate 
55237c478bd9Sstevel@tonic-gate 		/*
55247c478bd9Sstevel@tonic-gate 		 * Upgrade to exclusive lock on page
55257c478bd9Sstevel@tonic-gate 		 */
55267c478bd9Sstevel@tonic-gate 		if (!page_tryupgrade(pp)) {
55277c478bd9Sstevel@tonic-gate 			page_unlock(pp);
55287c478bd9Sstevel@tonic-gate 			va += pgsz;
55297c478bd9Sstevel@tonic-gate 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
55307c478bd9Sstevel@tonic-gate 			    btop(pgsz));
55317c478bd9Sstevel@tonic-gate 			continue;
55327c478bd9Sstevel@tonic-gate 		}
55337c478bd9Sstevel@tonic-gate 
55347c478bd9Sstevel@tonic-gate 		/*
55357c478bd9Sstevel@tonic-gate 		 * Remember pages locked exclusively and how many
55367c478bd9Sstevel@tonic-gate 		 */
55377c478bd9Sstevel@tonic-gate 		ppa[0] = pp;
55387c478bd9Sstevel@tonic-gate 		nlocked = 1;
55397c478bd9Sstevel@tonic-gate 
55407c478bd9Sstevel@tonic-gate 		/*
55417c478bd9Sstevel@tonic-gate 		 * Lock constituent pages if this is large page
55427c478bd9Sstevel@tonic-gate 		 */
55437c478bd9Sstevel@tonic-gate 		if (pages > 1) {
55447c478bd9Sstevel@tonic-gate 			/*
55457c478bd9Sstevel@tonic-gate 			 * Lock all constituents except root page, since it
55467c478bd9Sstevel@tonic-gate 			 * should be locked already.
55477c478bd9Sstevel@tonic-gate 			 */
55487c478bd9Sstevel@tonic-gate 			for (i = 1; i < pages; i++) {
5549affbd3ccSkchow 				pp++;
55507c478bd9Sstevel@tonic-gate 				if (!page_trylock(pp, SE_EXCL)) {
55517c478bd9Sstevel@tonic-gate 					break;
55527c478bd9Sstevel@tonic-gate 				}
55537c478bd9Sstevel@tonic-gate 				if (PP_ISFREE(pp) ||
55547c478bd9Sstevel@tonic-gate 				    pp->p_szc != pszc) {
55557c478bd9Sstevel@tonic-gate 					/*
55567c478bd9Sstevel@tonic-gate 					 * hat_page_demote() raced in with us.
55577c478bd9Sstevel@tonic-gate 					 */
55587c478bd9Sstevel@tonic-gate 					ASSERT(!IS_SWAPFSVP(curvp));
55597c478bd9Sstevel@tonic-gate 					page_unlock(pp);
55607c478bd9Sstevel@tonic-gate 					break;
55617c478bd9Sstevel@tonic-gate 				}
55627c478bd9Sstevel@tonic-gate 				ppa[nlocked] = pp;
55637c478bd9Sstevel@tonic-gate 				nlocked++;
55647c478bd9Sstevel@tonic-gate 			}
55657c478bd9Sstevel@tonic-gate 		}
55667c478bd9Sstevel@tonic-gate 
55677c478bd9Sstevel@tonic-gate 		/*
55687c478bd9Sstevel@tonic-gate 		 * If all constituent pages couldn't be locked,
55697c478bd9Sstevel@tonic-gate 		 * unlock pages locked so far and skip to next page.
55707c478bd9Sstevel@tonic-gate 		 */
55717c478bd9Sstevel@tonic-gate 		if (nlocked != pages) {
55727c478bd9Sstevel@tonic-gate 			for (i = 0; i < nlocked; i++)
55737c478bd9Sstevel@tonic-gate 				page_unlock(ppa[i]);
55747c478bd9Sstevel@tonic-gate 			va += pgsz;
55757c478bd9Sstevel@tonic-gate 			lgrp_stat_add(from->lgrp_id, LGRP_PMM_FAIL_PGS,
55767c478bd9Sstevel@tonic-gate 			    btop(pgsz));
55777c478bd9Sstevel@tonic-gate 			continue;
55787c478bd9Sstevel@tonic-gate 		}
55797c478bd9Sstevel@tonic-gate 
55807c478bd9Sstevel@tonic-gate 		/*
55817c478bd9Sstevel@tonic-gate 		 * hat_page_demote() can no longer happen
55827c478bd9Sstevel@tonic-gate 		 * since last cons page had the right p_szc after
55837c478bd9Sstevel@tonic-gate 		 * all cons pages were locked. all cons pages
55847c478bd9Sstevel@tonic-gate 		 * should now have the same p_szc.
55857c478bd9Sstevel@tonic-gate 		 */
55867c478bd9Sstevel@tonic-gate 
55877c478bd9Sstevel@tonic-gate 		/*
55887c478bd9Sstevel@tonic-gate 		 * All constituent pages locked successfully, so mark
55897c478bd9Sstevel@tonic-gate 		 * large page for migration and unload the mappings of
55907c478bd9Sstevel@tonic-gate 		 * constituent pages, so a fault will occur on any part of the
55917c478bd9Sstevel@tonic-gate 		 * large page
55927c478bd9Sstevel@tonic-gate 		 */
55937c478bd9Sstevel@tonic-gate 		PP_SETMIGRATE(ppa[0]);
55947c478bd9Sstevel@tonic-gate 		for (i = 0; i < nlocked; i++) {
55957c478bd9Sstevel@tonic-gate 			pp = ppa[i];
55967c478bd9Sstevel@tonic-gate 			(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
55977c478bd9Sstevel@tonic-gate 			ASSERT(hat_page_getshare(pp) == 0);
55987c478bd9Sstevel@tonic-gate 			page_unlock(pp);
55997c478bd9Sstevel@tonic-gate 		}
56007c478bd9Sstevel@tonic-gate 		lgrp_stat_add(from->lgrp_id, LGRP_PMM_PGS, nlocked);
56017c478bd9Sstevel@tonic-gate 
56027c478bd9Sstevel@tonic-gate 		va += pgsz;
56037c478bd9Sstevel@tonic-gate 	}
56047c478bd9Sstevel@tonic-gate 	kmem_free(ppa, ppa_nentries * sizeof (page_t *));
56057c478bd9Sstevel@tonic-gate }
56067c478bd9Sstevel@tonic-gate 
56077c478bd9Sstevel@tonic-gate /*
56087c478bd9Sstevel@tonic-gate  * Migrate any pages that have been marked for migration in the given range
56097c478bd9Sstevel@tonic-gate  */
56107c478bd9Sstevel@tonic-gate void
56117c478bd9Sstevel@tonic-gate page_migrate(
56127c478bd9Sstevel@tonic-gate 	struct seg	*seg,
56137c478bd9Sstevel@tonic-gate 	caddr_t		addr,
56147c478bd9Sstevel@tonic-gate 	page_t		**ppa,
56157c478bd9Sstevel@tonic-gate 	pgcnt_t		npages)
56167c478bd9Sstevel@tonic-gate {
56177c478bd9Sstevel@tonic-gate 	lgrp_t		*from;
56187c478bd9Sstevel@tonic-gate 	lgrp_t		*to;
56197c478bd9Sstevel@tonic-gate 	page_t		*newpp;
56207c478bd9Sstevel@tonic-gate 	page_t		*pp;
56217c478bd9Sstevel@tonic-gate 	pfn_t		pfn;
56227c478bd9Sstevel@tonic-gate 	size_t		pgsz;
56237c478bd9Sstevel@tonic-gate 	spgcnt_t	page_cnt;
56247c478bd9Sstevel@tonic-gate 	spgcnt_t	i;
56257c478bd9Sstevel@tonic-gate 	uint_t		pszc;
56267c478bd9Sstevel@tonic-gate 
56277c478bd9Sstevel@tonic-gate 	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
56287c478bd9Sstevel@tonic-gate 
56297c478bd9Sstevel@tonic-gate 	while (npages > 0) {
56307c478bd9Sstevel@tonic-gate 		pp = *ppa;
56317c478bd9Sstevel@tonic-gate 		pszc = pp->p_szc;
56327c478bd9Sstevel@tonic-gate 		pgsz = page_get_pagesize(pszc);
56337c478bd9Sstevel@tonic-gate 		page_cnt = btop(pgsz);
56347c478bd9Sstevel@tonic-gate 
56357c478bd9Sstevel@tonic-gate 		/*
56367c478bd9Sstevel@tonic-gate 		 * Check to see whether this page is marked for migration
56377c478bd9Sstevel@tonic-gate 		 *
56387c478bd9Sstevel@tonic-gate 		 * Assume that root page of large page is marked for
56397c478bd9Sstevel@tonic-gate 		 * migration and none of the other constituent pages
56407c478bd9Sstevel@tonic-gate 		 * are marked.  This really simplifies clearing the
56417c478bd9Sstevel@tonic-gate 		 * migrate bit by not having to clear it from each
56427c478bd9Sstevel@tonic-gate 		 * constituent page.
56437c478bd9Sstevel@tonic-gate 		 *
56447c478bd9Sstevel@tonic-gate 		 * note we don't want to relocate an entire large page if
56457c478bd9Sstevel@tonic-gate 		 * someone is only using one subpage.
56467c478bd9Sstevel@tonic-gate 		 */
56477c478bd9Sstevel@tonic-gate 		if (npages < page_cnt)
56487c478bd9Sstevel@tonic-gate 			break;
56497c478bd9Sstevel@tonic-gate 
56507c478bd9Sstevel@tonic-gate 		/*
56517c478bd9Sstevel@tonic-gate 		 * Is it marked for migration?
56527c478bd9Sstevel@tonic-gate 		 */
56537c478bd9Sstevel@tonic-gate 		if (!PP_ISMIGRATE(pp))
56547c478bd9Sstevel@tonic-gate 			goto next;
56557c478bd9Sstevel@tonic-gate 
56567c478bd9Sstevel@tonic-gate 		/*
56577c478bd9Sstevel@tonic-gate 		 * Determine lgroups that page is being migrated between
56587c478bd9Sstevel@tonic-gate 		 */
56597c478bd9Sstevel@tonic-gate 		pfn = page_pptonum(pp);
56607c478bd9Sstevel@tonic-gate 		if (!IS_P2ALIGNED(pfn, page_cnt)) {
56617c478bd9Sstevel@tonic-gate 			break;
56627c478bd9Sstevel@tonic-gate 		}
56637c478bd9Sstevel@tonic-gate 		from = lgrp_pfn_to_lgrp(pfn);
56647c478bd9Sstevel@tonic-gate 		to = lgrp_mem_choose(seg, addr, pgsz);
56657c478bd9Sstevel@tonic-gate 
56667c478bd9Sstevel@tonic-gate 		/*
56677c478bd9Sstevel@tonic-gate 		 * Check to see whether we are trying to migrate page to lgroup
56687c478bd9Sstevel@tonic-gate 		 * where it is allocated already
56697c478bd9Sstevel@tonic-gate 		 */
56707c478bd9Sstevel@tonic-gate 		if (to == from) {
56717c478bd9Sstevel@tonic-gate 			PP_CLRMIGRATE(pp);
56727c478bd9Sstevel@tonic-gate 			goto next;
56737c478bd9Sstevel@tonic-gate 		}
56747c478bd9Sstevel@tonic-gate 
56757c478bd9Sstevel@tonic-gate 		/*
56767c478bd9Sstevel@tonic-gate 		 * Need to get exclusive lock's to migrate
56777c478bd9Sstevel@tonic-gate 		 */
56787c478bd9Sstevel@tonic-gate 		for (i = 0; i < page_cnt; i++) {
56797c478bd9Sstevel@tonic-gate 			ASSERT(PAGE_LOCKED(ppa[i]));
56807c478bd9Sstevel@tonic-gate 			if (page_pptonum(ppa[i]) != pfn + i ||
56817c478bd9Sstevel@tonic-gate 			    ppa[i]->p_szc != pszc) {
56827c478bd9Sstevel@tonic-gate 				break;
56837c478bd9Sstevel@tonic-gate 			}
56847c478bd9Sstevel@tonic-gate 			if (!page_tryupgrade(ppa[i])) {
56857c478bd9Sstevel@tonic-gate 				lgrp_stat_add(from->lgrp_id,
56867c478bd9Sstevel@tonic-gate 				    LGRP_PM_FAIL_LOCK_PGS,
56877c478bd9Sstevel@tonic-gate 				    page_cnt);
56887c478bd9Sstevel@tonic-gate 				break;
56897c478bd9Sstevel@tonic-gate 			}
56907c478bd9Sstevel@tonic-gate 		}
56917c478bd9Sstevel@tonic-gate 		if (i != page_cnt) {
56927c478bd9Sstevel@tonic-gate 			while (--i != -1) {
56937c478bd9Sstevel@tonic-gate 				page_downgrade(ppa[i]);
56947c478bd9Sstevel@tonic-gate 			}
56957c478bd9Sstevel@tonic-gate 			goto next;
56967c478bd9Sstevel@tonic-gate 		}
56977c478bd9Sstevel@tonic-gate 
56987c478bd9Sstevel@tonic-gate 		(void) page_create_wait(page_cnt, PG_WAIT);
56997c478bd9Sstevel@tonic-gate 		newpp = page_get_replacement_page(pp, to, PGR_SAMESZC);
57007c478bd9Sstevel@tonic-gate 		if (newpp == NULL) {
57017c478bd9Sstevel@tonic-gate 			page_create_putback(page_cnt);
57027c478bd9Sstevel@tonic-gate 			for (i = 0; i < page_cnt; i++) {
57037c478bd9Sstevel@tonic-gate 				page_downgrade(ppa[i]);
57047c478bd9Sstevel@tonic-gate 			}
57057c478bd9Sstevel@tonic-gate 			lgrp_stat_add(to->lgrp_id, LGRP_PM_FAIL_ALLOC_PGS,
57067c478bd9Sstevel@tonic-gate 			    page_cnt);
57077c478bd9Sstevel@tonic-gate 			goto next;
57087c478bd9Sstevel@tonic-gate 		}
57097c478bd9Sstevel@tonic-gate 		ASSERT(newpp->p_szc == pszc);
57107c478bd9Sstevel@tonic-gate 		/*
57117c478bd9Sstevel@tonic-gate 		 * Clear migrate bit and relocate page
57127c478bd9Sstevel@tonic-gate 		 */
57137c478bd9Sstevel@tonic-gate 		PP_CLRMIGRATE(pp);
57147c478bd9Sstevel@tonic-gate 		if (page_relocate(&pp, &newpp, 0, 1, &page_cnt, to)) {
57157c478bd9Sstevel@tonic-gate 			panic("page_migrate: page_relocate failed");
57167c478bd9Sstevel@tonic-gate 		}
57177c478bd9Sstevel@tonic-gate 		ASSERT(page_cnt * PAGESIZE == pgsz);
57187c478bd9Sstevel@tonic-gate 
57197c478bd9Sstevel@tonic-gate 		/*
57207c478bd9Sstevel@tonic-gate 		 * Keep stats for number of pages migrated from and to
57217c478bd9Sstevel@tonic-gate 		 * each lgroup
57227c478bd9Sstevel@tonic-gate 		 */
57237c478bd9Sstevel@tonic-gate 		lgrp_stat_add(from->lgrp_id, LGRP_PM_SRC_PGS, page_cnt);
57247c478bd9Sstevel@tonic-gate 		lgrp_stat_add(to->lgrp_id, LGRP_PM_DEST_PGS, page_cnt);
57257c478bd9Sstevel@tonic-gate 		/*
57267c478bd9Sstevel@tonic-gate 		 * update the page_t array we were passed in and
57277c478bd9Sstevel@tonic-gate 		 * unlink constituent pages of a large page.
57287c478bd9Sstevel@tonic-gate 		 */
57297c478bd9Sstevel@tonic-gate 		for (i = 0; i < page_cnt; ++i, ++pp) {
57307c478bd9Sstevel@tonic-gate 			ASSERT(PAGE_EXCL(newpp));
57317c478bd9Sstevel@tonic-gate 			ASSERT(newpp->p_szc == pszc);
57327c478bd9Sstevel@tonic-gate 			ppa[i] = newpp;
57337c478bd9Sstevel@tonic-gate 			pp = newpp;
57347c478bd9Sstevel@tonic-gate 			page_sub(&newpp, pp);
57357c478bd9Sstevel@tonic-gate 			page_downgrade(pp);
57367c478bd9Sstevel@tonic-gate 		}
57377c478bd9Sstevel@tonic-gate 		ASSERT(newpp == NULL);
57387c478bd9Sstevel@tonic-gate next:
57397c478bd9Sstevel@tonic-gate 		addr += pgsz;
57407c478bd9Sstevel@tonic-gate 		ppa += page_cnt;
57417c478bd9Sstevel@tonic-gate 		npages -= page_cnt;
57427c478bd9Sstevel@tonic-gate 	}
57437c478bd9Sstevel@tonic-gate }
57447c478bd9Sstevel@tonic-gate 
57457c478bd9Sstevel@tonic-gate ulong_t mem_waiters 	= 0;
57467c478bd9Sstevel@tonic-gate ulong_t	max_count 	= 20;
57477c478bd9Sstevel@tonic-gate #define	MAX_DELAY	0x1ff
57487c478bd9Sstevel@tonic-gate 
57497c478bd9Sstevel@tonic-gate /*
57507c478bd9Sstevel@tonic-gate  * Check if enough memory is available to proceed.
57517c478bd9Sstevel@tonic-gate  * Depending on system configuration and how much memory is
57527c478bd9Sstevel@tonic-gate  * reserved for swap we need to check against two variables.
57537c478bd9Sstevel@tonic-gate  * e.g. on systems with little physical swap availrmem can be
57547c478bd9Sstevel@tonic-gate  * more reliable indicator of how much memory is available.
57557c478bd9Sstevel@tonic-gate  * On systems with large phys swap freemem can be better indicator.
57567c478bd9Sstevel@tonic-gate  * If freemem drops below threshold level don't return an error
57577c478bd9Sstevel@tonic-gate  * immediately but wake up pageout to free memory and block.
57587c478bd9Sstevel@tonic-gate  * This is done number of times. If pageout is not able to free
57597c478bd9Sstevel@tonic-gate  * memory within certain time return an error.
57607c478bd9Sstevel@tonic-gate  * The same applies for availrmem but kmem_reap is used to
57617c478bd9Sstevel@tonic-gate  * free memory.
57627c478bd9Sstevel@tonic-gate  */
57637c478bd9Sstevel@tonic-gate int
57647c478bd9Sstevel@tonic-gate page_mem_avail(pgcnt_t npages)
57657c478bd9Sstevel@tonic-gate {
57667c478bd9Sstevel@tonic-gate 	ulong_t count;
57677c478bd9Sstevel@tonic-gate 
57687c478bd9Sstevel@tonic-gate #if defined(__i386)
57697c478bd9Sstevel@tonic-gate 	if (freemem > desfree + npages &&
57707c478bd9Sstevel@tonic-gate 	    availrmem > swapfs_reserve + npages &&
57717c478bd9Sstevel@tonic-gate 	    btop(vmem_size(heap_arena, VMEM_FREE)) > tune.t_minarmem +
57727c478bd9Sstevel@tonic-gate 	    npages)
57737c478bd9Sstevel@tonic-gate 		return (1);
57747c478bd9Sstevel@tonic-gate #else
57757c478bd9Sstevel@tonic-gate 	if (freemem > desfree + npages &&
57767c478bd9Sstevel@tonic-gate 	    availrmem > swapfs_reserve + npages)
57777c478bd9Sstevel@tonic-gate 		return (1);
57787c478bd9Sstevel@tonic-gate #endif
57797c478bd9Sstevel@tonic-gate 
57807c478bd9Sstevel@tonic-gate 	count = max_count;
57817c478bd9Sstevel@tonic-gate 	atomic_add_long(&mem_waiters, 1);
57827c478bd9Sstevel@tonic-gate 
57837c478bd9Sstevel@tonic-gate 	while (freemem < desfree + npages && --count) {
57847c478bd9Sstevel@tonic-gate 		cv_signal(&proc_pageout->p_cv);
57857c478bd9Sstevel@tonic-gate 		if (delay_sig(hz + (mem_waiters & MAX_DELAY))) {
57867c478bd9Sstevel@tonic-gate 			atomic_add_long(&mem_waiters, -1);
57877c478bd9Sstevel@tonic-gate 			return (0);
57887c478bd9Sstevel@tonic-gate 		}
57897c478bd9Sstevel@tonic-gate 	}
57907c478bd9Sstevel@tonic-gate 	if (count == 0) {
57917c478bd9Sstevel@tonic-gate 		atomic_add_long(&mem_waiters, -1);
57927c478bd9Sstevel@tonic-gate 		return (0);
57937c478bd9Sstevel@tonic-gate 	}
57947c478bd9Sstevel@tonic-gate 
57957c478bd9Sstevel@tonic-gate 	count = max_count;
57967c478bd9Sstevel@tonic-gate 	while (availrmem < swapfs_reserve + npages && --count) {
57977c478bd9Sstevel@tonic-gate 		kmem_reap();
57987c478bd9Sstevel@tonic-gate 		if (delay_sig(hz + (mem_waiters & MAX_DELAY))) {
57997c478bd9Sstevel@tonic-gate 			atomic_add_long(&mem_waiters, -1);
58007c478bd9Sstevel@tonic-gate 			return (0);
58017c478bd9Sstevel@tonic-gate 		}
58027c478bd9Sstevel@tonic-gate 	}
58037c478bd9Sstevel@tonic-gate 	atomic_add_long(&mem_waiters, -1);
58047c478bd9Sstevel@tonic-gate 	if (count == 0)
58057c478bd9Sstevel@tonic-gate 		return (0);
58067c478bd9Sstevel@tonic-gate 
58077c478bd9Sstevel@tonic-gate #if defined(__i386)
58087c478bd9Sstevel@tonic-gate 	if (btop(vmem_size(heap_arena, VMEM_FREE)) <
58097c478bd9Sstevel@tonic-gate 	    tune.t_minarmem + npages)
58107c478bd9Sstevel@tonic-gate 		return (0);
58117c478bd9Sstevel@tonic-gate #endif
58127c478bd9Sstevel@tonic-gate 	return (1);
58137c478bd9Sstevel@tonic-gate }
58147c478bd9Sstevel@tonic-gate 
58153cff2f43Sstans #define	MAX_CNT	60	/* max num of iterations */
58163cff2f43Sstans /*
58173cff2f43Sstans  * Reclaim/reserve availrmem for npages.
58183cff2f43Sstans  * If there is not enough memory start reaping seg, kmem caches.
58193cff2f43Sstans  * Start pageout scanner (via page_needfree()).
58203cff2f43Sstans  * Exit after ~ MAX_CNT s regardless of how much memory has been released.
58213cff2f43Sstans  * Note: There is no guarantee that any availrmem will be freed as
58223cff2f43Sstans  * this memory typically is locked (kernel heap) or reserved for swap.
58233cff2f43Sstans  * Also due to memory fragmentation kmem allocator may not be able
58243cff2f43Sstans  * to free any memory (single user allocated buffer will prevent
58253cff2f43Sstans  * freeing slab or a page).
58263cff2f43Sstans  */
58273cff2f43Sstans int
58283cff2f43Sstans page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
58293cff2f43Sstans {
58303cff2f43Sstans 	int	i = 0;
58313cff2f43Sstans 	int	ret = 0;
58323cff2f43Sstans 	pgcnt_t	deficit;
58333cff2f43Sstans 	pgcnt_t old_availrmem;
58343cff2f43Sstans 
58353cff2f43Sstans 	mutex_enter(&freemem_lock);
58363cff2f43Sstans 	old_availrmem = availrmem - 1;
58373cff2f43Sstans 	while ((availrmem < tune.t_minarmem + npages + epages) &&
58383cff2f43Sstans 	    (old_availrmem < availrmem) && (i++ < MAX_CNT)) {
58393cff2f43Sstans 		old_availrmem = availrmem;
58403cff2f43Sstans 		deficit = tune.t_minarmem + npages + epages - availrmem;
58413cff2f43Sstans 		mutex_exit(&freemem_lock);
58423cff2f43Sstans 		page_needfree(deficit);
58433cff2f43Sstans 		seg_preap();
58443cff2f43Sstans 		kmem_reap();
58453cff2f43Sstans 		delay(hz);
58463cff2f43Sstans 		page_needfree(-(spgcnt_t)deficit);
58473cff2f43Sstans 		mutex_enter(&freemem_lock);
58483cff2f43Sstans 	}
58493cff2f43Sstans 
58503cff2f43Sstans 	if (adjust && (availrmem >= tune.t_minarmem + npages + epages)) {
58513cff2f43Sstans 		availrmem -= npages;
58523cff2f43Sstans 		ret = 1;
58533cff2f43Sstans 	}
58543cff2f43Sstans 
58553cff2f43Sstans 	mutex_exit(&freemem_lock);
58563cff2f43Sstans 
58573cff2f43Sstans 	return (ret);
58583cff2f43Sstans }
58597c478bd9Sstevel@tonic-gate 
58607c478bd9Sstevel@tonic-gate /*
58617c478bd9Sstevel@tonic-gate  * Search the memory segments to locate the desired page.  Within a
58627c478bd9Sstevel@tonic-gate  * segment, pages increase linearly with one page structure per
58637c478bd9Sstevel@tonic-gate  * physical page frame (size PAGESIZE).  The search begins
58647c478bd9Sstevel@tonic-gate  * with the segment that was accessed last, to take advantage of locality.
58657c478bd9Sstevel@tonic-gate  * If the hint misses, we start from the beginning of the sorted memseg list
58667c478bd9Sstevel@tonic-gate  */
58677c478bd9Sstevel@tonic-gate 
58687c478bd9Sstevel@tonic-gate 
58697c478bd9Sstevel@tonic-gate /*
58707c478bd9Sstevel@tonic-gate  * Some data structures for pfn to pp lookup.
58717c478bd9Sstevel@tonic-gate  */
58727c478bd9Sstevel@tonic-gate ulong_t mhash_per_slot;
58737c478bd9Sstevel@tonic-gate struct memseg *memseg_hash[N_MEM_SLOTS];
58747c478bd9Sstevel@tonic-gate 
58757c478bd9Sstevel@tonic-gate page_t *
58767c478bd9Sstevel@tonic-gate page_numtopp_nolock(pfn_t pfnum)
58777c478bd9Sstevel@tonic-gate {
58787c478bd9Sstevel@tonic-gate 	struct memseg *seg;
58797c478bd9Sstevel@tonic-gate 	page_t *pp;
5880affbd3ccSkchow 	vm_cpu_data_t *vc = CPU->cpu_vm_data;
58817c478bd9Sstevel@tonic-gate 
5882affbd3ccSkchow 	ASSERT(vc != NULL);
58837c478bd9Sstevel@tonic-gate 
58847c478bd9Sstevel@tonic-gate 	MEMSEG_STAT_INCR(nsearch);
58857c478bd9Sstevel@tonic-gate 
58867c478bd9Sstevel@tonic-gate 	/* Try last winner first */
5887affbd3ccSkchow 	if (((seg = vc->vc_pnum_memseg) != NULL) &&
58887c478bd9Sstevel@tonic-gate 		(pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
58897c478bd9Sstevel@tonic-gate 		MEMSEG_STAT_INCR(nlastwon);
58907c478bd9Sstevel@tonic-gate 		pp = seg->pages + (pfnum - seg->pages_base);
58917c478bd9Sstevel@tonic-gate 		if (pp->p_pagenum == pfnum)
58927c478bd9Sstevel@tonic-gate 			return ((page_t *)pp);
58937c478bd9Sstevel@tonic-gate 	}
58947c478bd9Sstevel@tonic-gate 
58957c478bd9Sstevel@tonic-gate 	/* Else Try hash */
58967c478bd9Sstevel@tonic-gate 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
58977c478bd9Sstevel@tonic-gate 		(pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
58987c478bd9Sstevel@tonic-gate 		MEMSEG_STAT_INCR(nhashwon);
5899affbd3ccSkchow 		vc->vc_pnum_memseg = seg;
59007c478bd9Sstevel@tonic-gate 		pp = seg->pages + (pfnum - seg->pages_base);
59017c478bd9Sstevel@tonic-gate 		if (pp->p_pagenum == pfnum)
59027c478bd9Sstevel@tonic-gate 			return ((page_t *)pp);
59037c478bd9Sstevel@tonic-gate 	}
59047c478bd9Sstevel@tonic-gate 
59057c478bd9Sstevel@tonic-gate 	/* Else Brute force */
59067c478bd9Sstevel@tonic-gate 	for (seg = memsegs; seg != NULL; seg = seg->next) {
59077c478bd9Sstevel@tonic-gate 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
5908affbd3ccSkchow 			vc->vc_pnum_memseg = seg;
59097c478bd9Sstevel@tonic-gate 			pp = seg->pages + (pfnum - seg->pages_base);
59107c478bd9Sstevel@tonic-gate 			return ((page_t *)pp);
59117c478bd9Sstevel@tonic-gate 		}
59127c478bd9Sstevel@tonic-gate 	}
5913affbd3ccSkchow 	vc->vc_pnum_memseg = NULL;
59147c478bd9Sstevel@tonic-gate 	MEMSEG_STAT_INCR(nnotfound);
59157c478bd9Sstevel@tonic-gate 	return ((page_t *)NULL);
59167c478bd9Sstevel@tonic-gate 
59177c478bd9Sstevel@tonic-gate }
59187c478bd9Sstevel@tonic-gate 
59197c478bd9Sstevel@tonic-gate struct memseg *
59207c478bd9Sstevel@tonic-gate page_numtomemseg_nolock(pfn_t pfnum)
59217c478bd9Sstevel@tonic-gate {
59227c478bd9Sstevel@tonic-gate 	struct memseg *seg;
59237c478bd9Sstevel@tonic-gate 	page_t *pp;
59247c478bd9Sstevel@tonic-gate 
59257c478bd9Sstevel@tonic-gate 	/* Try hash */
59267c478bd9Sstevel@tonic-gate 	if (((seg = memseg_hash[MEMSEG_PFN_HASH(pfnum)]) != NULL) &&
59277c478bd9Sstevel@tonic-gate 		(pfnum >= seg->pages_base) && (pfnum < seg->pages_end)) {
59287c478bd9Sstevel@tonic-gate 		pp = seg->pages + (pfnum - seg->pages_base);
59297c478bd9Sstevel@tonic-gate 		if (pp->p_pagenum == pfnum)
59307c478bd9Sstevel@tonic-gate 			return (seg);
59317c478bd9Sstevel@tonic-gate 	}
59327c478bd9Sstevel@tonic-gate 
59337c478bd9Sstevel@tonic-gate 	/* Else Brute force */
59347c478bd9Sstevel@tonic-gate 	for (seg = memsegs; seg != NULL; seg = seg->next) {
59357c478bd9Sstevel@tonic-gate 		if (pfnum >= seg->pages_base && pfnum < seg->pages_end) {
59367c478bd9Sstevel@tonic-gate 			return (seg);
59377c478bd9Sstevel@tonic-gate 		}
59387c478bd9Sstevel@tonic-gate 	}
59397c478bd9Sstevel@tonic-gate 	return ((struct memseg *)NULL);
59407c478bd9Sstevel@tonic-gate }
59417c478bd9Sstevel@tonic-gate 
59427c478bd9Sstevel@tonic-gate /*
59437c478bd9Sstevel@tonic-gate  * Given a page and a count return the page struct that is
59447c478bd9Sstevel@tonic-gate  * n structs away from the current one in the global page
59457c478bd9Sstevel@tonic-gate  * list.
59467c478bd9Sstevel@tonic-gate  *
59477c478bd9Sstevel@tonic-gate  * This function wraps to the first page upon
59487c478bd9Sstevel@tonic-gate  * reaching the end of the memseg list.
59497c478bd9Sstevel@tonic-gate  */
59507c478bd9Sstevel@tonic-gate page_t *
59517c478bd9Sstevel@tonic-gate page_nextn(page_t *pp, ulong_t n)
59527c478bd9Sstevel@tonic-gate {
59537c478bd9Sstevel@tonic-gate 	struct memseg *seg;
59547c478bd9Sstevel@tonic-gate 	page_t *ppn;
5955affbd3ccSkchow 	vm_cpu_data_t *vc = (vm_cpu_data_t *)CPU->cpu_vm_data;
59567c478bd9Sstevel@tonic-gate 
5957affbd3ccSkchow 	ASSERT(vc != NULL);
5958affbd3ccSkchow 
5959affbd3ccSkchow 	if (((seg = vc->vc_pnext_memseg) == NULL) ||
59607c478bd9Sstevel@tonic-gate 	    (seg->pages_base == seg->pages_end) ||
59617c478bd9Sstevel@tonic-gate 	    !(pp >= seg->pages && pp < seg->epages)) {
59627c478bd9Sstevel@tonic-gate 
59637c478bd9Sstevel@tonic-gate 		for (seg = memsegs; seg; seg = seg->next) {
59647c478bd9Sstevel@tonic-gate 			if (pp >= seg->pages && pp < seg->epages)
59657c478bd9Sstevel@tonic-gate 				break;
59667c478bd9Sstevel@tonic-gate 		}
59677c478bd9Sstevel@tonic-gate 
59687c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
59697c478bd9Sstevel@tonic-gate 			/* Memory delete got in, return something valid. */
59707c478bd9Sstevel@tonic-gate 			/* TODO: fix me. */
59717c478bd9Sstevel@tonic-gate 			seg = memsegs;
59727c478bd9Sstevel@tonic-gate 			pp = seg->pages;
59737c478bd9Sstevel@tonic-gate 		}
59747c478bd9Sstevel@tonic-gate 	}
59757c478bd9Sstevel@tonic-gate 
59767c478bd9Sstevel@tonic-gate 	/* check for wraparound - possible if n is large */
59777c478bd9Sstevel@tonic-gate 	while ((ppn = (pp + n)) >= seg->epages || ppn < pp) {
59787c478bd9Sstevel@tonic-gate 		n -= seg->epages - pp;
59797c478bd9Sstevel@tonic-gate 		seg = seg->next;
59807c478bd9Sstevel@tonic-gate 		if (seg == NULL)
59817c478bd9Sstevel@tonic-gate 			seg = memsegs;
59827c478bd9Sstevel@tonic-gate 		pp = seg->pages;
59837c478bd9Sstevel@tonic-gate 	}
5984affbd3ccSkchow 	vc->vc_pnext_memseg = seg;
59857c478bd9Sstevel@tonic-gate 	return (ppn);
59867c478bd9Sstevel@tonic-gate }
59877c478bd9Sstevel@tonic-gate 
59887c478bd9Sstevel@tonic-gate /*
59897c478bd9Sstevel@tonic-gate  * Initialize for a loop using page_next_scan_large().
59907c478bd9Sstevel@tonic-gate  */
59917c478bd9Sstevel@tonic-gate page_t *
59927c478bd9Sstevel@tonic-gate page_next_scan_init(void **cookie)
59937c478bd9Sstevel@tonic-gate {
59947c478bd9Sstevel@tonic-gate 	ASSERT(cookie != NULL);
59957c478bd9Sstevel@tonic-gate 	*cookie = (void *)memsegs;
59967c478bd9Sstevel@tonic-gate 	return ((page_t *)memsegs->pages);
59977c478bd9Sstevel@tonic-gate }
59987c478bd9Sstevel@tonic-gate 
59997c478bd9Sstevel@tonic-gate /*
60007c478bd9Sstevel@tonic-gate  * Return the next page in a scan of page_t's, assuming we want
60017c478bd9Sstevel@tonic-gate  * to skip over sub-pages within larger page sizes.
60027c478bd9Sstevel@tonic-gate  *
60037c478bd9Sstevel@tonic-gate  * The cookie is used to keep track of the current memseg.
60047c478bd9Sstevel@tonic-gate  */
60057c478bd9Sstevel@tonic-gate page_t *
60067c478bd9Sstevel@tonic-gate page_next_scan_large(
60077c478bd9Sstevel@tonic-gate 	page_t		*pp,
60087c478bd9Sstevel@tonic-gate 	ulong_t		*n,
60097c478bd9Sstevel@tonic-gate 	void		**cookie)
60107c478bd9Sstevel@tonic-gate {
60117c478bd9Sstevel@tonic-gate 	struct memseg	*seg = (struct memseg *)*cookie;
60127c478bd9Sstevel@tonic-gate 	page_t		*new_pp;
60137c478bd9Sstevel@tonic-gate 	ulong_t		cnt;
60147c478bd9Sstevel@tonic-gate 	pfn_t		pfn;
60157c478bd9Sstevel@tonic-gate 
60167c478bd9Sstevel@tonic-gate 
60177c478bd9Sstevel@tonic-gate 	/*
60187c478bd9Sstevel@tonic-gate 	 * get the count of page_t's to skip based on the page size
60197c478bd9Sstevel@tonic-gate 	 */
60207c478bd9Sstevel@tonic-gate 	ASSERT(pp != NULL);
60217c478bd9Sstevel@tonic-gate 	if (pp->p_szc == 0) {
60227c478bd9Sstevel@tonic-gate 		cnt = 1;
60237c478bd9Sstevel@tonic-gate 	} else {
60247c478bd9Sstevel@tonic-gate 		pfn = page_pptonum(pp);
60257c478bd9Sstevel@tonic-gate 		cnt = page_get_pagecnt(pp->p_szc);
60267c478bd9Sstevel@tonic-gate 		cnt -= pfn & (cnt - 1);
60277c478bd9Sstevel@tonic-gate 	}
60287c478bd9Sstevel@tonic-gate 	*n += cnt;
60297c478bd9Sstevel@tonic-gate 	new_pp = pp + cnt;
60307c478bd9Sstevel@tonic-gate 
60317c478bd9Sstevel@tonic-gate 	/*
60327c478bd9Sstevel@tonic-gate 	 * Catch if we went past the end of the current memory segment. If so,
60337c478bd9Sstevel@tonic-gate 	 * just move to the next segment with pages.
60347c478bd9Sstevel@tonic-gate 	 */
60357c478bd9Sstevel@tonic-gate 	if (new_pp >= seg->epages) {
60367c478bd9Sstevel@tonic-gate 		do {
60377c478bd9Sstevel@tonic-gate 			seg = seg->next;
60387c478bd9Sstevel@tonic-gate 			if (seg == NULL)
60397c478bd9Sstevel@tonic-gate 				seg = memsegs;
60407c478bd9Sstevel@tonic-gate 		} while (seg->pages == seg->epages);
60417c478bd9Sstevel@tonic-gate 		new_pp = seg->pages;
60427c478bd9Sstevel@tonic-gate 		*cookie = (void *)seg;
60437c478bd9Sstevel@tonic-gate 	}
60447c478bd9Sstevel@tonic-gate 
60457c478bd9Sstevel@tonic-gate 	return (new_pp);
60467c478bd9Sstevel@tonic-gate }
60477c478bd9Sstevel@tonic-gate 
60487c478bd9Sstevel@tonic-gate 
60497c478bd9Sstevel@tonic-gate /*
60507c478bd9Sstevel@tonic-gate  * Returns next page in list. Note: this function wraps
60517c478bd9Sstevel@tonic-gate  * to the first page in the list upon reaching the end
60527c478bd9Sstevel@tonic-gate  * of the list. Callers should be aware of this fact.
60537c478bd9Sstevel@tonic-gate  */
60547c478bd9Sstevel@tonic-gate 
60557c478bd9Sstevel@tonic-gate /* We should change this be a #define */
60567c478bd9Sstevel@tonic-gate 
60577c478bd9Sstevel@tonic-gate page_t *
60587c478bd9Sstevel@tonic-gate page_next(page_t *pp)
60597c478bd9Sstevel@tonic-gate {
60607c478bd9Sstevel@tonic-gate 	return (page_nextn(pp, 1));
60617c478bd9Sstevel@tonic-gate }
60627c478bd9Sstevel@tonic-gate 
60637c478bd9Sstevel@tonic-gate page_t *
60647c478bd9Sstevel@tonic-gate page_first()
60657c478bd9Sstevel@tonic-gate {
60667c478bd9Sstevel@tonic-gate 	return ((page_t *)memsegs->pages);
60677c478bd9Sstevel@tonic-gate }
60687c478bd9Sstevel@tonic-gate 
60697c478bd9Sstevel@tonic-gate 
60707c478bd9Sstevel@tonic-gate /*
60717c478bd9Sstevel@tonic-gate  * This routine is called at boot with the initial memory configuration
60727c478bd9Sstevel@tonic-gate  * and when memory is added or removed.
60737c478bd9Sstevel@tonic-gate  */
60747c478bd9Sstevel@tonic-gate void
60757c478bd9Sstevel@tonic-gate build_pfn_hash()
60767c478bd9Sstevel@tonic-gate {
60777c478bd9Sstevel@tonic-gate 	pfn_t cur;
60787c478bd9Sstevel@tonic-gate 	pgcnt_t index;
60797c478bd9Sstevel@tonic-gate 	struct memseg *pseg;
60807c478bd9Sstevel@tonic-gate 	int	i;
60817c478bd9Sstevel@tonic-gate 
60827c478bd9Sstevel@tonic-gate 	/*
60837c478bd9Sstevel@tonic-gate 	 * Clear memseg_hash array.
60847c478bd9Sstevel@tonic-gate 	 * Since memory add/delete is designed to operate concurrently
60857c478bd9Sstevel@tonic-gate 	 * with normal operation, the hash rebuild must be able to run
60867c478bd9Sstevel@tonic-gate 	 * concurrently with page_numtopp_nolock(). To support this
60877c478bd9Sstevel@tonic-gate 	 * functionality, assignments to memseg_hash array members must
60887c478bd9Sstevel@tonic-gate 	 * be done atomically.
60897c478bd9Sstevel@tonic-gate 	 *
60907c478bd9Sstevel@tonic-gate 	 * NOTE: bzero() does not currently guarantee this for kernel
60917c478bd9Sstevel@tonic-gate 	 * threads, and cannot be used here.
60927c478bd9Sstevel@tonic-gate 	 */
60937c478bd9Sstevel@tonic-gate 	for (i = 0; i < N_MEM_SLOTS; i++)
60947c478bd9Sstevel@tonic-gate 		memseg_hash[i] = NULL;
60957c478bd9Sstevel@tonic-gate 
60967c478bd9Sstevel@tonic-gate 	hat_kpm_mseghash_clear(N_MEM_SLOTS);
60977c478bd9Sstevel@tonic-gate 
60987c478bd9Sstevel@tonic-gate 	/*
60997c478bd9Sstevel@tonic-gate 	 * Physmax is the last valid pfn.
61007c478bd9Sstevel@tonic-gate 	 */
61017c478bd9Sstevel@tonic-gate 	mhash_per_slot = (physmax + 1) >> MEM_HASH_SHIFT;
61027c478bd9Sstevel@tonic-gate 	for (pseg = memsegs; pseg != NULL; pseg = pseg->next) {
61037c478bd9Sstevel@tonic-gate 		index = MEMSEG_PFN_HASH(pseg->pages_base);
61047c478bd9Sstevel@tonic-gate 		cur = pseg->pages_base;
61057c478bd9Sstevel@tonic-gate 		do {
61067c478bd9Sstevel@tonic-gate 			if (index >= N_MEM_SLOTS)
61077c478bd9Sstevel@tonic-gate 				index = MEMSEG_PFN_HASH(cur);
61087c478bd9Sstevel@tonic-gate 
61097c478bd9Sstevel@tonic-gate 			if (memseg_hash[index] == NULL ||
61107c478bd9Sstevel@tonic-gate 			    memseg_hash[index]->pages_base > pseg->pages_base) {
61117c478bd9Sstevel@tonic-gate 				memseg_hash[index] = pseg;
61127c478bd9Sstevel@tonic-gate 				hat_kpm_mseghash_update(index, pseg);
61137c478bd9Sstevel@tonic-gate 			}
61147c478bd9Sstevel@tonic-gate 			cur += mhash_per_slot;
61157c478bd9Sstevel@tonic-gate 			index++;
61167c478bd9Sstevel@tonic-gate 		} while (cur < pseg->pages_end);
61177c478bd9Sstevel@tonic-gate 	}
61187c478bd9Sstevel@tonic-gate }
61197c478bd9Sstevel@tonic-gate 
61207c478bd9Sstevel@tonic-gate /*
61217c478bd9Sstevel@tonic-gate  * Return the pagenum for the pp
61227c478bd9Sstevel@tonic-gate  */
61237c478bd9Sstevel@tonic-gate pfn_t
61247c478bd9Sstevel@tonic-gate page_pptonum(page_t *pp)
61257c478bd9Sstevel@tonic-gate {
61267c478bd9Sstevel@tonic-gate 	return (pp->p_pagenum);
61277c478bd9Sstevel@tonic-gate }
61287c478bd9Sstevel@tonic-gate 
61297c478bd9Sstevel@tonic-gate /*
61307c478bd9Sstevel@tonic-gate  * interface to the referenced and modified etc bits
61317c478bd9Sstevel@tonic-gate  * in the PSM part of the page struct
61327c478bd9Sstevel@tonic-gate  * when no locking is desired.
61337c478bd9Sstevel@tonic-gate  */
61347c478bd9Sstevel@tonic-gate void
61357c478bd9Sstevel@tonic-gate page_set_props(page_t *pp, uint_t flags)
61367c478bd9Sstevel@tonic-gate {
61377c478bd9Sstevel@tonic-gate 	ASSERT((flags & ~(P_MOD | P_REF | P_RO)) == 0);
61387c478bd9Sstevel@tonic-gate 	pp->p_nrm |= (uchar_t)flags;
61397c478bd9Sstevel@tonic-gate }
61407c478bd9Sstevel@tonic-gate 
61417c478bd9Sstevel@tonic-gate void
61427c478bd9Sstevel@tonic-gate page_clr_all_props(page_t *pp)
61437c478bd9Sstevel@tonic-gate {
61447c478bd9Sstevel@tonic-gate 	pp->p_nrm = 0;
61457c478bd9Sstevel@tonic-gate }
61467c478bd9Sstevel@tonic-gate 
61477c478bd9Sstevel@tonic-gate /*
6148db874c57Selowe  * Clear p_lckcnt and p_cowcnt, adjusting freemem if required.
61497c478bd9Sstevel@tonic-gate  */
61507c478bd9Sstevel@tonic-gate int
6151db874c57Selowe page_clear_lck_cow(page_t *pp, int adjust)
61527c478bd9Sstevel@tonic-gate {
6153db874c57Selowe 	int	f_amount;
61547c478bd9Sstevel@tonic-gate 
6155db874c57Selowe 	ASSERT(PAGE_EXCL(pp));
61567c478bd9Sstevel@tonic-gate 
61577c478bd9Sstevel@tonic-gate 	/*
6158db874c57Selowe 	 * The page_struct_lock need not be acquired here since
6159db874c57Selowe 	 * we require the caller hold the page exclusively locked.
61607c478bd9Sstevel@tonic-gate 	 */
6161db874c57Selowe 	f_amount = 0;
6162db874c57Selowe 	if (pp->p_lckcnt) {
6163db874c57Selowe 		f_amount = 1;
6164db874c57Selowe 		pp->p_lckcnt = 0;
6165db874c57Selowe 	}
6166db874c57Selowe 	if (pp->p_cowcnt) {
6167db874c57Selowe 		f_amount += pp->p_cowcnt;
6168db874c57Selowe 		pp->p_cowcnt = 0;
6169db874c57Selowe 	}
61707c478bd9Sstevel@tonic-gate 
6171db874c57Selowe 	if (adjust && f_amount) {
6172db874c57Selowe 		mutex_enter(&freemem_lock);
6173db874c57Selowe 		availrmem += f_amount;
6174db874c57Selowe 		mutex_exit(&freemem_lock);
61757c478bd9Sstevel@tonic-gate 	}
61767c478bd9Sstevel@tonic-gate 
6177db874c57Selowe 	return (f_amount);
61787c478bd9Sstevel@tonic-gate }
61797c478bd9Sstevel@tonic-gate 
61807c478bd9Sstevel@tonic-gate /*
6181db874c57Selowe  * The following functions is called from free_vp_pages()
6182db874c57Selowe  * for an inexact estimate of a newly free'd page...
61837c478bd9Sstevel@tonic-gate  */
6184db874c57Selowe ulong_t
6185db874c57Selowe page_share_cnt(page_t *pp)
6186db874c57Selowe {
6187db874c57Selowe 	return (hat_page_getshare(pp));
6188db874c57Selowe }
6189db874c57Selowe 
61907c478bd9Sstevel@tonic-gate int
61917c478bd9Sstevel@tonic-gate page_isshared(page_t *pp)
61927c478bd9Sstevel@tonic-gate {
61937c478bd9Sstevel@tonic-gate 	return (hat_page_getshare(pp) > 1);
61947c478bd9Sstevel@tonic-gate }
61957c478bd9Sstevel@tonic-gate 
61967c478bd9Sstevel@tonic-gate int
61977c478bd9Sstevel@tonic-gate page_isfree(page_t *pp)
61987c478bd9Sstevel@tonic-gate {
61997c478bd9Sstevel@tonic-gate 	return (PP_ISFREE(pp));
62007c478bd9Sstevel@tonic-gate }
62017c478bd9Sstevel@tonic-gate 
62027c478bd9Sstevel@tonic-gate int
62037c478bd9Sstevel@tonic-gate page_isref(page_t *pp)
62047c478bd9Sstevel@tonic-gate {
62057c478bd9Sstevel@tonic-gate 	return (hat_page_getattr(pp, P_REF));
62067c478bd9Sstevel@tonic-gate }
62077c478bd9Sstevel@tonic-gate 
62087c478bd9Sstevel@tonic-gate int
62097c478bd9Sstevel@tonic-gate page_ismod(page_t *pp)
62107c478bd9Sstevel@tonic-gate {
62117c478bd9Sstevel@tonic-gate 	return (hat_page_getattr(pp, P_MOD));
62127c478bd9Sstevel@tonic-gate }
62138b464eb8Smec 
62148b464eb8Smec /*
62158b464eb8Smec  * Reclaim the given constituent page from the freelist, regardless of it's
62168b464eb8Smec  * size.  The page will be demoted as required.
62178b464eb8Smec  * Returns 1 on success or 0 on failure.
62188b464eb8Smec  *
62198b464eb8Smec  * The page is unlocked if it can't be reclaimed (when freemem == 0).
62208b464eb8Smec  * If `lock' is non-null, it will be dropped and re-acquired if
62218b464eb8Smec  * the routine must wait while freemem is 0.
62228b464eb8Smec  */
62238b464eb8Smec int
62248b464eb8Smec page_reclaim_page(page_t *pp, kmutex_t *lock)
62258b464eb8Smec {
62268b464eb8Smec 	struct pcf	*p;
62278b464eb8Smec 	uint_t		pcf_index;
62288b464eb8Smec 	struct cpu	*cpup;
62298b464eb8Smec 	uint_t		i;
62308b464eb8Smec 	pgcnt_t		collected = 0;
62318b464eb8Smec 
62328b464eb8Smec 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
62338b464eb8Smec 	ASSERT(PAGE_EXCL(pp) && PP_ISFREE(pp));
62348b464eb8Smec 
62358b464eb8Smec 	/*
62368b464eb8Smec 	 * If `freemem' is 0, we cannot reclaim this page from the
62378b464eb8Smec 	 * freelist, so release every lock we might hold: the page,
62388b464eb8Smec 	 * and the `lock' before blocking.
62398b464eb8Smec 	 *
62408b464eb8Smec 	 * The only way `freemem' can become 0 while there are pages
62418b464eb8Smec 	 * marked free (have their p->p_free bit set) is when the
62428b464eb8Smec 	 * system is low on memory and doing a page_create().  In
62438b464eb8Smec 	 * order to guarantee that once page_create() starts acquiring
62448b464eb8Smec 	 * pages it will be able to get all that it needs since `freemem'
62458b464eb8Smec 	 * was decreased by the requested amount.  So, we need to release
62468b464eb8Smec 	 * this page, and let page_create() have it.
62478b464eb8Smec 	 *
62488b464eb8Smec 	 * Since `freemem' being zero is not supposed to happen, just
62498b464eb8Smec 	 * use the usual hash stuff as a starting point.  If that bucket
62508b464eb8Smec 	 * is empty, then assume the worst, and start at the beginning
62518b464eb8Smec 	 * of the pcf array.  If we always start at the beginning
62528b464eb8Smec 	 * when acquiring more than one pcf lock, there won't be any
62538b464eb8Smec 	 * deadlock problems.
62548b464eb8Smec 	 */
62558b464eb8Smec 
62568b464eb8Smec 	/* TODO: Do we need to test kcage_freemem if PG_NORELOC(pp)? */
62578b464eb8Smec 
62588b464eb8Smec 	if (freemem <= throttlefree && !page_create_throttle(1, 0)) {
62598b464eb8Smec 		pcf_acquire_all();
62608b464eb8Smec 		goto page_reclaim_nomem;
62618b464eb8Smec 	}
62628b464eb8Smec 
62638b464eb8Smec 	pcf_index = PCF_INDEX();
62648b464eb8Smec 	p = &pcf[pcf_index];
62658b464eb8Smec 	mutex_enter(&p->pcf_lock);
62668b464eb8Smec 	if (p->pcf_count > 0) {
62678b464eb8Smec 		collected = 1;
62688b464eb8Smec 		p->pcf_count -= 1;
62698b464eb8Smec 	}
62708b464eb8Smec 	mutex_exit(&p->pcf_lock);
62718b464eb8Smec 
62728b464eb8Smec 	if (!collected) {
62738b464eb8Smec 		VM_STAT_ADD(page_reclaim_zero);
62748b464eb8Smec 		/*
62758b464eb8Smec 		 * Check again. Its possible that some other thread
62768b464eb8Smec 		 * could have been right behind us, and added one
62778b464eb8Smec 		 * to a list somewhere.  Acquire each of the pcf locks
62788b464eb8Smec 		 * until we find a page.
62798b464eb8Smec 		 */
62808b464eb8Smec 		p = pcf;
62818b464eb8Smec 		for (i = 0; i < PCF_FANOUT; i++) {
62828b464eb8Smec 			mutex_enter(&p->pcf_lock);
62838b464eb8Smec 			if (p->pcf_count) {
62848b464eb8Smec 				if (p->pcf_count > 0) {
62858b464eb8Smec 					p->pcf_count -= 1;
62868b464eb8Smec 					collected = 1;
62878b464eb8Smec 					break;
62888b464eb8Smec 				}
62898b464eb8Smec 			}
62908b464eb8Smec 			p++;
62918b464eb8Smec 		}
62928b464eb8Smec 
62938b464eb8Smec 		if (!collected) {
62948b464eb8Smec page_reclaim_nomem:
62958b464eb8Smec 			/*
62968b464eb8Smec 			 * We really can't have page `pp'.
62978b464eb8Smec 			 * Time for the no-memory dance with
62988b464eb8Smec 			 * page_free().  This is just like
62998b464eb8Smec 			 * page_create_wait().  Plus the added
63008b464eb8Smec 			 * attraction of releasing whatever mutex
63018b464eb8Smec 			 * we held when we were called with in `lock'.
63028b464eb8Smec 			 * Page_unlock() will wakeup any thread
63038b464eb8Smec 			 * waiting around for this page.
63048b464eb8Smec 			 */
63058b464eb8Smec 			if (lock) {
63068b464eb8Smec 				VM_STAT_ADD(page_reclaim_zero_locked);
63078b464eb8Smec 				mutex_exit(lock);
63088b464eb8Smec 			}
63098b464eb8Smec 			page_unlock(pp);
63108b464eb8Smec 
63118b464eb8Smec 			/*
63128b464eb8Smec 			 * get this before we drop all the pcf locks.
63138b464eb8Smec 			 */
63148b464eb8Smec 			mutex_enter(&new_freemem_lock);
63158b464eb8Smec 
63168b464eb8Smec 			p = pcf;
63178b464eb8Smec 			for (i = 0; i < PCF_FANOUT; i++) {
63188b464eb8Smec 				p->pcf_wait++;
63198b464eb8Smec 				mutex_exit(&p->pcf_lock);
63208b464eb8Smec 				p++;
63218b464eb8Smec 			}
63228b464eb8Smec 
63238b464eb8Smec 			freemem_wait++;
63248b464eb8Smec 			cv_wait(&freemem_cv, &new_freemem_lock);
63258b464eb8Smec 			freemem_wait--;
63268b464eb8Smec 
63278b464eb8Smec 			mutex_exit(&new_freemem_lock);
63288b464eb8Smec 
63298b464eb8Smec 			if (lock) {
63308b464eb8Smec 				mutex_enter(lock);
63318b464eb8Smec 			}
63328b464eb8Smec 			return (0);
63338b464eb8Smec 		}
63348b464eb8Smec 
63358b464eb8Smec 		/*
63368b464eb8Smec 		 * We beat the PCF bins over the head until
63378b464eb8Smec 		 * we got the memory that we wanted.
63388b464eb8Smec 		 * The pcf accounting has been done,
63398b464eb8Smec 		 * though none of the pcf_wait flags have been set,
63408b464eb8Smec 		 * drop the locks and continue on.
63418b464eb8Smec 		 */
63428b464eb8Smec 		ASSERT(collected == 1);
63438b464eb8Smec 		while (p >= pcf) {
63448b464eb8Smec 			mutex_exit(&p->pcf_lock);
63458b464eb8Smec 			p--;
63468b464eb8Smec 		}
63478b464eb8Smec 	}
63488b464eb8Smec 
63498b464eb8Smec 	/*
63508b464eb8Smec 	 * freemem is not protected by any lock. Thus, we cannot
63518b464eb8Smec 	 * have any assertion containing freemem here.
63528b464eb8Smec 	 */
63538b464eb8Smec 	freemem -= 1;
63548b464eb8Smec 
63558b464eb8Smec 	VM_STAT_ADD(pagecnt.pc_reclaim);
63568b464eb8Smec 	if (PP_ISAGED(pp)) {
63578b464eb8Smec 		page_list_sub(pp, PG_FREE_LIST);
63588b464eb8Smec 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_FREE,
63598b464eb8Smec 		    "page_reclaim_page_free:pp %p", pp);
63608b464eb8Smec 	} else {
63618b464eb8Smec 		page_list_sub(pp, PG_CACHE_LIST);
63628b464eb8Smec 		TRACE_1(TR_FAC_VM, TR_PAGE_UNFREE_CACHE,
63638b464eb8Smec 		    "page_reclaim_page_cache:pp %p", pp);
63648b464eb8Smec 	}
63658b464eb8Smec 
63668b464eb8Smec 	/*
63678b464eb8Smec 	 * The page we took off the freelist must be szc 0 as
63688b464eb8Smec 	 * we used page_list_sub which will demote the page if needed.
63698b464eb8Smec 	 */
63708b464eb8Smec 	ASSERT(pp->p_szc == 0);
63718b464eb8Smec 
63728b464eb8Smec 	/*
63738b464eb8Smec 	 * clear the p_free & p_age bits since this page is no longer
63748b464eb8Smec 	 * on the free list.  Notice that there was a brief time where
63758b464eb8Smec 	 * a page is marked as free, but is not on the list.
63768b464eb8Smec 	 *
63778b464eb8Smec 	 * Set the reference bit to protect against immediate pageout.
63788b464eb8Smec 	 */
63798b464eb8Smec 	PP_CLRFREE(pp);
63808b464eb8Smec 	PP_CLRAGED(pp);
63818b464eb8Smec 	page_set_props(pp, P_REF);
63828b464eb8Smec 
63838b464eb8Smec 	CPU_STATS_ENTER_K();
63848b464eb8Smec 	cpup = CPU;	/* get cpup now that CPU cannot change */
63858b464eb8Smec 	CPU_STATS_ADDQ(cpup, vm, pgrec, 1);
63868b464eb8Smec 	CPU_STATS_ADDQ(cpup, vm, pgfrec, 1);
63878b464eb8Smec 	CPU_STATS_EXIT_K();
63888b464eb8Smec 
63898b464eb8Smec 	return (1);
63908b464eb8Smec }
63918b464eb8Smec 
63928b464eb8Smec /*
63938b464eb8Smec  * The following code all currently relates to the page capture logic:
63948b464eb8Smec  *
63958b464eb8Smec  * This logic is used for cases where there is a desire to claim a certain
63968b464eb8Smec  * physical page in the system for the caller.  As it may not be possible
63978b464eb8Smec  * to capture the page immediately, the p_toxic bits are used in the page
63988b464eb8Smec  * structure to indicate that someone wants to capture this page.  When the
63998b464eb8Smec  * page gets unlocked, the toxic flag will be noted and an attempt to capture
64008b464eb8Smec  * the page will be made.  If it is successful, the original callers callback
64018b464eb8Smec  * will be called with the page to do with it what they please.
64028b464eb8Smec  *
64038b464eb8Smec  * There is also an async thread which wakes up to attempt to capture
64048b464eb8Smec  * pages occasionally which have the capture bit set.  All of the pages which
64058b464eb8Smec  * need to be captured asynchronously have been inserted into the
64068b464eb8Smec  * page_capture_hash and thus this thread walks that hash list.  Items in the
64078b464eb8Smec  * hash have an expiration time so this thread handles that as well by removing
64088b464eb8Smec  * the item from the hash if it has expired.
64098b464eb8Smec  *
64108b464eb8Smec  * Some important things to note are:
64118b464eb8Smec  * - if the PR_CAPTURE bit is set on a page, then the page is in the
64128b464eb8Smec  *   page_capture_hash.  The page_capture_hash_head.pchh_mutex is needed
64138b464eb8Smec  *   to set and clear this bit, and while the lock is held is the only time
64148b464eb8Smec  *   you can add or remove an entry from the hash.
64158b464eb8Smec  * - the PR_CAPTURE bit can only be set and cleared while holding the
64168b464eb8Smec  *   page_capture_hash_head.pchh_mutex
64178b464eb8Smec  * - the t_flag field of the thread struct is used with the T_CAPTURING
64188b464eb8Smec  *   flag to prevent recursion while dealing with large pages.
64198b464eb8Smec  * - pages which need to be retired never expire on the page_capture_hash.
64208b464eb8Smec  */
64218b464eb8Smec 
64228b464eb8Smec static void page_capture_thread(void);
64238b464eb8Smec static kthread_t *pc_thread_id;
64248b464eb8Smec kcondvar_t pc_cv;
64258b464eb8Smec static kmutex_t pc_thread_mutex;
64268b464eb8Smec static clock_t pc_thread_shortwait;
64278b464eb8Smec static clock_t pc_thread_longwait;
6428*cee1d74bSjfrank static int pc_thread_ism_retry;
64298b464eb8Smec 
64308b464eb8Smec struct page_capture_callback pc_cb[PC_NUM_CALLBACKS];
64318b464eb8Smec 
64328b464eb8Smec /* Note that this is a circular linked list */
64338b464eb8Smec typedef struct page_capture_hash_bucket {
64348b464eb8Smec 	page_t *pp;
64358b464eb8Smec 	uint_t szc;
64368b464eb8Smec 	uint_t flags;
64378b464eb8Smec 	clock_t expires;	/* lbolt at which this request expires. */
64388b464eb8Smec 	void *datap;		/* Cached data passed in for callback */
64398b464eb8Smec 	struct page_capture_hash_bucket *next;
64408b464eb8Smec 	struct page_capture_hash_bucket *prev;
64418b464eb8Smec } page_capture_hash_bucket_t;
64428b464eb8Smec 
64438b464eb8Smec /*
64448b464eb8Smec  * Each hash bucket will have it's own mutex and two lists which are:
64458b464eb8Smec  * active (0):	represents requests which have not been processed by
64468b464eb8Smec  *		the page_capture async thread yet.
64478b464eb8Smec  * walked (1):	represents requests which have been processed by the
64488b464eb8Smec  *		page_capture async thread within it's given walk of this bucket.
64498b464eb8Smec  *
64508b464eb8Smec  * These are all needed so that we can synchronize all async page_capture
64518b464eb8Smec  * events.  When the async thread moves to a new bucket, it will append the
64528b464eb8Smec  * walked list to the active list and walk each item one at a time, moving it
64538b464eb8Smec  * from the active list to the walked list.  Thus if there is an async request
64548b464eb8Smec  * outstanding for a given page, it will always be in one of the two lists.
64558b464eb8Smec  * New requests will always be added to the active list.
64568b464eb8Smec  * If we were not able to capture a page before the request expired, we'd free
64578b464eb8Smec  * up the request structure which would indicate to page_capture that there is
64588b464eb8Smec  * no longer a need for the given page, and clear the PR_CAPTURE flag if
64598b464eb8Smec  * possible.
64608b464eb8Smec  */
64618b464eb8Smec typedef struct page_capture_hash_head {
64628b464eb8Smec 	kmutex_t pchh_mutex;
64638b464eb8Smec 	uint_t num_pages;
64648b464eb8Smec 	page_capture_hash_bucket_t lists[2]; /* sentinel nodes */
64658b464eb8Smec } page_capture_hash_head_t;
64668b464eb8Smec 
64678b464eb8Smec #ifdef DEBUG
64688b464eb8Smec #define	NUM_PAGE_CAPTURE_BUCKETS 4
64698b464eb8Smec #else
64708b464eb8Smec #define	NUM_PAGE_CAPTURE_BUCKETS 64
64718b464eb8Smec #endif
64728b464eb8Smec 
64738b464eb8Smec page_capture_hash_head_t page_capture_hash[NUM_PAGE_CAPTURE_BUCKETS];
64748b464eb8Smec 
64758b464eb8Smec /* for now use a very simple hash based upon the size of a page struct */
64768b464eb8Smec #define	PAGE_CAPTURE_HASH(pp)	\
64778b464eb8Smec 	((int)(((uintptr_t)pp >> 7) & (NUM_PAGE_CAPTURE_BUCKETS - 1)))
64788b464eb8Smec 
64798b464eb8Smec extern pgcnt_t swapfs_minfree;
64808b464eb8Smec 
64818b464eb8Smec int page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap);
64828b464eb8Smec 
64838b464eb8Smec /*
64848b464eb8Smec  * a callback function is required for page capture requests.
64858b464eb8Smec  */
64868b464eb8Smec void
64878b464eb8Smec page_capture_register_callback(uint_t index, clock_t duration,
64888b464eb8Smec     int (*cb_func)(page_t *, void *, uint_t))
64898b464eb8Smec {
64908b464eb8Smec 	ASSERT(pc_cb[index].cb_active == 0);
64918b464eb8Smec 	ASSERT(cb_func != NULL);
64928b464eb8Smec 	rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
64938b464eb8Smec 	pc_cb[index].duration = duration;
64948b464eb8Smec 	pc_cb[index].cb_func = cb_func;
64958b464eb8Smec 	pc_cb[index].cb_active = 1;
64968b464eb8Smec 	rw_exit(&pc_cb[index].cb_rwlock);
64978b464eb8Smec }
64988b464eb8Smec 
64998b464eb8Smec void
65008b464eb8Smec page_capture_unregister_callback(uint_t index)
65018b464eb8Smec {
65028b464eb8Smec 	int i, j;
65038b464eb8Smec 	struct page_capture_hash_bucket *bp1;
65048b464eb8Smec 	struct page_capture_hash_bucket *bp2;
65058b464eb8Smec 	struct page_capture_hash_bucket *head = NULL;
65068b464eb8Smec 	uint_t flags = (1 << index);
65078b464eb8Smec 
65088b464eb8Smec 	rw_enter(&pc_cb[index].cb_rwlock, RW_WRITER);
65098b464eb8Smec 	ASSERT(pc_cb[index].cb_active == 1);
65108b464eb8Smec 	pc_cb[index].duration = 0;	/* Paranoia */
65118b464eb8Smec 	pc_cb[index].cb_func = NULL;	/* Paranoia */
65128b464eb8Smec 	pc_cb[index].cb_active = 0;
65138b464eb8Smec 	rw_exit(&pc_cb[index].cb_rwlock);
65148b464eb8Smec 
65158b464eb8Smec 	/*
65168b464eb8Smec 	 * Just move all the entries to a private list which we can walk
65178b464eb8Smec 	 * through without the need to hold any locks.
65188b464eb8Smec 	 * No more requests can get added to the hash lists for this consumer
65198b464eb8Smec 	 * as the cb_active field for the callback has been cleared.
65208b464eb8Smec 	 */
65218b464eb8Smec 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
65228b464eb8Smec 		mutex_enter(&page_capture_hash[i].pchh_mutex);
65238b464eb8Smec 		for (j = 0; j < 2; j++) {
65248b464eb8Smec 			bp1 = page_capture_hash[i].lists[j].next;
65258b464eb8Smec 			/* walk through all but first (sentinel) element */
65268b464eb8Smec 			while (bp1 != &page_capture_hash[i].lists[j]) {
65278b464eb8Smec 				bp2 = bp1;
65288b464eb8Smec 				if (bp2->flags & flags) {
65298b464eb8Smec 					bp1 = bp2->next;
65308b464eb8Smec 					bp1->prev = bp2->prev;
65318b464eb8Smec 					bp2->prev->next = bp1;
65328b464eb8Smec 					bp2->next = head;
65338b464eb8Smec 					head = bp2;
65348b464eb8Smec 					/*
65358b464eb8Smec 					 * Clear the PR_CAPTURE bit as we
65368b464eb8Smec 					 * hold appropriate locks here.
65378b464eb8Smec 					 */
65388b464eb8Smec 					page_clrtoxic(head->pp, PR_CAPTURE);
65398b464eb8Smec 					page_capture_hash[i].num_pages--;
65408b464eb8Smec 					continue;
65418b464eb8Smec 				}
65428b464eb8Smec 				bp1 = bp1->next;
65438b464eb8Smec 			}
65448b464eb8Smec 		}
65458b464eb8Smec 		mutex_exit(&page_capture_hash[i].pchh_mutex);
65468b464eb8Smec 	}
65478b464eb8Smec 
65488b464eb8Smec 	while (head != NULL) {
65498b464eb8Smec 		bp1 = head;
65508b464eb8Smec 		head = head->next;
65518b464eb8Smec 		kmem_free(bp1, sizeof (*bp1));
65528b464eb8Smec 	}
65538b464eb8Smec }
65548b464eb8Smec 
65558b464eb8Smec 
65568b464eb8Smec /*
65578b464eb8Smec  * Find pp in the active list and move it to the walked list if it
65588b464eb8Smec  * exists.
65598b464eb8Smec  * Note that most often pp should be at the front of the active list
65608b464eb8Smec  * as it is currently used and thus there is no other sort of optimization
65618b464eb8Smec  * being done here as this is a linked list data structure.
65628b464eb8Smec  * Returns 1 on successful move or 0 if page could not be found.
65638b464eb8Smec  */
65648b464eb8Smec static int
65658b464eb8Smec page_capture_move_to_walked(page_t *pp)
65668b464eb8Smec {
65678b464eb8Smec 	page_capture_hash_bucket_t *bp;
65688b464eb8Smec 	int index;
65698b464eb8Smec 
65708b464eb8Smec 	index = PAGE_CAPTURE_HASH(pp);
65718b464eb8Smec 
65728b464eb8Smec 	mutex_enter(&page_capture_hash[index].pchh_mutex);
65738b464eb8Smec 	bp = page_capture_hash[index].lists[0].next;
65748b464eb8Smec 	while (bp != &page_capture_hash[index].lists[0]) {
65758b464eb8Smec 		if (bp->pp == pp) {
65768b464eb8Smec 			/* Remove from old list */
65778b464eb8Smec 			bp->next->prev = bp->prev;
65788b464eb8Smec 			bp->prev->next = bp->next;
65798b464eb8Smec 
65808b464eb8Smec 			/* Add to new list */
65818b464eb8Smec 			bp->next = page_capture_hash[index].lists[1].next;
65828b464eb8Smec 			bp->prev = &page_capture_hash[index].lists[1];
65838b464eb8Smec 			page_capture_hash[index].lists[1].next = bp;
65848b464eb8Smec 			bp->next->prev = bp;
65858b464eb8Smec 			mutex_exit(&page_capture_hash[index].pchh_mutex);
65868b464eb8Smec 
65878b464eb8Smec 			return (1);
65888b464eb8Smec 		}
65898b464eb8Smec 		bp = bp->next;
65908b464eb8Smec 	}
65918b464eb8Smec 	mutex_exit(&page_capture_hash[index].pchh_mutex);
65928b464eb8Smec 	return (0);
65938b464eb8Smec }
65948b464eb8Smec 
65958b464eb8Smec /*
65968b464eb8Smec  * Add a new entry to the page capture hash.  The only case where a new
65978b464eb8Smec  * entry is not added is when the page capture consumer is no longer registered.
65988b464eb8Smec  * In this case, we'll silently not add the page to the hash.  We know that
65998b464eb8Smec  * page retire will always be registered for the case where we are currently
66008b464eb8Smec  * unretiring a page and thus there are no conflicts.
66018b464eb8Smec  */
66028b464eb8Smec static void
66038b464eb8Smec page_capture_add_hash(page_t *pp, uint_t szc, uint_t flags, void *datap)
66048b464eb8Smec {
66058b464eb8Smec 	page_capture_hash_bucket_t *bp1;
66068b464eb8Smec 	page_capture_hash_bucket_t *bp2;
66078b464eb8Smec 	int index;
66088b464eb8Smec 	int cb_index;
66098b464eb8Smec 	int i;
66108b464eb8Smec #ifdef DEBUG
66118b464eb8Smec 	page_capture_hash_bucket_t *tp1;
66128b464eb8Smec 	int l;
66138b464eb8Smec #endif
66148b464eb8Smec 
66158b464eb8Smec 	ASSERT(!(flags & CAPTURE_ASYNC));
66168b464eb8Smec 
66178b464eb8Smec 	bp1 = kmem_alloc(sizeof (struct page_capture_hash_bucket), KM_SLEEP);
66188b464eb8Smec 
66198b464eb8Smec 	bp1->pp = pp;
66208b464eb8Smec 	bp1->szc = szc;
66218b464eb8Smec 	bp1->flags = flags;
66228b464eb8Smec 	bp1->datap = datap;
66238b464eb8Smec 
66248b464eb8Smec 	for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
66258b464eb8Smec 		if ((flags >> cb_index) & 1) {
66268b464eb8Smec 			break;
66278b464eb8Smec 		}
66288b464eb8Smec 	}
66298b464eb8Smec 
66308b464eb8Smec 	ASSERT(cb_index != PC_NUM_CALLBACKS);
66318b464eb8Smec 
66328b464eb8Smec 	rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
66338b464eb8Smec 	if (pc_cb[cb_index].cb_active) {
66348b464eb8Smec 		if (pc_cb[cb_index].duration == -1) {
66358b464eb8Smec 			bp1->expires = (clock_t)-1;
66368b464eb8Smec 		} else {
66378b464eb8Smec 			bp1->expires = lbolt + pc_cb[cb_index].duration;
66388b464eb8Smec 		}
66398b464eb8Smec 	} else {
66408b464eb8Smec 		/* There's no callback registered so don't add to the hash */
66418b464eb8Smec 		rw_exit(&pc_cb[cb_index].cb_rwlock);
66428b464eb8Smec 		kmem_free(bp1, sizeof (*bp1));
66438b464eb8Smec 		return;
66448b464eb8Smec 	}
66458b464eb8Smec 
66468b464eb8Smec 	index = PAGE_CAPTURE_HASH(pp);
66478b464eb8Smec 
66488b464eb8Smec 	/*
66498b464eb8Smec 	 * Only allow capture flag to be modified under this mutex.
66508b464eb8Smec 	 * Prevents multiple entries for same page getting added.
66518b464eb8Smec 	 */
66528b464eb8Smec 	mutex_enter(&page_capture_hash[index].pchh_mutex);
66538b464eb8Smec 
66548b464eb8Smec 	/*
66558b464eb8Smec 	 * if not already on the hash, set capture bit and add to the hash
66568b464eb8Smec 	 */
66578b464eb8Smec 	if (!(pp->p_toxic & PR_CAPTURE)) {
66588b464eb8Smec #ifdef DEBUG
66598b464eb8Smec 		/* Check for duplicate entries */
66608b464eb8Smec 		for (l = 0; l < 2; l++) {
66618b464eb8Smec 			tp1 = page_capture_hash[index].lists[l].next;
66628b464eb8Smec 			while (tp1 != &page_capture_hash[index].lists[l]) {
66638b464eb8Smec 				if (tp1->pp == pp) {
66648b464eb8Smec 					panic("page pp 0x%p already on hash "
66658b464eb8Smec 					    "at 0x%p\n", pp, tp1);
66668b464eb8Smec 				}
66678b464eb8Smec 				tp1 = tp1->next;
66688b464eb8Smec 			}
66698b464eb8Smec 		}
66708b464eb8Smec 
66718b464eb8Smec #endif
66728b464eb8Smec 		page_settoxic(pp, PR_CAPTURE);
66738b464eb8Smec 		bp1->next = page_capture_hash[index].lists[0].next;
66748b464eb8Smec 		bp1->prev = &page_capture_hash[index].lists[0];
66758b464eb8Smec 		bp1->next->prev = bp1;
66768b464eb8Smec 		page_capture_hash[index].lists[0].next = bp1;
66778b464eb8Smec 		page_capture_hash[index].num_pages++;
6678*cee1d74bSjfrank 		if (flags & CAPTURE_RETIRE) {
6679*cee1d74bSjfrank 			page_retire_incr_pend_count();
6680*cee1d74bSjfrank 		}
66818b464eb8Smec 		mutex_exit(&page_capture_hash[index].pchh_mutex);
66828b464eb8Smec 		rw_exit(&pc_cb[cb_index].cb_rwlock);
66838b464eb8Smec 		cv_signal(&pc_cv);
66848b464eb8Smec 		return;
66858b464eb8Smec 	}
66868b464eb8Smec 
66878b464eb8Smec 	/*
66888b464eb8Smec 	 * A page retire request will replace any other request.
66898b464eb8Smec 	 * A second physmem request which is for a different process than
66908b464eb8Smec 	 * the currently registered one will be dropped as there is
66918b464eb8Smec 	 * no way to hold the private data for both calls.
66928b464eb8Smec 	 * In the future, once there are more callers, this will have to
66938b464eb8Smec 	 * be worked out better as there needs to be private storage for
66948b464eb8Smec 	 * at least each type of caller (maybe have datap be an array of
66958b464eb8Smec 	 * *void's so that we can index based upon callers index).
66968b464eb8Smec 	 */
66978b464eb8Smec 
66988b464eb8Smec 	/* walk hash list to update expire time */
66998b464eb8Smec 	for (i = 0; i < 2; i++) {
67008b464eb8Smec 		bp2 = page_capture_hash[index].lists[i].next;
67018b464eb8Smec 		while (bp2 != &page_capture_hash[index].lists[i]) {
67028b464eb8Smec 			if (bp2->pp == pp) {
67038b464eb8Smec 				if (flags & CAPTURE_RETIRE) {
67048b464eb8Smec 					if (!(bp2->flags & CAPTURE_RETIRE)) {
6705*cee1d74bSjfrank 						page_retire_incr_pend_count();
67068b464eb8Smec 						bp2->flags = flags;
67078b464eb8Smec 						bp2->expires = bp1->expires;
67088b464eb8Smec 						bp2->datap = datap;
67098b464eb8Smec 					}
67108b464eb8Smec 				} else {
67118b464eb8Smec 					ASSERT(flags & CAPTURE_PHYSMEM);
67128b464eb8Smec 					if (!(bp2->flags & CAPTURE_RETIRE) &&
67138b464eb8Smec 					    (datap == bp2->datap)) {
67148b464eb8Smec 						bp2->expires = bp1->expires;
67158b464eb8Smec 					}
67168b464eb8Smec 				}
67178b464eb8Smec 				mutex_exit(&page_capture_hash[index].
67188b464eb8Smec 				    pchh_mutex);
67198b464eb8Smec 				rw_exit(&pc_cb[cb_index].cb_rwlock);
67208b464eb8Smec 				kmem_free(bp1, sizeof (*bp1));
67218b464eb8Smec 				return;
67228b464eb8Smec 			}
67238b464eb8Smec 			bp2 = bp2->next;
67248b464eb8Smec 		}
67258b464eb8Smec 	}
67268b464eb8Smec 
67278b464eb8Smec 	/*
67288b464eb8Smec 	 * the PR_CAPTURE flag is protected by the page_capture_hash mutexes
67298b464eb8Smec 	 * and thus it either has to be set or not set and can't change
67308b464eb8Smec 	 * while holding the mutex above.
67318b464eb8Smec 	 */
67328b464eb8Smec 	panic("page_capture_add_hash, PR_CAPTURE flag set on pp %p\n", pp);
67338b464eb8Smec }
67348b464eb8Smec 
67358b464eb8Smec /*
67368b464eb8Smec  * We have a page in our hands, lets try and make it ours by turning
67378b464eb8Smec  * it into a clean page like it had just come off the freelists.
67388b464eb8Smec  *
67398b464eb8Smec  * Returns 0 on success, with the page still EXCL locked.
67408b464eb8Smec  * On failure, the page will be unlocked, and returns EAGAIN
67418b464eb8Smec  */
67428b464eb8Smec static int
67438b464eb8Smec page_capture_clean_page(page_t *pp)
67448b464eb8Smec {
67458b464eb8Smec 	page_t *newpp;
67468b464eb8Smec 	int skip_unlock = 0;
67478b464eb8Smec 	spgcnt_t count;
67488b464eb8Smec 	page_t *tpp;
67498b464eb8Smec 	int ret = 0;
67508b464eb8Smec 	int extra;
67518b464eb8Smec 
67528b464eb8Smec 	ASSERT(PAGE_EXCL(pp));
67538b464eb8Smec 	ASSERT(!PP_RETIRED(pp));
67548b464eb8Smec 	ASSERT(curthread->t_flag & T_CAPTURING);
67558b464eb8Smec 
67568b464eb8Smec 	if (PP_ISFREE(pp)) {
67578b464eb8Smec 		if (!page_reclaim_page(pp, NULL)) {
67588b464eb8Smec 			skip_unlock = 1;
67598b464eb8Smec 			ret = EAGAIN;
67608b464eb8Smec 			goto cleanup;
67618b464eb8Smec 		}
67628b464eb8Smec 		if (pp->p_vnode != NULL) {
67638b464eb8Smec 			/*
67648b464eb8Smec 			 * Since this page came from the
67658b464eb8Smec 			 * cachelist, we must destroy the
67668b464eb8Smec 			 * old vnode association.
67678b464eb8Smec 			 */
67688b464eb8Smec 			page_hashout(pp, NULL);
67698b464eb8Smec 		}
67708b464eb8Smec 		goto cleanup;
67718b464eb8Smec 	}
67728b464eb8Smec 
67738b464eb8Smec 	/*
67748b464eb8Smec 	 * If we know page_relocate will fail, skip it
67758b464eb8Smec 	 * It could still fail due to a UE on another page but we
67768b464eb8Smec 	 * can't do anything about that.
67778b464eb8Smec 	 */
67788b464eb8Smec 	if (pp->p_toxic & PR_UE) {
67798b464eb8Smec 		goto skip_relocate;
67808b464eb8Smec 	}
67818b464eb8Smec 
67828b464eb8Smec 	/*
67838b464eb8Smec 	 * It's possible that pages can not have a vnode as fsflush comes
67848b464eb8Smec 	 * through and cleans up these pages.  It's ugly but that's how it is.
67858b464eb8Smec 	 */
67868b464eb8Smec 	if (pp->p_vnode == NULL) {
67878b464eb8Smec 		goto skip_relocate;
67888b464eb8Smec 	}
67898b464eb8Smec 
67908b464eb8Smec 	/*
67918b464eb8Smec 	 * Page was not free, so lets try to relocate it.
67928b464eb8Smec 	 * page_relocate only works with root pages, so if this is not a root
67938b464eb8Smec 	 * page, we need to demote it to try and relocate it.
67948b464eb8Smec 	 * Unfortunately this is the best we can do right now.
67958b464eb8Smec 	 */
67968b464eb8Smec 	newpp = NULL;
67978b464eb8Smec 	if ((pp->p_szc > 0) && (pp != PP_PAGEROOT(pp))) {
67988b464eb8Smec 		if (page_try_demote_pages(pp) == 0) {
67998b464eb8Smec 			ret = EAGAIN;
68008b464eb8Smec 			goto cleanup;
68018b464eb8Smec 		}
68028b464eb8Smec 	}
68038b464eb8Smec 	ret = page_relocate(&pp, &newpp, 1, 0, &count, NULL);
68048b464eb8Smec 	if (ret == 0) {
68058b464eb8Smec 		page_t *npp;
68068b464eb8Smec 		/* unlock the new page(s) */
68078b464eb8Smec 		while (count-- > 0) {
68088b464eb8Smec 			ASSERT(newpp != NULL);
68098b464eb8Smec 			npp = newpp;
68108b464eb8Smec 			page_sub(&newpp, npp);
68118b464eb8Smec 			page_unlock(npp);
68128b464eb8Smec 		}
68138b464eb8Smec 		ASSERT(newpp == NULL);
68148b464eb8Smec 		/*
68158b464eb8Smec 		 * Check to see if the page we have is too large.
68168b464eb8Smec 		 * If so, demote it freeing up the extra pages.
68178b464eb8Smec 		 */
68188b464eb8Smec 		if (pp->p_szc > 0) {
68198b464eb8Smec 			/* For now demote extra pages to szc == 0 */
68208b464eb8Smec 			extra = page_get_pagecnt(pp->p_szc) - 1;
68218b464eb8Smec 			while (extra > 0) {
68228b464eb8Smec 				tpp = pp->p_next;
68238b464eb8Smec 				page_sub(&pp, tpp);
68248b464eb8Smec 				tpp->p_szc = 0;
68258b464eb8Smec 				page_free(tpp, 1);
68268b464eb8Smec 				extra--;
68278b464eb8Smec 			}
68288b464eb8Smec 			/* Make sure to set our page to szc 0 as well */
68298b464eb8Smec 			ASSERT(pp->p_next == pp && pp->p_prev == pp);
68308b464eb8Smec 			pp->p_szc = 0;
68318b464eb8Smec 		}
68328b464eb8Smec 		goto cleanup;
68338b464eb8Smec 	} else if (ret == EIO) {
68348b464eb8Smec 		ret = EAGAIN;
68358b464eb8Smec 		goto cleanup;
68368b464eb8Smec 	} else {
68378b464eb8Smec 		/*
68388b464eb8Smec 		 * Need to reset return type as we failed to relocate the page
68398b464eb8Smec 		 * but that does not mean that some of the next steps will not
68408b464eb8Smec 		 * work.
68418b464eb8Smec 		 */
68428b464eb8Smec 		ret = 0;
68438b464eb8Smec 	}
68448b464eb8Smec 
68458b464eb8Smec skip_relocate:
68468b464eb8Smec 
68478b464eb8Smec 	if (pp->p_szc > 0) {
68488b464eb8Smec 		if (page_try_demote_pages(pp) == 0) {
68498b464eb8Smec 			ret = EAGAIN;
68508b464eb8Smec 			goto cleanup;
68518b464eb8Smec 		}
68528b464eb8Smec 	}
68538b464eb8Smec 
68548b464eb8Smec 	ASSERT(pp->p_szc == 0);
68558b464eb8Smec 
68568b464eb8Smec 	if (hat_ismod(pp)) {
68578b464eb8Smec 		ret = EAGAIN;
68588b464eb8Smec 		goto cleanup;
68598b464eb8Smec 	}
6860ad23a2dbSjohansen 	if (PP_ISKAS(pp)) {
68618b464eb8Smec 		ret = EAGAIN;
68628b464eb8Smec 		goto cleanup;
68638b464eb8Smec 	}
68648b464eb8Smec 	if (pp->p_lckcnt || pp->p_cowcnt) {
68658b464eb8Smec 		ret = EAGAIN;
68668b464eb8Smec 		goto cleanup;
68678b464eb8Smec 	}
68688b464eb8Smec 
68698b464eb8Smec 	(void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD);
68708b464eb8Smec 	ASSERT(!hat_page_is_mapped(pp));
68718b464eb8Smec 
68728b464eb8Smec 	if (hat_ismod(pp)) {
68738b464eb8Smec 		/*
68748b464eb8Smec 		 * This is a semi-odd case as the page is now modified but not
68758b464eb8Smec 		 * mapped as we just unloaded the mappings above.
68768b464eb8Smec 		 */
68778b464eb8Smec 		ret = EAGAIN;
68788b464eb8Smec 		goto cleanup;
68798b464eb8Smec 	}
68808b464eb8Smec 	if (pp->p_vnode != NULL) {
68818b464eb8Smec 		page_hashout(pp, NULL);
68828b464eb8Smec 	}
68838b464eb8Smec 
68848b464eb8Smec 	/*
68858b464eb8Smec 	 * At this point, the page should be in a clean state and
68868b464eb8Smec 	 * we can do whatever we want with it.
68878b464eb8Smec 	 */
68888b464eb8Smec 
68898b464eb8Smec cleanup:
68908b464eb8Smec 	if (ret != 0) {
68918b464eb8Smec 		if (!skip_unlock) {
68928b464eb8Smec 			page_unlock(pp);
68938b464eb8Smec 		}
68948b464eb8Smec 	} else {
68958b464eb8Smec 		ASSERT(pp->p_szc == 0);
68968b464eb8Smec 		ASSERT(PAGE_EXCL(pp));
68978b464eb8Smec 
68988b464eb8Smec 		pp->p_next = pp;
68998b464eb8Smec 		pp->p_prev = pp;
69008b464eb8Smec 	}
69018b464eb8Smec 	return (ret);
69028b464eb8Smec }
69038b464eb8Smec 
69048b464eb8Smec /*
69058b464eb8Smec  * Various callers of page_trycapture() can have different restrictions upon
69068b464eb8Smec  * what memory they have access to.
69078b464eb8Smec  * Returns 0 on success, with the following error codes on failure:
69088b464eb8Smec  *      EPERM - The requested page is long term locked, and thus repeated
69098b464eb8Smec  *              requests to capture this page will likely fail.
69108b464eb8Smec  *      ENOMEM - There was not enough free memory in the system to safely
69118b464eb8Smec  *              map the requested page.
69128b464eb8Smec  *      ENOENT - The requested page was inside the kernel cage, and the
69138b464eb8Smec  *              PHYSMEM_CAGE flag was not set.
69148b464eb8Smec  */
69158b464eb8Smec int
69168b464eb8Smec page_capture_pre_checks(page_t *pp, uint_t flags)
69178b464eb8Smec {
69188b464eb8Smec #if defined(__sparc)
69198b464eb8Smec 	extern struct vnode prom_ppages;
69208b464eb8Smec #endif /* __sparc */
69218b464eb8Smec 
69228b464eb8Smec 	ASSERT(pp != NULL);
69238b464eb8Smec 
69248b464eb8Smec 	/* only physmem currently has restrictions */
69258b464eb8Smec 	if (!(flags & CAPTURE_PHYSMEM)) {
69268b464eb8Smec 		return (0);
69278b464eb8Smec 	}
69288b464eb8Smec 
69298b464eb8Smec #if defined(__sparc)
69308b464eb8Smec 	if (pp->p_vnode == &prom_ppages) {
69318b464eb8Smec 		return (EPERM);
69328b464eb8Smec 	}
69338b464eb8Smec 
69348b464eb8Smec 	if (PP_ISNORELOC(pp) && !(flags & CAPTURE_GET_CAGE)) {
69358b464eb8Smec 		return (ENOENT);
69368b464eb8Smec 	}
69378b464eb8Smec 
69388b464eb8Smec 	if (PP_ISNORELOCKERNEL(pp)) {
69398b464eb8Smec 		return (EPERM);
69408b464eb8Smec 	}
69418b464eb8Smec #else
6942ad23a2dbSjohansen 	if (PP_ISKAS(pp)) {
69438b464eb8Smec 		return (EPERM);
69448b464eb8Smec 	}
69458b464eb8Smec #endif /* __sparc */
69468b464eb8Smec 
69478b464eb8Smec 	if (availrmem < swapfs_minfree) {
69488b464eb8Smec 		/*
69498b464eb8Smec 		 * We won't try to capture this page as we are
69508b464eb8Smec 		 * running low on memory.
69518b464eb8Smec 		 */
69528b464eb8Smec 		return (ENOMEM);
69538b464eb8Smec 	}
69548b464eb8Smec 	return (0);
69558b464eb8Smec }
69568b464eb8Smec 
69578b464eb8Smec /*
69588b464eb8Smec  * Once we have a page in our mits, go ahead and complete the capture
69598b464eb8Smec  * operation.
69608b464eb8Smec  * Returns 1 on failure where page is no longer needed
69618b464eb8Smec  * Returns 0 on success
69628b464eb8Smec  * Returns -1 if there was a transient failure.
69638b464eb8Smec  * Failure cases must release the SE_EXCL lock on pp (usually via page_free).
69648b464eb8Smec  */
69658b464eb8Smec int
69668b464eb8Smec page_capture_take_action(page_t *pp, uint_t flags, void *datap)
69678b464eb8Smec {
69688b464eb8Smec 	int cb_index;
69698b464eb8Smec 	int ret = 0;
69708b464eb8Smec 	page_capture_hash_bucket_t *bp1;
69718b464eb8Smec 	page_capture_hash_bucket_t *bp2;
69728b464eb8Smec 	int index;
69738b464eb8Smec 	int found = 0;
69748b464eb8Smec 	int i;
69758b464eb8Smec 
69768b464eb8Smec 	ASSERT(PAGE_EXCL(pp));
69778b464eb8Smec 	ASSERT(curthread->t_flag & T_CAPTURING);
69788b464eb8Smec 
69798b464eb8Smec 	for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
69808b464eb8Smec 		if ((flags >> cb_index) & 1) {
69818b464eb8Smec 			break;
69828b464eb8Smec 		}
69838b464eb8Smec 	}
69848b464eb8Smec 	ASSERT(cb_index < PC_NUM_CALLBACKS);
69858b464eb8Smec 
69868b464eb8Smec 	/*
69878b464eb8Smec 	 * Remove the entry from the page_capture hash, but don't free it yet
69888b464eb8Smec 	 * as we may need to put it back.
69898b464eb8Smec 	 * Since we own the page at this point in time, we should find it
69908b464eb8Smec 	 * in the hash if this is an ASYNC call.  If we don't it's likely
69918b464eb8Smec 	 * that the page_capture_async() thread decided that this request
69928b464eb8Smec 	 * had expired, in which case we just continue on.
69938b464eb8Smec 	 */
69948b464eb8Smec 	if (flags & CAPTURE_ASYNC) {
69958b464eb8Smec 
69968b464eb8Smec 		index = PAGE_CAPTURE_HASH(pp);
69978b464eb8Smec 
69988b464eb8Smec 		mutex_enter(&page_capture_hash[index].pchh_mutex);
69998b464eb8Smec 		for (i = 0; i < 2 && !found; i++) {
70008b464eb8Smec 			bp1 = page_capture_hash[index].lists[i].next;
70018b464eb8Smec 			while (bp1 != &page_capture_hash[index].lists[i]) {
70028b464eb8Smec 				if (bp1->pp == pp) {
70038b464eb8Smec 					bp1->next->prev = bp1->prev;
70048b464eb8Smec 					bp1->prev->next = bp1->next;
70058b464eb8Smec 					page_capture_hash[index].num_pages--;
70068b464eb8Smec 					page_clrtoxic(pp, PR_CAPTURE);
70078b464eb8Smec 					found = 1;
70088b464eb8Smec 					break;
70098b464eb8Smec 				}
70108b464eb8Smec 				bp1 = bp1->next;
70118b464eb8Smec 			}
70128b464eb8Smec 		}
70138b464eb8Smec 		mutex_exit(&page_capture_hash[index].pchh_mutex);
70148b464eb8Smec 	}
70158b464eb8Smec 
70168b464eb8Smec 	/* Synchronize with the unregister func. */
70178b464eb8Smec 	rw_enter(&pc_cb[cb_index].cb_rwlock, RW_READER);
70188b464eb8Smec 	if (!pc_cb[cb_index].cb_active) {
70198b464eb8Smec 		page_free(pp, 1);
70208b464eb8Smec 		rw_exit(&pc_cb[cb_index].cb_rwlock);
70218b464eb8Smec 		if (found) {
70228b464eb8Smec 			kmem_free(bp1, sizeof (*bp1));
70238b464eb8Smec 		}
70248b464eb8Smec 		return (1);
70258b464eb8Smec 	}
70268b464eb8Smec 
70278b464eb8Smec 	/*
70288b464eb8Smec 	 * We need to remove the entry from the page capture hash and turn off
70298b464eb8Smec 	 * the PR_CAPTURE bit before calling the callback.  We'll need to cache
70308b464eb8Smec 	 * the entry here, and then based upon the return value, cleanup
70318b464eb8Smec 	 * appropriately or re-add it to the hash, making sure that someone else
70328b464eb8Smec 	 * hasn't already done so.
70338b464eb8Smec 	 * It should be rare for the callback to fail and thus it's ok for
70348b464eb8Smec 	 * the failure path to be a bit complicated as the success path is
70358b464eb8Smec 	 * cleaner and the locking rules are easier to follow.
70368b464eb8Smec 	 */
70378b464eb8Smec 
70388b464eb8Smec 	ret = pc_cb[cb_index].cb_func(pp, datap, flags);
70398b464eb8Smec 
70408b464eb8Smec 	rw_exit(&pc_cb[cb_index].cb_rwlock);
70418b464eb8Smec 
70428b464eb8Smec 	/*
70438b464eb8Smec 	 * If this was an ASYNC request, we need to cleanup the hash if the
70448b464eb8Smec 	 * callback was successful or if the request was no longer valid.
70458b464eb8Smec 	 * For non-ASYNC requests, we return failure to map and the caller
70468b464eb8Smec 	 * will take care of adding the request to the hash.
70478b464eb8Smec 	 * Note also that the callback itself is responsible for the page
70488b464eb8Smec 	 * at this point in time in terms of locking ...  The most common
70498b464eb8Smec 	 * case for the failure path should just be a page_free.
70508b464eb8Smec 	 */
70518b464eb8Smec 	if (ret >= 0) {
70528b464eb8Smec 		if (found) {
7053*cee1d74bSjfrank 			if (bp1->flags & CAPTURE_RETIRE) {
7054*cee1d74bSjfrank 				page_retire_decr_pend_count();
7055*cee1d74bSjfrank 			}
70568b464eb8Smec 			kmem_free(bp1, sizeof (*bp1));
70578b464eb8Smec 		}
70588b464eb8Smec 		return (ret);
70598b464eb8Smec 	}
70608b464eb8Smec 	if (!found) {
70618b464eb8Smec 		return (ret);
70628b464eb8Smec 	}
70638b464eb8Smec 
70648b464eb8Smec 	ASSERT(flags & CAPTURE_ASYNC);
70658b464eb8Smec 
70668b464eb8Smec 	/*
70678b464eb8Smec 	 * Check for expiration time first as we can just free it up if it's
70688b464eb8Smec 	 * expired.
70698b464eb8Smec 	 */
70708b464eb8Smec 	if (lbolt > bp1->expires && bp1->expires != -1) {
70718b464eb8Smec 		kmem_free(bp1, sizeof (*bp1));
70728b464eb8Smec 		return (ret);
70738b464eb8Smec 	}
70748b464eb8Smec 
70758b464eb8Smec 	/*
70768b464eb8Smec 	 * The callback failed and there used to be an entry in the hash for
70778b464eb8Smec 	 * this page, so we need to add it back to the hash.
70788b464eb8Smec 	 */
70798b464eb8Smec 	mutex_enter(&page_capture_hash[index].pchh_mutex);
70808b464eb8Smec 	if (!(pp->p_toxic & PR_CAPTURE)) {
70818b464eb8Smec 		/* just add bp1 back to head of walked list */
70828b464eb8Smec 		page_settoxic(pp, PR_CAPTURE);
70838b464eb8Smec 		bp1->next = page_capture_hash[index].lists[1].next;
70848b464eb8Smec 		bp1->prev = &page_capture_hash[index].lists[1];
70858b464eb8Smec 		bp1->next->prev = bp1;
70868b464eb8Smec 		page_capture_hash[index].lists[1].next = bp1;
70878b464eb8Smec 		page_capture_hash[index].num_pages++;
70888b464eb8Smec 		mutex_exit(&page_capture_hash[index].pchh_mutex);
70898b464eb8Smec 		return (ret);
70908b464eb8Smec 	}
70918b464eb8Smec 
70928b464eb8Smec 	/*
70938b464eb8Smec 	 * Otherwise there was a new capture request added to list
70948b464eb8Smec 	 * Need to make sure that our original data is represented if
70958b464eb8Smec 	 * appropriate.
70968b464eb8Smec 	 */
70978b464eb8Smec 	for (i = 0; i < 2; i++) {
70988b464eb8Smec 		bp2 = page_capture_hash[index].lists[i].next;
70998b464eb8Smec 		while (bp2 != &page_capture_hash[index].lists[i]) {
71008b464eb8Smec 			if (bp2->pp == pp) {
71018b464eb8Smec 				if (bp1->flags & CAPTURE_RETIRE) {
71028b464eb8Smec 					if (!(bp2->flags & CAPTURE_RETIRE)) {
71038b464eb8Smec 						bp2->szc = bp1->szc;
71048b464eb8Smec 						bp2->flags = bp1->flags;
71058b464eb8Smec 						bp2->expires = bp1->expires;
71068b464eb8Smec 						bp2->datap = bp1->datap;
71078b464eb8Smec 					}
71088b464eb8Smec 				} else {
71098b464eb8Smec 					ASSERT(bp1->flags & CAPTURE_PHYSMEM);
71108b464eb8Smec 					if (!(bp2->flags & CAPTURE_RETIRE)) {
71118b464eb8Smec 						bp2->szc = bp1->szc;
71128b464eb8Smec 						bp2->flags = bp1->flags;
71138b464eb8Smec 						bp2->expires = bp1->expires;
71148b464eb8Smec 						bp2->datap = bp1->datap;
71158b464eb8Smec 					}
71168b464eb8Smec 				}
71178b464eb8Smec 				mutex_exit(&page_capture_hash[index].
71188b464eb8Smec 				    pchh_mutex);
71198b464eb8Smec 				kmem_free(bp1, sizeof (*bp1));
71208b464eb8Smec 				return (ret);
71218b464eb8Smec 			}
71228b464eb8Smec 			bp2 = bp2->next;
71238b464eb8Smec 		}
71248b464eb8Smec 	}
71258b464eb8Smec 	panic("PR_CAPTURE set but not on hash for pp 0x%p\n", pp);
71268b464eb8Smec 	/*NOTREACHED*/
71278b464eb8Smec }
71288b464eb8Smec 
71298b464eb8Smec /*
71308b464eb8Smec  * Try to capture the given page for the caller specified in the flags
71318b464eb8Smec  * parameter.  The page will either be captured and handed over to the
71328b464eb8Smec  * appropriate callback, or will be queued up in the page capture hash
71338b464eb8Smec  * to be captured asynchronously.
71348b464eb8Smec  * If the current request is due to an async capture, the page must be
71358b464eb8Smec  * exclusively locked before calling this function.
71368b464eb8Smec  * Currently szc must be 0 but in the future this should be expandable to
71378b464eb8Smec  * other page sizes.
71388b464eb8Smec  * Returns 0 on success, with the following error codes on failure:
71398b464eb8Smec  *      EPERM - The requested page is long term locked, and thus repeated
71408b464eb8Smec  *              requests to capture this page will likely fail.
71418b464eb8Smec  *      ENOMEM - There was not enough free memory in the system to safely
71428b464eb8Smec  *              map the requested page.
71438b464eb8Smec  *      ENOENT - The requested page was inside the kernel cage, and the
71448b464eb8Smec  *              CAPTURE_GET_CAGE flag was not set.
71458b464eb8Smec  *	EAGAIN - The requested page could not be capturead at this point in
71468b464eb8Smec  *		time but future requests will likely work.
71478b464eb8Smec  *	EBUSY - The requested page is retired and the CAPTURE_GET_RETIRED flag
71488b464eb8Smec  *		was not set.
71498b464eb8Smec  */
71508b464eb8Smec int
71518b464eb8Smec page_itrycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
71528b464eb8Smec {
71538b464eb8Smec 	int ret;
71548b464eb8Smec 	int cb_index;
71558b464eb8Smec 
71568b464eb8Smec 	if (flags & CAPTURE_ASYNC) {
71578b464eb8Smec 		ASSERT(PAGE_EXCL(pp));
71588b464eb8Smec 		goto async;
71598b464eb8Smec 	}
71608b464eb8Smec 
71618b464eb8Smec 	/* Make sure there's enough availrmem ... */
71628b464eb8Smec 	ret = page_capture_pre_checks(pp, flags);
71638b464eb8Smec 	if (ret != 0) {
71648b464eb8Smec 		return (ret);
71658b464eb8Smec 	}
71668b464eb8Smec 
71678b464eb8Smec 	if (!page_trylock(pp, SE_EXCL)) {
71688b464eb8Smec 		for (cb_index = 0; cb_index < PC_NUM_CALLBACKS; cb_index++) {
71698b464eb8Smec 			if ((flags >> cb_index) & 1) {
71708b464eb8Smec 				break;
71718b464eb8Smec 			}
71728b464eb8Smec 		}
71738b464eb8Smec 		ASSERT(cb_index < PC_NUM_CALLBACKS);
71748b464eb8Smec 		ret = EAGAIN;
71758b464eb8Smec 		/* Special case for retired pages */
71768b464eb8Smec 		if (PP_RETIRED(pp)) {
71778b464eb8Smec 			if (flags & CAPTURE_GET_RETIRED) {
71788b464eb8Smec 				if (!page_unretire_pp(pp, PR_UNR_TEMP)) {
71798b464eb8Smec 					/*
71808b464eb8Smec 					 * Need to set capture bit and add to
71818b464eb8Smec 					 * hash so that the page will be
71828b464eb8Smec 					 * retired when freed.
71838b464eb8Smec 					 */
71848b464eb8Smec 					page_capture_add_hash(pp, szc,
71858b464eb8Smec 					    CAPTURE_RETIRE, NULL);
71868b464eb8Smec 					ret = 0;
71878b464eb8Smec 					goto own_page;
71888b464eb8Smec 				}
71898b464eb8Smec 			} else {
71908b464eb8Smec 				return (EBUSY);
71918b464eb8Smec 			}
71928b464eb8Smec 		}
71938b464eb8Smec 		page_capture_add_hash(pp, szc, flags, datap);
71948b464eb8Smec 		return (ret);
71958b464eb8Smec 	}
71968b464eb8Smec 
71978b464eb8Smec async:
71988b464eb8Smec 	ASSERT(PAGE_EXCL(pp));
71998b464eb8Smec 
72008b464eb8Smec 	/* Need to check for physmem async requests that availrmem is sane */
72018b464eb8Smec 	if ((flags & (CAPTURE_ASYNC | CAPTURE_PHYSMEM)) ==
72028b464eb8Smec 	    (CAPTURE_ASYNC | CAPTURE_PHYSMEM) &&
72038b464eb8Smec 	    (availrmem < swapfs_minfree)) {
72048b464eb8Smec 		page_unlock(pp);
72058b464eb8Smec 		return (ENOMEM);
72068b464eb8Smec 	}
72078b464eb8Smec 
72088b464eb8Smec 	ret = page_capture_clean_page(pp);
72098b464eb8Smec 
72108b464eb8Smec 	if (ret != 0) {
72118b464eb8Smec 		/* We failed to get the page, so lets add it to the hash */
72128b464eb8Smec 		if (!(flags & CAPTURE_ASYNC)) {
72138b464eb8Smec 			page_capture_add_hash(pp, szc, flags, datap);
72148b464eb8Smec 		}
72158b464eb8Smec 		return (ret);
72168b464eb8Smec 	}
72178b464eb8Smec 
72188b464eb8Smec own_page:
72198b464eb8Smec 	ASSERT(PAGE_EXCL(pp));
72208b464eb8Smec 	ASSERT(pp->p_szc == 0);
72218b464eb8Smec 
72228b464eb8Smec 	/* Call the callback */
72238b464eb8Smec 	ret = page_capture_take_action(pp, flags, datap);
72248b464eb8Smec 
72258b464eb8Smec 	if (ret == 0) {
72268b464eb8Smec 		return (0);
72278b464eb8Smec 	}
72288b464eb8Smec 
72298b464eb8Smec 	/*
72308b464eb8Smec 	 * Note that in the failure cases from page_capture_take_action, the
72318b464eb8Smec 	 * EXCL lock will have already been dropped.
72328b464eb8Smec 	 */
72338b464eb8Smec 	if ((ret == -1) && (!(flags & CAPTURE_ASYNC))) {
72348b464eb8Smec 		page_capture_add_hash(pp, szc, flags, datap);
72358b464eb8Smec 	}
72368b464eb8Smec 	return (EAGAIN);
72378b464eb8Smec }
72388b464eb8Smec 
72398b464eb8Smec int
72408b464eb8Smec page_trycapture(page_t *pp, uint_t szc, uint_t flags, void *datap)
72418b464eb8Smec {
72428b464eb8Smec 	int ret;
72438b464eb8Smec 
72448b464eb8Smec 	curthread->t_flag |= T_CAPTURING;
72458b464eb8Smec 	ret = page_itrycapture(pp, szc, flags, datap);
72468b464eb8Smec 	curthread->t_flag &= ~T_CAPTURING; /* xor works as we know its set */
72478b464eb8Smec 	return (ret);
72488b464eb8Smec }
72498b464eb8Smec 
72508b464eb8Smec /*
72518b464eb8Smec  * When unlocking a page which has the PR_CAPTURE bit set, this routine
72528b464eb8Smec  * gets called to try and capture the page.
72538b464eb8Smec  */
72548b464eb8Smec void
72558b464eb8Smec page_unlock_capture(page_t *pp)
72568b464eb8Smec {
72578b464eb8Smec 	page_capture_hash_bucket_t *bp;
72588b464eb8Smec 	int index;
72598b464eb8Smec 	int i;
72608b464eb8Smec 	uint_t szc;
72618b464eb8Smec 	uint_t flags = 0;
72628b464eb8Smec 	void *datap;
72638b464eb8Smec 	kmutex_t *mp;
72648b464eb8Smec 	extern vnode_t retired_pages;
72658b464eb8Smec 
72668b464eb8Smec 	/*
72678b464eb8Smec 	 * We need to protect against a possible deadlock here where we own
72688b464eb8Smec 	 * the vnode page hash mutex and want to acquire it again as there
72698b464eb8Smec 	 * are locations in the code, where we unlock a page while holding
72708b464eb8Smec 	 * the mutex which can lead to the page being captured and eventually
72718b464eb8Smec 	 * end up here.  As we may be hashing out the old page and hashing into
72728b464eb8Smec 	 * the retire vnode, we need to make sure we don't own them.
72738b464eb8Smec 	 * Other callbacks who do hash operations also need to make sure that
72748b464eb8Smec 	 * before they hashin to a vnode that they do not currently own the
72758b464eb8Smec 	 * vphm mutex otherwise there will be a panic.
72768b464eb8Smec 	 */
72778b464eb8Smec 	if (mutex_owned(page_vnode_mutex(&retired_pages))) {
727822addef7Smec 		page_unlock_nocapture(pp);
72798b464eb8Smec 		return;
72808b464eb8Smec 	}
72818b464eb8Smec 	if (pp->p_vnode != NULL && mutex_owned(page_vnode_mutex(pp->p_vnode))) {
728222addef7Smec 		page_unlock_nocapture(pp);
72838b464eb8Smec 		return;
72848b464eb8Smec 	}
72858b464eb8Smec 
72868b464eb8Smec 	index = PAGE_CAPTURE_HASH(pp);
72878b464eb8Smec 
72888b464eb8Smec 	mp = &page_capture_hash[index].pchh_mutex;
72898b464eb8Smec 	mutex_enter(mp);
72908b464eb8Smec 	for (i = 0; i < 2; i++) {
72918b464eb8Smec 		bp = page_capture_hash[index].lists[i].next;
72928b464eb8Smec 		while (bp != &page_capture_hash[index].lists[i]) {
72938b464eb8Smec 			if (bp->pp == pp) {
72948b464eb8Smec 				szc = bp->szc;
72958b464eb8Smec 				flags = bp->flags | CAPTURE_ASYNC;
72968b464eb8Smec 				datap = bp->datap;
72978b464eb8Smec 				mutex_exit(mp);
72988b464eb8Smec 				(void) page_trycapture(pp, szc, flags, datap);
72998b464eb8Smec 				return;
73008b464eb8Smec 			}
73018b464eb8Smec 			bp = bp->next;
73028b464eb8Smec 		}
73038b464eb8Smec 	}
73048b464eb8Smec 
73058b464eb8Smec 	/* Failed to find page in hash so clear flags and unlock it. */
73068b464eb8Smec 	page_clrtoxic(pp, PR_CAPTURE);
73078b464eb8Smec 	page_unlock(pp);
73088b464eb8Smec 
73098b464eb8Smec 	mutex_exit(mp);
73108b464eb8Smec }
73118b464eb8Smec 
73128b464eb8Smec void
73138b464eb8Smec page_capture_init()
73148b464eb8Smec {
73158b464eb8Smec 	int i;
73168b464eb8Smec 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
73178b464eb8Smec 		page_capture_hash[i].lists[0].next =
73188b464eb8Smec 		    &page_capture_hash[i].lists[0];
73198b464eb8Smec 		page_capture_hash[i].lists[0].prev =
73208b464eb8Smec 		    &page_capture_hash[i].lists[0];
73218b464eb8Smec 		page_capture_hash[i].lists[1].next =
73228b464eb8Smec 		    &page_capture_hash[i].lists[1];
73238b464eb8Smec 		page_capture_hash[i].lists[1].prev =
73248b464eb8Smec 		    &page_capture_hash[i].lists[1];
73258b464eb8Smec 	}
73268b464eb8Smec 
73278b464eb8Smec 	pc_thread_shortwait = 23 * hz;
73288b464eb8Smec 	pc_thread_longwait = 1201 * hz;
7329*cee1d74bSjfrank 	pc_thread_ism_retry = 3;
73308b464eb8Smec 	mutex_init(&pc_thread_mutex, NULL, MUTEX_DEFAULT, NULL);
73318b464eb8Smec 	cv_init(&pc_cv, NULL, CV_DEFAULT, NULL);
73328b464eb8Smec 	pc_thread_id = thread_create(NULL, 0, page_capture_thread, NULL, 0, &p0,
73338b464eb8Smec 	    TS_RUN, minclsyspri);
73348b464eb8Smec }
73358b464eb8Smec 
73368b464eb8Smec /*
73378b464eb8Smec  * It is necessary to scrub any failing pages prior to reboot in order to
73388b464eb8Smec  * prevent a latent error trap from occurring on the next boot.
73398b464eb8Smec  */
73408b464eb8Smec void
73418b464eb8Smec page_retire_mdboot()
73428b464eb8Smec {
73438b464eb8Smec 	page_t *pp;
73448b464eb8Smec 	int i, j;
73458b464eb8Smec 	page_capture_hash_bucket_t *bp;
73468b464eb8Smec 
73478b464eb8Smec 	/* walk lists looking for pages to scrub */
73488b464eb8Smec 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
73498b464eb8Smec 		if (page_capture_hash[i].num_pages == 0)
73508b464eb8Smec 			continue;
73518b464eb8Smec 
73528b464eb8Smec 		mutex_enter(&page_capture_hash[i].pchh_mutex);
73538b464eb8Smec 
73548b464eb8Smec 		for (j = 0; j < 2; j++) {
73558b464eb8Smec 			bp = page_capture_hash[i].lists[j].next;
73568b464eb8Smec 			while (bp != &page_capture_hash[i].lists[j]) {
73578b464eb8Smec 				pp = bp->pp;
7358ad23a2dbSjohansen 				if (!PP_ISKAS(pp) && PP_TOXIC(pp)) {
73598b464eb8Smec 					pp->p_selock = -1;  /* pacify ASSERTs */
73608b464eb8Smec 					PP_CLRFREE(pp);
73618b464eb8Smec 					pagescrub(pp, 0, PAGESIZE);
73628b464eb8Smec 					pp->p_selock = 0;
73638b464eb8Smec 				}
73648b464eb8Smec 				bp = bp->next;
73658b464eb8Smec 			}
73668b464eb8Smec 		}
73678b464eb8Smec 		mutex_exit(&page_capture_hash[i].pchh_mutex);
73688b464eb8Smec 	}
73698b464eb8Smec }
73708b464eb8Smec 
73718b464eb8Smec /*
73728b464eb8Smec  * Walk the page_capture_hash trying to capture pages and also cleanup old
73738b464eb8Smec  * entries which have expired.
73748b464eb8Smec  */
73758b464eb8Smec void
73768b464eb8Smec page_capture_async()
73778b464eb8Smec {
73788b464eb8Smec 	page_t *pp;
73798b464eb8Smec 	int i;
73808b464eb8Smec 	int ret;
73818b464eb8Smec 	page_capture_hash_bucket_t *bp1, *bp2;
73828b464eb8Smec 	uint_t szc;
73838b464eb8Smec 	uint_t flags;
73848b464eb8Smec 	void *datap;
73858b464eb8Smec 
73868b464eb8Smec 	/* If there are outstanding pages to be captured, get to work */
73878b464eb8Smec 	for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++) {
73888b464eb8Smec 		if (page_capture_hash[i].num_pages == 0)
73898b464eb8Smec 			continue;
73908b464eb8Smec 		/* Append list 1 to list 0 and then walk through list 0 */
73918b464eb8Smec 		mutex_enter(&page_capture_hash[i].pchh_mutex);
73928b464eb8Smec 		bp1 = &page_capture_hash[i].lists[1];
73938b464eb8Smec 		bp2 = bp1->next;
73948b464eb8Smec 		if (bp1 != bp2) {
73958b464eb8Smec 			bp1->prev->next = page_capture_hash[i].lists[0].next;
73968b464eb8Smec 			bp2->prev = &page_capture_hash[i].lists[0];
73978b464eb8Smec 			page_capture_hash[i].lists[0].next->prev = bp1->prev;
73988b464eb8Smec 			page_capture_hash[i].lists[0].next = bp2;
73998b464eb8Smec 			bp1->next = bp1;
74008b464eb8Smec 			bp1->prev = bp1;
74018b464eb8Smec 		}
74028b464eb8Smec 
74038b464eb8Smec 		/* list[1] will be empty now */
74048b464eb8Smec 
74058b464eb8Smec 		bp1 = page_capture_hash[i].lists[0].next;
74068b464eb8Smec 		while (bp1 != &page_capture_hash[i].lists[0]) {
74078b464eb8Smec 			/* Check expiration time */
74088b464eb8Smec 			if ((lbolt > bp1->expires && bp1->expires != -1) ||
74098b464eb8Smec 			    page_deleted(bp1->pp)) {
74108b464eb8Smec 				page_capture_hash[i].lists[0].next = bp1->next;
74118b464eb8Smec 				bp1->next->prev =
74128b464eb8Smec 				    &page_capture_hash[i].lists[0];
74138b464eb8Smec 				page_capture_hash[i].num_pages--;
74148b464eb8Smec 
74158b464eb8Smec 				/*
74168b464eb8Smec 				 * We can safely remove the PR_CAPTURE bit
74178b464eb8Smec 				 * without holding the EXCL lock on the page
74188b464eb8Smec 				 * as the PR_CAPTURE bit requres that the
74198b464eb8Smec 				 * page_capture_hash[].pchh_mutex be held
74208b464eb8Smec 				 * to modify it.
74218b464eb8Smec 				 */
74228b464eb8Smec 				page_clrtoxic(bp1->pp, PR_CAPTURE);
74238b464eb8Smec 				mutex_exit(&page_capture_hash[i].pchh_mutex);
74248b464eb8Smec 				kmem_free(bp1, sizeof (*bp1));
74258b464eb8Smec 				mutex_enter(&page_capture_hash[i].pchh_mutex);
74268b464eb8Smec 				bp1 = page_capture_hash[i].lists[0].next;
74278b464eb8Smec 				continue;
74288b464eb8Smec 			}
74298b464eb8Smec 			pp = bp1->pp;
74308b464eb8Smec 			szc = bp1->szc;
74318b464eb8Smec 			flags = bp1->flags;
74328b464eb8Smec 			datap = bp1->datap;
74338b464eb8Smec 			mutex_exit(&page_capture_hash[i].pchh_mutex);
74348b464eb8Smec 			if (page_trylock(pp, SE_EXCL)) {
74358b464eb8Smec 				ret = page_trycapture(pp, szc,
74368b464eb8Smec 				    flags | CAPTURE_ASYNC, datap);
74378b464eb8Smec 			} else {
74388b464eb8Smec 				ret = 1;	/* move to walked hash */
74398b464eb8Smec 			}
74408b464eb8Smec 
74418b464eb8Smec 			if (ret != 0) {
74428b464eb8Smec 				/* Move to walked hash */
74438b464eb8Smec 				(void) page_capture_move_to_walked(pp);
74448b464eb8Smec 			}
74458b464eb8Smec 			mutex_enter(&page_capture_hash[i].pchh_mutex);
74468b464eb8Smec 			bp1 = page_capture_hash[i].lists[0].next;
74478b464eb8Smec 		}
74488b464eb8Smec 
74498b464eb8Smec 		mutex_exit(&page_capture_hash[i].pchh_mutex);
74508b464eb8Smec 	}
74518b464eb8Smec }
74528b464eb8Smec 
7453*cee1d74bSjfrank /*
7454*cee1d74bSjfrank  * This function is called by the page_capture_thread, and is needed in
7455*cee1d74bSjfrank  * in order to initiate aio cleanup, so that pages used in aio
7456*cee1d74bSjfrank  * will be unlocked and subsequently retired by page_capture_thread.
7457*cee1d74bSjfrank  */
7458*cee1d74bSjfrank static int
7459*cee1d74bSjfrank do_aio_cleanup(void)
7460*cee1d74bSjfrank {
7461*cee1d74bSjfrank 	proc_t *procp;
7462*cee1d74bSjfrank 	int (*aio_cleanup_dr_delete_memory)(proc_t *);
7463*cee1d74bSjfrank 	int cleaned = 0;
7464*cee1d74bSjfrank 
7465*cee1d74bSjfrank 	if (modload("sys", "kaio") == -1) {
7466*cee1d74bSjfrank 		cmn_err(CE_WARN, "do_aio_cleanup: cannot load kaio");
7467*cee1d74bSjfrank 		return (0);
7468*cee1d74bSjfrank 	}
7469*cee1d74bSjfrank 	/*
7470*cee1d74bSjfrank 	 * We use the aio_cleanup_dr_delete_memory function to
7471*cee1d74bSjfrank 	 * initiate the actual clean up; this function will wake
7472*cee1d74bSjfrank 	 * up the per-process aio_cleanup_thread.
7473*cee1d74bSjfrank 	 */
7474*cee1d74bSjfrank 	aio_cleanup_dr_delete_memory = (int (*)(proc_t *))
7475*cee1d74bSjfrank 	    modgetsymvalue("aio_cleanup_dr_delete_memory", 0);
7476*cee1d74bSjfrank 	if (aio_cleanup_dr_delete_memory == NULL) {
7477*cee1d74bSjfrank 		cmn_err(CE_WARN,
7478*cee1d74bSjfrank 	    "aio_cleanup_dr_delete_memory not found in kaio");
7479*cee1d74bSjfrank 		return (0);
7480*cee1d74bSjfrank 	}
7481*cee1d74bSjfrank 	mutex_enter(&pidlock);
7482*cee1d74bSjfrank 	for (procp = practive; (procp != NULL); procp = procp->p_next) {
7483*cee1d74bSjfrank 		mutex_enter(&procp->p_lock);
7484*cee1d74bSjfrank 		if (procp->p_aio != NULL) {
7485*cee1d74bSjfrank 			/* cleanup proc's outstanding kaio */
7486*cee1d74bSjfrank 			cleaned += (*aio_cleanup_dr_delete_memory)(procp);
7487*cee1d74bSjfrank 		}
7488*cee1d74bSjfrank 		mutex_exit(&procp->p_lock);
7489*cee1d74bSjfrank 	}
7490*cee1d74bSjfrank 	mutex_exit(&pidlock);
7491*cee1d74bSjfrank 	return (cleaned);
7492*cee1d74bSjfrank }
7493*cee1d74bSjfrank 
7494*cee1d74bSjfrank /*
7495*cee1d74bSjfrank  * helper function for page_capture_thread
7496*cee1d74bSjfrank  */
7497*cee1d74bSjfrank static void
7498*cee1d74bSjfrank page_capture_handle_outstanding(void)
7499*cee1d74bSjfrank {
7500*cee1d74bSjfrank 	extern size_t spt_used;
7501*cee1d74bSjfrank 	int ntry;
7502*cee1d74bSjfrank 
7503*cee1d74bSjfrank 	if (!page_retire_pend_count()) {
7504*cee1d74bSjfrank 		/*
7505*cee1d74bSjfrank 		 * Do we really want to be this aggressive
7506*cee1d74bSjfrank 		 * for things other than page_retire?
7507*cee1d74bSjfrank 		 * Maybe have a counter for each callback
7508*cee1d74bSjfrank 		 * type to guide how aggressive we should
7509*cee1d74bSjfrank 		 * be here.  Thus if there's at least one
7510*cee1d74bSjfrank 		 * page for page_retire we go ahead and reap
7511*cee1d74bSjfrank 		 * like this.
7512*cee1d74bSjfrank 		 */
7513*cee1d74bSjfrank 		kmem_reap();
7514*cee1d74bSjfrank 		seg_preap();
7515*cee1d74bSjfrank 		page_capture_async();
7516*cee1d74bSjfrank 	} else {
7517*cee1d74bSjfrank 		/*
7518*cee1d74bSjfrank 		 * There are pages pending retirement, so
7519*cee1d74bSjfrank 		 * we reap prior to attempting to capture.
7520*cee1d74bSjfrank 		 */
7521*cee1d74bSjfrank 		kmem_reap();
7522*cee1d74bSjfrank 		/*
7523*cee1d74bSjfrank 		 * When ISM is in use, we need to disable and
7524*cee1d74bSjfrank 		 * purge the seg_pcache, and initiate aio
7525*cee1d74bSjfrank 		 * cleanup in order to release page locks and
7526*cee1d74bSjfrank 		 * subsquently retire pages in need of
7527*cee1d74bSjfrank 		 * retirement.
7528*cee1d74bSjfrank 		 */
7529*cee1d74bSjfrank 		if (spt_used) {
7530*cee1d74bSjfrank 			/* disable and purge seg_pcache */
7531*cee1d74bSjfrank 			(void) seg_p_disable();
7532*cee1d74bSjfrank 			for (ntry = 0; ntry < pc_thread_ism_retry; ntry++) {
7533*cee1d74bSjfrank 				if (!page_retire_pend_count())
7534*cee1d74bSjfrank 					break;
7535*cee1d74bSjfrank 				if (do_aio_cleanup()) {
7536*cee1d74bSjfrank 					/*
7537*cee1d74bSjfrank 					 * allow the apps cleanup threads
7538*cee1d74bSjfrank 					 * to run
7539*cee1d74bSjfrank 					 */
7540*cee1d74bSjfrank 					delay(pc_thread_shortwait);
7541*cee1d74bSjfrank 				}
7542*cee1d74bSjfrank 				page_capture_async();
7543*cee1d74bSjfrank 			}
7544*cee1d74bSjfrank 			/* reenable seg_pcache */
7545*cee1d74bSjfrank 			seg_p_enable();
7546*cee1d74bSjfrank 		} else {
7547*cee1d74bSjfrank 			seg_preap();
7548*cee1d74bSjfrank 			page_capture_async();
7549*cee1d74bSjfrank 		}
7550*cee1d74bSjfrank 	}
7551*cee1d74bSjfrank }
7552*cee1d74bSjfrank 
75538b464eb8Smec /*
75548b464eb8Smec  * The page_capture_thread loops forever, looking to see if there are
75558b464eb8Smec  * pages still waiting to be captured.
75568b464eb8Smec  */
75578b464eb8Smec static void
75588b464eb8Smec page_capture_thread(void)
75598b464eb8Smec {
75608b464eb8Smec 	callb_cpr_t c;
75618b464eb8Smec 	int outstanding;
75628b464eb8Smec 	int i;
75638b464eb8Smec 
75648b464eb8Smec 	CALLB_CPR_INIT(&c, &pc_thread_mutex, callb_generic_cpr, "page_capture");
75658b464eb8Smec 
75668b464eb8Smec 	mutex_enter(&pc_thread_mutex);
75678b464eb8Smec 	for (;;) {
75688b464eb8Smec 		outstanding = 0;
75698b464eb8Smec 		for (i = 0; i < NUM_PAGE_CAPTURE_BUCKETS; i++)
75708b464eb8Smec 			outstanding += page_capture_hash[i].num_pages;
75718b464eb8Smec 		if (outstanding) {
7572*cee1d74bSjfrank 			page_capture_handle_outstanding();
75738b464eb8Smec 			CALLB_CPR_SAFE_BEGIN(&c);
75748b464eb8Smec 			(void) cv_timedwait(&pc_cv, &pc_thread_mutex,
75758b464eb8Smec 			    lbolt + pc_thread_shortwait);
75768b464eb8Smec 			CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
75778b464eb8Smec 		} else {
75788b464eb8Smec 			CALLB_CPR_SAFE_BEGIN(&c);
75798b464eb8Smec 			(void) cv_timedwait(&pc_cv, &pc_thread_mutex,
75808b464eb8Smec 			    lbolt + pc_thread_longwait);
75818b464eb8Smec 			CALLB_CPR_SAFE_END(&c, &pc_thread_mutex);
75828b464eb8Smec 		}
75838b464eb8Smec 	}
75848b464eb8Smec 	/*NOTREACHED*/
75858b464eb8Smec }
7586