xref: /illumos-gate/usr/src/uts/common/vm/page_lock.c (revision 04909c8c)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
58bc68872Selowe  * Common Development and Distribution License (the "License").
68bc68872Selowe  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22cb15d5d9SPeter Rival  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
236a0b1217SPatrick Mooney  * Copyright 2019 Joyent, Inc.
247c478bd9Sstevel@tonic-gate  */
277c478bd9Sstevel@tonic-gate /*
287c478bd9Sstevel@tonic-gate  * VM - page locking primitives
297c478bd9Sstevel@tonic-gate  */
307c478bd9Sstevel@tonic-gate #include <sys/param.h>
317c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
327c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
337c478bd9Sstevel@tonic-gate #include <sys/debug.h>
347c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
357c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
367c478bd9Sstevel@tonic-gate #include <sys/lockstat.h>
37d7d93655Sblakej #include <sys/sysmacros.h>
387c478bd9Sstevel@tonic-gate #include <sys/condvar_impl.h>
397c478bd9Sstevel@tonic-gate #include <vm/page.h>
407c478bd9Sstevel@tonic-gate #include <vm/seg_enum.h>
417c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h>
42af4c679fSSean McEnroe #include <vm/seg_kmem.h>
447c478bd9Sstevel@tonic-gate /*
45cb15d5d9SPeter Rival  * This global mutex array is for logical page locking.
467c478bd9Sstevel@tonic-gate  * The following fields in the page structure are protected
477c478bd9Sstevel@tonic-gate  * by this lock:
487c478bd9Sstevel@tonic-gate  *
497c478bd9Sstevel@tonic-gate  *	p_lckcnt
507c478bd9Sstevel@tonic-gate  *	p_cowcnt
517c478bd9Sstevel@tonic-gate  */
52cb15d5d9SPeter Rival pad_mutex_t page_llocks[8 * NCPU_P2];
547c478bd9Sstevel@tonic-gate /*
557c478bd9Sstevel@tonic-gate  * This is a global lock for the logical page free list.  The
567c478bd9Sstevel@tonic-gate  * logical free list, in this implementation, is maintained as two
577c478bd9Sstevel@tonic-gate  * separate physical lists - the cache list and the free list.
587c478bd9Sstevel@tonic-gate  */
597c478bd9Sstevel@tonic-gate kmutex_t  page_freelock;
617c478bd9Sstevel@tonic-gate /*
627c478bd9Sstevel@tonic-gate  * The hash table, page_hash[], the p_selock fields, and the
637c478bd9Sstevel@tonic-gate  * list of pages associated with vnodes are protected by arrays of mutexes.
647c478bd9Sstevel@tonic-gate  *
657c478bd9Sstevel@tonic-gate  * Unless the hashes are changed radically, the table sizes must be
667c478bd9Sstevel@tonic-gate  * a power of two.  Also, we typically need more mutexes for the
677c478bd9Sstevel@tonic-gate  * vnodes since these locks are occasionally held for long periods.
687c478bd9Sstevel@tonic-gate  * And since there seem to be two special vnodes (kvp and swapvp),
697c478bd9Sstevel@tonic-gate  * we make room for private mutexes for them.
707c478bd9Sstevel@tonic-gate  *
717c478bd9Sstevel@tonic-gate  * The pse_mutex[] array holds the mutexes to protect the p_selock
727c478bd9Sstevel@tonic-gate  * fields of all page_t structures.
737c478bd9Sstevel@tonic-gate  *
747c478bd9Sstevel@tonic-gate  * PAGE_SE_MUTEX(pp) returns the address of the appropriate mutex
757c478bd9Sstevel@tonic-gate  * when given a pointer to a page_t.
767c478bd9Sstevel@tonic-gate  *
77d7d93655Sblakej  * PIO_TABLE_SIZE must be a power of two.  One could argue that we
787c478bd9Sstevel@tonic-gate  * should go to the trouble of setting it up at run time and base it
797c478bd9Sstevel@tonic-gate  * on memory size rather than the number of compile time CPUs.
807c478bd9Sstevel@tonic-gate  *
81d7d93655Sblakej  * XX64	We should be using physmem size to calculate PIO_SHIFT.
827c478bd9Sstevel@tonic-gate  *
837c478bd9Sstevel@tonic-gate  *	These might break in 64 bit world.
847c478bd9Sstevel@tonic-gate  */
85d7d93655Sblakej #define	PIO_SHIFT	7	/* log2(sizeof(page_t)) */
86d7d93655Sblakej #define	PIO_TABLE_SIZE	128	/* number of io mutexes to have */
887c478bd9Sstevel@tonic-gate pad_mutex_t	ph_mutex[PH_TABLE_SIZE];
897c478bd9Sstevel@tonic-gate kmutex_t	pio_mutex[PIO_TABLE_SIZE];
917c478bd9Sstevel@tonic-gate #define	PAGE_IO_MUTEX(pp) \
927c478bd9Sstevel@tonic-gate 	    &pio_mutex[(((uintptr_t)pp) >> PIO_SHIFT) & (PIO_TABLE_SIZE - 1)]
94d7d93655Sblakej /*
95d7d93655Sblakej  * The pse_mutex[] array is allocated in the platform startup code
96d7d93655Sblakej  * based on the size of the machine at startup.
97d7d93655Sblakej  */
98d7d93655Sblakej extern pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
99d7d93655Sblakej extern size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
100d7d93655Sblakej extern int pse_shift;			/* log2(pse_table_size) */
101d7d93655Sblakej #define	PAGE_SE_MUTEX(pp)	&pse_mutex[				\
102d7d93655Sblakej 	((((uintptr_t)(pp) >> pse_shift) ^ ((uintptr_t)(pp))) >> 7) &	\
103d7d93655Sblakej 	(pse_table_size - 1)].pad_mutex
1057c478bd9Sstevel@tonic-gate #define	PSZC_MTX_TABLE_SIZE	128
1067c478bd9Sstevel@tonic-gate #define	PSZC_MTX_TABLE_SHIFT	7
1087c478bd9Sstevel@tonic-gate static pad_mutex_t	pszc_mutex[PSZC_MTX_TABLE_SIZE];
1107c478bd9Sstevel@tonic-gate #define	PAGE_SZC_MUTEX(_pp) \
1117c478bd9Sstevel@tonic-gate 	    &pszc_mutex[((((uintptr_t)(_pp) >> PSZC_MTX_TABLE_SHIFT) ^ \
1127c478bd9Sstevel@tonic-gate 		((uintptr_t)(_pp) >> (PSZC_MTX_TABLE_SHIFT << 1)) ^ \
1137c478bd9Sstevel@tonic-gate 		((uintptr_t)(_pp) >> (3 * PSZC_MTX_TABLE_SHIFT))) & \
1147c478bd9Sstevel@tonic-gate 		(PSZC_MTX_TABLE_SIZE - 1))].pad_mutex
1167c478bd9Sstevel@tonic-gate /*
1177c478bd9Sstevel@tonic-gate  * The vph_mutex[] array  holds the mutexes to protect the vnode chains,
1187c478bd9Sstevel@tonic-gate  * (i.e., the list of pages anchored by v_pages and connected via p_vpprev
1197c478bd9Sstevel@tonic-gate  * and p_vpnext).
1207c478bd9Sstevel@tonic-gate  *
1217c478bd9Sstevel@tonic-gate  * The page_vnode_mutex(vp) function returns the address of the appropriate
1227c478bd9Sstevel@tonic-gate  * mutex from this array given a pointer to a vnode.  It is complicated
1237c478bd9Sstevel@tonic-gate  * by the fact that the kernel's vnode and the swapfs vnode are referenced
1247c478bd9Sstevel@tonic-gate  * frequently enough to warrent their own mutexes.
1257c478bd9Sstevel@tonic-gate  *
1267c478bd9Sstevel@tonic-gate  * The VP_HASH_FUNC returns the index into the vph_mutex array given
1277c478bd9Sstevel@tonic-gate  * an address of a vnode.
1287c478bd9Sstevel@tonic-gate  */
130ac52b00eSqiao #if defined(_LP64)
131cb15d5d9SPeter Rival #define	VPH_TABLE_SIZE  (8 * NCPU_P2)
132ac52b00eSqiao #else	/* 32 bits */
133cb15d5d9SPeter Rival #define	VPH_TABLE_SIZE	(2 * NCPU_P2)
134ac52b00eSqiao #endif
1367c478bd9Sstevel@tonic-gate #define	VP_HASH_FUNC(vp) \
1377c478bd9Sstevel@tonic-gate 	((((uintptr_t)(vp) >> 6) + \
1387c478bd9Sstevel@tonic-gate 	    ((uintptr_t)(vp) >> 8) + \
1397c478bd9Sstevel@tonic-gate 	    ((uintptr_t)(vp) >> 10) + \
1407c478bd9Sstevel@tonic-gate 	    ((uintptr_t)(vp) >> 12)) \
1417c478bd9Sstevel@tonic-gate 	    & (VPH_TABLE_SIZE - 1))
143ad23a2dbSjohansen /*
144*04909c8cSJohn Levon  * Two slots after VPH_TABLE_SIZE are reserved in vph_mutex for kernel vnodes,
145*04909c8cSJohn Levon  * one for kvps[KV_ZVP], and one for other kvps[] users.
146ad23a2dbSjohansen  */
1487c478bd9Sstevel@tonic-gate kmutex_t	vph_mutex[VPH_TABLE_SIZE + 2];
1507c478bd9Sstevel@tonic-gate /*
1517c478bd9Sstevel@tonic-gate  * Initialize the locks used by the Virtual Memory Management system.
1527c478bd9Sstevel@tonic-gate  */
1537c478bd9Sstevel@tonic-gate void
page_lock_init()1547c478bd9Sstevel@tonic-gate page_lock_init()
1557c478bd9Sstevel@tonic-gate {
1567c478bd9Sstevel@tonic-gate }
158d7d93655Sblakej /*
159d7d93655Sblakej  * Return a value for pse_shift based on npg (the number of physical pages)
160d7d93655Sblakej  * and ncpu (the maximum number of CPUs).  This is called by platform startup
161d7d93655Sblakej  * code.
162d7d93655Sblakej  *
163d7d93655Sblakej  * Lockstat data from TPC-H runs showed that contention on the pse_mutex[]
164d7d93655Sblakej  * locks grew approximately as the square of the number of threads executing.
165d7d93655Sblakej  * So the primary scaling factor used is NCPU^2.  The size of the machine in
166d7d93655Sblakej  * megabytes is used as an upper bound, particularly for sun4v machines which
167d7d93655Sblakej  * all claim to have 256 CPUs maximum, and the old value of PSE_TABLE_SIZE
168d7d93655Sblakej  * (128) is used as a minimum.  Since the size of the table has to be a power
169d7d93655Sblakej  * of two, the calculated size is rounded up to the next power of two.
170d7d93655Sblakej  */
171d7d93655Sblakej /*ARGSUSED*/
172d7d93655Sblakej int
size_pse_array(pgcnt_t npg,int ncpu)173d7d93655Sblakej size_pse_array(pgcnt_t npg, int ncpu)
174d7d93655Sblakej {
175d7d93655Sblakej 	size_t size;
176d7d93655Sblakej 	pgcnt_t pp_per_mb = (1024 * 1024) / PAGESIZE;
178d7d93655Sblakej 	size = MAX(128, MIN(npg / pp_per_mb, 2 * ncpu * ncpu));
179d7d93655Sblakej 	size += (1 << (highbit(size) - 1)) - 1;
180d7d93655Sblakej 	return (highbit(size) - 1);
181d7d93655Sblakej }
1837c478bd9Sstevel@tonic-gate /*
1847c478bd9Sstevel@tonic-gate  * At present we only use page ownership to aid debugging, so it's
1857c478bd9Sstevel@tonic-gate  * OK if the owner field isn't exact.  In the 32-bit world two thread ids
1867c478bd9Sstevel@tonic-gate  * can map to the same owner because we just 'or' in 0x80000000 and
1877c478bd9Sstevel@tonic-gate  * then clear the second highest bit, so that (for example) 0x2faced00
1887c478bd9Sstevel@tonic-gate  * and 0xafaced00 both map to 0xafaced00.
1897c478bd9Sstevel@tonic-gate  * In the 64-bit world, p_selock may not be large enough to hold a full
1907c478bd9Sstevel@tonic-gate  * thread pointer.  If we ever need precise ownership (e.g. if we implement
1917c478bd9Sstevel@tonic-gate  * priority inheritance for page locks) then p_selock should become a
1927c478bd9Sstevel@tonic-gate  * uintptr_t and SE_WRITER should be -((uintptr_t)curthread >> 2).
1937c478bd9Sstevel@tonic-gate  */
1947c478bd9Sstevel@tonic-gate #define	SE_WRITER	(((selock_t)(ulong_t)curthread | INT_MIN) & ~SE_EWANTED)
1957c478bd9Sstevel@tonic-gate #define	SE_READER	1
1977c478bd9Sstevel@tonic-gate /*
1987c478bd9Sstevel@tonic-gate  * A page that is deleted must be marked as such using the
1997c478bd9Sstevel@tonic-gate  * page_lock_delete() function. The page must be exclusively locked.
2007c478bd9Sstevel@tonic-gate  * The SE_DELETED marker is put in p_selock when this function is called.
2017c478bd9Sstevel@tonic-gate  * SE_DELETED must be distinct from any SE_WRITER value.
2027c478bd9Sstevel@tonic-gate  */
2037c478bd9Sstevel@tonic-gate #define	SE_DELETED	(1 | INT_MIN)
2057c478bd9Sstevel@tonic-gate #ifdef VM_STATS
2067c478bd9Sstevel@tonic-gate uint_t	vph_kvp_count;
2077c478bd9Sstevel@tonic-gate uint_t	vph_swapfsvp_count;
2087c478bd9Sstevel@tonic-gate uint_t	vph_other;
2097c478bd9Sstevel@tonic-gate #endif /* VM_STATS */
2117c478bd9Sstevel@tonic-gate #ifdef VM_STATS
2127c478bd9Sstevel@tonic-gate uint_t	page_lock_count;
2137c478bd9Sstevel@tonic-gate uint_t	page_lock_miss;
2147c478bd9Sstevel@tonic-gate uint_t	page_lock_miss_lock;
2157c478bd9Sstevel@tonic-gate uint_t	page_lock_reclaim;
2167c478bd9Sstevel@tonic-gate uint_t	page_lock_bad_reclaim;
2177c478bd9Sstevel@tonic-gate uint_t	page_lock_same_page;
2187c478bd9Sstevel@tonic-gate uint_t	page_lock_upgrade;
219db874c57Selowe uint_t	page_lock_retired;
2207c478bd9Sstevel@tonic-gate uint_t	page_lock_upgrade_failed;
2217c478bd9Sstevel@tonic-gate uint_t	page_lock_deleted;
2237c478bd9Sstevel@tonic-gate uint_t	page_trylock_locked;
224db874c57Selowe uint_t	page_trylock_failed;
2257c478bd9Sstevel@tonic-gate uint_t	page_trylock_missed;
2277c478bd9Sstevel@tonic-gate uint_t	page_try_reclaim_upgrade;
2287c478bd9Sstevel@tonic-gate #endif /* VM_STATS */
2307c478bd9Sstevel@tonic-gate /*
2317c478bd9Sstevel@tonic-gate  * Acquire the "shared/exclusive" lock on a page.
2327c478bd9Sstevel@tonic-gate  *
2337c478bd9Sstevel@tonic-gate  * Returns 1 on success and locks the page appropriately.
2347c478bd9Sstevel@tonic-gate  *	   0 on failure and does not lock the page.
2357c478bd9Sstevel@tonic-gate  *
2367c478bd9Sstevel@tonic-gate  * If `lock' is non-NULL, it will be dropped and reacquired in the
2377c478bd9Sstevel@tonic-gate  * failure case.  This routine can block, and if it does
2387c478bd9Sstevel@tonic-gate  * it will always return a failure since the page identity [vp, off]
2397c478bd9Sstevel@tonic-gate  * or state may have changed.
2407c478bd9Sstevel@tonic-gate  */
2427c478bd9Sstevel@tonic-gate int
page_lock(page_t * pp,se_t se,kmutex_t * lock,reclaim_t reclaim)2437c478bd9Sstevel@tonic-gate page_lock(page_t *pp, se_t se, kmutex_t *lock, reclaim_t reclaim)
2447c478bd9Sstevel@tonic-gate {
2457c478bd9Sstevel@tonic-gate 	return (page_lock_es(pp, se, lock, reclaim, 0));
2467c478bd9Sstevel@tonic-gate }
2487c478bd9Sstevel@tonic-gate /*
2497c478bd9Sstevel@tonic-gate  * With the addition of reader-writer lock semantics to page_lock_es,
2507c478bd9Sstevel@tonic-gate  * callers wanting an exclusive (writer) lock may prevent shared-lock
2517c478bd9Sstevel@tonic-gate  * (reader) starvation by setting the es parameter to SE_EXCL_WANTED.
2527c478bd9Sstevel@tonic-gate  * In this case, when an exclusive lock cannot be acquired, p_selock's
253db874c57Selowe  * SE_EWANTED bit is set. Shared-lock (reader) requests are also denied
254db874c57Selowe  * if the page is slated for retirement.
255db874c57Selowe  *
256db874c57Selowe  * The se and es parameters determine if the lock should be granted
257db874c57Selowe  * based on the following decision table:
258db874c57Selowe  *
259db874c57Selowe  * Lock wanted   es flags     p_selock/SE_EWANTED  Action
260db874c57Selowe  * ----------- -------------- -------------------  ---------
261db874c57Selowe  * SE_EXCL        any [1][2]   unlocked/any        grant lock, clear SE_EWANTED
262db874c57Selowe  * SE_EXCL        SE_EWANTED   any lock/any        deny, set SE_EWANTED
263db874c57Selowe  * SE_EXCL        none         any lock/any        deny
2648bc68872Selowe  * SE_SHARED      n/a [2]        shared/0          grant
2658bc68872Selowe  * SE_SHARED      n/a [2]      unlocked/0          grant
266db874c57Selowe  * SE_SHARED      n/a            shared/1          deny
267db874c57Selowe  * SE_SHARED      n/a          unlocked/1          deny
268db874c57Selowe  * SE_SHARED      n/a              excl/any        deny
269db874c57Selowe  *
270db874c57Selowe  * Notes:
271db874c57Selowe  * [1] The code grants an exclusive lock to the caller and clears the bit
272db874c57Selowe  *   SE_EWANTED whenever p_selock is unlocked, regardless of the SE_EWANTED
273db874c57Selowe  *   bit's value.  This was deemed acceptable as we are not concerned about
274db874c57Selowe  *   exclusive-lock starvation. If this ever becomes an issue, a priority or
275db874c57Selowe  *   fifo mechanism should also be implemented. Meantime, the thread that
276db874c57Selowe  *   set SE_EWANTED should be prepared to catch this condition and reset it
277db874c57Selowe  *
278db874c57Selowe  * [2] Retired pages may not be locked at any time, regardless of the
279db874c57Selowe  *   dispostion of se, unless the es parameter has SE_RETIRED flag set.
2807c478bd9Sstevel@tonic-gate  *
281db874c57Selowe  * Notes on values of "es":
282db874c57Selowe  *
283db874c57Selowe  *   es & 1: page_lookup_create will attempt page relocation
284db874c57Selowe  *   es & SE_EXCL_WANTED: caller wants SE_EWANTED set (eg. delete
285db874c57Selowe  *       memory thread); this prevents reader-starvation of waiting
286db874c57Selowe  *       writer thread(s) by giving priority to writers over readers.
287db874c57Selowe  *   es & SE_RETIRED: caller wants to lock pages even if they are
288db874c57Selowe  *       retired.  Default is to deny the lock if the page is retired.
289db874c57Selowe  *
290db874c57Selowe  * And yes, we know, the semantics of this function are too complicated.
291db874c57Selowe  * It's on the list to be cleaned up.
2927c478bd9Sstevel@tonic-gate  */
2937c478bd9Sstevel@tonic-gate int
page_lock_es(page_t * pp,se_t se,kmutex_t * lock,reclaim_t reclaim,int es)2947c478bd9Sstevel@tonic-gate page_lock_es(page_t *pp, se_t se, kmutex_t *lock, reclaim_t reclaim, int es)
2957c478bd9Sstevel@tonic-gate {
2967c478bd9Sstevel@tonic-gate 	int		retval;
2977c478bd9Sstevel@tonic-gate 	kmutex_t	*pse = PAGE_SE_MUTEX(pp);
2987c478bd9Sstevel@tonic-gate 	int		upgraded;
2997c478bd9Sstevel@tonic-gate 	int		reclaim_it;
3017c478bd9Sstevel@tonic-gate 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
3037c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(page_lock_count);
3057c478bd9Sstevel@tonic-gate 	upgraded = 0;
3067c478bd9Sstevel@tonic-gate 	reclaim_it = 0;
3087c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
3107c478bd9Sstevel@tonic-gate 	ASSERT(((es & SE_EXCL_WANTED) == 0) ||
311db874c57Selowe 	    ((es & SE_EXCL_WANTED) && (se == SE_EXCL)));
313db874c57Selowe 	if (PP_RETIRED(pp) && !(es & SE_RETIRED)) {
314db874c57Selowe 		mutex_exit(pse);
315db874c57Selowe 		VM_STAT_ADD(page_lock_retired);
316db874c57Selowe 		return (0);
317db874c57Selowe 	}
3197c478bd9Sstevel@tonic-gate 	if (se == SE_SHARED && es == 1 && pp->p_selock == 0) {
3207c478bd9Sstevel@tonic-gate 		se = SE_EXCL;
3217c478bd9Sstevel@tonic-gate 	}
3237c478bd9Sstevel@tonic-gate 	if ((reclaim == P_RECLAIM) && (PP_ISFREE(pp))) {
3257c478bd9Sstevel@tonic-gate 		reclaim_it = 1;
3267c478bd9Sstevel@tonic-gate 		if (se == SE_SHARED) {
3277c478bd9Sstevel@tonic-gate 			/*
3287c478bd9Sstevel@tonic-gate 			 * This is an interesting situation.
3297c478bd9Sstevel@tonic-gate 			 *
3307c478bd9Sstevel@tonic-gate 			 * Remember that p_free can only change if
3317c478bd9Sstevel@tonic-gate 			 * p_selock < 0.
3327c478bd9Sstevel@tonic-gate 			 * p_free does not depend on our holding `pse'.
3337c478bd9Sstevel@tonic-gate 			 * And, since we hold `pse', p_selock can not change.
3347c478bd9Sstevel@tonic-gate 			 * So, if p_free changes on us, the page is already
3357c478bd9Sstevel@tonic-gate 			 * exclusively held, and we would fail to get p_selock
3367c478bd9Sstevel@tonic-gate 			 * regardless.
3377c478bd9Sstevel@tonic-gate 			 *
3387c478bd9Sstevel@tonic-gate 			 * We want to avoid getting the share
3397c478bd9Sstevel@tonic-gate 			 * lock on a free page that needs to be reclaimed.
3407c478bd9Sstevel@tonic-gate 			 * It is possible that some other thread has the share
3417c478bd9Sstevel@tonic-gate 			 * lock and has left the free page on the cache list.
3427c478bd9Sstevel@tonic-gate 			 * pvn_vplist_dirty() does this for brief periods.
3437c478bd9Sstevel@tonic-gate 			 * If the se_share is currently SE_EXCL, we will fail
3447c478bd9Sstevel@tonic-gate 			 * to acquire p_selock anyway.  Blocking is the
3457c478bd9Sstevel@tonic-gate 			 * right thing to do.
3467c478bd9Sstevel@tonic-gate 			 * If we need to reclaim this page, we must get
3477c478bd9Sstevel@tonic-gate 			 * exclusive access to it, force the upgrade now.
3487c478bd9Sstevel@tonic-gate 			 * Again, we will fail to acquire p_selock if the
3497c478bd9Sstevel@tonic-gate 			 * page is not free and block.
3507c478bd9Sstevel@tonic-gate 			 */
3517c478bd9Sstevel@tonic-gate 			upgraded = 1;
3527c478bd9Sstevel@tonic-gate 			se = SE_EXCL;
3537c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lock_upgrade);
3547c478bd9Sstevel@tonic-gate 		}
3557c478bd9Sstevel@tonic-gate 	}
3577c478bd9Sstevel@tonic-gate 	if (se == SE_EXCL) {
358db874c57Selowe 		if (!(es & SE_EXCL_WANTED) && (pp->p_selock & SE_EWANTED)) {
3597c478bd9Sstevel@tonic-gate 			/*
3607c478bd9Sstevel@tonic-gate 			 * if the caller wants a writer lock (but did not
3617c478bd9Sstevel@tonic-gate 			 * specify exclusive access), and there is a pending
3627c478bd9Sstevel@tonic-gate 			 * writer that wants exclusive access, return failure
3637c478bd9Sstevel@tonic-gate 			 */
3647c478bd9Sstevel@tonic-gate 			retval = 0;
3657c478bd9Sstevel@tonic-gate 		} else if ((pp->p_selock & ~SE_EWANTED) == 0) {
3667c478bd9Sstevel@tonic-gate 			/* no reader/writer lock held */
3677c478bd9Sstevel@tonic-gate 			/* this clears our setting of the SE_EWANTED bit */
3687c478bd9Sstevel@tonic-gate 			pp->p_selock = SE_WRITER;
3697c478bd9Sstevel@tonic-gate 			retval = 1;
3707c478bd9Sstevel@tonic-gate 		} else {
3717c478bd9Sstevel@tonic-gate 			/* page is locked */
372db874c57Selowe 			if (es & SE_EXCL_WANTED) {
3737c478bd9Sstevel@tonic-gate 				/* set the SE_EWANTED bit */
3747c478bd9Sstevel@tonic-gate 				pp->p_selock |= SE_EWANTED;
3757c478bd9Sstevel@tonic-gate 			}
3767c478bd9Sstevel@tonic-gate 			retval = 0;
3777c478bd9Sstevel@tonic-gate 		}
3787c478bd9Sstevel@tonic-gate 	} else {
3797c478bd9Sstevel@tonic-gate 		retval = 0;
3807c478bd9Sstevel@tonic-gate 		if (pp->p_selock >= 0) {
381db874c57Selowe 			if ((pp->p_selock & SE_EWANTED) == 0) {
3828bc68872Selowe 				pp->p_selock += SE_READER;
3838bc68872Selowe 				retval = 1;
3847c478bd9Sstevel@tonic-gate 			}
3857c478bd9Sstevel@tonic-gate 		}
3867c478bd9Sstevel@tonic-gate 	}
3887c478bd9Sstevel@tonic-gate 	if (retval == 0) {
3897c478bd9Sstevel@tonic-gate 		if ((pp->p_selock & ~SE_EWANTED) == SE_DELETED) {
3907c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lock_deleted);
3917c478bd9Sstevel@tonic-gate 			mutex_exit(pse);
3927c478bd9Sstevel@tonic-gate 			return (retval);
3937c478bd9Sstevel@tonic-gate 		}
3957c478bd9Sstevel@tonic-gate #ifdef VM_STATS
3967c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_lock_miss);
3977c478bd9Sstevel@tonic-gate 		if (upgraded) {
3987c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lock_upgrade_failed);
3997c478bd9Sstevel@tonic-gate 		}
4007c478bd9Sstevel@tonic-gate #endif
4017c478bd9Sstevel@tonic-gate 		if (lock) {
4027c478bd9Sstevel@tonic-gate 			VM_STAT_ADD(page_lock_miss_lock);
4037c478bd9Sstevel@tonic-gate 			mutex_exit(lock);
4047c478bd9Sstevel@tonic-gate 		}
4067c478bd9Sstevel@tonic-gate 		/*
4077c478bd9Sstevel@tonic-gate 		 * Now, wait for the page to be unlocked and
4087c478bd9Sstevel@tonic-gate 		 * release the lock protecting p_cv and p_selock.
4097c478bd9Sstevel@tonic-gate 		 */
4107c478bd9Sstevel@tonic-gate 		cv_wait(&pp->p_cv, pse);
4117c478bd9Sstevel@tonic-gate 		mutex_exit(pse);
4137c478bd9Sstevel@tonic-gate 		/*
4147c478bd9Sstevel@tonic-gate 		 * The page identity may have changed while we were
4157c478bd9Sstevel@tonic-gate 		 * blocked.  If we are willing to depend on "pp"
4167c478bd9Sstevel@tonic-gate 		 * still pointing to a valid page structure (i.e.,
4177c478bd9Sstevel@tonic-gate 		 * assuming page structures are not dynamically allocated
4187c478bd9Sstevel@tonic-gate 		 * or freed), we could try to lock the page if its
4197c478bd9Sstevel@tonic-gate 		 * identity hasn't changed.
4207c478bd9Sstevel@tonic-gate 		 *
4217c478bd9Sstevel@tonic-gate 		 * This needs to be measured, since we come back from
4227c478bd9Sstevel@tonic-gate 		 * cv_wait holding pse (the expensive part of this
4237c478bd9Sstevel@tonic-gate 		 * operation) we might as well try the cheap part.
4247c478bd9Sstevel@tonic-gate 		 * Though we would also have to confirm that dropping
4257c478bd9Sstevel@tonic-gate 		 * `lock' did not cause any grief to the callers.
4267c478bd9Sstevel@tonic-gate 		 */
4277c478bd9Sstevel@tonic-gate 		if (lock) {
4287c478bd9Sstevel@tonic-gate 			mutex_enter(lock);
4297c478bd9Sstevel@tonic-gate 		}
4307c478bd9Sstevel@tonic-gate 	} else {
4317c478bd9Sstevel@tonic-gate 		/*
4327c478bd9Sstevel@tonic-gate 		 * We have the page lock.
4337c478bd9Sstevel@tonic-gate 		 * If we needed to reclaim the page, and the page
4347c478bd9Sstevel@tonic-gate 		 * needed reclaiming (ie, it was free), then we
4357c478bd9Sstevel@tonic-gate 		 * have the page exclusively locked.  We may need
4367c478bd9Sstevel@tonic-gate 		 * to downgrade the page.
4377c478bd9Sstevel@tonic-gate 		 */
4387c478bd9Sstevel@tonic-gate 		ASSERT((upgraded) ?
4397c478bd9Sstevel@tonic-gate 		    ((PP_ISFREE(pp)) && PAGE_EXCL(pp)) : 1);
4407c478bd9Sstevel@tonic-gate 		mutex_exit(pse);
4427c478bd9Sstevel@tonic-gate 		/*
4437c478bd9Sstevel@tonic-gate 		 * We now hold this page's lock, either shared or
4447c478bd9Sstevel@tonic-gate 		 * exclusive.  This will prevent its identity from changing.
4457c478bd9Sstevel@tonic-gate 		 * The page, however, may or may not be free.  If the caller
4467c478bd9Sstevel@tonic-gate 		 * requested, and it is free, go reclaim it from the
4477c478bd9Sstevel@tonic-gate 		 * free list.  If the page can't be reclaimed, return failure
4487c478bd9Sstevel@tonic-gate 		 * so that the caller can start all over again.
4497c478bd9Sstevel@tonic-gate 		 *
4507c478bd9Sstevel@tonic-gate 		 * NOTE:page_reclaim() releases the page lock (p_selock)
4517c478bd9Sstevel@tonic-gate 		 *	if it can't be reclaimed.
4527c478bd9Sstevel@tonic-gate 		 */
4537c478bd9Sstevel@tonic-gate 		if (reclaim_it) {
4547c478bd9Sstevel@tonic-gate 			if (!page_reclaim(pp, lock)) {
4557c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lock_bad_reclaim);
4567c478bd9Sstevel@tonic-gate 				retval = 0;
4577c478bd9Sstevel@tonic-gate 			} else {
4587c478bd9Sstevel@tonic-gate 				VM_STAT_ADD(page_lock_reclaim);
4597c478bd9Sstevel@tonic-gate 				if (upgraded) {
4607c478bd9Sstevel@tonic-gate 					page_downgrade(pp);
4617c478bd9Sstevel@tonic-gate 				}
4627c478bd9Sstevel@tonic-gate 			}
4637c478bd9Sstevel@tonic-gate 		}
4647c478bd9Sstevel@tonic-gate 	}
4657c478bd9Sstevel@tonic-gate 	return (retval);
4667c478bd9Sstevel@tonic-gate }
4687c478bd9Sstevel@tonic-gate /*
4697c478bd9Sstevel@tonic-gate  * Clear the SE_EWANTED bit from p_selock.  This function allows
4707c478bd9Sstevel@tonic-gate  * callers of page_lock_es and page_try_reclaim_lock to clear
4717c478bd9Sstevel@tonic-gate  * their setting of this bit if they decide they no longer wish
4727c478bd9Sstevel@tonic-gate  * to gain exclusive access to the page.  Currently only
4737c478bd9Sstevel@tonic-gate  * delete_memory_thread uses this when the delete memory
4747c478bd9Sstevel@tonic-gate  * operation is cancelled.
4757c478bd9Sstevel@tonic-gate  */
4767c478bd9Sstevel@tonic-gate void
page_lock_clr_exclwanted(page_t * pp)4777c478bd9Sstevel@tonic-gate page_lock_clr_exclwanted(page_t *pp)
4787c478bd9Sstevel@tonic-gate {
4797c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
4817c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
4827c478bd9Sstevel@tonic-gate 	pp->p_selock &= ~SE_EWANTED;
4837c478bd9Sstevel@tonic-gate 	if (CV_HAS_WAITERS(&pp->p_cv))
4847c478bd9Sstevel@tonic-gate 		cv_broadcast(&pp->p_cv);
4857c478bd9Sstevel@tonic-gate 	mutex_exit(pse);
4867c478bd9Sstevel@tonic-gate }
4887c478bd9Sstevel@tonic-gate /*
4897c478bd9Sstevel@tonic-gate  * Read the comments inside of page_lock_es() carefully.
4907c478bd9Sstevel@tonic-gate  *
4917c478bd9Sstevel@tonic-gate  * SE_EXCL callers specifying es == SE_EXCL_WANTED will cause the
4927c478bd9Sstevel@tonic-gate  * SE_EWANTED bit of p_selock to be set when the lock cannot be obtained.
4937c478bd9Sstevel@tonic-gate  * This is used by threads subject to reader-starvation (eg. memory delete).
4947c478bd9Sstevel@tonic-gate  *
4957c478bd9Sstevel@tonic-gate  * When a thread using SE_EXCL_WANTED does not obtain the SE_EXCL lock,
4967c478bd9Sstevel@tonic-gate  * it is expected that it will retry at a later time.  Threads that will
4977c478bd9Sstevel@tonic-gate  * not retry the lock *must* call page_lock_clr_exclwanted to clear the
4987c478bd9Sstevel@tonic-gate  * SE_EWANTED bit.  (When a thread using SE_EXCL_WANTED obtains the lock,
4997c478bd9Sstevel@tonic-gate  * the bit is cleared.)
5007c478bd9Sstevel@tonic-gate  */
5017c478bd9Sstevel@tonic-gate int
page_try_reclaim_lock(page_t * pp,se_t se,int es)5027c478bd9Sstevel@tonic-gate page_try_reclaim_lock(page_t *pp, se_t se, int es)
5037c478bd9Sstevel@tonic-gate {
5047c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
5057c478bd9Sstevel@tonic-gate 	selock_t old;
5077c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
5097c478bd9Sstevel@tonic-gate 	old = pp->p_selock;
5117c478bd9Sstevel@tonic-gate 	ASSERT(((es & SE_EXCL_WANTED) == 0) ||
512db874c57Selowe 	    ((es & SE_EXCL_WANTED) && (se == SE_EXCL)));
514db874c57Selowe 	if (PP_RETIRED(pp) && !(es & SE_RETIRED)) {
515db874c57Selowe 		mutex_exit(pse);
516db874c57Selowe 		VM_STAT_ADD(page_trylock_failed);
517db874c57Selowe 		return (0);
518db874c57Selowe 	}
5207c478bd9Sstevel@tonic-gate 	if (se == SE_SHARED && es == 1 && old == 0) {
5217c478bd9Sstevel@tonic-gate 		se = SE_EXCL;
5227c478bd9Sstevel@tonic-gate 	}
5247c478bd9Sstevel@tonic-gate 	if (se == SE_SHARED) {
5257c478bd9Sstevel@tonic-gate 		if (!PP_ISFREE(pp)) {
5267c478bd9Sstevel@tonic-gate 			if (old >= 0) {
527db874c57Selowe 				/*
528db874c57Selowe 				 * Readers are not allowed when excl wanted
529db874c57Selowe 				 */
530db874c57Selowe 				if ((old & SE_EWANTED) == 0) {
5318bc68872Selowe 					pp->p_selock = old + SE_READER;
5328bc68872Selowe 					mutex_exit(pse);
5338bc68872Selowe 					return (1);
5347c478bd9Sstevel@tonic-gate 				}
5357c478bd9Sstevel@tonic-gate 			}
5367c478bd9Sstevel@tonic-gate 			mutex_exit(pse);
5377c478bd9Sstevel@tonic-gate 			return (0);
5387c478bd9Sstevel@tonic-gate 		}
5397c478bd9Sstevel@tonic-gate 		/*
5407c478bd9Sstevel@tonic-gate 		 * The page is free, so we really want SE_EXCL (below)
5417c478bd9Sstevel@tonic-gate 		 */
5427c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(page_try_reclaim_upgrade);
5437c478bd9Sstevel@tonic-gate 	}
5457c478bd9Sstevel@tonic-gate 	/*
5467c478bd9Sstevel@tonic-gate 	 * The caller wants a writer lock.  We try for it only if
5477c478bd9Sstevel@tonic-gate 	 * SE_EWANTED is not set, or if the caller specified
5487c478bd9Sstevel@tonic-gate 	 * SE_EXCL_WANTED.
5497c478bd9Sstevel@tonic-gate 	 */
550db874c57Selowe 	if (!(old & SE_EWANTED) || (es & SE_EXCL_WANTED)) {
5517c478bd9Sstevel@tonic-gate 		if ((old & ~SE_EWANTED) == 0) {
5527c478bd9Sstevel@tonic-gate 			/* no reader/writer lock held */
5537c478bd9Sstevel@tonic-gate 			/* this clears out our setting of the SE_EWANTED bit */
5547c478bd9Sstevel@tonic-gate 			pp->p_selock = SE_WRITER;
5557c478bd9Sstevel@tonic-gate 			mutex_exit(pse);
5567c478bd9Sstevel@tonic-gate 			return (1);
5577c478bd9Sstevel@tonic-gate 		}
5587c478bd9Sstevel@tonic-gate 	}
559db874c57Selowe 	if (es & SE_EXCL_WANTED) {
5607c478bd9Sstevel@tonic-gate 		/* page is locked, set the SE_EWANTED bit */
5617c478bd9Sstevel@tonic-gate 		pp->p_selock |= SE_EWANTED;
5627c478bd9Sstevel@tonic-gate 	}
5637c478bd9Sstevel@tonic-gate 	mutex_exit(pse);
5647c478bd9Sstevel@tonic-gate 	return (0);
5657c478bd9Sstevel@tonic-gate }
5677c478bd9Sstevel@tonic-gate /*
5687c478bd9Sstevel@tonic-gate  * Acquire a page's "shared/exclusive" lock, but never block.
5697c478bd9Sstevel@tonic-gate  * Returns 1 on success, 0 on failure.
5707c478bd9Sstevel@tonic-gate  */
5717c478bd9Sstevel@tonic-gate int
page_trylock(page_t * pp,se_t se)5727c478bd9Sstevel@tonic-gate page_trylock(page_t *pp, se_t se)
5737c478bd9Sstevel@tonic-gate {
5747c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
5767c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
577db874c57Selowe 	if (pp->p_selock & SE_EWANTED || PP_RETIRED(pp) ||
57824e9c58bSelowe 	    (se == SE_SHARED && PP_PR_NOSHARE(pp))) {
579db874c57Selowe 		/*
580db874c57Selowe 		 * Fail if a thread wants exclusive access and page is
581db874c57Selowe 		 * retired, if the page is slated for retirement, or a
582db874c57Selowe 		 * share lock is requested.
583db874c57Selowe 		 */
5847c478bd9Sstevel@tonic-gate 		mutex_exit(pse);
585db874c57Selowe 		VM_STAT_ADD(page_trylock_failed);
5867c478bd9Sstevel@tonic-gate 		return (0);
5877c478bd9Sstevel@tonic-gate 	}
5897c478bd9Sstevel@tonic-gate 	if (se == SE_EXCL) {
5907c478bd9Sstevel@tonic-gate 		if (pp->p_selock == 0) {
5917c478bd9Sstevel@tonic-gate 			pp->p_selock = SE_WRITER;
5927c478bd9Sstevel@tonic-gate 			mutex_exit(pse);
5937c478bd9Sstevel@tonic-gate 			return (1);
5947c478bd9Sstevel@tonic-gate 		}
5957c478bd9Sstevel@tonic-gate 	} else {
5967c478bd9Sstevel@tonic-gate 		if (pp->p_selock >= 0) {
5977c478bd9Sstevel@tonic-gate 			pp->p_selock += SE_READER;
5987c478bd9Sstevel@tonic-gate 			mutex_exit(pse);
5997c478bd9Sstevel@tonic-gate 			return (1);
6007c478bd9Sstevel@tonic-gate 		}
6017c478bd9Sstevel@tonic-gate 	}
6027c478bd9Sstevel@tonic-gate 	mutex_exit(pse);
6037c478bd9Sstevel@tonic-gate 	return (0);
6047c478bd9Sstevel@tonic-gate }
606db874c57Selowe /*
607db874c57Selowe  * Variant of page_unlock() specifically for the page freelist
608db874c57Selowe  * code. The mere existence of this code is a vile hack that
609db874c57Selowe  * has resulted due to the backwards locking order of the page
610db874c57Selowe  * freelist manager; please don't call it.
611db874c57Selowe  */
612db874c57Selowe void
page_unlock_nocapture(page_t * pp)6138b464eb8Smec page_unlock_nocapture(page_t *pp)
614db874c57Selowe {
615db874c57Selowe 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
616db874c57Selowe 	selock_t old;
618db874c57Selowe 	mutex_enter(pse);
620db874c57Selowe 	old = pp->p_selock;
621db874c57Selowe 	if ((old & ~SE_EWANTED) == SE_READER) {
622db874c57Selowe 		pp->p_selock = old & ~SE_READER;
623db874c57Selowe 		if (CV_HAS_WAITERS(&pp->p_cv))
624db874c57Selowe 			cv_broadcast(&pp->p_cv);
625db874c57Selowe 	} else if ((old & ~SE_EWANTED) == SE_DELETED) {
626903a11ebSrh 		panic("page_unlock_nocapture: page %p is deleted", (void *)pp);
627db874c57Selowe 	} else if (old < 0) {
628db874c57Selowe 		pp->p_selock &= SE_EWANTED;
629db874c57Selowe 		if (CV_HAS_WAITERS(&pp->p_cv))
630db874c57Selowe 			cv_broadcast(&pp->p_cv);
631db874c57Selowe 	} else if ((old & ~SE_EWANTED) > SE_READER) {
632db874c57Selowe 		pp->p_selock = old - SE_READER;
633db874c57Selowe 	} else {
634903a11ebSrh 		panic("page_unlock_nocapture: page %p is not locked",
635903a11ebSrh 		    (void *)pp);
636db874c57Selowe 	}
638db874c57Selowe 	mutex_exit(pse);
639db874c57Selowe }
6417c478bd9Sstevel@tonic-gate /*
6427c478bd9Sstevel@tonic-gate  * Release the page's "shared/exclusive" lock and wake up anyone
6437c478bd9Sstevel@tonic-gate  * who might be waiting for it.
6447c478bd9Sstevel@tonic-gate  */
6457c478bd9Sstevel@tonic-gate void
page_unlock(page_t * pp)6467c478bd9Sstevel@tonic-gate page_unlock(page_t *pp)
6477c478bd9Sstevel@tonic-gate {
6487c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
6497c478bd9Sstevel@tonic-gate 	selock_t old;
6517c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
6537c478bd9Sstevel@tonic-gate 	old = pp->p_selock;
6547c478bd9Sstevel@tonic-gate 	if ((old & ~SE_EWANTED) == SE_READER) {
6557c478bd9Sstevel@tonic-gate 		pp->p_selock = old & ~SE_READER;
6567c478bd9Sstevel@tonic-gate 		if (CV_HAS_WAITERS(&pp->p_cv))
6577c478bd9Sstevel@tonic-gate 			cv_broadcast(&pp->p_cv);
6587c478bd9Sstevel@tonic-gate 	} else if ((old & ~SE_EWANTED) == SE_DELETED) {
659903a11ebSrh 		panic("page_unlock: page %p is deleted", (void *)pp);
6607c478bd9Sstevel@tonic-gate 	} else if (old < 0) {
6617c478bd9Sstevel@tonic-gate 		pp->p_selock &= SE_EWANTED;
6627c478bd9Sstevel@tonic-gate 		if (CV_HAS_WAITERS(&pp->p_cv))
6637c478bd9Sstevel@tonic-gate 			cv_broadcast(&pp->p_cv);
6647c478bd9Sstevel@tonic-gate 	} else if ((old & ~SE_EWANTED) > SE_READER) {
6657c478bd9Sstevel@tonic-gate 		pp->p_selock = old - SE_READER;
6667c478bd9Sstevel@tonic-gate 	} else {
667903a11ebSrh 		panic("page_unlock: page %p is not locked", (void *)pp);
6687c478bd9Sstevel@tonic-gate 	}
6708b464eb8Smec 	if (pp->p_selock == 0) {
671db874c57Selowe 		/*
6728b464eb8Smec 		 * If the T_CAPTURING bit is set, that means that we should
6738b464eb8Smec 		 * not try and capture the page again as we could recurse
6748b464eb8Smec 		 * which could lead to a stack overflow panic or spending a
6758b464eb8Smec 		 * relatively long time in the kernel making no progress.
676db874c57Selowe 		 */
6778b464eb8Smec 		if ((pp->p_toxic & PR_CAPTURE) &&
6788b464eb8Smec 		    !(curthread->t_flag & T_CAPTURING) &&
6798b464eb8Smec 		    !PP_RETIRED(pp)) {
680db874c57Selowe 			pp->p_selock = SE_WRITER;
681db874c57Selowe 			mutex_exit(pse);
6828b464eb8Smec 			page_unlock_capture(pp);
683db874c57Selowe 		} else {
684db874c57Selowe 			mutex_exit(pse);
685db874c57Selowe 		}
686db874c57Selowe 	} else {
687db874c57Selowe 		mutex_exit(pse);
688db874c57Selowe 	}
6897c478bd9Sstevel@tonic-gate }
6917c478bd9Sstevel@tonic-gate /*
6927c478bd9Sstevel@tonic-gate  * Try to upgrade the lock on the page from a "shared" to an
6937c478bd9Sstevel@tonic-gate  * "exclusive" lock.  Since this upgrade operation is done while
6947c478bd9Sstevel@tonic-gate  * holding the mutex protecting this page, no one else can acquire this page's
6957c478bd9Sstevel@tonic-gate  * lock and change the page. Thus, it is safe to drop the "shared"
6967c478bd9Sstevel@tonic-gate  * lock and attempt to acquire the "exclusive" lock.
6977c478bd9Sstevel@tonic-gate  *
6987c478bd9Sstevel@tonic-gate  * Returns 1 on success, 0 on failure.
6997c478bd9Sstevel@tonic-gate  */
7007c478bd9Sstevel@tonic-gate int
page_tryupgrade(page_t * pp)7017c478bd9Sstevel@tonic-gate page_tryupgrade(page_t *pp)
7027c478bd9Sstevel@tonic-gate {
7037c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
7057c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
7067c478bd9Sstevel@tonic-gate 	if (!(pp->p_selock & SE_EWANTED)) {
7077c478bd9Sstevel@tonic-gate 		/* no threads want exclusive access, try upgrade */
7087c478bd9Sstevel@tonic-gate 		if (pp->p_selock == SE_READER) {
7097c478bd9Sstevel@tonic-gate 			/* convert to exclusive lock */
7107c478bd9Sstevel@tonic-gate 			pp->p_selock = SE_WRITER;
7117c478bd9Sstevel@tonic-gate 			mutex_exit(pse);
7127c478bd9Sstevel@tonic-gate 			return (1);
7137c478bd9Sstevel@tonic-gate 		}
7147c478bd9Sstevel@tonic-gate 	}
7157c478bd9Sstevel@tonic-gate 	mutex_exit(pse);
7167c478bd9Sstevel@tonic-gate 	return (0);
7177c478bd9Sstevel@tonic-gate }
7197c478bd9Sstevel@tonic-gate /*
7207c478bd9Sstevel@tonic-gate  * Downgrade the "exclusive" lock on the page to a "shared" lock
7217c478bd9Sstevel@tonic-gate  * while holding the mutex protecting this page's p_selock field.
7227c478bd9Sstevel@tonic-gate  */
7237c478bd9Sstevel@tonic-gate void
page_downgrade(page_t * pp)7247c478bd9Sstevel@tonic-gate page_downgrade(page_t *pp)
7257c478bd9Sstevel@tonic-gate {
7267c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
7277c478bd9Sstevel@tonic-gate 	int excl_waiting;
7297c478bd9Sstevel@tonic-gate 	ASSERT((pp->p_selock & ~SE_EWANTED) != SE_DELETED);
7307c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp));
7327c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
7337c478bd9Sstevel@tonic-gate 	excl_waiting =  pp->p_selock & SE_EWANTED;
7347c478bd9Sstevel@tonic-gate 	pp->p_selock = SE_READER | excl_waiting;
7357c478bd9Sstevel@tonic-gate 	if (CV_HAS_WAITERS(&pp->p_cv))
7367c478bd9Sstevel@tonic-gate 		cv_broadcast(&pp->p_cv);
7377c478bd9Sstevel@tonic-gate 	mutex_exit(pse);
7387c478bd9Sstevel@tonic-gate }
7407c478bd9Sstevel@tonic-gate void
page_lock_delete(page_t * pp)7417c478bd9Sstevel@tonic-gate page_lock_delete(page_t *pp)
7427c478bd9Sstevel@tonic-gate {
7437c478bd9Sstevel@tonic-gate 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
7457c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_EXCL(pp));
7467c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode == NULL);
7477c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_offset == (u_offset_t)-1);
7487c478bd9Sstevel@tonic-gate 	ASSERT(!PP_ISFREE(pp));
7507c478bd9Sstevel@tonic-gate 	mutex_enter(pse);
7517c478bd9Sstevel@tonic-gate 	pp->p_selock = SE_DELETED;
7527c478bd9Sstevel@tonic-gate 	if (CV_HAS_WAITERS(&pp->p_cv))
7537c478bd9Sstevel@tonic-gate 		cv_broadcast(&pp->p_cv);
7547c478bd9Sstevel@tonic-gate 	mutex_exit(pse);
7557c478bd9Sstevel@tonic-gate }
7578b464eb8Smec int
page_deleted(page_t * pp)7588b464eb8Smec page_deleted(page_t *pp)
7598b464eb8Smec {
7608b464eb8Smec 	return (pp->p_selock == SE_DELETED);
7618b464eb8Smec }
7637c478bd9Sstevel@tonic-gate /*
7647c478bd9Sstevel@tonic-gate  * Implement the io lock for pages
7657c478bd9Sstevel@tonic-gate  */
7667c478bd9Sstevel@tonic-gate void
page_iolock_init(page_t * pp)7677c478bd9Sstevel@tonic-gate page_iolock_init(page_t *pp)
7687c478bd9Sstevel@tonic-gate {
7697c478bd9Sstevel@tonic-gate 	pp->p_iolock_state = 0;
7707c478bd9Sstevel@tonic-gate 	cv_init(&pp->p_io_cv, NULL, CV_DEFAULT, NULL);
7717c478bd9Sstevel@tonic-gate }
7737c478bd9Sstevel@tonic-gate /*
7747c478bd9Sstevel@tonic-gate  * Acquire the i/o lock on a page.
7757c478bd9Sstevel@tonic-gate  */
7767c478bd9Sstevel@tonic-gate void
page_io_lock(page_t * pp)7777c478bd9Sstevel@tonic-gate page_io_lock(page_t *pp)
7787c478bd9Sstevel@tonic-gate {
7797c478bd9Sstevel@tonic-gate 	kmutex_t *pio;
7817c478bd9Sstevel@tonic-gate 	pio = PAGE_IO_MUTEX(pp);
7827c478bd9Sstevel@tonic-gate 	mutex_enter(pio);
7837c478bd9Sstevel@tonic-gate 	while (pp->p_iolock_state & PAGE_IO_INUSE) {
7847c478bd9Sstevel@tonic-gate 		cv_wait(&(pp->p_io_cv), pio);
7857c478bd9Sstevel@tonic-gate 	}
7867c478bd9Sstevel@tonic-gate 	pp->p_iolock_state |= PAGE_IO_INUSE;
7877c478bd9Sstevel@tonic-gate 	mutex_exit(pio);
7887c478bd9Sstevel@tonic-gate }
7907c478bd9Sstevel@tonic-gate /*
7917c478bd9Sstevel@tonic-gate  * Release the i/o lock on a page.
7927c478bd9Sstevel@tonic-gate  */
7937c478bd9Sstevel@tonic-gate void
page_io_unlock(page_t * pp)7947c478bd9Sstevel@tonic-gate page_io_unlock(page_t *pp)
7957c478bd9Sstevel@tonic-gate {
7967c478bd9Sstevel@tonic-gate 	kmutex_t *pio;
7987c478bd9Sstevel@tonic-gate 	pio = PAGE_IO_MUTEX(pp);
7997c478bd9Sstevel@tonic-gate 	mutex_enter(pio);
800a71e32b6Sstans 	cv_broadcast(&pp->p_io_cv);
8017c478bd9Sstevel@tonic-gate 	pp->p_iolock_state &= ~PAGE_IO_INUSE;
8027c478bd9Sstevel@tonic-gate 	mutex_exit(pio);
8037c478bd9Sstevel@tonic-gate }
8057c478bd9Sstevel@tonic-gate /*
8067c478bd9Sstevel@tonic-gate  * Try to acquire the i/o lock on a page without blocking.
8077c478bd9Sstevel@tonic-gate  * Returns 1 on success, 0 on failure.
8087c478bd9Sstevel@tonic-gate  */
8097c478bd9Sstevel@tonic-gate int
page_io_trylock(page_t * pp)8107c478bd9Sstevel@tonic-gate page_io_trylock(page_t *pp)
8117c478bd9Sstevel@tonic-gate {
8127c478bd9Sstevel@tonic-gate 	kmutex_t *pio;
8147c478bd9Sstevel@tonic-gate 	if (pp->p_iolock_state & PAGE_IO_INUSE)
8157c478bd9Sstevel@tonic-gate 		return (0);
8177c478bd9Sstevel@tonic-gate 	pio = PAGE_IO_MUTEX(pp);
8187c478bd9Sstevel@tonic-gate 	mutex_enter(pio);
8207c478bd9Sstevel@tonic-gate 	if (pp->p_iolock_state & PAGE_IO_INUSE) {
8217c478bd9Sstevel@tonic-gate 		mutex_exit(pio);
8227c478bd9Sstevel@tonic-gate 		return (0);
8237c478bd9Sstevel@tonic-gate 	}
8247c478bd9Sstevel@tonic-gate 	pp->p_iolock_state |= PAGE_IO_INUSE;
8257c478bd9Sstevel@tonic-gate 	mutex_exit(pio);
8277c478bd9Sstevel@tonic-gate 	return (1);
8287c478bd9Sstevel@tonic-gate }
830a71e32b6Sstans /*
831a71e32b6Sstans  * Wait until the i/o lock is not held.
832a71e32b6Sstans  */
833a71e32b6Sstans void
page_io_wait(page_t * pp)834a71e32b6Sstans page_io_wait(page_t *pp)
835a71e32b6Sstans {
836a71e32b6Sstans 	kmutex_t *pio;
838a71e32b6Sstans 	pio = PAGE_IO_MUTEX(pp);
839a71e32b6Sstans 	mutex_enter(pio);
840a71e32b6Sstans 	while (pp->p_iolock_state & PAGE_IO_INUSE) {
841a71e32b6Sstans 		cv_wait(&(pp->p_io_cv), pio);
842a71e32b6Sstans 	}
843a71e32b6Sstans 	mutex_exit(pio);
844a71e32b6Sstans }
846a71e32b6Sstans /*
847a71e32b6Sstans  * Returns 1 on success, 0 on failure.
848a71e32b6Sstans  */
849a71e32b6Sstans int
page_io_locked(page_t * pp)850a71e32b6Sstans page_io_locked(page_t *pp)
851a71e32b6Sstans {
852a71e32b6Sstans 	return (pp->p_iolock_state & PAGE_IO_INUSE);
853a71e32b6Sstans }
8557c478bd9Sstevel@tonic-gate /*
8567c478bd9Sstevel@tonic-gate  * Assert that the i/o lock on a page is held.
8577c478bd9Sstevel@tonic-gate  * Returns 1 on success, 0 on failure.
8587c478bd9Sstevel@tonic-gate  */
8597c478bd9Sstevel@tonic-gate int
page_iolock_assert(page_t * pp)8607c478bd9Sstevel@tonic-gate page_iolock_assert(page_t *pp)
8617c478bd9Sstevel@tonic-gate {
862a71e32b6Sstans 	return (page_io_locked(pp));
8637c478bd9Sstevel@tonic-gate }
8657c478bd9Sstevel@tonic-gate /*
8667c478bd9Sstevel@tonic-gate  * Wrapper exported to kernel routines that are built
8677c478bd9Sstevel@tonic-gate  * platform-independent (the macro is platform-dependent;
8687c478bd9Sstevel@tonic-gate  * the size of vph_mutex[] is based on NCPU).
8697c478bd9Sstevel@tonic-gate  *
8707c478bd9Sstevel@tonic-gate  * Note that you can do stress testing on this by setting the
8717c478bd9Sstevel@tonic-gate  * variable page_vnode_mutex_stress to something other than
8727c478bd9Sstevel@tonic-gate  * zero in a DEBUG kernel in a debugger after loading the kernel.
8737c478bd9Sstevel@tonic-gate  * Setting it after the kernel is running may not work correctly.
8747c478bd9Sstevel@tonic-gate  */
8757c478bd9Sstevel@tonic-gate #ifdef DEBUG
8767c478bd9Sstevel@tonic-gate static int page_vnode_mutex_stress = 0;
8777c478bd9Sstevel@tonic-gate #endif
8797c478bd9Sstevel@tonic-gate kmutex_t *
page_vnode_mutex(vnode_t * vp)8807c478bd9Sstevel@tonic-gate page_vnode_mutex(vnode_t *vp)
8817c478bd9Sstevel@tonic-gate {
882*04909c8cSJohn Levon 	if (vp == &kvp || vp == &kvps[KV_VVP])
8837c478bd9Sstevel@tonic-gate 		return (&vph_mutex[VPH_TABLE_SIZE + 0]);
885*04909c8cSJohn Levon 	if (vp == &kvps[KV_ZVP])
886ad23a2dbSjohansen 		return (&vph_mutex[VPH_TABLE_SIZE + 1]);
8877c478bd9Sstevel@tonic-gate #ifdef DEBUG
8887c478bd9Sstevel@tonic-gate 	if (page_vnode_mutex_stress != 0)
8897c478bd9Sstevel@tonic-gate 		return (&vph_mutex[0]);
8907c478bd9Sstevel@tonic-gate #endif
8927c478bd9Sstevel@tonic-gate 	return (&vph_mutex[VP_HASH_FUNC(vp)]);
8937c478bd9Sstevel@tonic-gate }
8957c478bd9Sstevel@tonic-gate kmutex_t *
page_se_mutex(page_t * pp)8967c478bd9Sstevel@tonic-gate page_se_mutex(page_t *pp)
8977c478bd9Sstevel@tonic-gate {
8987c478bd9Sstevel@tonic-gate 	return (PAGE_SE_MUTEX(pp));
8997c478bd9Sstevel@tonic-gate }
9017c478bd9Sstevel@tonic-gate #ifdef VM_STATS
9027c478bd9Sstevel@tonic-gate uint_t pszclck_stat[4];
9037c478bd9Sstevel@tonic-gate #endif
9047c478bd9Sstevel@tonic-gate /*
9057c478bd9Sstevel@tonic-gate  * Find, take and return a mutex held by hat_page_demote().
9067c478bd9Sstevel@tonic-gate  * Called by page_demote_vp_pages() before hat_page_demote() call and by
9077c478bd9Sstevel@tonic-gate  * routines that want to block hat_page_demote() but can't do it
9087c478bd9Sstevel@tonic-gate  * via locking all constituent pages.
9097c478bd9Sstevel@tonic-gate  *
9107c478bd9Sstevel@tonic-gate  * Return NULL if p_szc is 0.
9117c478bd9Sstevel@tonic-gate  *
9127c478bd9Sstevel@tonic-gate  * It should only be used for pages that can be demoted by hat_page_demote()
9137c478bd9Sstevel@tonic-gate  * i.e. non swapfs file system pages.  The logic here is lifted from
9147c478bd9Sstevel@tonic-gate  * sfmmu_mlspl_enter() except there's no need to worry about p_szc increase
9157c478bd9Sstevel@tonic-gate  * since the page is locked and not free.
9167c478bd9Sstevel@tonic-gate  *
9177c478bd9Sstevel@tonic-gate  * Hash of the root page is used to find the lock.
9187c478bd9Sstevel@tonic-gate  * To find the root in the presense of hat_page_demote() chageing the location
9197c478bd9Sstevel@tonic-gate  * of the root this routine relies on the fact that hat_page_demote() changes
9207c478bd9Sstevel@tonic-gate  * root last.
9217c478bd9Sstevel@tonic-gate  *
9227c478bd9Sstevel@tonic-gate  * If NULL is returned pp's p_szc is guaranteed to be 0. If non NULL is
9237c478bd9Sstevel@tonic-gate  * returned pp's p_szc may be any value.
9247c478bd9Sstevel@tonic-gate  */
9257c478bd9Sstevel@tonic-gate kmutex_t *
page_szc_lock(page_t * pp)9267c478bd9Sstevel@tonic-gate page_szc_lock(page_t *pp)
9277c478bd9Sstevel@tonic-gate {
9287c478bd9Sstevel@tonic-gate 	kmutex_t	*mtx;
9297c478bd9Sstevel@tonic-gate 	page_t		*rootpp;
9307c478bd9Sstevel@tonic-gate 	uint_t		szc;
9317c478bd9Sstevel@tonic-gate 	uint_t		rszc;
9327c478bd9Sstevel@tonic-gate 	uint_t		pszc = pp->p_szc;
9347c478bd9Sstevel@tonic-gate 	ASSERT(pp != NULL);
9357c478bd9Sstevel@tonic-gate 	ASSERT(PAGE_LOCKED(pp));
9367c478bd9Sstevel@tonic-gate 	ASSERT(!PP_ISFREE(pp));
9377c478bd9Sstevel@tonic-gate 	ASSERT(pp->p_vnode != NULL);
9387c478bd9Sstevel@tonic-gate 	ASSERT(!IS_SWAPFSVP(pp->p_vnode));
939ad23a2dbSjohansen 	ASSERT(!PP_ISKAS(pp));
9417c478bd9Sstevel@tonic-gate again:
9427c478bd9Sstevel@tonic-gate 	if (pszc == 0) {
9437c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pszclck_stat[0]);
9447c478bd9Sstevel@tonic-gate 		return (NULL);
9457c478bd9Sstevel@tonic-gate 	}
9477c478bd9Sstevel@tonic-gate 	/* The lock lives in the root page */
9497c478bd9Sstevel@tonic-gate 	rootpp = PP_GROUPLEADER(pp, pszc);
9507c478bd9Sstevel@tonic-gate 	mtx = PAGE_SZC_MUTEX(rootpp);
9517c478bd9Sstevel@tonic-gate 	mutex_enter(mtx);
9537c478bd9Sstevel@tonic-gate 	/*
9547c478bd9Sstevel@tonic-gate 	 * since p_szc can only decrease if pp == rootpp
9557c478bd9Sstevel@tonic-gate 	 * rootpp will be always the same i.e we have the right root
9567c478bd9Sstevel@tonic-gate 	 * regardless of rootpp->p_szc.
9577c478bd9Sstevel@tonic-gate 	 * If location of pp's root didn't change after we took
9587c478bd9Sstevel@tonic-gate 	 * the lock we have the right root. return mutex hashed off it.
9597c478bd9Sstevel@tonic-gate 	 */
9607c478bd9Sstevel@tonic-gate 	if (pp == rootpp || (rszc = rootpp->p_szc) == pszc) {
9617c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pszclck_stat[1]);
9627c478bd9Sstevel@tonic-gate 		return (mtx);
9637c478bd9Sstevel@tonic-gate 	}
9657c478bd9Sstevel@tonic-gate 	/*
9667c478bd9Sstevel@tonic-gate 	 * root location changed because page got demoted.
9677c478bd9Sstevel@tonic-gate 	 * locate the new root.
9687c478bd9Sstevel@tonic-gate 	 */
9697c478bd9Sstevel@tonic-gate 	if (rszc < pszc) {
9707c478bd9Sstevel@tonic-gate 		szc = pp->p_szc;
9717c478bd9Sstevel@tonic-gate 		ASSERT(szc < pszc);
9727c478bd9Sstevel@tonic-gate 		mutex_exit(mtx);
9737c478bd9Sstevel@tonic-gate 		pszc = szc;
9747c478bd9Sstevel@tonic-gate 		VM_STAT_ADD(pszclck_stat[2]);
9757c478bd9Sstevel@tonic-gate 		goto again;
9767c478bd9Sstevel@tonic-gate 	}
9787c478bd9Sstevel@tonic-gate 	VM_STAT_ADD(pszclck_stat[3]);
9797c478bd9Sstevel@tonic-gate 	/*
9807c478bd9Sstevel@tonic-gate 	 * current hat_page_demote not done yet.
9817c478bd9Sstevel@tonic-gate 	 * wait for it to finish.
9827c478bd9Sstevel@tonic-gate 	 */
9837c478bd9Sstevel@tonic-gate 	mutex_exit(mtx);
9847c478bd9Sstevel@tonic-gate 	rootpp = PP_GROUPLEADER(rootpp, rszc);
9857c478bd9Sstevel@tonic-gate 	mtx = PAGE_SZC_MUTEX(rootpp);
9867c478bd9Sstevel@tonic-gate 	mutex_enter(mtx);
9877c478bd9Sstevel@tonic-gate 	mutex_exit(mtx);
9887c478bd9Sstevel@tonic-gate 	ASSERT(rootpp->p_szc < rszc);
9897c478bd9Sstevel@tonic-gate 	goto again;
9907c478bd9Sstevel@tonic-gate }
9927c478bd9Sstevel@tonic-gate int
page_szc_lock_assert(page_t * pp)9937c478bd9Sstevel@tonic-gate page_szc_lock_assert(page_t *pp)
9947c478bd9Sstevel@tonic-gate {
9957c478bd9Sstevel@tonic-gate 	page_t *rootpp = PP_PAGEROOT(pp);
9967c478bd9Sstevel@tonic-gate 	kmutex_t *mtx = PAGE_SZC_MUTEX(rootpp);
9987c478bd9Sstevel@tonic-gate 	return (MUTEX_HELD(mtx));
9997c478bd9Sstevel@tonic-gate }
1001ae115bc7Smrj /*