xref: /illumos-gate/usr/src/uts/common/vm/page_lock.c (revision 04909c8c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2019 Joyent, Inc.
24  */
25 
26 
27 /*
28  * VM - page locking primitives
29  */
30 #include <sys/param.h>
31 #include <sys/t_lock.h>
32 #include <sys/vtrace.h>
33 #include <sys/debug.h>
34 #include <sys/cmn_err.h>
35 #include <sys/bitmap.h>
36 #include <sys/lockstat.h>
37 #include <sys/sysmacros.h>
38 #include <sys/condvar_impl.h>
39 #include <vm/page.h>
40 #include <vm/seg_enum.h>
41 #include <vm/vm_dep.h>
42 #include <vm/seg_kmem.h>
43 
44 /*
45  * This global mutex array is for logical page locking.
46  * The following fields in the page structure are protected
47  * by this lock:
48  *
49  *	p_lckcnt
50  *	p_cowcnt
51  */
52 pad_mutex_t page_llocks[8 * NCPU_P2];
53 
54 /*
55  * This is a global lock for the logical page free list.  The
56  * logical free list, in this implementation, is maintained as two
57  * separate physical lists - the cache list and the free list.
58  */
59 kmutex_t  page_freelock;
60 
61 /*
62  * The hash table, page_hash[], the p_selock fields, and the
63  * list of pages associated with vnodes are protected by arrays of mutexes.
64  *
65  * Unless the hashes are changed radically, the table sizes must be
66  * a power of two.  Also, we typically need more mutexes for the
67  * vnodes since these locks are occasionally held for long periods.
68  * And since there seem to be two special vnodes (kvp and swapvp),
69  * we make room for private mutexes for them.
70  *
71  * The pse_mutex[] array holds the mutexes to protect the p_selock
72  * fields of all page_t structures.
73  *
74  * PAGE_SE_MUTEX(pp) returns the address of the appropriate mutex
75  * when given a pointer to a page_t.
76  *
77  * PIO_TABLE_SIZE must be a power of two.  One could argue that we
78  * should go to the trouble of setting it up at run time and base it
79  * on memory size rather than the number of compile time CPUs.
80  *
81  * XX64	We should be using physmem size to calculate PIO_SHIFT.
82  *
83  *	These might break in 64 bit world.
84  */
85 #define	PIO_SHIFT	7	/* log2(sizeof(page_t)) */
86 #define	PIO_TABLE_SIZE	128	/* number of io mutexes to have */
87 
88 pad_mutex_t	ph_mutex[PH_TABLE_SIZE];
89 kmutex_t	pio_mutex[PIO_TABLE_SIZE];
90 
91 #define	PAGE_IO_MUTEX(pp) \
92 	    &pio_mutex[(((uintptr_t)pp) >> PIO_SHIFT) & (PIO_TABLE_SIZE - 1)]
93 
94 /*
95  * The pse_mutex[] array is allocated in the platform startup code
96  * based on the size of the machine at startup.
97  */
98 extern pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */
99 extern size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */
100 extern int pse_shift;			/* log2(pse_table_size) */
101 #define	PAGE_SE_MUTEX(pp)	&pse_mutex[				\
102 	((((uintptr_t)(pp) >> pse_shift) ^ ((uintptr_t)(pp))) >> 7) &	\
103 	(pse_table_size - 1)].pad_mutex
104 
105 #define	PSZC_MTX_TABLE_SIZE	128
106 #define	PSZC_MTX_TABLE_SHIFT	7
107 
108 static pad_mutex_t	pszc_mutex[PSZC_MTX_TABLE_SIZE];
109 
110 #define	PAGE_SZC_MUTEX(_pp) \
111 	    &pszc_mutex[((((uintptr_t)(_pp) >> PSZC_MTX_TABLE_SHIFT) ^ \
112 		((uintptr_t)(_pp) >> (PSZC_MTX_TABLE_SHIFT << 1)) ^ \
113 		((uintptr_t)(_pp) >> (3 * PSZC_MTX_TABLE_SHIFT))) & \
114 		(PSZC_MTX_TABLE_SIZE - 1))].pad_mutex
115 
116 /*
117  * The vph_mutex[] array  holds the mutexes to protect the vnode chains,
118  * (i.e., the list of pages anchored by v_pages and connected via p_vpprev
119  * and p_vpnext).
120  *
121  * The page_vnode_mutex(vp) function returns the address of the appropriate
122  * mutex from this array given a pointer to a vnode.  It is complicated
123  * by the fact that the kernel's vnode and the swapfs vnode are referenced
124  * frequently enough to warrent their own mutexes.
125  *
126  * The VP_HASH_FUNC returns the index into the vph_mutex array given
127  * an address of a vnode.
128  */
129 
130 #if defined(_LP64)
131 #define	VPH_TABLE_SIZE  (8 * NCPU_P2)
132 #else	/* 32 bits */
133 #define	VPH_TABLE_SIZE	(2 * NCPU_P2)
134 #endif
135 
136 #define	VP_HASH_FUNC(vp) \
137 	((((uintptr_t)(vp) >> 6) + \
138 	    ((uintptr_t)(vp) >> 8) + \
139 	    ((uintptr_t)(vp) >> 10) + \
140 	    ((uintptr_t)(vp) >> 12)) \
141 	    & (VPH_TABLE_SIZE - 1))
142 
143 /*
144  * Two slots after VPH_TABLE_SIZE are reserved in vph_mutex for kernel vnodes,
145  * one for kvps[KV_ZVP], and one for other kvps[] users.
146  */
147 
148 kmutex_t	vph_mutex[VPH_TABLE_SIZE + 2];
149 
150 /*
151  * Initialize the locks used by the Virtual Memory Management system.
152  */
153 void
page_lock_init()154 page_lock_init()
155 {
156 }
157 
158 /*
159  * Return a value for pse_shift based on npg (the number of physical pages)
160  * and ncpu (the maximum number of CPUs).  This is called by platform startup
161  * code.
162  *
163  * Lockstat data from TPC-H runs showed that contention on the pse_mutex[]
164  * locks grew approximately as the square of the number of threads executing.
165  * So the primary scaling factor used is NCPU^2.  The size of the machine in
166  * megabytes is used as an upper bound, particularly for sun4v machines which
167  * all claim to have 256 CPUs maximum, and the old value of PSE_TABLE_SIZE
168  * (128) is used as a minimum.  Since the size of the table has to be a power
169  * of two, the calculated size is rounded up to the next power of two.
170  */
171 /*ARGSUSED*/
172 int
size_pse_array(pgcnt_t npg,int ncpu)173 size_pse_array(pgcnt_t npg, int ncpu)
174 {
175 	size_t size;
176 	pgcnt_t pp_per_mb = (1024 * 1024) / PAGESIZE;
177 
178 	size = MAX(128, MIN(npg / pp_per_mb, 2 * ncpu * ncpu));
179 	size += (1 << (highbit(size) - 1)) - 1;
180 	return (highbit(size) - 1);
181 }
182 
183 /*
184  * At present we only use page ownership to aid debugging, so it's
185  * OK if the owner field isn't exact.  In the 32-bit world two thread ids
186  * can map to the same owner because we just 'or' in 0x80000000 and
187  * then clear the second highest bit, so that (for example) 0x2faced00
188  * and 0xafaced00 both map to 0xafaced00.
189  * In the 64-bit world, p_selock may not be large enough to hold a full
190  * thread pointer.  If we ever need precise ownership (e.g. if we implement
191  * priority inheritance for page locks) then p_selock should become a
192  * uintptr_t and SE_WRITER should be -((uintptr_t)curthread >> 2).
193  */
194 #define	SE_WRITER	(((selock_t)(ulong_t)curthread | INT_MIN) & ~SE_EWANTED)
195 #define	SE_READER	1
196 
197 /*
198  * A page that is deleted must be marked as such using the
199  * page_lock_delete() function. The page must be exclusively locked.
200  * The SE_DELETED marker is put in p_selock when this function is called.
201  * SE_DELETED must be distinct from any SE_WRITER value.
202  */
203 #define	SE_DELETED	(1 | INT_MIN)
204 
205 #ifdef VM_STATS
206 uint_t	vph_kvp_count;
207 uint_t	vph_swapfsvp_count;
208 uint_t	vph_other;
209 #endif /* VM_STATS */
210 
211 #ifdef VM_STATS
212 uint_t	page_lock_count;
213 uint_t	page_lock_miss;
214 uint_t	page_lock_miss_lock;
215 uint_t	page_lock_reclaim;
216 uint_t	page_lock_bad_reclaim;
217 uint_t	page_lock_same_page;
218 uint_t	page_lock_upgrade;
219 uint_t	page_lock_retired;
220 uint_t	page_lock_upgrade_failed;
221 uint_t	page_lock_deleted;
222 
223 uint_t	page_trylock_locked;
224 uint_t	page_trylock_failed;
225 uint_t	page_trylock_missed;
226 
227 uint_t	page_try_reclaim_upgrade;
228 #endif /* VM_STATS */
229 
230 /*
231  * Acquire the "shared/exclusive" lock on a page.
232  *
233  * Returns 1 on success and locks the page appropriately.
234  *	   0 on failure and does not lock the page.
235  *
236  * If `lock' is non-NULL, it will be dropped and reacquired in the
237  * failure case.  This routine can block, and if it does
238  * it will always return a failure since the page identity [vp, off]
239  * or state may have changed.
240  */
241 
242 int
page_lock(page_t * pp,se_t se,kmutex_t * lock,reclaim_t reclaim)243 page_lock(page_t *pp, se_t se, kmutex_t *lock, reclaim_t reclaim)
244 {
245 	return (page_lock_es(pp, se, lock, reclaim, 0));
246 }
247 
248 /*
249  * With the addition of reader-writer lock semantics to page_lock_es,
250  * callers wanting an exclusive (writer) lock may prevent shared-lock
251  * (reader) starvation by setting the es parameter to SE_EXCL_WANTED.
252  * In this case, when an exclusive lock cannot be acquired, p_selock's
253  * SE_EWANTED bit is set. Shared-lock (reader) requests are also denied
254  * if the page is slated for retirement.
255  *
256  * The se and es parameters determine if the lock should be granted
257  * based on the following decision table:
258  *
259  * Lock wanted   es flags     p_selock/SE_EWANTED  Action
260  * ----------- -------------- -------------------  ---------
261  * SE_EXCL        any [1][2]   unlocked/any        grant lock, clear SE_EWANTED
262  * SE_EXCL        SE_EWANTED   any lock/any        deny, set SE_EWANTED
263  * SE_EXCL        none         any lock/any        deny
264  * SE_SHARED      n/a [2]        shared/0          grant
265  * SE_SHARED      n/a [2]      unlocked/0          grant
266  * SE_SHARED      n/a            shared/1          deny
267  * SE_SHARED      n/a          unlocked/1          deny
268  * SE_SHARED      n/a              excl/any        deny
269  *
270  * Notes:
271  * [1] The code grants an exclusive lock to the caller and clears the bit
272  *   SE_EWANTED whenever p_selock is unlocked, regardless of the SE_EWANTED
273  *   bit's value.  This was deemed acceptable as we are not concerned about
274  *   exclusive-lock starvation. If this ever becomes an issue, a priority or
275  *   fifo mechanism should also be implemented. Meantime, the thread that
276  *   set SE_EWANTED should be prepared to catch this condition and reset it
277  *
278  * [2] Retired pages may not be locked at any time, regardless of the
279  *   dispostion of se, unless the es parameter has SE_RETIRED flag set.
280  *
281  * Notes on values of "es":
282  *
283  *   es & 1: page_lookup_create will attempt page relocation
284  *   es & SE_EXCL_WANTED: caller wants SE_EWANTED set (eg. delete
285  *       memory thread); this prevents reader-starvation of waiting
286  *       writer thread(s) by giving priority to writers over readers.
287  *   es & SE_RETIRED: caller wants to lock pages even if they are
288  *       retired.  Default is to deny the lock if the page is retired.
289  *
290  * And yes, we know, the semantics of this function are too complicated.
291  * It's on the list to be cleaned up.
292  */
293 int
page_lock_es(page_t * pp,se_t se,kmutex_t * lock,reclaim_t reclaim,int es)294 page_lock_es(page_t *pp, se_t se, kmutex_t *lock, reclaim_t reclaim, int es)
295 {
296 	int		retval;
297 	kmutex_t	*pse = PAGE_SE_MUTEX(pp);
298 	int		upgraded;
299 	int		reclaim_it;
300 
301 	ASSERT(lock != NULL ? MUTEX_HELD(lock) : 1);
302 
303 	VM_STAT_ADD(page_lock_count);
304 
305 	upgraded = 0;
306 	reclaim_it = 0;
307 
308 	mutex_enter(pse);
309 
310 	ASSERT(((es & SE_EXCL_WANTED) == 0) ||
311 	    ((es & SE_EXCL_WANTED) && (se == SE_EXCL)));
312 
313 	if (PP_RETIRED(pp) && !(es & SE_RETIRED)) {
314 		mutex_exit(pse);
315 		VM_STAT_ADD(page_lock_retired);
316 		return (0);
317 	}
318 
319 	if (se == SE_SHARED && es == 1 && pp->p_selock == 0) {
320 		se = SE_EXCL;
321 	}
322 
323 	if ((reclaim == P_RECLAIM) && (PP_ISFREE(pp))) {
324 
325 		reclaim_it = 1;
326 		if (se == SE_SHARED) {
327 			/*
328 			 * This is an interesting situation.
329 			 *
330 			 * Remember that p_free can only change if
331 			 * p_selock < 0.
332 			 * p_free does not depend on our holding `pse'.
333 			 * And, since we hold `pse', p_selock can not change.
334 			 * So, if p_free changes on us, the page is already
335 			 * exclusively held, and we would fail to get p_selock
336 			 * regardless.
337 			 *
338 			 * We want to avoid getting the share
339 			 * lock on a free page that needs to be reclaimed.
340 			 * It is possible that some other thread has the share
341 			 * lock and has left the free page on the cache list.
342 			 * pvn_vplist_dirty() does this for brief periods.
343 			 * If the se_share is currently SE_EXCL, we will fail
344 			 * to acquire p_selock anyway.  Blocking is the
345 			 * right thing to do.
346 			 * If we need to reclaim this page, we must get
347 			 * exclusive access to it, force the upgrade now.
348 			 * Again, we will fail to acquire p_selock if the
349 			 * page is not free and block.
350 			 */
351 			upgraded = 1;
352 			se = SE_EXCL;
353 			VM_STAT_ADD(page_lock_upgrade);
354 		}
355 	}
356 
357 	if (se == SE_EXCL) {
358 		if (!(es & SE_EXCL_WANTED) && (pp->p_selock & SE_EWANTED)) {
359 			/*
360 			 * if the caller wants a writer lock (but did not
361 			 * specify exclusive access), and there is a pending
362 			 * writer that wants exclusive access, return failure
363 			 */
364 			retval = 0;
365 		} else if ((pp->p_selock & ~SE_EWANTED) == 0) {
366 			/* no reader/writer lock held */
367 			/* this clears our setting of the SE_EWANTED bit */
368 			pp->p_selock = SE_WRITER;
369 			retval = 1;
370 		} else {
371 			/* page is locked */
372 			if (es & SE_EXCL_WANTED) {
373 				/* set the SE_EWANTED bit */
374 				pp->p_selock |= SE_EWANTED;
375 			}
376 			retval = 0;
377 		}
378 	} else {
379 		retval = 0;
380 		if (pp->p_selock >= 0) {
381 			if ((pp->p_selock & SE_EWANTED) == 0) {
382 				pp->p_selock += SE_READER;
383 				retval = 1;
384 			}
385 		}
386 	}
387 
388 	if (retval == 0) {
389 		if ((pp->p_selock & ~SE_EWANTED) == SE_DELETED) {
390 			VM_STAT_ADD(page_lock_deleted);
391 			mutex_exit(pse);
392 			return (retval);
393 		}
394 
395 #ifdef VM_STATS
396 		VM_STAT_ADD(page_lock_miss);
397 		if (upgraded) {
398 			VM_STAT_ADD(page_lock_upgrade_failed);
399 		}
400 #endif
401 		if (lock) {
402 			VM_STAT_ADD(page_lock_miss_lock);
403 			mutex_exit(lock);
404 		}
405 
406 		/*
407 		 * Now, wait for the page to be unlocked and
408 		 * release the lock protecting p_cv and p_selock.
409 		 */
410 		cv_wait(&pp->p_cv, pse);
411 		mutex_exit(pse);
412 
413 		/*
414 		 * The page identity may have changed while we were
415 		 * blocked.  If we are willing to depend on "pp"
416 		 * still pointing to a valid page structure (i.e.,
417 		 * assuming page structures are not dynamically allocated
418 		 * or freed), we could try to lock the page if its
419 		 * identity hasn't changed.
420 		 *
421 		 * This needs to be measured, since we come back from
422 		 * cv_wait holding pse (the expensive part of this
423 		 * operation) we might as well try the cheap part.
424 		 * Though we would also have to confirm that dropping
425 		 * `lock' did not cause any grief to the callers.
426 		 */
427 		if (lock) {
428 			mutex_enter(lock);
429 		}
430 	} else {
431 		/*
432 		 * We have the page lock.
433 		 * If we needed to reclaim the page, and the page
434 		 * needed reclaiming (ie, it was free), then we
435 		 * have the page exclusively locked.  We may need
436 		 * to downgrade the page.
437 		 */
438 		ASSERT((upgraded) ?
439 		    ((PP_ISFREE(pp)) && PAGE_EXCL(pp)) : 1);
440 		mutex_exit(pse);
441 
442 		/*
443 		 * We now hold this page's lock, either shared or
444 		 * exclusive.  This will prevent its identity from changing.
445 		 * The page, however, may or may not be free.  If the caller
446 		 * requested, and it is free, go reclaim it from the
447 		 * free list.  If the page can't be reclaimed, return failure
448 		 * so that the caller can start all over again.
449 		 *
450 		 * NOTE:page_reclaim() releases the page lock (p_selock)
451 		 *	if it can't be reclaimed.
452 		 */
453 		if (reclaim_it) {
454 			if (!page_reclaim(pp, lock)) {
455 				VM_STAT_ADD(page_lock_bad_reclaim);
456 				retval = 0;
457 			} else {
458 				VM_STAT_ADD(page_lock_reclaim);
459 				if (upgraded) {
460 					page_downgrade(pp);
461 				}
462 			}
463 		}
464 	}
465 	return (retval);
466 }
467 
468 /*
469  * Clear the SE_EWANTED bit from p_selock.  This function allows
470  * callers of page_lock_es and page_try_reclaim_lock to clear
471  * their setting of this bit if they decide they no longer wish
472  * to gain exclusive access to the page.  Currently only
473  * delete_memory_thread uses this when the delete memory
474  * operation is cancelled.
475  */
476 void
page_lock_clr_exclwanted(page_t * pp)477 page_lock_clr_exclwanted(page_t *pp)
478 {
479 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
480 
481 	mutex_enter(pse);
482 	pp->p_selock &= ~SE_EWANTED;
483 	if (CV_HAS_WAITERS(&pp->p_cv))
484 		cv_broadcast(&pp->p_cv);
485 	mutex_exit(pse);
486 }
487 
488 /*
489  * Read the comments inside of page_lock_es() carefully.
490  *
491  * SE_EXCL callers specifying es == SE_EXCL_WANTED will cause the
492  * SE_EWANTED bit of p_selock to be set when the lock cannot be obtained.
493  * This is used by threads subject to reader-starvation (eg. memory delete).
494  *
495  * When a thread using SE_EXCL_WANTED does not obtain the SE_EXCL lock,
496  * it is expected that it will retry at a later time.  Threads that will
497  * not retry the lock *must* call page_lock_clr_exclwanted to clear the
498  * SE_EWANTED bit.  (When a thread using SE_EXCL_WANTED obtains the lock,
499  * the bit is cleared.)
500  */
501 int
page_try_reclaim_lock(page_t * pp,se_t se,int es)502 page_try_reclaim_lock(page_t *pp, se_t se, int es)
503 {
504 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
505 	selock_t old;
506 
507 	mutex_enter(pse);
508 
509 	old = pp->p_selock;
510 
511 	ASSERT(((es & SE_EXCL_WANTED) == 0) ||
512 	    ((es & SE_EXCL_WANTED) && (se == SE_EXCL)));
513 
514 	if (PP_RETIRED(pp) && !(es & SE_RETIRED)) {
515 		mutex_exit(pse);
516 		VM_STAT_ADD(page_trylock_failed);
517 		return (0);
518 	}
519 
520 	if (se == SE_SHARED && es == 1 && old == 0) {
521 		se = SE_EXCL;
522 	}
523 
524 	if (se == SE_SHARED) {
525 		if (!PP_ISFREE(pp)) {
526 			if (old >= 0) {
527 				/*
528 				 * Readers are not allowed when excl wanted
529 				 */
530 				if ((old & SE_EWANTED) == 0) {
531 					pp->p_selock = old + SE_READER;
532 					mutex_exit(pse);
533 					return (1);
534 				}
535 			}
536 			mutex_exit(pse);
537 			return (0);
538 		}
539 		/*
540 		 * The page is free, so we really want SE_EXCL (below)
541 		 */
542 		VM_STAT_ADD(page_try_reclaim_upgrade);
543 	}
544 
545 	/*
546 	 * The caller wants a writer lock.  We try for it only if
547 	 * SE_EWANTED is not set, or if the caller specified
548 	 * SE_EXCL_WANTED.
549 	 */
550 	if (!(old & SE_EWANTED) || (es & SE_EXCL_WANTED)) {
551 		if ((old & ~SE_EWANTED) == 0) {
552 			/* no reader/writer lock held */
553 			/* this clears out our setting of the SE_EWANTED bit */
554 			pp->p_selock = SE_WRITER;
555 			mutex_exit(pse);
556 			return (1);
557 		}
558 	}
559 	if (es & SE_EXCL_WANTED) {
560 		/* page is locked, set the SE_EWANTED bit */
561 		pp->p_selock |= SE_EWANTED;
562 	}
563 	mutex_exit(pse);
564 	return (0);
565 }
566 
567 /*
568  * Acquire a page's "shared/exclusive" lock, but never block.
569  * Returns 1 on success, 0 on failure.
570  */
571 int
page_trylock(page_t * pp,se_t se)572 page_trylock(page_t *pp, se_t se)
573 {
574 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
575 
576 	mutex_enter(pse);
577 	if (pp->p_selock & SE_EWANTED || PP_RETIRED(pp) ||
578 	    (se == SE_SHARED && PP_PR_NOSHARE(pp))) {
579 		/*
580 		 * Fail if a thread wants exclusive access and page is
581 		 * retired, if the page is slated for retirement, or a
582 		 * share lock is requested.
583 		 */
584 		mutex_exit(pse);
585 		VM_STAT_ADD(page_trylock_failed);
586 		return (0);
587 	}
588 
589 	if (se == SE_EXCL) {
590 		if (pp->p_selock == 0) {
591 			pp->p_selock = SE_WRITER;
592 			mutex_exit(pse);
593 			return (1);
594 		}
595 	} else {
596 		if (pp->p_selock >= 0) {
597 			pp->p_selock += SE_READER;
598 			mutex_exit(pse);
599 			return (1);
600 		}
601 	}
602 	mutex_exit(pse);
603 	return (0);
604 }
605 
606 /*
607  * Variant of page_unlock() specifically for the page freelist
608  * code. The mere existence of this code is a vile hack that
609  * has resulted due to the backwards locking order of the page
610  * freelist manager; please don't call it.
611  */
612 void
page_unlock_nocapture(page_t * pp)613 page_unlock_nocapture(page_t *pp)
614 {
615 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
616 	selock_t old;
617 
618 	mutex_enter(pse);
619 
620 	old = pp->p_selock;
621 	if ((old & ~SE_EWANTED) == SE_READER) {
622 		pp->p_selock = old & ~SE_READER;
623 		if (CV_HAS_WAITERS(&pp->p_cv))
624 			cv_broadcast(&pp->p_cv);
625 	} else if ((old & ~SE_EWANTED) == SE_DELETED) {
626 		panic("page_unlock_nocapture: page %p is deleted", (void *)pp);
627 	} else if (old < 0) {
628 		pp->p_selock &= SE_EWANTED;
629 		if (CV_HAS_WAITERS(&pp->p_cv))
630 			cv_broadcast(&pp->p_cv);
631 	} else if ((old & ~SE_EWANTED) > SE_READER) {
632 		pp->p_selock = old - SE_READER;
633 	} else {
634 		panic("page_unlock_nocapture: page %p is not locked",
635 		    (void *)pp);
636 	}
637 
638 	mutex_exit(pse);
639 }
640 
641 /*
642  * Release the page's "shared/exclusive" lock and wake up anyone
643  * who might be waiting for it.
644  */
645 void
page_unlock(page_t * pp)646 page_unlock(page_t *pp)
647 {
648 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
649 	selock_t old;
650 
651 	mutex_enter(pse);
652 
653 	old = pp->p_selock;
654 	if ((old & ~SE_EWANTED) == SE_READER) {
655 		pp->p_selock = old & ~SE_READER;
656 		if (CV_HAS_WAITERS(&pp->p_cv))
657 			cv_broadcast(&pp->p_cv);
658 	} else if ((old & ~SE_EWANTED) == SE_DELETED) {
659 		panic("page_unlock: page %p is deleted", (void *)pp);
660 	} else if (old < 0) {
661 		pp->p_selock &= SE_EWANTED;
662 		if (CV_HAS_WAITERS(&pp->p_cv))
663 			cv_broadcast(&pp->p_cv);
664 	} else if ((old & ~SE_EWANTED) > SE_READER) {
665 		pp->p_selock = old - SE_READER;
666 	} else {
667 		panic("page_unlock: page %p is not locked", (void *)pp);
668 	}
669 
670 	if (pp->p_selock == 0) {
671 		/*
672 		 * If the T_CAPTURING bit is set, that means that we should
673 		 * not try and capture the page again as we could recurse
674 		 * which could lead to a stack overflow panic or spending a
675 		 * relatively long time in the kernel making no progress.
676 		 */
677 		if ((pp->p_toxic & PR_CAPTURE) &&
678 		    !(curthread->t_flag & T_CAPTURING) &&
679 		    !PP_RETIRED(pp)) {
680 			pp->p_selock = SE_WRITER;
681 			mutex_exit(pse);
682 			page_unlock_capture(pp);
683 		} else {
684 			mutex_exit(pse);
685 		}
686 	} else {
687 		mutex_exit(pse);
688 	}
689 }
690 
691 /*
692  * Try to upgrade the lock on the page from a "shared" to an
693  * "exclusive" lock.  Since this upgrade operation is done while
694  * holding the mutex protecting this page, no one else can acquire this page's
695  * lock and change the page. Thus, it is safe to drop the "shared"
696  * lock and attempt to acquire the "exclusive" lock.
697  *
698  * Returns 1 on success, 0 on failure.
699  */
700 int
page_tryupgrade(page_t * pp)701 page_tryupgrade(page_t *pp)
702 {
703 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
704 
705 	mutex_enter(pse);
706 	if (!(pp->p_selock & SE_EWANTED)) {
707 		/* no threads want exclusive access, try upgrade */
708 		if (pp->p_selock == SE_READER) {
709 			/* convert to exclusive lock */
710 			pp->p_selock = SE_WRITER;
711 			mutex_exit(pse);
712 			return (1);
713 		}
714 	}
715 	mutex_exit(pse);
716 	return (0);
717 }
718 
719 /*
720  * Downgrade the "exclusive" lock on the page to a "shared" lock
721  * while holding the mutex protecting this page's p_selock field.
722  */
723 void
page_downgrade(page_t * pp)724 page_downgrade(page_t *pp)
725 {
726 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
727 	int excl_waiting;
728 
729 	ASSERT((pp->p_selock & ~SE_EWANTED) != SE_DELETED);
730 	ASSERT(PAGE_EXCL(pp));
731 
732 	mutex_enter(pse);
733 	excl_waiting =  pp->p_selock & SE_EWANTED;
734 	pp->p_selock = SE_READER | excl_waiting;
735 	if (CV_HAS_WAITERS(&pp->p_cv))
736 		cv_broadcast(&pp->p_cv);
737 	mutex_exit(pse);
738 }
739 
740 void
page_lock_delete(page_t * pp)741 page_lock_delete(page_t *pp)
742 {
743 	kmutex_t *pse = PAGE_SE_MUTEX(pp);
744 
745 	ASSERT(PAGE_EXCL(pp));
746 	ASSERT(pp->p_vnode == NULL);
747 	ASSERT(pp->p_offset == (u_offset_t)-1);
748 	ASSERT(!PP_ISFREE(pp));
749 
750 	mutex_enter(pse);
751 	pp->p_selock = SE_DELETED;
752 	if (CV_HAS_WAITERS(&pp->p_cv))
753 		cv_broadcast(&pp->p_cv);
754 	mutex_exit(pse);
755 }
756 
757 int
page_deleted(page_t * pp)758 page_deleted(page_t *pp)
759 {
760 	return (pp->p_selock == SE_DELETED);
761 }
762 
763 /*
764  * Implement the io lock for pages
765  */
766 void
page_iolock_init(page_t * pp)767 page_iolock_init(page_t *pp)
768 {
769 	pp->p_iolock_state = 0;
770 	cv_init(&pp->p_io_cv, NULL, CV_DEFAULT, NULL);
771 }
772 
773 /*
774  * Acquire the i/o lock on a page.
775  */
776 void
page_io_lock(page_t * pp)777 page_io_lock(page_t *pp)
778 {
779 	kmutex_t *pio;
780 
781 	pio = PAGE_IO_MUTEX(pp);
782 	mutex_enter(pio);
783 	while (pp->p_iolock_state & PAGE_IO_INUSE) {
784 		cv_wait(&(pp->p_io_cv), pio);
785 	}
786 	pp->p_iolock_state |= PAGE_IO_INUSE;
787 	mutex_exit(pio);
788 }
789 
790 /*
791  * Release the i/o lock on a page.
792  */
793 void
page_io_unlock(page_t * pp)794 page_io_unlock(page_t *pp)
795 {
796 	kmutex_t *pio;
797 
798 	pio = PAGE_IO_MUTEX(pp);
799 	mutex_enter(pio);
800 	cv_broadcast(&pp->p_io_cv);
801 	pp->p_iolock_state &= ~PAGE_IO_INUSE;
802 	mutex_exit(pio);
803 }
804 
805 /*
806  * Try to acquire the i/o lock on a page without blocking.
807  * Returns 1 on success, 0 on failure.
808  */
809 int
page_io_trylock(page_t * pp)810 page_io_trylock(page_t *pp)
811 {
812 	kmutex_t *pio;
813 
814 	if (pp->p_iolock_state & PAGE_IO_INUSE)
815 		return (0);
816 
817 	pio = PAGE_IO_MUTEX(pp);
818 	mutex_enter(pio);
819 
820 	if (pp->p_iolock_state & PAGE_IO_INUSE) {
821 		mutex_exit(pio);
822 		return (0);
823 	}
824 	pp->p_iolock_state |= PAGE_IO_INUSE;
825 	mutex_exit(pio);
826 
827 	return (1);
828 }
829 
830 /*
831  * Wait until the i/o lock is not held.
832  */
833 void
page_io_wait(page_t * pp)834 page_io_wait(page_t *pp)
835 {
836 	kmutex_t *pio;
837 
838 	pio = PAGE_IO_MUTEX(pp);
839 	mutex_enter(pio);
840 	while (pp->p_iolock_state & PAGE_IO_INUSE) {
841 		cv_wait(&(pp->p_io_cv), pio);
842 	}
843 	mutex_exit(pio);
844 }
845 
846 /*
847  * Returns 1 on success, 0 on failure.
848  */
849 int
page_io_locked(page_t * pp)850 page_io_locked(page_t *pp)
851 {
852 	return (pp->p_iolock_state & PAGE_IO_INUSE);
853 }
854 
855 /*
856  * Assert that the i/o lock on a page is held.
857  * Returns 1 on success, 0 on failure.
858  */
859 int
page_iolock_assert(page_t * pp)860 page_iolock_assert(page_t *pp)
861 {
862 	return (page_io_locked(pp));
863 }
864 
865 /*
866  * Wrapper exported to kernel routines that are built
867  * platform-independent (the macro is platform-dependent;
868  * the size of vph_mutex[] is based on NCPU).
869  *
870  * Note that you can do stress testing on this by setting the
871  * variable page_vnode_mutex_stress to something other than
872  * zero in a DEBUG kernel in a debugger after loading the kernel.
873  * Setting it after the kernel is running may not work correctly.
874  */
875 #ifdef DEBUG
876 static int page_vnode_mutex_stress = 0;
877 #endif
878 
879 kmutex_t *
page_vnode_mutex(vnode_t * vp)880 page_vnode_mutex(vnode_t *vp)
881 {
882 	if (vp == &kvp || vp == &kvps[KV_VVP])
883 		return (&vph_mutex[VPH_TABLE_SIZE + 0]);
884 
885 	if (vp == &kvps[KV_ZVP])
886 		return (&vph_mutex[VPH_TABLE_SIZE + 1]);
887 #ifdef DEBUG
888 	if (page_vnode_mutex_stress != 0)
889 		return (&vph_mutex[0]);
890 #endif
891 
892 	return (&vph_mutex[VP_HASH_FUNC(vp)]);
893 }
894 
895 kmutex_t *
page_se_mutex(page_t * pp)896 page_se_mutex(page_t *pp)
897 {
898 	return (PAGE_SE_MUTEX(pp));
899 }
900 
901 #ifdef VM_STATS
902 uint_t pszclck_stat[4];
903 #endif
904 /*
905  * Find, take and return a mutex held by hat_page_demote().
906  * Called by page_demote_vp_pages() before hat_page_demote() call and by
907  * routines that want to block hat_page_demote() but can't do it
908  * via locking all constituent pages.
909  *
910  * Return NULL if p_szc is 0.
911  *
912  * It should only be used for pages that can be demoted by hat_page_demote()
913  * i.e. non swapfs file system pages.  The logic here is lifted from
914  * sfmmu_mlspl_enter() except there's no need to worry about p_szc increase
915  * since the page is locked and not free.
916  *
917  * Hash of the root page is used to find the lock.
918  * To find the root in the presense of hat_page_demote() chageing the location
919  * of the root this routine relies on the fact that hat_page_demote() changes
920  * root last.
921  *
922  * If NULL is returned pp's p_szc is guaranteed to be 0. If non NULL is
923  * returned pp's p_szc may be any value.
924  */
925 kmutex_t *
page_szc_lock(page_t * pp)926 page_szc_lock(page_t *pp)
927 {
928 	kmutex_t	*mtx;
929 	page_t		*rootpp;
930 	uint_t		szc;
931 	uint_t		rszc;
932 	uint_t		pszc = pp->p_szc;
933 
934 	ASSERT(pp != NULL);
935 	ASSERT(PAGE_LOCKED(pp));
936 	ASSERT(!PP_ISFREE(pp));
937 	ASSERT(pp->p_vnode != NULL);
938 	ASSERT(!IS_SWAPFSVP(pp->p_vnode));
939 	ASSERT(!PP_ISKAS(pp));
940 
941 again:
942 	if (pszc == 0) {
943 		VM_STAT_ADD(pszclck_stat[0]);
944 		return (NULL);
945 	}
946 
947 	/* The lock lives in the root page */
948 
949 	rootpp = PP_GROUPLEADER(pp, pszc);
950 	mtx = PAGE_SZC_MUTEX(rootpp);
951 	mutex_enter(mtx);
952 
953 	/*
954 	 * since p_szc can only decrease if pp == rootpp
955 	 * rootpp will be always the same i.e we have the right root
956 	 * regardless of rootpp->p_szc.
957 	 * If location of pp's root didn't change after we took
958 	 * the lock we have the right root. return mutex hashed off it.
959 	 */
960 	if (pp == rootpp || (rszc = rootpp->p_szc) == pszc) {
961 		VM_STAT_ADD(pszclck_stat[1]);
962 		return (mtx);
963 	}
964 
965 	/*
966 	 * root location changed because page got demoted.
967 	 * locate the new root.
968 	 */
969 	if (rszc < pszc) {
970 		szc = pp->p_szc;
971 		ASSERT(szc < pszc);
972 		mutex_exit(mtx);
973 		pszc = szc;
974 		VM_STAT_ADD(pszclck_stat[2]);
975 		goto again;
976 	}
977 
978 	VM_STAT_ADD(pszclck_stat[3]);
979 	/*
980 	 * current hat_page_demote not done yet.
981 	 * wait for it to finish.
982 	 */
983 	mutex_exit(mtx);
984 	rootpp = PP_GROUPLEADER(rootpp, rszc);
985 	mtx = PAGE_SZC_MUTEX(rootpp);
986 	mutex_enter(mtx);
987 	mutex_exit(mtx);
988 	ASSERT(rootpp->p_szc < rszc);
989 	goto again;
990 }
991 
992 int
page_szc_lock_assert(page_t * pp)993 page_szc_lock_assert(page_t *pp)
994 {
995 	page_t *rootpp = PP_PAGEROOT(pp);
996 	kmutex_t *mtx = PAGE_SZC_MUTEX(rootpp);
997 
998 	return (MUTEX_HELD(mtx));
999 }
1000 
1001 /*
1002  * memseg locking
1003  */
1004 static krwlock_t memsegslock;
1005 
1006 /*
1007  * memlist (phys_install, phys_avail) locking.
1008  */
1009 static krwlock_t memlists_lock;
1010 
1011 int
memsegs_trylock(int writer)1012 memsegs_trylock(int writer)
1013 {
1014 	return (rw_tryenter(&memsegslock, writer ? RW_WRITER : RW_READER));
1015 }
1016 
1017 void
memsegs_lock(int writer)1018 memsegs_lock(int writer)
1019 {
1020 	rw_enter(&memsegslock, writer ? RW_WRITER : RW_READER);
1021 }
1022 
1023 /*ARGSUSED*/
1024 void
memsegs_unlock(int writer)1025 memsegs_unlock(int writer)
1026 {
1027 	rw_exit(&memsegslock);
1028 }
1029 
1030 int
memsegs_lock_held(void)1031 memsegs_lock_held(void)
1032 {
1033 	return (RW_LOCK_HELD(&memsegslock));
1034 }
1035 
1036 void
memlist_read_lock(void)1037 memlist_read_lock(void)
1038 {
1039 	rw_enter(&memlists_lock, RW_READER);
1040 }
1041 
1042 void
memlist_read_unlock(void)1043 memlist_read_unlock(void)
1044 {
1045 	rw_exit(&memlists_lock);
1046 }
1047 
1048 void
memlist_write_lock(void)1049 memlist_write_lock(void)
1050 {
1051 	rw_enter(&memlists_lock, RW_WRITER);
1052 }
1053 
1054 void
memlist_write_unlock(void)1055 memlist_write_unlock(void)
1056 {
1057 	rw_exit(&memlists_lock);
1058 }
1059