xref: /illumos-gate/usr/src/uts/common/vm/anon.h (revision 8905f42c)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
507b65a64Saguzovsk  * Common Development and Distribution License (the "License").
607b65a64Saguzovsk  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2223d9e5acSMichael Corcoran  * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23*8905f42cSBryan Cantrill  * Copyright (c) 2015, Joyent, Inc. All rights reserved.
247c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
277c478bd9Sstevel@tonic-gate /*	 All Rights Reserved   */
297c478bd9Sstevel@tonic-gate /*
307c478bd9Sstevel@tonic-gate  * University Copyright- Copyright (c) 1982, 1986, 1988
317c478bd9Sstevel@tonic-gate  * The Regents of the University of California
327c478bd9Sstevel@tonic-gate  * All Rights Reserved
337c478bd9Sstevel@tonic-gate  *
347c478bd9Sstevel@tonic-gate  * University Acknowledgment- Portions of this document are derived from
357c478bd9Sstevel@tonic-gate  * software developed by the University of California, Berkeley, and its
367c478bd9Sstevel@tonic-gate  * contributors.
377c478bd9Sstevel@tonic-gate  */
397c478bd9Sstevel@tonic-gate #ifndef	_VM_ANON_H
407c478bd9Sstevel@tonic-gate #define	_VM_ANON_H
427c478bd9Sstevel@tonic-gate #include <sys/cred.h>
430209230bSgjelinek #include <sys/zone.h>
447c478bd9Sstevel@tonic-gate #include <vm/seg.h>
457c478bd9Sstevel@tonic-gate #include <vm/vpage.h>
477c478bd9Sstevel@tonic-gate #ifdef	__cplusplus
487c478bd9Sstevel@tonic-gate extern "C" {
497c478bd9Sstevel@tonic-gate #endif
517c478bd9Sstevel@tonic-gate /*
527c478bd9Sstevel@tonic-gate  * VM - Anonymous pages.
537c478bd9Sstevel@tonic-gate  */
557c478bd9Sstevel@tonic-gate typedef	unsigned long anoff_t;		/* anon offsets */
577c478bd9Sstevel@tonic-gate /*
587c478bd9Sstevel@tonic-gate  *	Each anonymous page, either in memory or in swap, has an anon structure.
597c478bd9Sstevel@tonic-gate  * The structure (slot) provides a level of indirection between anonymous pages
607c478bd9Sstevel@tonic-gate  * and their backing store.
617c478bd9Sstevel@tonic-gate  *
627c478bd9Sstevel@tonic-gate  *	(an_vp, an_off) names the vnode of the anonymous page for this slot.
637c478bd9Sstevel@tonic-gate  *
647c478bd9Sstevel@tonic-gate  * 	(an_pvp, an_poff) names the location of the physical backing store
657c478bd9Sstevel@tonic-gate  * 	for the page this slot represents. If the name is null there is no
667c478bd9Sstevel@tonic-gate  * 	associated physical store. The physical backing store location can
677c478bd9Sstevel@tonic-gate  *	change while the slot is in use.
687c478bd9Sstevel@tonic-gate  *
697c478bd9Sstevel@tonic-gate  *	an_hash is a hash list of anon slots. The list is hashed by
707c478bd9Sstevel@tonic-gate  * 	(an_vp, an_off) of the associated anonymous page and provides a
717c478bd9Sstevel@tonic-gate  *	method of going from the name of an anonymous page to its
727c478bd9Sstevel@tonic-gate  * 	associated anon slot.
737c478bd9Sstevel@tonic-gate  *
747c478bd9Sstevel@tonic-gate  *	an_refcnt holds a reference count which is the number of separate
757c478bd9Sstevel@tonic-gate  * 	copies that will need to be created in case of copy-on-write.
767c478bd9Sstevel@tonic-gate  *	A refcnt > 0 protects the existence of the slot. The refcnt is
777c478bd9Sstevel@tonic-gate  * 	initialized to 1 when the anon slot is created in anon_alloc().
787c478bd9Sstevel@tonic-gate  *	If a client obtains an anon slot and allows multiple threads to
797c478bd9Sstevel@tonic-gate  * 	share it, then it is the client's responsibility to insure that
807c478bd9Sstevel@tonic-gate  *	it does not allow one thread to try to reference the slot at the
817c478bd9Sstevel@tonic-gate  *	same time as another is trying to decrement the last count and
827c478bd9Sstevel@tonic-gate  *	destroy the anon slot. E.g., the seg_vn segment type protects
837c478bd9Sstevel@tonic-gate  *	against this with higher level locks.
847c478bd9Sstevel@tonic-gate  */
867c478bd9Sstevel@tonic-gate struct anon {
877c478bd9Sstevel@tonic-gate 	struct vnode *an_vp;	/* vnode of anon page */
887c478bd9Sstevel@tonic-gate 	struct vnode *an_pvp;	/* vnode of physical backing store */
897c478bd9Sstevel@tonic-gate 	anoff_t an_off;		/* offset of anon page */
907c478bd9Sstevel@tonic-gate 	anoff_t an_poff;	/* offset in vnode */
917c478bd9Sstevel@tonic-gate 	struct anon *an_hash;	/* hash table of anon slots */
927c478bd9Sstevel@tonic-gate 	int an_refcnt;		/* # of people sharing slot */
937c478bd9Sstevel@tonic-gate };
95cb15d5d9SPeter Rival #define	AN_CACHE_ALIGN_LOG2	4	/* log2(AN_CACHE_ALIGN) */
96cb15d5d9SPeter Rival #define	AN_CACHE_ALIGN	(1U << AN_CACHE_ALIGN_LOG2) /* anon address aligned */
97cb15d5d9SPeter Rival 						/* 16 bytes */
98cb15d5d9SPeter Rival 
99cb15d5d9SPeter Rival 
1007c478bd9Sstevel@tonic-gate #ifdef _KERNEL
1017c478bd9Sstevel@tonic-gate /*
1027c478bd9Sstevel@tonic-gate  * The swapinfo_lock protects:
1037c478bd9Sstevel@tonic-gate  *		swapinfo list
1047c478bd9Sstevel@tonic-gate  *		individual swapinfo structures
1057c478bd9Sstevel@tonic-gate  *
1067c478bd9Sstevel@tonic-gate  * The anoninfo_lock protects:
1077c478bd9Sstevel@tonic-gate  *		anoninfo counters
1087c478bd9Sstevel@tonic-gate  *
1097c478bd9Sstevel@tonic-gate  * The anonhash_lock protects:
1107c478bd9Sstevel@tonic-gate  *		anon hash lists
1117c478bd9Sstevel@tonic-gate  *		anon slot fields
1127c478bd9Sstevel@tonic-gate  *
1137c478bd9Sstevel@tonic-gate  * Fields in the anon slot which are read-only for the life of the slot
1147c478bd9Sstevel@tonic-gate  * (an_vp, an_off) do not require the anonhash_lock be held to access them.
1157c478bd9Sstevel@tonic-gate  * If you access a field without the anonhash_lock held you must be holding
1167c478bd9Sstevel@tonic-gate  * the slot with an_refcnt to make sure it isn't destroyed.
1177c478bd9Sstevel@tonic-gate  * To write (an_pvp, an_poff) in a given slot you must also hold the
1187c478bd9Sstevel@tonic-gate  * p_iolock of the anonymous page for slot.
1197c478bd9Sstevel@tonic-gate  */
1207c478bd9Sstevel@tonic-gate extern kmutex_t anoninfo_lock;
1217c478bd9Sstevel@tonic-gate extern kmutex_t swapinfo_lock;
12223d9e5acSMichael Corcoran extern pad_mutex_t *anonhash_lock;
1237c478bd9Sstevel@tonic-gate extern pad_mutex_t anon_array_lock[];
1247c478bd9Sstevel@tonic-gate extern kcondvar_t anon_array_cv[];
1267c478bd9Sstevel@tonic-gate /*
1277c478bd9Sstevel@tonic-gate  * Global hash table to provide a function from (vp, off) -> ap
1287c478bd9Sstevel@tonic-gate  */
1297c478bd9Sstevel@tonic-gate extern size_t anon_hash_size;
130cb15d5d9SPeter Rival extern unsigned int anon_hash_shift;
1317c478bd9Sstevel@tonic-gate extern struct anon **anon_hash;
1327c478bd9Sstevel@tonic-gate #define	ANON_HASH_SIZE	anon_hash_size
1337c478bd9Sstevel@tonic-gate #define	ANON_HASHAVELEN	4
134cb15d5d9SPeter Rival /*
135cb15d5d9SPeter Rival  * Try to use as many bits of randomness from both vp and off as we can.
136cb15d5d9SPeter Rival  * This should help spreading evenly for a variety of workloads.  See comments
137cb15d5d9SPeter Rival  * for PAGE_HASH_FUNC for more explanation.
138cb15d5d9SPeter Rival  */
139cb15d5d9SPeter Rival #define	ANON_HASH(vp, off)	\
140cb15d5d9SPeter Rival 	(((((uintptr_t)(off) >> PAGESHIFT) ^ \
141cb15d5d9SPeter Rival 		((uintptr_t)(off) >> (PAGESHIFT + anon_hash_shift))) ^ \
142cb15d5d9SPeter Rival 		(((uintptr_t)(vp) >> 3) ^ \
143cb15d5d9SPeter Rival 		((uintptr_t)(vp) >> (3 + anon_hash_shift)) ^ \
144cb15d5d9SPeter Rival 		((uintptr_t)(vp) >> (3 + 2 * anon_hash_shift)) ^ \
145cb15d5d9SPeter Rival 		((uintptr_t)(vp) << \
146cb15d5d9SPeter Rival 		    (anon_hash_shift - AN_VPSHIFT - VNODE_ALIGN_LOG2)))) & \
147cb15d5d9SPeter Rival 		(anon_hash_size - 1))
14923d9e5acSMichael Corcoran #define	AH_LOCK_SIZE	(2 << NCPU_LOG2)
15023d9e5acSMichael Corcoran 
15123d9e5acSMichael Corcoran #define	AH_MUTEX(vp, off)				\
15223d9e5acSMichael Corcoran 	(&anonhash_lock[(ANON_HASH((vp), (off)) &	\
15323d9e5acSMichael Corcoran 	    (AH_LOCK_SIZE - 1))].pad_mutex)
1557c478bd9Sstevel@tonic-gate #endif	/* _KERNEL */
1577c478bd9Sstevel@tonic-gate /*
1587c478bd9Sstevel@tonic-gate  * Declaration for the Global counters to accurately
1597c478bd9Sstevel@tonic-gate  * track the kernel foot print in memory.
1607c478bd9Sstevel@tonic-gate  */
1617c478bd9Sstevel@tonic-gate extern  pgcnt_t pages_locked;
1627c478bd9Sstevel@tonic-gate extern  pgcnt_t pages_claimed;
1637c478bd9Sstevel@tonic-gate extern  pgcnt_t pages_useclaim;
1647c478bd9Sstevel@tonic-gate extern  pgcnt_t obp_pages;
1667c478bd9Sstevel@tonic-gate /*
1677c478bd9Sstevel@tonic-gate  * Anonymous backing store accounting structure for swapctl.
1687c478bd9Sstevel@tonic-gate  *
1697c478bd9Sstevel@tonic-gate  * ani_max = maximum amount of swap space
1707c478bd9Sstevel@tonic-gate  *	(including potentially available physical memory)
1717c478bd9Sstevel@tonic-gate  * ani_free = amount of unallocated anonymous memory
1727c478bd9Sstevel@tonic-gate  *	(some of which might be reserved and including
1737c478bd9Sstevel@tonic-gate  *	potentially available physical memory)
1747c478bd9Sstevel@tonic-gate  * ani_resv = amount of claimed (reserved) anonymous memory
1757c478bd9Sstevel@tonic-gate  *
1767c478bd9Sstevel@tonic-gate  * The swap data can be aquired more efficiently through the
1777c478bd9Sstevel@tonic-gate  * kstats interface.
1787c478bd9Sstevel@tonic-gate  * Total slots currently available for reservation =
1797c478bd9Sstevel@tonic-gate  *	MAX(ani_max - ani_resv, 0) + (availrmem - swapfs_minfree)
1807c478bd9Sstevel@tonic-gate  */
1817c478bd9Sstevel@tonic-gate struct anoninfo {
1827c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_max;
1837c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_free;
1847c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_resv;
1857c478bd9Sstevel@tonic-gate };
1877c478bd9Sstevel@tonic-gate #ifdef _SYSCALL32
1887c478bd9Sstevel@tonic-gate struct anoninfo32 {
1897c478bd9Sstevel@tonic-gate 	size32_t ani_max;
1907c478bd9Sstevel@tonic-gate 	size32_t ani_free;
1917c478bd9Sstevel@tonic-gate 	size32_t ani_resv;
1927c478bd9Sstevel@tonic-gate };
1937c478bd9Sstevel@tonic-gate #endif /* _SYSCALL32 */
1957c478bd9Sstevel@tonic-gate /*
1967c478bd9Sstevel@tonic-gate  * Define the NCPU pool of the ani_free counters. Update the counter
1977c478bd9Sstevel@tonic-gate  * of the cpu on which the thread is running and in every clock intr
1987c478bd9Sstevel@tonic-gate  * sync anoninfo.ani_free with the current total off all the NCPU entries.
1997c478bd9Sstevel@tonic-gate  */
2017c478bd9Sstevel@tonic-gate typedef	struct	ani_free {
2027c478bd9Sstevel@tonic-gate 	pgcnt_t		ani_count;
203b52a336eSPavel Tatashin 	uchar_t		pad[64 - sizeof (pgcnt_t)];
2047c478bd9Sstevel@tonic-gate 			/* XXX 64 = cacheline size */
2057c478bd9Sstevel@tonic-gate } ani_free_t;
207b52a336eSPavel Tatashin #define	ANI_MAX_POOL	(NCPU_P2)
208b52a336eSPavel Tatashin extern	ani_free_t	*ani_free_pool;
210b52a336eSPavel Tatashin /*
211b52a336eSPavel Tatashin  * Since each CPU has its own bucket in ani_free_pool, there should be no
212b52a336eSPavel Tatashin  * contention here.
213b52a336eSPavel Tatashin  */
2147c478bd9Sstevel@tonic-gate #define	ANI_ADD(inc)	{ \
215b52a336eSPavel Tatashin 	pgcnt_t	*ani_countp; \
216b52a336eSPavel Tatashin 	int	index; \
217b52a336eSPavel Tatashin 	index = (CPU->cpu_seqid & (ANI_MAX_POOL - 1)); \
218b52a336eSPavel Tatashin 	ani_countp = &ani_free_pool[index].ani_count; \
219b52a336eSPavel Tatashin 	atomic_add_long(ani_countp, inc); \
2207c478bd9Sstevel@tonic-gate }
222b52a336eSPavel Tatashin extern void	set_anoninfo(void);
223b52a336eSPavel Tatashin 
2247c478bd9Sstevel@tonic-gate /*
2257c478bd9Sstevel@tonic-gate  * Anon array pointers are allocated in chunks. Each chunk
2267c478bd9Sstevel@tonic-gate  * has PAGESIZE/sizeof(u_long *) of anon pointers.
2277c478bd9Sstevel@tonic-gate  * There are two levels of arrays for anon array pointers larger
2287c478bd9Sstevel@tonic-gate  * than a chunk. The first level points to anon array chunks.
2297c478bd9Sstevel@tonic-gate  * The second level consists of chunks of anon pointers.
2307c478bd9Sstevel@tonic-gate  *
2317c478bd9Sstevel@tonic-gate  * If anon array is smaller than a chunk then the whole anon array
2327c478bd9Sstevel@tonic-gate  * is created (memory is allocated for whole anon array).
2337c478bd9Sstevel@tonic-gate  * If anon array is larger than a chunk only first level array is
2347c478bd9Sstevel@tonic-gate  * allocated. Then other arrays (chunks) are allocated only when
2357c478bd9Sstevel@tonic-gate  * they are initialized with anon pointers.
2367c478bd9Sstevel@tonic-gate  */
2377c478bd9Sstevel@tonic-gate struct anon_hdr {
2387c478bd9Sstevel@tonic-gate 	kmutex_t serial_lock;	/* serialize array chunk allocation */
2397c478bd9Sstevel@tonic-gate 	pgcnt_t	size;		/* number of pointers to (anon) pages */
2407c478bd9Sstevel@tonic-gate 	void	**array_chunk;	/* pointers to anon pointers or chunks of */
2417c478bd9Sstevel@tonic-gate 				/* anon pointers */
2427c478bd9Sstevel@tonic-gate 	int	flags;		/* ANON_ALLOC_FORCE force preallocation of */
2437c478bd9Sstevel@tonic-gate 				/* whole anon array	*/
2447c478bd9Sstevel@tonic-gate };
2467c478bd9Sstevel@tonic-gate #ifdef	_LP64
2477c478bd9Sstevel@tonic-gate #define	ANON_PTRSHIFT	3
2487c478bd9Sstevel@tonic-gate #define	ANON_PTRMASK	~7
2497c478bd9Sstevel@tonic-gate #else
2507c478bd9Sstevel@tonic-gate #define	ANON_PTRSHIFT	2
2517c478bd9Sstevel@tonic-gate #define	ANON_PTRMASK	~3
2527c478bd9Sstevel@tonic-gate #endif
2547c478bd9Sstevel@tonic-gate #define	ANON_CHUNK_SIZE		(PAGESIZE >> ANON_PTRSHIFT)
2557c478bd9Sstevel@tonic-gate #define	ANON_CHUNK_SHIFT	(PAGESHIFT - ANON_PTRSHIFT)
2567c478bd9Sstevel@tonic-gate #define	ANON_CHUNK_OFF		(ANON_CHUNK_SIZE - 1)
2587c478bd9Sstevel@tonic-gate /*
2597c478bd9Sstevel@tonic-gate  * Anon flags.
2607c478bd9Sstevel@tonic-gate  */
2617c478bd9Sstevel@tonic-gate #define	ANON_SLEEP		0x0	/* ok to block */
2627c478bd9Sstevel@tonic-gate #define	ANON_NOSLEEP		0x1	/* non-blocking call */
2637c478bd9Sstevel@tonic-gate #define	ANON_ALLOC_FORCE	0x2	/* force single level anon array */
2647c478bd9Sstevel@tonic-gate #define	ANON_GROWDOWN		0x4	/* anon array should grow downward */
266c6939658Ssl struct kshmid;
2687c478bd9Sstevel@tonic-gate /*
2697c478bd9Sstevel@tonic-gate  * The anon_map structure is used by various clients of the anon layer to
2707c478bd9Sstevel@tonic-gate  * manage anonymous memory.   When anonymous memory is shared,
2717c478bd9Sstevel@tonic-gate  * then the different clients sharing it will point to the
2727c478bd9Sstevel@tonic-gate  * same anon_map structure.  Also, if a segment is unmapped
2737c478bd9Sstevel@tonic-gate  * in the middle where an anon_map structure exists, the
2747c478bd9Sstevel@tonic-gate  * newly created segment will also share the anon_map structure,
2757c478bd9Sstevel@tonic-gate  * although the two segments will use different ranges of the
2767c478bd9Sstevel@tonic-gate  * anon array.  When mappings are private (or shared with
2777c478bd9Sstevel@tonic-gate  * a reference count of 1), an unmap operation will free up
2787c478bd9Sstevel@tonic-gate  * a range of anon slots in the array given by the anon_map
2797c478bd9Sstevel@tonic-gate  * structure.  Because of fragmentation due to this unmapping,
2807c478bd9Sstevel@tonic-gate  * we have to store the size of the anon array in the anon_map
2817c478bd9Sstevel@tonic-gate  * structure so that we can free everything when the referernce
2827c478bd9Sstevel@tonic-gate  * count goes to zero.
2837c478bd9Sstevel@tonic-gate  *
2847c478bd9Sstevel@tonic-gate  * A new rangelock scheme is introduced to make the anon layer scale.
2857c478bd9Sstevel@tonic-gate  * A reader/writer lock per anon_amp and an array of system-wide hash
2867c478bd9Sstevel@tonic-gate  * locks, anon_array_lock[] are introduced to replace serial_lock and
2877c478bd9Sstevel@tonic-gate  * anonmap lock.  The writer lock is held when we want to singlethreaD
2887c478bd9Sstevel@tonic-gate  * the reference to the anon array pointers or when references to
2897c478bd9Sstevel@tonic-gate  * anon_map's members, whereas reader lock and anon_array_lock are
2907c478bd9Sstevel@tonic-gate  * held to allows multiple threads to reference different part of
2917c478bd9Sstevel@tonic-gate  * anon array.  A global set of condition variables, anon_array_cv,
2927c478bd9Sstevel@tonic-gate  * are used with anon_array_lock[] to make the hold time of the locks
2937c478bd9Sstevel@tonic-gate  * short.
2947c478bd9Sstevel@tonic-gate  *
2957c478bd9Sstevel@tonic-gate  * szc is used to calculate the index of hash locks and cv's.  We
2967c478bd9Sstevel@tonic-gate  * could've just used seg->s_szc if not for the possible sharing of
2977c478bd9Sstevel@tonic-gate  * anon_amp between SYSV shared memory and ISM, so now we introduce
2987c478bd9Sstevel@tonic-gate  * szc in the anon_map structure.  For MAP_SHARED, the amp->szc is either
2997c478bd9Sstevel@tonic-gate  * 0 (base page size) or page_num_pagesizes() - 1, while MAP_PRIVATE
3007c478bd9Sstevel@tonic-gate  * the amp->szc could be anything in [0, page_num_pagesizes() - 1].
3017c478bd9Sstevel@tonic-gate  */
302a98e9dbfSaguzovsk typedef struct anon_map {
3037c478bd9Sstevel@tonic-gate 	krwlock_t a_rwlock;	/* protect anon_map and anon array */
3047c478bd9Sstevel@tonic-gate 	size_t	size;		/* size in bytes mapped by the anon array */
3057c478bd9Sstevel@tonic-gate 	struct	anon_hdr *ahp; 	/* anon array header pointer, containing */
3067c478bd9Sstevel@tonic-gate 				/* anon pointer array(s) */
3077c478bd9Sstevel@tonic-gate 	size_t	swresv;		/* swap space reserved for this anon_map */
30807b65a64Saguzovsk 	ulong_t	refcnt;		/* reference count on this structure */
3097c478bd9Sstevel@tonic-gate 	ushort_t a_szc;		/* max szc among shared processes */
3107c478bd9Sstevel@tonic-gate 	void	*locality;	/* lgroup locality info */
311c6939658Ssl 	struct kshmid *a_sp;	/* kshmid if amp backs sysV, or NULL */
312a98e9dbfSaguzovsk 	int	a_purgewait;	/* somebody waits for slocks to go away */
313a98e9dbfSaguzovsk 	kcondvar_t a_purgecv;	/* cv for waiting for slocks to go away */
314a98e9dbfSaguzovsk 	kmutex_t a_purgemtx;	/* mutex for anonmap_purge() */
315a98e9dbfSaguzovsk 	spgcnt_t a_softlockcnt; /* number of pages locked in pcache */
316a98e9dbfSaguzovsk 	kmutex_t a_pmtx;	/* protects amp's pcache list */
317a98e9dbfSaguzovsk 	pcache_link_t a_phead;	/* head of amp's pcache list */
318a98e9dbfSaguzovsk } amp_t;
3207c478bd9Sstevel@tonic-gate #ifdef _KERNEL
3227c478bd9Sstevel@tonic-gate #define	ANON_BUSY		0x1
3237c478bd9Sstevel@tonic-gate #define	ANON_ISBUSY(slot)	(*(slot) & ANON_BUSY)
3247c478bd9Sstevel@tonic-gate #define	ANON_SETBUSY(slot)	(*(slot) |= ANON_BUSY)
3257c478bd9Sstevel@tonic-gate #define	ANON_CLRBUSY(slot)	(*(slot) &= ~ANON_BUSY)
3277c478bd9Sstevel@tonic-gate #define	ANON_MAP_SHIFT		6	/* log2(sizeof (struct anon_map)) */
3287c478bd9Sstevel@tonic-gate #define	ANON_ARRAY_SHIFT	7	/* log2(ANON_LOCKSIZE) */
3297c478bd9Sstevel@tonic-gate #define	ANON_LOCKSIZE		128
3317c478bd9Sstevel@tonic-gate #define	ANON_LOCK_ENTER(lock, type)	rw_enter((lock), (type))
3327c478bd9Sstevel@tonic-gate #define	ANON_LOCK_EXIT(lock)		rw_exit((lock))
333a98e9dbfSaguzovsk #define	ANON_LOCK_HELD(lock)		RW_LOCK_HELD((lock))
334a98e9dbfSaguzovsk #define	ANON_READ_HELD(lock)		RW_READ_HELD((lock))
335a98e9dbfSaguzovsk #define	ANON_WRITE_HELD(lock)		RW_WRITE_HELD((lock))
3377c478bd9Sstevel@tonic-gate #define	ANON_ARRAY_HASH(amp, idx)\
3387c478bd9Sstevel@tonic-gate 	((((idx) + ((idx) >> ANON_ARRAY_SHIFT) +\
3397c478bd9Sstevel@tonic-gate 	((idx) >> (ANON_ARRAY_SHIFT << 1)) +\
3407c478bd9Sstevel@tonic-gate 	((idx) >> (ANON_ARRAY_SHIFT + (ANON_ARRAY_SHIFT << 1)))) ^\
3417c478bd9Sstevel@tonic-gate 	((uintptr_t)(amp) >> ANON_MAP_SHIFT)) & (ANON_LOCKSIZE - 1))
3437c478bd9Sstevel@tonic-gate typedef struct anon_sync_obj {
3447c478bd9Sstevel@tonic-gate 	kmutex_t	*sync_mutex;
3457c478bd9Sstevel@tonic-gate 	kcondvar_t	*sync_cv;
3467c478bd9Sstevel@tonic-gate 	ulong_t		*sync_data;
3477c478bd9Sstevel@tonic-gate } anon_sync_obj_t;
3497c478bd9Sstevel@tonic-gate /*
3507c478bd9Sstevel@tonic-gate  * Anonymous backing store accounting structure for kernel.
3517c478bd9Sstevel@tonic-gate  * ani_max = total reservable slots on physical (disk-backed) swap
3527c478bd9Sstevel@tonic-gate  * ani_phys_resv = total phys slots reserved for use by clients
3537c478bd9Sstevel@tonic-gate  * ani_mem_resv = total mem slots reserved for use by clients
3547c478bd9Sstevel@tonic-gate  * ani_free = # unallocated physical slots + # of reserved unallocated
3557c478bd9Sstevel@tonic-gate  * memory slots
3567c478bd9Sstevel@tonic-gate  */
3587c478bd9Sstevel@tonic-gate /*
3597c478bd9Sstevel@tonic-gate  * Initial total swap slots available for reservation
3607c478bd9Sstevel@tonic-gate  */
3617c478bd9Sstevel@tonic-gate #define	TOTAL_AVAILABLE_SWAP \
3627c478bd9Sstevel@tonic-gate 	(k_anoninfo.ani_max + MAX((spgcnt_t)(availrmem - swapfs_minfree), 0))
3647c478bd9Sstevel@tonic-gate /*
3657c478bd9Sstevel@tonic-gate  * Swap slots currently available for reservation
3667c478bd9Sstevel@tonic-gate  */
367a98e9dbfSaguzovsk #define	CURRENT_TOTAL_AVAILABLE_SWAP				\
3687c478bd9Sstevel@tonic-gate 	((k_anoninfo.ani_max - k_anoninfo.ani_phys_resv) +	\
369a98e9dbfSaguzovsk 	    MAX((spgcnt_t)(availrmem - swapfs_minfree), 0))
3717c478bd9Sstevel@tonic-gate struct k_anoninfo {
3727c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_max;	/* total reservable slots on phys */
3737c478bd9Sstevel@tonic-gate 					/* (disk) swap */
3747c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_free;	/* # of unallocated phys and mem slots */
3757c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_phys_resv;	/* # of reserved phys (disk) slots */
3767c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_mem_resv;	/* # of reserved mem slots */
3777c478bd9Sstevel@tonic-gate 	pgcnt_t	ani_locked_swap; /* # of swap slots locked in reserved */
3787c478bd9Sstevel@tonic-gate 				/* mem swap */
3797c478bd9Sstevel@tonic-gate };
3817c478bd9Sstevel@tonic-gate extern	struct k_anoninfo k_anoninfo;
3837c478bd9Sstevel@tonic-gate extern void	anon_init(void);
3847c478bd9Sstevel@tonic-gate extern struct	anon *anon_alloc(struct vnode *, anoff_t);
3857c478bd9Sstevel@tonic-gate extern void	anon_dup(struct anon_hdr *, ulong_t,
3867c478bd9Sstevel@tonic-gate 		    struct anon_hdr *, ulong_t, size_t);
3877c478bd9Sstevel@tonic-gate extern void	anon_dup_fill_holes(struct anon_hdr *, ulong_t,
3887c478bd9Sstevel@tonic-gate 		    struct anon_hdr *, ulong_t, size_t, uint_t, int);
3897c478bd9Sstevel@tonic-gate extern int	anon_fill_cow_holes(struct seg *, caddr_t, struct anon_hdr *,
3907c478bd9Sstevel@tonic-gate 		    ulong_t, struct vnode *, u_offset_t, size_t, uint_t,
3917c478bd9Sstevel@tonic-gate 		    uint_t, struct vpage [], struct cred *);
3927c478bd9Sstevel@tonic-gate extern void	anon_free(struct anon_hdr *, ulong_t, size_t);
3937c478bd9Sstevel@tonic-gate extern void	anon_free_pages(struct anon_hdr *, ulong_t, size_t, uint_t);
394*8905f42cSBryan Cantrill extern int	anon_disclaim(struct anon_map *,
395*8905f42cSBryan Cantrill 		    ulong_t, size_t, uint_t, pgcnt_t *);
3967c478bd9Sstevel@tonic-gate extern int	anon_getpage(struct anon **, uint_t *, struct page **,
3977c478bd9Sstevel@tonic-gate 		    size_t, struct seg *, caddr_t, enum seg_rw, struct cred *);
3987c478bd9Sstevel@tonic-gate extern int	swap_getconpage(struct vnode *, u_offset_t, size_t,
39907b65a64Saguzovsk 		    uint_t *, page_t *[], size_t, page_t *, uint_t *,
4007c478bd9Sstevel@tonic-gate 		    spgcnt_t *, struct seg *, caddr_t,
4017c478bd9Sstevel@tonic-gate 		    enum seg_rw, struct cred *);
4027c478bd9Sstevel@tonic-gate extern int	anon_map_getpages(struct anon_map *, ulong_t,
4037c478bd9Sstevel@tonic-gate 		    uint_t, struct seg *, caddr_t, uint_t,
4047c478bd9Sstevel@tonic-gate 		    uint_t *, page_t *[], uint_t *,
4052cb27123Saguzovsk 		    struct vpage [], enum seg_rw, int, int, int, struct cred *);
4067c478bd9Sstevel@tonic-gate extern int	anon_map_privatepages(struct anon_map *, ulong_t,
4077c478bd9Sstevel@tonic-gate 		    uint_t, struct seg *, caddr_t, uint_t,
4082cb27123Saguzovsk 		    page_t *[], struct vpage [], int, int, struct cred *);
4097c478bd9Sstevel@tonic-gate extern struct	page *anon_private(struct anon **, struct seg *,
4107c478bd9Sstevel@tonic-gate 		    caddr_t, uint_t, struct page *,
4117c478bd9Sstevel@tonic-gate 		    int, struct cred *);
4127c478bd9Sstevel@tonic-gate extern struct	page *anon_zero(struct seg *, caddr_t,
4137c478bd9Sstevel@tonic-gate 		    struct anon **, struct cred *);
4147c478bd9Sstevel@tonic-gate extern int	anon_map_createpages(struct anon_map *, ulong_t,
4157c478bd9Sstevel@tonic-gate 		    size_t, struct page **,
4167c478bd9Sstevel@tonic-gate 		    struct seg *, caddr_t,
4177c478bd9Sstevel@tonic-gate 		    enum seg_rw, struct cred *);
4187c478bd9Sstevel@tonic-gate extern int	anon_map_demotepages(struct anon_map *, ulong_t,
4197c478bd9Sstevel@tonic-gate 		    struct seg *, caddr_t, uint_t,
4207c478bd9Sstevel@tonic-gate 		    struct vpage [], struct cred *);
42107b65a64Saguzovsk extern void	anon_shmap_free_pages(struct anon_map *, ulong_t, size_t);
4222cb27123Saguzovsk extern int	anon_resvmem(size_t, boolean_t, zone_t *, int);
4230209230bSgjelinek extern void	anon_unresvmem(size_t, zone_t *);
4242cb27123Saguzovsk extern struct	anon_map *anonmap_alloc(size_t, size_t, int);
4257c478bd9Sstevel@tonic-gate extern void	anonmap_free(struct anon_map *);
426a98e9dbfSaguzovsk extern void	anonmap_purge(struct anon_map *);
427a98e9dbfSaguzovsk extern void	anon_swap_free(struct anon *, struct page *);
4287c478bd9Sstevel@tonic-gate extern void	anon_decref(struct anon *);
4297c478bd9Sstevel@tonic-gate extern int	non_anon(struct anon_hdr *, ulong_t, u_offset_t *, size_t *);
4307c478bd9Sstevel@tonic-gate extern pgcnt_t	anon_pages(struct anon_hdr *, ulong_t, pgcnt_t);
431e0cb4e8dSOndrej Kubecka extern int	anon_swap_adjust(pgcnt_t);
4327c478bd9Sstevel@tonic-gate extern void	anon_swap_restore(pgcnt_t);
4337c478bd9Sstevel@tonic-gate extern struct	anon_hdr *anon_create(pgcnt_t, int);
4347c478bd9Sstevel@tonic-gate extern void	anon_release(struct anon_hdr *, pgcnt_t);
4357c478bd9Sstevel@tonic-gate extern struct	anon *anon_get_ptr(struct anon_hdr *, ulong_t);
4367c478bd9Sstevel@tonic-gate extern ulong_t	*anon_get_slot(struct anon_hdr *, ulong_t);
4377c478bd9Sstevel@tonic-gate extern struct	anon *anon_get_next_ptr(struct anon_hdr *, ulong_t *);
4387c478bd9Sstevel@tonic-gate extern int	anon_set_ptr(struct anon_hdr *, ulong_t, struct anon *, int);
4397c478bd9Sstevel@tonic-gate extern int 	anon_copy_ptr(struct anon_hdr *, ulong_t,
4407c478bd9Sstevel@tonic-gate 		    struct anon_hdr *, ulong_t, pgcnt_t, int);
4417c478bd9Sstevel@tonic-gate extern pgcnt_t	anon_grow(struct anon_hdr *, ulong_t *, pgcnt_t, pgcnt_t, int);
4427c478bd9Sstevel@tonic-gate extern void	anon_array_enter(struct anon_map *, ulong_t,
4437c478bd9Sstevel@tonic-gate 			anon_sync_obj_t *);
44487015465Scwb extern int	anon_array_try_enter(struct anon_map *, ulong_t,
44587015465Scwb 			anon_sync_obj_t *);
4467c478bd9Sstevel@tonic-gate extern void	anon_array_exit(anon_sync_obj_t *);
4487c478bd9Sstevel@tonic-gate /*
4497c478bd9Sstevel@tonic-gate  * anon_resv checks to see if there is enough swap space to fulfill a
4507c478bd9Sstevel@tonic-gate  * request and if so, reserves the appropriate anonymous memory resources.
4517c478bd9Sstevel@tonic-gate  * anon_checkspace just checks to see if there is space to fulfill the request,
4527c478bd9Sstevel@tonic-gate  * without taking any resources.  Both return 1 if successful and 0 if not.
4530209230bSgjelinek  *
4540209230bSgjelinek  * Macros are provided as anon reservation is usually charged to the zone of
4550209230bSgjelinek  * the current process.  In some cases (such as anon reserved by tmpfs), a
4560209230bSgjelinek  * zone pointer is needed to charge the appropriate zone.
4577c478bd9Sstevel@tonic-gate  */
4580209230bSgjelinek #define	anon_unresv(size)		anon_unresvmem(size, curproc->p_zone)
4590209230bSgjelinek #define	anon_unresv_zone(size, zone)	anon_unresvmem(size, zone)
4602cb27123Saguzovsk #define	anon_resv(size)			\
4612cb27123Saguzovsk 	anon_resvmem((size), 1, curproc->p_zone, 1)
4622cb27123Saguzovsk #define	anon_resv_zone(size, zone)	anon_resvmem((size), 1, zone, 1)
4638c06a490Sjj #define	anon_checkspace(size, zone)	anon_resvmem((size), 0, zone, 0)
4642cb27123Saguzovsk #define	anon_try_resv_zone(size, zone)	anon_resvmem((size), 1, zone, 0)
4667c478bd9Sstevel@tonic-gate /*
4677c478bd9Sstevel@tonic-gate  * Flags to anon_private
4687c478bd9Sstevel@tonic-gate  */
4697c478bd9Sstevel@tonic-gate #define	STEAL_PAGE	0x1	/* page can be stolen */
4707c478bd9Sstevel@tonic-gate #define	LOCK_PAGE	0x2	/* page must be ``logically'' locked */
4727c478bd9Sstevel@tonic-gate /*
4737c478bd9Sstevel@tonic-gate  * SEGKP ANON pages that are locked are assumed to be LWP stack pages
4747c478bd9Sstevel@tonic-gate  * and thus count towards the user pages locked count.
4757c478bd9Sstevel@tonic-gate  * This value is protected by the same lock as availrmem.
4767c478bd9Sstevel@tonic-gate  */
4777c478bd9Sstevel@tonic-gate extern pgcnt_t anon_segkp_pages_locked;
4797c478bd9Sstevel@tonic-gate extern int anon_debug;
4817c478bd9Sstevel@tonic-gate #ifdef ANON_DEBUG
4837c478bd9Sstevel@tonic-gate #define	A_ANON	0x01
4847c478bd9Sstevel@tonic-gate #define	A_RESV	0x02
4857c478bd9Sstevel@tonic-gate #define	A_MRESV	0x04
4877c478bd9Sstevel@tonic-gate /* vararg-like debugging macro. */
4887c478bd9Sstevel@tonic-gate #define	ANON_PRINT(f, printf_args) \
4897c478bd9Sstevel@tonic-gate 		if (anon_debug & f) \
4907c478bd9Sstevel@tonic-gate 			printf printf_args
4927c478bd9Sstevel@tonic-gate #else	/* ANON_DEBUG */
4947c478bd9Sstevel@tonic-gate #define	ANON_PRINT(f, printf_args)
4967c478bd9Sstevel@tonic-gate #endif	/* ANON_DEBUG */
4987c478bd9Sstevel@tonic-gate #endif	/* _KERNEL */
5007c478bd9Sstevel@tonic-gate #ifdef	__cplusplus
5017c478bd9Sstevel@tonic-gate }
5027c478bd9Sstevel@tonic-gate #endif
5047c478bd9Sstevel@tonic-gate #endif	/* _VM_ANON_H */