17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 507b65a64Saguzovsk * Common Development and Distribution License (the "License"). 607b65a64Saguzovsk * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 2223d9e5acSMichael Corcoran * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved. 23*8905f42cSBryan Cantrill * Copyright (c) 2015, Joyent, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * University Copyright- Copyright (c) 1982, 1986, 1988 317c478bd9Sstevel@tonic-gate * The Regents of the University of California 327c478bd9Sstevel@tonic-gate * All Rights Reserved 337c478bd9Sstevel@tonic-gate * 347c478bd9Sstevel@tonic-gate * University Acknowledgment- Portions of this document are derived from 357c478bd9Sstevel@tonic-gate * software developed by the University of California, Berkeley, and its 367c478bd9Sstevel@tonic-gate * contributors. 377c478bd9Sstevel@tonic-gate */ 387c478bd9Sstevel@tonic-gate 397c478bd9Sstevel@tonic-gate #ifndef _VM_ANON_H 407c478bd9Sstevel@tonic-gate #define _VM_ANON_H 417c478bd9Sstevel@tonic-gate 427c478bd9Sstevel@tonic-gate #include <sys/cred.h> 430209230bSgjelinek #include <sys/zone.h> 447c478bd9Sstevel@tonic-gate #include <vm/seg.h> 457c478bd9Sstevel@tonic-gate #include <vm/vpage.h> 467c478bd9Sstevel@tonic-gate 477c478bd9Sstevel@tonic-gate #ifdef __cplusplus 487c478bd9Sstevel@tonic-gate extern "C" { 497c478bd9Sstevel@tonic-gate #endif 507c478bd9Sstevel@tonic-gate 517c478bd9Sstevel@tonic-gate /* 527c478bd9Sstevel@tonic-gate * VM - Anonymous pages. 537c478bd9Sstevel@tonic-gate */ 547c478bd9Sstevel@tonic-gate 557c478bd9Sstevel@tonic-gate typedef unsigned long anoff_t; /* anon offsets */ 567c478bd9Sstevel@tonic-gate 577c478bd9Sstevel@tonic-gate /* 587c478bd9Sstevel@tonic-gate * Each anonymous page, either in memory or in swap, has an anon structure. 597c478bd9Sstevel@tonic-gate * The structure (slot) provides a level of indirection between anonymous pages 607c478bd9Sstevel@tonic-gate * and their backing store. 617c478bd9Sstevel@tonic-gate * 627c478bd9Sstevel@tonic-gate * (an_vp, an_off) names the vnode of the anonymous page for this slot. 637c478bd9Sstevel@tonic-gate * 647c478bd9Sstevel@tonic-gate * (an_pvp, an_poff) names the location of the physical backing store 657c478bd9Sstevel@tonic-gate * for the page this slot represents. If the name is null there is no 667c478bd9Sstevel@tonic-gate * associated physical store. The physical backing store location can 677c478bd9Sstevel@tonic-gate * change while the slot is in use. 687c478bd9Sstevel@tonic-gate * 697c478bd9Sstevel@tonic-gate * an_hash is a hash list of anon slots. The list is hashed by 707c478bd9Sstevel@tonic-gate * (an_vp, an_off) of the associated anonymous page and provides a 717c478bd9Sstevel@tonic-gate * method of going from the name of an anonymous page to its 727c478bd9Sstevel@tonic-gate * associated anon slot. 737c478bd9Sstevel@tonic-gate * 747c478bd9Sstevel@tonic-gate * an_refcnt holds a reference count which is the number of separate 757c478bd9Sstevel@tonic-gate * copies that will need to be created in case of copy-on-write. 767c478bd9Sstevel@tonic-gate * A refcnt > 0 protects the existence of the slot. The refcnt is 777c478bd9Sstevel@tonic-gate * initialized to 1 when the anon slot is created in anon_alloc(). 787c478bd9Sstevel@tonic-gate * If a client obtains an anon slot and allows multiple threads to 797c478bd9Sstevel@tonic-gate * share it, then it is the client's responsibility to insure that 807c478bd9Sstevel@tonic-gate * it does not allow one thread to try to reference the slot at the 817c478bd9Sstevel@tonic-gate * same time as another is trying to decrement the last count and 827c478bd9Sstevel@tonic-gate * destroy the anon slot. E.g., the seg_vn segment type protects 837c478bd9Sstevel@tonic-gate * against this with higher level locks. 847c478bd9Sstevel@tonic-gate */ 857c478bd9Sstevel@tonic-gate 867c478bd9Sstevel@tonic-gate struct anon { 877c478bd9Sstevel@tonic-gate struct vnode *an_vp; /* vnode of anon page */ 887c478bd9Sstevel@tonic-gate struct vnode *an_pvp; /* vnode of physical backing store */ 897c478bd9Sstevel@tonic-gate anoff_t an_off; /* offset of anon page */ 907c478bd9Sstevel@tonic-gate anoff_t an_poff; /* offset in vnode */ 917c478bd9Sstevel@tonic-gate struct anon *an_hash; /* hash table of anon slots */ 927c478bd9Sstevel@tonic-gate int an_refcnt; /* # of people sharing slot */ 937c478bd9Sstevel@tonic-gate }; 947c478bd9Sstevel@tonic-gate 95cb15d5d9SPeter Rival #define AN_CACHE_ALIGN_LOG2 4 /* log2(AN_CACHE_ALIGN) */ 96cb15d5d9SPeter Rival #define AN_CACHE_ALIGN (1U << AN_CACHE_ALIGN_LOG2) /* anon address aligned */ 97cb15d5d9SPeter Rival /* 16 bytes */ 98cb15d5d9SPeter Rival 99cb15d5d9SPeter Rival 1007c478bd9Sstevel@tonic-gate #ifdef _KERNEL 1017c478bd9Sstevel@tonic-gate /* 1027c478bd9Sstevel@tonic-gate * The swapinfo_lock protects: 1037c478bd9Sstevel@tonic-gate * swapinfo list 1047c478bd9Sstevel@tonic-gate * individual swapinfo structures 1057c478bd9Sstevel@tonic-gate * 1067c478bd9Sstevel@tonic-gate * The anoninfo_lock protects: 1077c478bd9Sstevel@tonic-gate * anoninfo counters 1087c478bd9Sstevel@tonic-gate * 1097c478bd9Sstevel@tonic-gate * The anonhash_lock protects: 1107c478bd9Sstevel@tonic-gate * anon hash lists 1117c478bd9Sstevel@tonic-gate * anon slot fields 1127c478bd9Sstevel@tonic-gate * 1137c478bd9Sstevel@tonic-gate * Fields in the anon slot which are read-only for the life of the slot 1147c478bd9Sstevel@tonic-gate * (an_vp, an_off) do not require the anonhash_lock be held to access them. 1157c478bd9Sstevel@tonic-gate * If you access a field without the anonhash_lock held you must be holding 1167c478bd9Sstevel@tonic-gate * the slot with an_refcnt to make sure it isn't destroyed. 1177c478bd9Sstevel@tonic-gate * To write (an_pvp, an_poff) in a given slot you must also hold the 1187c478bd9Sstevel@tonic-gate * p_iolock of the anonymous page for slot. 1197c478bd9Sstevel@tonic-gate */ 1207c478bd9Sstevel@tonic-gate extern kmutex_t anoninfo_lock; 1217c478bd9Sstevel@tonic-gate extern kmutex_t swapinfo_lock; 12223d9e5acSMichael Corcoran extern pad_mutex_t *anonhash_lock; 1237c478bd9Sstevel@tonic-gate extern pad_mutex_t anon_array_lock[]; 1247c478bd9Sstevel@tonic-gate extern kcondvar_t anon_array_cv[]; 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate /* 1277c478bd9Sstevel@tonic-gate * Global hash table to provide a function from (vp, off) -> ap 1287c478bd9Sstevel@tonic-gate */ 1297c478bd9Sstevel@tonic-gate extern size_t anon_hash_size; 130cb15d5d9SPeter Rival extern unsigned int anon_hash_shift; 1317c478bd9Sstevel@tonic-gate extern struct anon **anon_hash; 1327c478bd9Sstevel@tonic-gate #define ANON_HASH_SIZE anon_hash_size 1337c478bd9Sstevel@tonic-gate #define ANON_HASHAVELEN 4 134cb15d5d9SPeter Rival /* 135cb15d5d9SPeter Rival * Try to use as many bits of randomness from both vp and off as we can. 136cb15d5d9SPeter Rival * This should help spreading evenly for a variety of workloads. See comments 137cb15d5d9SPeter Rival * for PAGE_HASH_FUNC for more explanation. 138cb15d5d9SPeter Rival */ 139cb15d5d9SPeter Rival #define ANON_HASH(vp, off) \ 140cb15d5d9SPeter Rival (((((uintptr_t)(off) >> PAGESHIFT) ^ \ 141cb15d5d9SPeter Rival ((uintptr_t)(off) >> (PAGESHIFT + anon_hash_shift))) ^ \ 142cb15d5d9SPeter Rival (((uintptr_t)(vp) >> 3) ^ \ 143cb15d5d9SPeter Rival ((uintptr_t)(vp) >> (3 + anon_hash_shift)) ^ \ 144cb15d5d9SPeter Rival ((uintptr_t)(vp) >> (3 + 2 * anon_hash_shift)) ^ \ 145cb15d5d9SPeter Rival ((uintptr_t)(vp) << \ 146cb15d5d9SPeter Rival (anon_hash_shift - AN_VPSHIFT - VNODE_ALIGN_LOG2)))) & \ 147cb15d5d9SPeter Rival (anon_hash_size - 1)) 1487c478bd9Sstevel@tonic-gate 14923d9e5acSMichael Corcoran #define AH_LOCK_SIZE (2 << NCPU_LOG2) 15023d9e5acSMichael Corcoran 15123d9e5acSMichael Corcoran #define AH_MUTEX(vp, off) \ 15223d9e5acSMichael Corcoran (&anonhash_lock[(ANON_HASH((vp), (off)) & \ 15323d9e5acSMichael Corcoran (AH_LOCK_SIZE - 1))].pad_mutex) 1547c478bd9Sstevel@tonic-gate 1557c478bd9Sstevel@tonic-gate #endif /* _KERNEL */ 1567c478bd9Sstevel@tonic-gate 1577c478bd9Sstevel@tonic-gate /* 1587c478bd9Sstevel@tonic-gate * Declaration for the Global counters to accurately 1597c478bd9Sstevel@tonic-gate * track the kernel foot print in memory. 1607c478bd9Sstevel@tonic-gate */ 1617c478bd9Sstevel@tonic-gate extern pgcnt_t pages_locked; 1627c478bd9Sstevel@tonic-gate extern pgcnt_t pages_claimed; 1637c478bd9Sstevel@tonic-gate extern pgcnt_t pages_useclaim; 1647c478bd9Sstevel@tonic-gate extern pgcnt_t obp_pages; 1657c478bd9Sstevel@tonic-gate 1667c478bd9Sstevel@tonic-gate /* 1677c478bd9Sstevel@tonic-gate * Anonymous backing store accounting structure for swapctl. 1687c478bd9Sstevel@tonic-gate * 1697c478bd9Sstevel@tonic-gate * ani_max = maximum amount of swap space 1707c478bd9Sstevel@tonic-gate * (including potentially available physical memory) 1717c478bd9Sstevel@tonic-gate * ani_free = amount of unallocated anonymous memory 1727c478bd9Sstevel@tonic-gate * (some of which might be reserved and including 1737c478bd9Sstevel@tonic-gate * potentially available physical memory) 1747c478bd9Sstevel@tonic-gate * ani_resv = amount of claimed (reserved) anonymous memory 1757c478bd9Sstevel@tonic-gate * 1767c478bd9Sstevel@tonic-gate * The swap data can be aquired more efficiently through the 1777c478bd9Sstevel@tonic-gate * kstats interface. 1787c478bd9Sstevel@tonic-gate * Total slots currently available for reservation = 1797c478bd9Sstevel@tonic-gate * MAX(ani_max - ani_resv, 0) + (availrmem - swapfs_minfree) 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate struct anoninfo { 1827c478bd9Sstevel@tonic-gate pgcnt_t ani_max; 1837c478bd9Sstevel@tonic-gate pgcnt_t ani_free; 1847c478bd9Sstevel@tonic-gate pgcnt_t ani_resv; 1857c478bd9Sstevel@tonic-gate }; 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate #ifdef _SYSCALL32 1887c478bd9Sstevel@tonic-gate struct anoninfo32 { 1897c478bd9Sstevel@tonic-gate size32_t ani_max; 1907c478bd9Sstevel@tonic-gate size32_t ani_free; 1917c478bd9Sstevel@tonic-gate size32_t ani_resv; 1927c478bd9Sstevel@tonic-gate }; 1937c478bd9Sstevel@tonic-gate #endif /* _SYSCALL32 */ 1947c478bd9Sstevel@tonic-gate 1957c478bd9Sstevel@tonic-gate /* 1967c478bd9Sstevel@tonic-gate * Define the NCPU pool of the ani_free counters. Update the counter 1977c478bd9Sstevel@tonic-gate * of the cpu on which the thread is running and in every clock intr 1987c478bd9Sstevel@tonic-gate * sync anoninfo.ani_free with the current total off all the NCPU entries. 1997c478bd9Sstevel@tonic-gate */ 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate typedef struct ani_free { 2027c478bd9Sstevel@tonic-gate pgcnt_t ani_count; 203b52a336eSPavel Tatashin uchar_t pad[64 - sizeof (pgcnt_t)]; 2047c478bd9Sstevel@tonic-gate /* XXX 64 = cacheline size */ 2057c478bd9Sstevel@tonic-gate } ani_free_t; 2067c478bd9Sstevel@tonic-gate 207b52a336eSPavel Tatashin #define ANI_MAX_POOL (NCPU_P2) 208b52a336eSPavel Tatashin extern ani_free_t *ani_free_pool; 2097c478bd9Sstevel@tonic-gate 210b52a336eSPavel Tatashin /* 211b52a336eSPavel Tatashin * Since each CPU has its own bucket in ani_free_pool, there should be no 212b52a336eSPavel Tatashin * contention here. 213b52a336eSPavel Tatashin */ 2147c478bd9Sstevel@tonic-gate #define ANI_ADD(inc) { \ 215b52a336eSPavel Tatashin pgcnt_t *ani_countp; \ 216b52a336eSPavel Tatashin int index; \ 217b52a336eSPavel Tatashin index = (CPU->cpu_seqid & (ANI_MAX_POOL - 1)); \ 218b52a336eSPavel Tatashin ani_countp = &ani_free_pool[index].ani_count; \ 219b52a336eSPavel Tatashin atomic_add_long(ani_countp, inc); \ 2207c478bd9Sstevel@tonic-gate } 2217c478bd9Sstevel@tonic-gate 222b52a336eSPavel Tatashin extern void set_anoninfo(void); 223b52a336eSPavel Tatashin 2247c478bd9Sstevel@tonic-gate /* 2257c478bd9Sstevel@tonic-gate * Anon array pointers are allocated in chunks. Each chunk 2267c478bd9Sstevel@tonic-gate * has PAGESIZE/sizeof(u_long *) of anon pointers. 2277c478bd9Sstevel@tonic-gate * There are two levels of arrays for anon array pointers larger 2287c478bd9Sstevel@tonic-gate * than a chunk. The first level points to anon array chunks. 2297c478bd9Sstevel@tonic-gate * The second level consists of chunks of anon pointers. 2307c478bd9Sstevel@tonic-gate * 2317c478bd9Sstevel@tonic-gate * If anon array is smaller than a chunk then the whole anon array 2327c478bd9Sstevel@tonic-gate * is created (memory is allocated for whole anon array). 2337c478bd9Sstevel@tonic-gate * If anon array is larger than a chunk only first level array is 2347c478bd9Sstevel@tonic-gate * allocated. Then other arrays (chunks) are allocated only when 2357c478bd9Sstevel@tonic-gate * they are initialized with anon pointers. 2367c478bd9Sstevel@tonic-gate */ 2377c478bd9Sstevel@tonic-gate struct anon_hdr { 2387c478bd9Sstevel@tonic-gate kmutex_t serial_lock; /* serialize array chunk allocation */ 2397c478bd9Sstevel@tonic-gate pgcnt_t size; /* number of pointers to (anon) pages */ 2407c478bd9Sstevel@tonic-gate void **array_chunk; /* pointers to anon pointers or chunks of */ 2417c478bd9Sstevel@tonic-gate /* anon pointers */ 2427c478bd9Sstevel@tonic-gate int flags; /* ANON_ALLOC_FORCE force preallocation of */ 2437c478bd9Sstevel@tonic-gate /* whole anon array */ 2447c478bd9Sstevel@tonic-gate }; 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate #ifdef _LP64 2477c478bd9Sstevel@tonic-gate #define ANON_PTRSHIFT 3 2487c478bd9Sstevel@tonic-gate #define ANON_PTRMASK ~7 2497c478bd9Sstevel@tonic-gate #else 2507c478bd9Sstevel@tonic-gate #define ANON_PTRSHIFT 2 2517c478bd9Sstevel@tonic-gate #define ANON_PTRMASK ~3 2527c478bd9Sstevel@tonic-gate #endif 2537c478bd9Sstevel@tonic-gate 2547c478bd9Sstevel@tonic-gate #define ANON_CHUNK_SIZE (PAGESIZE >> ANON_PTRSHIFT) 2557c478bd9Sstevel@tonic-gate #define ANON_CHUNK_SHIFT (PAGESHIFT - ANON_PTRSHIFT) 2567c478bd9Sstevel@tonic-gate #define ANON_CHUNK_OFF (ANON_CHUNK_SIZE - 1) 2577c478bd9Sstevel@tonic-gate 2587c478bd9Sstevel@tonic-gate /* 2597c478bd9Sstevel@tonic-gate * Anon flags. 2607c478bd9Sstevel@tonic-gate */ 2617c478bd9Sstevel@tonic-gate #define ANON_SLEEP 0x0 /* ok to block */ 2627c478bd9Sstevel@tonic-gate #define ANON_NOSLEEP 0x1 /* non-blocking call */ 2637c478bd9Sstevel@tonic-gate #define ANON_ALLOC_FORCE 0x2 /* force single level anon array */ 2647c478bd9Sstevel@tonic-gate #define ANON_GROWDOWN 0x4 /* anon array should grow downward */ 2657c478bd9Sstevel@tonic-gate 266c6939658Ssl struct kshmid; 267c6939658Ssl 2687c478bd9Sstevel@tonic-gate /* 2697c478bd9Sstevel@tonic-gate * The anon_map structure is used by various clients of the anon layer to 2707c478bd9Sstevel@tonic-gate * manage anonymous memory. When anonymous memory is shared, 2717c478bd9Sstevel@tonic-gate * then the different clients sharing it will point to the 2727c478bd9Sstevel@tonic-gate * same anon_map structure. Also, if a segment is unmapped 2737c478bd9Sstevel@tonic-gate * in the middle where an anon_map structure exists, the 2747c478bd9Sstevel@tonic-gate * newly created segment will also share the anon_map structure, 2757c478bd9Sstevel@tonic-gate * although the two segments will use different ranges of the 2767c478bd9Sstevel@tonic-gate * anon array. When mappings are private (or shared with 2777c478bd9Sstevel@tonic-gate * a reference count of 1), an unmap operation will free up 2787c478bd9Sstevel@tonic-gate * a range of anon slots in the array given by the anon_map 2797c478bd9Sstevel@tonic-gate * structure. Because of fragmentation due to this unmapping, 2807c478bd9Sstevel@tonic-gate * we have to store the size of the anon array in the anon_map 2817c478bd9Sstevel@tonic-gate * structure so that we can free everything when the referernce 2827c478bd9Sstevel@tonic-gate * count goes to zero. 2837c478bd9Sstevel@tonic-gate * 2847c478bd9Sstevel@tonic-gate * A new rangelock scheme is introduced to make the anon layer scale. 2857c478bd9Sstevel@tonic-gate * A reader/writer lock per anon_amp and an array of system-wide hash 2867c478bd9Sstevel@tonic-gate * locks, anon_array_lock[] are introduced to replace serial_lock and 2877c478bd9Sstevel@tonic-gate * anonmap lock. The writer lock is held when we want to singlethreaD 2887c478bd9Sstevel@tonic-gate * the reference to the anon array pointers or when references to 2897c478bd9Sstevel@tonic-gate * anon_map's members, whereas reader lock and anon_array_lock are 2907c478bd9Sstevel@tonic-gate * held to allows multiple threads to reference different part of 2917c478bd9Sstevel@tonic-gate * anon array. A global set of condition variables, anon_array_cv, 2927c478bd9Sstevel@tonic-gate * are used with anon_array_lock[] to make the hold time of the locks 2937c478bd9Sstevel@tonic-gate * short. 2947c478bd9Sstevel@tonic-gate * 2957c478bd9Sstevel@tonic-gate * szc is used to calculate the index of hash locks and cv's. We 2967c478bd9Sstevel@tonic-gate * could've just used seg->s_szc if not for the possible sharing of 2977c478bd9Sstevel@tonic-gate * anon_amp between SYSV shared memory and ISM, so now we introduce 2987c478bd9Sstevel@tonic-gate * szc in the anon_map structure. For MAP_SHARED, the amp->szc is either 2997c478bd9Sstevel@tonic-gate * 0 (base page size) or page_num_pagesizes() - 1, while MAP_PRIVATE 3007c478bd9Sstevel@tonic-gate * the amp->szc could be anything in [0, page_num_pagesizes() - 1]. 3017c478bd9Sstevel@tonic-gate */ 302a98e9dbfSaguzovsk typedef struct anon_map { 3037c478bd9Sstevel@tonic-gate krwlock_t a_rwlock; /* protect anon_map and anon array */ 3047c478bd9Sstevel@tonic-gate size_t size; /* size in bytes mapped by the anon array */ 3057c478bd9Sstevel@tonic-gate struct anon_hdr *ahp; /* anon array header pointer, containing */ 3067c478bd9Sstevel@tonic-gate /* anon pointer array(s) */ 3077c478bd9Sstevel@tonic-gate size_t swresv; /* swap space reserved for this anon_map */ 30807b65a64Saguzovsk ulong_t refcnt; /* reference count on this structure */ 3097c478bd9Sstevel@tonic-gate ushort_t a_szc; /* max szc among shared processes */ 3107c478bd9Sstevel@tonic-gate void *locality; /* lgroup locality info */ 311c6939658Ssl struct kshmid *a_sp; /* kshmid if amp backs sysV, or NULL */ 312a98e9dbfSaguzovsk int a_purgewait; /* somebody waits for slocks to go away */ 313a98e9dbfSaguzovsk kcondvar_t a_purgecv; /* cv for waiting for slocks to go away */ 314a98e9dbfSaguzovsk kmutex_t a_purgemtx; /* mutex for anonmap_purge() */ 315a98e9dbfSaguzovsk spgcnt_t a_softlockcnt; /* number of pages locked in pcache */ 316a98e9dbfSaguzovsk kmutex_t a_pmtx; /* protects amp's pcache list */ 317a98e9dbfSaguzovsk pcache_link_t a_phead; /* head of amp's pcache list */ 318a98e9dbfSaguzovsk } amp_t; 3197c478bd9Sstevel@tonic-gate 3207c478bd9Sstevel@tonic-gate #ifdef _KERNEL 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate #define ANON_BUSY 0x1 3237c478bd9Sstevel@tonic-gate #define ANON_ISBUSY(slot) (*(slot) & ANON_BUSY) 3247c478bd9Sstevel@tonic-gate #define ANON_SETBUSY(slot) (*(slot) |= ANON_BUSY) 3257c478bd9Sstevel@tonic-gate #define ANON_CLRBUSY(slot) (*(slot) &= ~ANON_BUSY) 3267c478bd9Sstevel@tonic-gate 3277c478bd9Sstevel@tonic-gate #define ANON_MAP_SHIFT 6 /* log2(sizeof (struct anon_map)) */ 3287c478bd9Sstevel@tonic-gate #define ANON_ARRAY_SHIFT 7 /* log2(ANON_LOCKSIZE) */ 3297c478bd9Sstevel@tonic-gate #define ANON_LOCKSIZE 128 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate #define ANON_LOCK_ENTER(lock, type) rw_enter((lock), (type)) 3327c478bd9Sstevel@tonic-gate #define ANON_LOCK_EXIT(lock) rw_exit((lock)) 333a98e9dbfSaguzovsk #define ANON_LOCK_HELD(lock) RW_LOCK_HELD((lock)) 334a98e9dbfSaguzovsk #define ANON_READ_HELD(lock) RW_READ_HELD((lock)) 335a98e9dbfSaguzovsk #define ANON_WRITE_HELD(lock) RW_WRITE_HELD((lock)) 3367c478bd9Sstevel@tonic-gate 3377c478bd9Sstevel@tonic-gate #define ANON_ARRAY_HASH(amp, idx)\ 3387c478bd9Sstevel@tonic-gate ((((idx) + ((idx) >> ANON_ARRAY_SHIFT) +\ 3397c478bd9Sstevel@tonic-gate ((idx) >> (ANON_ARRAY_SHIFT << 1)) +\ 3407c478bd9Sstevel@tonic-gate ((idx) >> (ANON_ARRAY_SHIFT + (ANON_ARRAY_SHIFT << 1)))) ^\ 3417c478bd9Sstevel@tonic-gate ((uintptr_t)(amp) >> ANON_MAP_SHIFT)) & (ANON_LOCKSIZE - 1)) 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate typedef struct anon_sync_obj { 3447c478bd9Sstevel@tonic-gate kmutex_t *sync_mutex; 3457c478bd9Sstevel@tonic-gate kcondvar_t *sync_cv; 3467c478bd9Sstevel@tonic-gate ulong_t *sync_data; 3477c478bd9Sstevel@tonic-gate } anon_sync_obj_t; 3487c478bd9Sstevel@tonic-gate 3497c478bd9Sstevel@tonic-gate /* 3507c478bd9Sstevel@tonic-gate * Anonymous backing store accounting structure for kernel. 3517c478bd9Sstevel@tonic-gate * ani_max = total reservable slots on physical (disk-backed) swap 3527c478bd9Sstevel@tonic-gate * ani_phys_resv = total phys slots reserved for use by clients 3537c478bd9Sstevel@tonic-gate * ani_mem_resv = total mem slots reserved for use by clients 3547c478bd9Sstevel@tonic-gate * ani_free = # unallocated physical slots + # of reserved unallocated 3557c478bd9Sstevel@tonic-gate * memory slots 3567c478bd9Sstevel@tonic-gate */ 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate /* 3597c478bd9Sstevel@tonic-gate * Initial total swap slots available for reservation 3607c478bd9Sstevel@tonic-gate */ 3617c478bd9Sstevel@tonic-gate #define TOTAL_AVAILABLE_SWAP \ 3627c478bd9Sstevel@tonic-gate (k_anoninfo.ani_max + MAX((spgcnt_t)(availrmem - swapfs_minfree), 0)) 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate /* 3657c478bd9Sstevel@tonic-gate * Swap slots currently available for reservation 3667c478bd9Sstevel@tonic-gate */ 367a98e9dbfSaguzovsk #define CURRENT_TOTAL_AVAILABLE_SWAP \ 3687c478bd9Sstevel@tonic-gate ((k_anoninfo.ani_max - k_anoninfo.ani_phys_resv) + \ 369a98e9dbfSaguzovsk MAX((spgcnt_t)(availrmem - swapfs_minfree), 0)) 3707c478bd9Sstevel@tonic-gate 3717c478bd9Sstevel@tonic-gate struct k_anoninfo { 3727c478bd9Sstevel@tonic-gate pgcnt_t ani_max; /* total reservable slots on phys */ 3737c478bd9Sstevel@tonic-gate /* (disk) swap */ 3747c478bd9Sstevel@tonic-gate pgcnt_t ani_free; /* # of unallocated phys and mem slots */ 3757c478bd9Sstevel@tonic-gate pgcnt_t ani_phys_resv; /* # of reserved phys (disk) slots */ 3767c478bd9Sstevel@tonic-gate pgcnt_t ani_mem_resv; /* # of reserved mem slots */ 3777c478bd9Sstevel@tonic-gate pgcnt_t ani_locked_swap; /* # of swap slots locked in reserved */ 3787c478bd9Sstevel@tonic-gate /* mem swap */ 3797c478bd9Sstevel@tonic-gate }; 3807c478bd9Sstevel@tonic-gate 3817c478bd9Sstevel@tonic-gate extern struct k_anoninfo k_anoninfo; 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate extern void anon_init(void); 3847c478bd9Sstevel@tonic-gate extern struct anon *anon_alloc(struct vnode *, anoff_t); 3857c478bd9Sstevel@tonic-gate extern void anon_dup(struct anon_hdr *, ulong_t, 3867c478bd9Sstevel@tonic-gate struct anon_hdr *, ulong_t, size_t); 3877c478bd9Sstevel@tonic-gate extern void anon_dup_fill_holes(struct anon_hdr *, ulong_t, 3887c478bd9Sstevel@tonic-gate struct anon_hdr *, ulong_t, size_t, uint_t, int); 3897c478bd9Sstevel@tonic-gate extern int anon_fill_cow_holes(struct seg *, caddr_t, struct anon_hdr *, 3907c478bd9Sstevel@tonic-gate ulong_t, struct vnode *, u_offset_t, size_t, uint_t, 3917c478bd9Sstevel@tonic-gate uint_t, struct vpage [], struct cred *); 3927c478bd9Sstevel@tonic-gate extern void anon_free(struct anon_hdr *, ulong_t, size_t); 3937c478bd9Sstevel@tonic-gate extern void anon_free_pages(struct anon_hdr *, ulong_t, size_t, uint_t); 394*8905f42cSBryan Cantrill extern int anon_disclaim(struct anon_map *, 395*8905f42cSBryan Cantrill ulong_t, size_t, uint_t, pgcnt_t *); 3967c478bd9Sstevel@tonic-gate extern int anon_getpage(struct anon **, uint_t *, struct page **, 3977c478bd9Sstevel@tonic-gate size_t, struct seg *, caddr_t, enum seg_rw, struct cred *); 3987c478bd9Sstevel@tonic-gate extern int swap_getconpage(struct vnode *, u_offset_t, size_t, 39907b65a64Saguzovsk uint_t *, page_t *[], size_t, page_t *, uint_t *, 4007c478bd9Sstevel@tonic-gate spgcnt_t *, struct seg *, caddr_t, 4017c478bd9Sstevel@tonic-gate enum seg_rw, struct cred *); 4027c478bd9Sstevel@tonic-gate extern int anon_map_getpages(struct anon_map *, ulong_t, 4037c478bd9Sstevel@tonic-gate uint_t, struct seg *, caddr_t, uint_t, 4047c478bd9Sstevel@tonic-gate uint_t *, page_t *[], uint_t *, 4052cb27123Saguzovsk struct vpage [], enum seg_rw, int, int, int, struct cred *); 4067c478bd9Sstevel@tonic-gate extern int anon_map_privatepages(struct anon_map *, ulong_t, 4077c478bd9Sstevel@tonic-gate uint_t, struct seg *, caddr_t, uint_t, 4082cb27123Saguzovsk page_t *[], struct vpage [], int, int, struct cred *); 4097c478bd9Sstevel@tonic-gate extern struct page *anon_private(struct anon **, struct seg *, 4107c478bd9Sstevel@tonic-gate caddr_t, uint_t, struct page *, 4117c478bd9Sstevel@tonic-gate int, struct cred *); 4127c478bd9Sstevel@tonic-gate extern struct page *anon_zero(struct seg *, caddr_t, 4137c478bd9Sstevel@tonic-gate struct anon **, struct cred *); 4147c478bd9Sstevel@tonic-gate extern int anon_map_createpages(struct anon_map *, ulong_t, 4157c478bd9Sstevel@tonic-gate size_t, struct page **, 4167c478bd9Sstevel@tonic-gate struct seg *, caddr_t, 4177c478bd9Sstevel@tonic-gate enum seg_rw, struct cred *); 4187c478bd9Sstevel@tonic-gate extern int anon_map_demotepages(struct anon_map *, ulong_t, 4197c478bd9Sstevel@tonic-gate struct seg *, caddr_t, uint_t, 4207c478bd9Sstevel@tonic-gate struct vpage [], struct cred *); 42107b65a64Saguzovsk extern void anon_shmap_free_pages(struct anon_map *, ulong_t, size_t); 4222cb27123Saguzovsk extern int anon_resvmem(size_t, boolean_t, zone_t *, int); 4230209230bSgjelinek extern void anon_unresvmem(size_t, zone_t *); 4242cb27123Saguzovsk extern struct anon_map *anonmap_alloc(size_t, size_t, int); 4257c478bd9Sstevel@tonic-gate extern void anonmap_free(struct anon_map *); 426a98e9dbfSaguzovsk extern void anonmap_purge(struct anon_map *); 427a98e9dbfSaguzovsk extern void anon_swap_free(struct anon *, struct page *); 4287c478bd9Sstevel@tonic-gate extern void anon_decref(struct anon *); 4297c478bd9Sstevel@tonic-gate extern int non_anon(struct anon_hdr *, ulong_t, u_offset_t *, size_t *); 4307c478bd9Sstevel@tonic-gate extern pgcnt_t anon_pages(struct anon_hdr *, ulong_t, pgcnt_t); 431e0cb4e8dSOndrej Kubecka extern int anon_swap_adjust(pgcnt_t); 4327c478bd9Sstevel@tonic-gate extern void anon_swap_restore(pgcnt_t); 4337c478bd9Sstevel@tonic-gate extern struct anon_hdr *anon_create(pgcnt_t, int); 4347c478bd9Sstevel@tonic-gate extern void anon_release(struct anon_hdr *, pgcnt_t); 4357c478bd9Sstevel@tonic-gate extern struct anon *anon_get_ptr(struct anon_hdr *, ulong_t); 4367c478bd9Sstevel@tonic-gate extern ulong_t *anon_get_slot(struct anon_hdr *, ulong_t); 4377c478bd9Sstevel@tonic-gate extern struct anon *anon_get_next_ptr(struct anon_hdr *, ulong_t *); 4387c478bd9Sstevel@tonic-gate extern int anon_set_ptr(struct anon_hdr *, ulong_t, struct anon *, int); 4397c478bd9Sstevel@tonic-gate extern int anon_copy_ptr(struct anon_hdr *, ulong_t, 4407c478bd9Sstevel@tonic-gate struct anon_hdr *, ulong_t, pgcnt_t, int); 4417c478bd9Sstevel@tonic-gate extern pgcnt_t anon_grow(struct anon_hdr *, ulong_t *, pgcnt_t, pgcnt_t, int); 4427c478bd9Sstevel@tonic-gate extern void anon_array_enter(struct anon_map *, ulong_t, 4437c478bd9Sstevel@tonic-gate anon_sync_obj_t *); 44487015465Scwb extern int anon_array_try_enter(struct anon_map *, ulong_t, 44587015465Scwb anon_sync_obj_t *); 4467c478bd9Sstevel@tonic-gate extern void anon_array_exit(anon_sync_obj_t *); 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate /* 4497c478bd9Sstevel@tonic-gate * anon_resv checks to see if there is enough swap space to fulfill a 4507c478bd9Sstevel@tonic-gate * request and if so, reserves the appropriate anonymous memory resources. 4517c478bd9Sstevel@tonic-gate * anon_checkspace just checks to see if there is space to fulfill the request, 4527c478bd9Sstevel@tonic-gate * without taking any resources. Both return 1 if successful and 0 if not. 4530209230bSgjelinek * 4540209230bSgjelinek * Macros are provided as anon reservation is usually charged to the zone of 4550209230bSgjelinek * the current process. In some cases (such as anon reserved by tmpfs), a 4560209230bSgjelinek * zone pointer is needed to charge the appropriate zone. 4577c478bd9Sstevel@tonic-gate */ 4580209230bSgjelinek #define anon_unresv(size) anon_unresvmem(size, curproc->p_zone) 4590209230bSgjelinek #define anon_unresv_zone(size, zone) anon_unresvmem(size, zone) 4602cb27123Saguzovsk #define anon_resv(size) \ 4612cb27123Saguzovsk anon_resvmem((size), 1, curproc->p_zone, 1) 4622cb27123Saguzovsk #define anon_resv_zone(size, zone) anon_resvmem((size), 1, zone, 1) 4638c06a490Sjj #define anon_checkspace(size, zone) anon_resvmem((size), 0, zone, 0) 4642cb27123Saguzovsk #define anon_try_resv_zone(size, zone) anon_resvmem((size), 1, zone, 0) 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate /* 4677c478bd9Sstevel@tonic-gate * Flags to anon_private 4687c478bd9Sstevel@tonic-gate */ 4697c478bd9Sstevel@tonic-gate #define STEAL_PAGE 0x1 /* page can be stolen */ 4707c478bd9Sstevel@tonic-gate #define LOCK_PAGE 0x2 /* page must be ``logically'' locked */ 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate /* 4737c478bd9Sstevel@tonic-gate * SEGKP ANON pages that are locked are assumed to be LWP stack pages 4747c478bd9Sstevel@tonic-gate * and thus count towards the user pages locked count. 4757c478bd9Sstevel@tonic-gate * This value is protected by the same lock as availrmem. 4767c478bd9Sstevel@tonic-gate */ 4777c478bd9Sstevel@tonic-gate extern pgcnt_t anon_segkp_pages_locked; 4787c478bd9Sstevel@tonic-gate 4797c478bd9Sstevel@tonic-gate extern int anon_debug; 4807c478bd9Sstevel@tonic-gate 4817c478bd9Sstevel@tonic-gate #ifdef ANON_DEBUG 4827c478bd9Sstevel@tonic-gate 4837c478bd9Sstevel@tonic-gate #define A_ANON 0x01 4847c478bd9Sstevel@tonic-gate #define A_RESV 0x02 4857c478bd9Sstevel@tonic-gate #define A_MRESV 0x04 4867c478bd9Sstevel@tonic-gate 4877c478bd9Sstevel@tonic-gate /* vararg-like debugging macro. */ 4887c478bd9Sstevel@tonic-gate #define ANON_PRINT(f, printf_args) \ 4897c478bd9Sstevel@tonic-gate if (anon_debug & f) \ 4907c478bd9Sstevel@tonic-gate printf printf_args 4917c478bd9Sstevel@tonic-gate 4927c478bd9Sstevel@tonic-gate #else /* ANON_DEBUG */ 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate #define ANON_PRINT(f, printf_args) 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate #endif /* ANON_DEBUG */ 4977c478bd9Sstevel@tonic-gate 4987c478bd9Sstevel@tonic-gate #endif /* _KERNEL */ 4997c478bd9Sstevel@tonic-gate 5007c478bd9Sstevel@tonic-gate #ifdef __cplusplus 5017c478bd9Sstevel@tonic-gate } 5027c478bd9Sstevel@tonic-gate #endif 5037c478bd9Sstevel@tonic-gate 5047c478bd9Sstevel@tonic-gate #endif /* _VM_ANON_H */ 505