17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50b5aa17bSmec * Common Development and Distribution License (the "License"). 60b5aa17bSmec * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 227196569bSSherry Moore * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 317c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate 357c478bd9Sstevel@tonic-gate /* 367c478bd9Sstevel@tonic-gate * This file contains common functions to access and manage the page lists. 377c478bd9Sstevel@tonic-gate * Many of these routines originated from platform dependent modules 387c478bd9Sstevel@tonic-gate * (sun4/vm/vm_dep.c, i86pc/vm/vm_machdep.c) and modified to function in 397c478bd9Sstevel@tonic-gate * a platform independent manner. 407c478bd9Sstevel@tonic-gate * 417c478bd9Sstevel@tonic-gate * vm/vm_dep.h provides for platform specific support. 427c478bd9Sstevel@tonic-gate */ 437c478bd9Sstevel@tonic-gate 447c478bd9Sstevel@tonic-gate #include <sys/types.h> 457c478bd9Sstevel@tonic-gate #include <sys/debug.h> 467c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 477c478bd9Sstevel@tonic-gate #include <sys/systm.h> 487c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 497c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 507c478bd9Sstevel@tonic-gate #include <vm/as.h> 517c478bd9Sstevel@tonic-gate #include <vm/page.h> 527c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 537c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 5478b03d3aSkchow #include <sys/vmsystm.h> 557c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 567c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 577c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 587c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 597c478bd9Sstevel@tonic-gate #include <sys/callb.h> 607c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 617c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 627c478bd9Sstevel@tonic-gate 637c478bd9Sstevel@tonic-gate extern uint_t vac_colors; 647c478bd9Sstevel@tonic-gate 656061ce8aSkchow #define MAX_PRAGMA_ALIGN 128 666061ce8aSkchow 676061ce8aSkchow /* vm_cpu_data0 for the boot cpu before kmem is initialized */ 686061ce8aSkchow 696061ce8aSkchow #if L2CACHE_ALIGN_MAX <= MAX_PRAGMA_ALIGN 70affbd3ccSkchow #pragma align L2CACHE_ALIGN_MAX(vm_cpu_data0) 716061ce8aSkchow #else 726061ce8aSkchow #pragma align MAX_PRAGMA_ALIGN(vm_cpu_data0) 736061ce8aSkchow #endif 74affbd3ccSkchow char vm_cpu_data0[VM_CPU_DATA_PADSIZE]; 75affbd3ccSkchow 767c478bd9Sstevel@tonic-gate /* 777c478bd9Sstevel@tonic-gate * number of page colors equivalent to reqested color in page_get routines. 787c478bd9Sstevel@tonic-gate * If set, keeps large pages intact longer and keeps MPO allocation 797c478bd9Sstevel@tonic-gate * from the local mnode in favor of acquiring the 'correct' page color from 807c478bd9Sstevel@tonic-gate * a demoted large page or from a remote mnode. 817c478bd9Sstevel@tonic-gate */ 825d07b933Sdp uint_t colorequiv; 835d07b933Sdp 845d07b933Sdp /* 855d07b933Sdp * color equivalency mask for each page size. 865d07b933Sdp * Mask is computed based on cpu L2$ way sizes and colorequiv global. 875d07b933Sdp * High 4 bits determine the number of high order bits of the color to ignore. 885d07b933Sdp * Low 4 bits determines number of low order bits of color to ignore (it's only 895d07b933Sdp * relevant for hashed index based page coloring). 905d07b933Sdp */ 915d07b933Sdp uchar_t colorequivszc[MMU_PAGE_SIZES]; 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate /* 947c478bd9Sstevel@tonic-gate * if set, specifies the percentage of large pages that are free from within 957c478bd9Sstevel@tonic-gate * a large page region before attempting to lock those pages for 967c478bd9Sstevel@tonic-gate * page_get_contig_pages processing. 977c478bd9Sstevel@tonic-gate * 987c478bd9Sstevel@tonic-gate * Should be turned on when kpr is available when page_trylock_contig_pages 997c478bd9Sstevel@tonic-gate * can be more selective. 1007c478bd9Sstevel@tonic-gate */ 1017c478bd9Sstevel@tonic-gate 1027c478bd9Sstevel@tonic-gate int ptcpthreshold; 1037c478bd9Sstevel@tonic-gate 1047c478bd9Sstevel@tonic-gate /* 1057c478bd9Sstevel@tonic-gate * Limit page get contig page search based on failure cnts in pgcpfailcnt[]. 10683f9b804Skchow * Enabled by default via pgcplimitsearch. 10783f9b804Skchow * 10883f9b804Skchow * pgcpfailcnt[] is bounded by PGCPFAILMAX (>= 1/2 of installed 10983f9b804Skchow * memory). When reached, pgcpfailcnt[] is reset to 1/2 of this upper 11083f9b804Skchow * bound. This upper bound range guarantees: 11183f9b804Skchow * - all large page 'slots' will be searched over time 11283f9b804Skchow * - the minimum (1) large page candidates considered on each pgcp call 11383f9b804Skchow * - count doesn't wrap around to 0 1147c478bd9Sstevel@tonic-gate */ 11583f9b804Skchow pgcnt_t pgcpfailcnt[MMU_PAGE_SIZES]; 1167c478bd9Sstevel@tonic-gate int pgcplimitsearch = 1; 1177c478bd9Sstevel@tonic-gate 11883f9b804Skchow #define PGCPFAILMAX (1 << (highbit(physinstalled) - 1)) 11983f9b804Skchow #define SETPGCPFAILCNT(szc) \ 12083f9b804Skchow if (++pgcpfailcnt[szc] >= PGCPFAILMAX) \ 12183f9b804Skchow pgcpfailcnt[szc] = PGCPFAILMAX / 2; 12283f9b804Skchow 1237c478bd9Sstevel@tonic-gate #ifdef VM_STATS 1247c478bd9Sstevel@tonic-gate struct vmm_vmstats_str vmm_vmstats; 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 1277c478bd9Sstevel@tonic-gate 1287c478bd9Sstevel@tonic-gate #if defined(__sparc) 1297c478bd9Sstevel@tonic-gate #define LPGCREATE 0 1307c478bd9Sstevel@tonic-gate #else 1317c478bd9Sstevel@tonic-gate /* enable page_get_contig_pages */ 1327c478bd9Sstevel@tonic-gate #define LPGCREATE 1 1337c478bd9Sstevel@tonic-gate #endif 1347c478bd9Sstevel@tonic-gate 1357c478bd9Sstevel@tonic-gate int pg_contig_disable; 1367c478bd9Sstevel@tonic-gate int pg_lpgcreate_nocage = LPGCREATE; 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate /* 13919397407SSherry Moore * page_freelist_split pfn flag to signify no lo or hi pfn requirement. 1407c478bd9Sstevel@tonic-gate */ 1417c478bd9Sstevel@tonic-gate #define PFNNULL 0 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate /* Flags involved in promotion and demotion routines */ 1447c478bd9Sstevel@tonic-gate #define PC_FREE 0x1 /* put page on freelist */ 1457c478bd9Sstevel@tonic-gate #define PC_ALLOC 0x2 /* return page for allocation */ 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate /* 1487c478bd9Sstevel@tonic-gate * Flag for page_demote to be used with PC_FREE to denote that we don't care 1497c478bd9Sstevel@tonic-gate * what the color is as the color parameter to the function is ignored. 1507c478bd9Sstevel@tonic-gate */ 1517c478bd9Sstevel@tonic-gate #define PC_NO_COLOR (-1) 1527c478bd9Sstevel@tonic-gate 1535d07b933Sdp /* mtype value for page_promote to use when mtype does not matter */ 1545d07b933Sdp #define PC_MTYPE_ANY (-1) 1555d07b933Sdp 1567c478bd9Sstevel@tonic-gate /* 1577c478bd9Sstevel@tonic-gate * page counters candidates info 1587c478bd9Sstevel@tonic-gate * See page_ctrs_cands comment below for more details. 1597c478bd9Sstevel@tonic-gate * fields are as follows: 1607c478bd9Sstevel@tonic-gate * pcc_pages_free: # pages which freelist coalesce can create 1617c478bd9Sstevel@tonic-gate * pcc_color_free: pointer to page free counts per color 1627c478bd9Sstevel@tonic-gate */ 1637c478bd9Sstevel@tonic-gate typedef struct pcc_info { 1647c478bd9Sstevel@tonic-gate pgcnt_t pcc_pages_free; 1657c478bd9Sstevel@tonic-gate pgcnt_t *pcc_color_free; 16606fb6a36Sdv uint_t pad[12]; 1677c478bd9Sstevel@tonic-gate } pcc_info_t; 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate /* 1707c478bd9Sstevel@tonic-gate * On big machines it can take a long time to check page_counters 1717c478bd9Sstevel@tonic-gate * arrays. page_ctrs_cands is a summary array whose elements are a dynamically 1727c478bd9Sstevel@tonic-gate * updated sum of all elements of the corresponding page_counters arrays. 1737c478bd9Sstevel@tonic-gate * page_freelist_coalesce() searches page_counters only if an appropriate 1747c478bd9Sstevel@tonic-gate * element of page_ctrs_cands array is greater than 0. 1757c478bd9Sstevel@tonic-gate * 1765d07b933Sdp * page_ctrs_cands is indexed by mutex (i), region (r), mnode (m), mrange (g) 1777c478bd9Sstevel@tonic-gate */ 1785d07b933Sdp pcc_info_t **page_ctrs_cands[NPC_MUTEX][MMU_PAGE_SIZES]; 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate /* 1817c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1825d07b933Sdp * for the given mnode (m), mrange (g), and region size (r) 1837c478bd9Sstevel@tonic-gate */ 1845d07b933Sdp #define PGCTRS_CANDS_GETVALUE(m, g, r, val) { \ 1857c478bd9Sstevel@tonic-gate int i; \ 1867c478bd9Sstevel@tonic-gate val = 0; \ 1877c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 1885d07b933Sdp val += page_ctrs_cands[i][(r)][(m)][(g)].pcc_pages_free; \ 1897c478bd9Sstevel@tonic-gate } \ 1907c478bd9Sstevel@tonic-gate } 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate /* 1937c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1945d07b933Sdp * for the given mnode (m), mrange (g), region size (r), and color (c) 1957c478bd9Sstevel@tonic-gate */ 1965d07b933Sdp #define PGCTRS_CANDS_GETVALUECOLOR(m, g, r, c, val) { \ 1977c478bd9Sstevel@tonic-gate int i; \ 1987c478bd9Sstevel@tonic-gate val = 0; \ 1995d07b933Sdp ASSERT((c) < PAGE_GET_PAGECOLORS(r)); \ 2007c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 2015d07b933Sdp val += \ 2025d07b933Sdp page_ctrs_cands[i][(r)][(m)][(g)].pcc_color_free[(c)]; \ 2037c478bd9Sstevel@tonic-gate } \ 2047c478bd9Sstevel@tonic-gate } 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate /* 2077c478bd9Sstevel@tonic-gate * We can only allow a single thread to update a counter within the physical 2087c478bd9Sstevel@tonic-gate * range of the largest supported page size. That is the finest granularity 2097c478bd9Sstevel@tonic-gate * possible since the counter values are dependent on each other 2107c478bd9Sstevel@tonic-gate * as you move accross region sizes. PP_CTR_LOCK_INDX is used to determine the 2117c478bd9Sstevel@tonic-gate * ctr_mutex lock index for a particular physical range. 2127c478bd9Sstevel@tonic-gate */ 2137c478bd9Sstevel@tonic-gate static kmutex_t *ctr_mutex[NPC_MUTEX]; 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate #define PP_CTR_LOCK_INDX(pp) \ 2165d07b933Sdp (((pp)->p_pagenum >> \ 2177c478bd9Sstevel@tonic-gate (PAGE_BSZS_SHIFT(mmu_page_sizes - 1))) & (NPC_MUTEX - 1)) 2187c478bd9Sstevel@tonic-gate 2195d07b933Sdp #define INVALID_COLOR 0xffffffff 2205d07b933Sdp #define INVALID_MASK 0xffffffff 2215d07b933Sdp 2227c478bd9Sstevel@tonic-gate /* 2237c478bd9Sstevel@tonic-gate * Local functions prototypes. 2247c478bd9Sstevel@tonic-gate */ 2257c478bd9Sstevel@tonic-gate 226affbd3ccSkchow void page_ctr_add(int, int, page_t *, int); 227affbd3ccSkchow void page_ctr_add_internal(int, int, page_t *, int); 228affbd3ccSkchow void page_ctr_sub(int, int, page_t *, int); 2295d07b933Sdp void page_ctr_sub_internal(int, int, page_t *, int); 2307c478bd9Sstevel@tonic-gate void page_freelist_lock(int); 2317c478bd9Sstevel@tonic-gate void page_freelist_unlock(int); 2325d07b933Sdp page_t *page_promote(int, pfn_t, uchar_t, int, int); 23319397407SSherry Moore page_t *page_demote(int, pfn_t, pfn_t, uchar_t, uchar_t, int, int); 2345d07b933Sdp page_t *page_freelist_split(uchar_t, 23519397407SSherry Moore uint_t, int, int, pfn_t, pfn_t, page_list_walker_t *); 2367c478bd9Sstevel@tonic-gate page_t *page_get_mnode_cachelist(uint_t, uint_t, int, int); 2377c478bd9Sstevel@tonic-gate static int page_trylock_cons(page_t *pp, se_t se); 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * The page_counters array below is used to keep track of free contiguous 2417c478bd9Sstevel@tonic-gate * physical memory. A hw_page_map_t will be allocated per mnode per szc. 2427c478bd9Sstevel@tonic-gate * This contains an array of counters, the size of the array, a shift value 2437c478bd9Sstevel@tonic-gate * used to convert a pagenum into a counter array index or vice versa, as 2447c478bd9Sstevel@tonic-gate * well as a cache of the last successful index to be promoted to a larger 2457c478bd9Sstevel@tonic-gate * page size. As an optimization, we keep track of the last successful index 2467c478bd9Sstevel@tonic-gate * to be promoted per page color for the given size region, and this is 2477c478bd9Sstevel@tonic-gate * allocated dynamically based upon the number of colors for a given 2487c478bd9Sstevel@tonic-gate * region size. 2497c478bd9Sstevel@tonic-gate * 2507c478bd9Sstevel@tonic-gate * Conceptually, the page counters are represented as: 2517c478bd9Sstevel@tonic-gate * 2527c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode] 2537c478bd9Sstevel@tonic-gate * 2547c478bd9Sstevel@tonic-gate * region_size: size code of a candidate larger page made up 2557c478bd9Sstevel@tonic-gate * of contiguous free smaller pages. 2567c478bd9Sstevel@tonic-gate * 2577c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode].hpm_counters[index]: 2587c478bd9Sstevel@tonic-gate * represents how many (region_size - 1) pages either 2597c478bd9Sstevel@tonic-gate * exist or can be created within the given index range. 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * Let's look at a sparc example: 2627c478bd9Sstevel@tonic-gate * If we want to create a free 512k page, we look at region_size 2 2637c478bd9Sstevel@tonic-gate * for the mnode we want. We calculate the index and look at a specific 2647c478bd9Sstevel@tonic-gate * hpm_counters location. If we see 8 (FULL_REGION_CNT on sparc) at 2657c478bd9Sstevel@tonic-gate * this location, it means that 8 64k pages either exist or can be created 2667c478bd9Sstevel@tonic-gate * from 8K pages in order to make a single free 512k page at the given 2677c478bd9Sstevel@tonic-gate * index. Note that when a region is full, it will contribute to the 2687c478bd9Sstevel@tonic-gate * counts in the region above it. Thus we will not know what page 2697c478bd9Sstevel@tonic-gate * size the free pages will be which can be promoted to this new free 2707c478bd9Sstevel@tonic-gate * page unless we look at all regions below the current region. 2717c478bd9Sstevel@tonic-gate */ 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate /* 2747c478bd9Sstevel@tonic-gate * Note: hpmctr_t is defined in platform vm_dep.h 2757c478bd9Sstevel@tonic-gate * hw_page_map_t contains all the information needed for the page_counters 2767c478bd9Sstevel@tonic-gate * logic. The fields are as follows: 2777c478bd9Sstevel@tonic-gate * 2787c478bd9Sstevel@tonic-gate * hpm_counters: dynamically allocated array to hold counter data 2797c478bd9Sstevel@tonic-gate * hpm_entries: entries in hpm_counters 2807c478bd9Sstevel@tonic-gate * hpm_shift: shift for pnum/array index conv 2817c478bd9Sstevel@tonic-gate * hpm_base: PFN mapped to counter index 0 2827c478bd9Sstevel@tonic-gate * hpm_color_current: last index in counter array for this color at 2837c478bd9Sstevel@tonic-gate * which we successfully created a large page 2847c478bd9Sstevel@tonic-gate */ 2857c478bd9Sstevel@tonic-gate typedef struct hw_page_map { 2867c478bd9Sstevel@tonic-gate hpmctr_t *hpm_counters; 2877c478bd9Sstevel@tonic-gate size_t hpm_entries; 2887c478bd9Sstevel@tonic-gate int hpm_shift; 2897c478bd9Sstevel@tonic-gate pfn_t hpm_base; 2905d07b933Sdp size_t *hpm_color_current[MAX_MNODE_MRANGES]; 29106fb6a36Sdv #if defined(__sparc) 29206fb6a36Sdv uint_t pad[4]; 29306fb6a36Sdv #endif 2947c478bd9Sstevel@tonic-gate } hw_page_map_t; 2957c478bd9Sstevel@tonic-gate 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * Element zero is not used, but is allocated for convenience. 2987c478bd9Sstevel@tonic-gate */ 2997c478bd9Sstevel@tonic-gate static hw_page_map_t *page_counters[MMU_PAGE_SIZES]; 3007c478bd9Sstevel@tonic-gate 3015d07b933Sdp /* 3025d07b933Sdp * Cached value of MNODE_RANGE_CNT(mnode). 3035d07b933Sdp * This is a function call in x86. 3045d07b933Sdp */ 3055d07b933Sdp static int mnode_nranges[MAX_MEM_NODES]; 3065d07b933Sdp static int mnode_maxmrange[MAX_MEM_NODES]; 3075d07b933Sdp 3087c478bd9Sstevel@tonic-gate /* 3097c478bd9Sstevel@tonic-gate * The following macros are convenient ways to get access to the individual 3107c478bd9Sstevel@tonic-gate * elements of the page_counters arrays. They can be used on both 3117c478bd9Sstevel@tonic-gate * the left side and right side of equations. 3127c478bd9Sstevel@tonic-gate */ 3137c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS(mnode, rg_szc, idx) \ 3147c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters[(idx)]) 3157c478bd9Sstevel@tonic-gate 3167c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_COUNTERS(mnode, rg_szc) \ 3177c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters) 3187c478bd9Sstevel@tonic-gate 3197c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_SHIFT(mnode, rg_szc) \ 3207c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_shift) 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_ENTRIES(mnode, rg_szc) \ 3237c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_entries) 3247c478bd9Sstevel@tonic-gate 3257c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_BASE(mnode, rg_szc) \ 3267c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_base) 3277c478bd9Sstevel@tonic-gate 3285d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, rg_szc, g) \ 3295d07b933Sdp (page_counters[(rg_szc)][(mnode)].hpm_color_current[(g)]) 3307c478bd9Sstevel@tonic-gate 3315d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR(mnode, rg_szc, color, mrange) \ 3325d07b933Sdp (page_counters[(rg_szc)][(mnode)]. \ 3335d07b933Sdp hpm_color_current[(mrange)][(color)]) 3347c478bd9Sstevel@tonic-gate 3357c478bd9Sstevel@tonic-gate #define PNUM_TO_IDX(mnode, rg_szc, pnum) \ 3367c478bd9Sstevel@tonic-gate (((pnum) - PAGE_COUNTERS_BASE((mnode), (rg_szc))) >> \ 3377c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT((mnode), (rg_szc))) 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate #define IDX_TO_PNUM(mnode, rg_szc, index) \ 3407c478bd9Sstevel@tonic-gate (PAGE_COUNTERS_BASE((mnode), (rg_szc)) + \ 3417c478bd9Sstevel@tonic-gate ((index) << PAGE_COUNTERS_SHIFT((mnode), (rg_szc)))) 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate /* 3447c478bd9Sstevel@tonic-gate * Protects the hpm_counters and hpm_color_current memory from changing while 3457c478bd9Sstevel@tonic-gate * looking at page counters information. 3467c478bd9Sstevel@tonic-gate * Grab the write lock to modify what these fields point at. 3477c478bd9Sstevel@tonic-gate * Grab the read lock to prevent any pointers from changing. 3487c478bd9Sstevel@tonic-gate * The write lock can not be held during memory allocation due to a possible 3497c478bd9Sstevel@tonic-gate * recursion deadlock with trying to grab the read lock while the 3507c478bd9Sstevel@tonic-gate * write lock is already held. 3517c478bd9Sstevel@tonic-gate */ 3527c478bd9Sstevel@tonic-gate krwlock_t page_ctrs_rwlock[MAX_MEM_NODES]; 3537c478bd9Sstevel@tonic-gate 354affbd3ccSkchow 355affbd3ccSkchow /* 356affbd3ccSkchow * initialize cpu_vm_data to point at cache aligned vm_cpu_data_t. 357affbd3ccSkchow */ 358affbd3ccSkchow void 359affbd3ccSkchow cpu_vm_data_init(struct cpu *cp) 360affbd3ccSkchow { 361affbd3ccSkchow if (cp == CPU0) { 362affbd3ccSkchow cp->cpu_vm_data = (void *)&vm_cpu_data0; 363affbd3ccSkchow } else { 364affbd3ccSkchow void *kmptr; 3656061ce8aSkchow int align; 3666061ce8aSkchow size_t sz; 367affbd3ccSkchow 3686061ce8aSkchow align = (L2CACHE_ALIGN) ? L2CACHE_ALIGN : L2CACHE_ALIGN_MAX; 3696061ce8aSkchow sz = P2ROUNDUP(sizeof (vm_cpu_data_t), align) + align; 3706061ce8aSkchow kmptr = kmem_zalloc(sz, KM_SLEEP); 371affbd3ccSkchow cp->cpu_vm_data = (void *) P2ROUNDUP((uintptr_t)kmptr, align); 372affbd3ccSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr = kmptr; 3736061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize = sz; 374affbd3ccSkchow } 375affbd3ccSkchow } 376affbd3ccSkchow 377affbd3ccSkchow /* 378affbd3ccSkchow * free cpu_vm_data 379affbd3ccSkchow */ 380affbd3ccSkchow void 381affbd3ccSkchow cpu_vm_data_destroy(struct cpu *cp) 382affbd3ccSkchow { 383affbd3ccSkchow if (cp->cpu_seqid && cp->cpu_vm_data) { 384affbd3ccSkchow ASSERT(cp != CPU0); 385affbd3ccSkchow kmem_free(((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr, 3866061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize); 387affbd3ccSkchow } 388affbd3ccSkchow cp->cpu_vm_data = NULL; 389affbd3ccSkchow } 390affbd3ccSkchow 391affbd3ccSkchow 3927c478bd9Sstevel@tonic-gate /* 3937c478bd9Sstevel@tonic-gate * page size to page size code 3947c478bd9Sstevel@tonic-gate */ 3957c478bd9Sstevel@tonic-gate int 3967c478bd9Sstevel@tonic-gate page_szc(size_t pagesize) 3977c478bd9Sstevel@tonic-gate { 3987c478bd9Sstevel@tonic-gate int i = 0; 3997c478bd9Sstevel@tonic-gate 4007c478bd9Sstevel@tonic-gate while (hw_page_array[i].hp_size) { 4017c478bd9Sstevel@tonic-gate if (pagesize == hw_page_array[i].hp_size) 4027c478bd9Sstevel@tonic-gate return (i); 4037c478bd9Sstevel@tonic-gate i++; 4047c478bd9Sstevel@tonic-gate } 4057c478bd9Sstevel@tonic-gate return (-1); 4067c478bd9Sstevel@tonic-gate } 4077c478bd9Sstevel@tonic-gate 4087c478bd9Sstevel@tonic-gate /* 4094abce959Smec * page size to page size code with the restriction that it be a supported 4104abce959Smec * user page size. If it's not a supported user page size, -1 will be returned. 4117c478bd9Sstevel@tonic-gate */ 4127c478bd9Sstevel@tonic-gate int 4134abce959Smec page_szc_user_filtered(size_t pagesize) 4147c478bd9Sstevel@tonic-gate { 4157c478bd9Sstevel@tonic-gate int szc = page_szc(pagesize); 4164abce959Smec if ((szc != -1) && (SZC_2_USERSZC(szc) != -1)) { 4174abce959Smec return (szc); 4184abce959Smec } 4197c478bd9Sstevel@tonic-gate return (-1); 4207c478bd9Sstevel@tonic-gate } 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate /* 4237c478bd9Sstevel@tonic-gate * Return how many page sizes are available for the user to use. This is 4247c478bd9Sstevel@tonic-gate * what the hardware supports and not based upon how the OS implements the 4257c478bd9Sstevel@tonic-gate * support of different page sizes. 42602bc52beSkchow * 42702bc52beSkchow * If legacy is non-zero, return the number of pagesizes available to legacy 42802bc52beSkchow * applications. The number of legacy page sizes might be less than the 42902bc52beSkchow * exported user page sizes. This is to prevent legacy applications that 43002bc52beSkchow * use the largest page size returned from getpagesizes(3c) from inadvertantly 43102bc52beSkchow * using the 'new' large pagesizes. 4327c478bd9Sstevel@tonic-gate */ 4337c478bd9Sstevel@tonic-gate uint_t 43402bc52beSkchow page_num_user_pagesizes(int legacy) 4357c478bd9Sstevel@tonic-gate { 43602bc52beSkchow if (legacy) 43702bc52beSkchow return (mmu_legacy_page_sizes); 4387c478bd9Sstevel@tonic-gate return (mmu_exported_page_sizes); 4397c478bd9Sstevel@tonic-gate } 4407c478bd9Sstevel@tonic-gate 4417c478bd9Sstevel@tonic-gate uint_t 4427c478bd9Sstevel@tonic-gate page_num_pagesizes(void) 4437c478bd9Sstevel@tonic-gate { 4447c478bd9Sstevel@tonic-gate return (mmu_page_sizes); 4457c478bd9Sstevel@tonic-gate } 4467c478bd9Sstevel@tonic-gate 4477c478bd9Sstevel@tonic-gate /* 4487c478bd9Sstevel@tonic-gate * returns the count of the number of base pagesize pages associated with szc 4497c478bd9Sstevel@tonic-gate */ 4507c478bd9Sstevel@tonic-gate pgcnt_t 4517c478bd9Sstevel@tonic-gate page_get_pagecnt(uint_t szc) 4527c478bd9Sstevel@tonic-gate { 4537c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4547c478bd9Sstevel@tonic-gate panic("page_get_pagecnt: out of range %d", szc); 4557c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_pgcnt); 4567c478bd9Sstevel@tonic-gate } 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate size_t 4597c478bd9Sstevel@tonic-gate page_get_pagesize(uint_t szc) 4607c478bd9Sstevel@tonic-gate { 4617c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4627c478bd9Sstevel@tonic-gate panic("page_get_pagesize: out of range %d", szc); 4637c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4647c478bd9Sstevel@tonic-gate } 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate /* 4677c478bd9Sstevel@tonic-gate * Return the size of a page based upon the index passed in. An index of 4687c478bd9Sstevel@tonic-gate * zero refers to the smallest page size in the system, and as index increases 4697c478bd9Sstevel@tonic-gate * it refers to the next larger supported page size in the system. 4707c478bd9Sstevel@tonic-gate * Note that szc and userszc may not be the same due to unsupported szc's on 4717c478bd9Sstevel@tonic-gate * some systems. 4727c478bd9Sstevel@tonic-gate */ 4737c478bd9Sstevel@tonic-gate size_t 4747c478bd9Sstevel@tonic-gate page_get_user_pagesize(uint_t userszc) 4757c478bd9Sstevel@tonic-gate { 4767c478bd9Sstevel@tonic-gate uint_t szc = USERSZC_2_SZC(userszc); 4777c478bd9Sstevel@tonic-gate 4787c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4797c478bd9Sstevel@tonic-gate panic("page_get_user_pagesize: out of range %d", szc); 4807c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4817c478bd9Sstevel@tonic-gate } 4827c478bd9Sstevel@tonic-gate 4837c478bd9Sstevel@tonic-gate uint_t 4847c478bd9Sstevel@tonic-gate page_get_shift(uint_t szc) 4857c478bd9Sstevel@tonic-gate { 4867c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4877c478bd9Sstevel@tonic-gate panic("page_get_shift: out of range %d", szc); 4885d07b933Sdp return (PAGE_GET_SHIFT(szc)); 4897c478bd9Sstevel@tonic-gate } 4907c478bd9Sstevel@tonic-gate 4917c478bd9Sstevel@tonic-gate uint_t 4927c478bd9Sstevel@tonic-gate page_get_pagecolors(uint_t szc) 4937c478bd9Sstevel@tonic-gate { 4945d07b933Sdp if (szc >= mmu_page_sizes) 4955d07b933Sdp panic("page_get_pagecolors: out of range %d", szc); 4965d07b933Sdp return (PAGE_GET_PAGECOLORS(szc)); 4975d07b933Sdp } 4985d07b933Sdp 4995d07b933Sdp /* 5005d07b933Sdp * this assigns the desired equivalent color after a split 5015d07b933Sdp */ 5025d07b933Sdp uint_t 5035d07b933Sdp page_correct_color(uchar_t szc, uchar_t nszc, uint_t color, 5045d07b933Sdp uint_t ncolor, uint_t ceq_mask) 5055d07b933Sdp { 5065d07b933Sdp ASSERT(nszc > szc); 5075d07b933Sdp ASSERT(szc < mmu_page_sizes); 5085d07b933Sdp ASSERT(color < PAGE_GET_PAGECOLORS(szc)); 5095d07b933Sdp ASSERT(ncolor < PAGE_GET_PAGECOLORS(nszc)); 5105d07b933Sdp 5115d07b933Sdp color &= ceq_mask; 512ce8eb11aSdp ncolor = PAGE_CONVERT_COLOR(ncolor, szc, nszc); 5135d07b933Sdp return (color | (ncolor & ~ceq_mask)); 5147c478bd9Sstevel@tonic-gate } 5157c478bd9Sstevel@tonic-gate 516ce8eb11aSdp /* 517ce8eb11aSdp * The interleaved_mnodes flag is set when mnodes overlap in 518ce8eb11aSdp * the physbase..physmax range, but have disjoint slices. 519ce8eb11aSdp * In this case hpm_counters is shared by all mnodes. 520ce8eb11aSdp * This flag is set dynamically by the platform. 521ce8eb11aSdp */ 522ce8eb11aSdp int interleaved_mnodes = 0; 523ce8eb11aSdp 5247c478bd9Sstevel@tonic-gate /* 5257c478bd9Sstevel@tonic-gate * Called by startup(). 5267c478bd9Sstevel@tonic-gate * Size up the per page size free list counters based on physmax 5277c478bd9Sstevel@tonic-gate * of each node and max_mem_nodes. 528ce8eb11aSdp * 529ce8eb11aSdp * If interleaved_mnodes is set we need to find the first mnode that 530ce8eb11aSdp * exists. hpm_counters for the first mnode will then be shared by 531ce8eb11aSdp * all other mnodes. If interleaved_mnodes is not set, just set 532ce8eb11aSdp * first=mnode each time. That means there will be no sharing. 5337c478bd9Sstevel@tonic-gate */ 5347c478bd9Sstevel@tonic-gate size_t 5357c478bd9Sstevel@tonic-gate page_ctrs_sz(void) 5367c478bd9Sstevel@tonic-gate { 5377c478bd9Sstevel@tonic-gate int r; /* region size */ 5387c478bd9Sstevel@tonic-gate int mnode; 539ce8eb11aSdp int firstmn; /* first mnode that exists */ 5405d07b933Sdp int nranges; 541ce8eb11aSdp pfn_t physbase; 542ce8eb11aSdp pfn_t physmax; 5437c478bd9Sstevel@tonic-gate uint_t ctrs_sz = 0; 5447c478bd9Sstevel@tonic-gate int i; 5457c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 5467c478bd9Sstevel@tonic-gate 5477c478bd9Sstevel@tonic-gate /* 5487c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 5497c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 5507c478bd9Sstevel@tonic-gate * arrays. 5517c478bd9Sstevel@tonic-gate */ 5525d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 5535d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 5547c478bd9Sstevel@tonic-gate } 5557c478bd9Sstevel@tonic-gate 556ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 5577c478bd9Sstevel@tonic-gate 5587c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 5597c478bd9Sstevel@tonic-gate pfn_t r_base; 5607c478bd9Sstevel@tonic-gate pgcnt_t r_align; 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 5637c478bd9Sstevel@tonic-gate continue; 5647c478bd9Sstevel@tonic-gate 565ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 5665d07b933Sdp nranges = MNODE_RANGE_CNT(mnode); 5675d07b933Sdp mnode_nranges[mnode] = nranges; 5685d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 5695d07b933Sdp 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * determine size needed for page counter arrays with 5727c478bd9Sstevel@tonic-gate * base aligned to large page size. 5737c478bd9Sstevel@tonic-gate */ 5747c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 575ce8eb11aSdp /* add in space for hpm_color_current */ 576ce8eb11aSdp ctrs_sz += sizeof (size_t) * 577ce8eb11aSdp colors_per_szc[r] * nranges; 578ce8eb11aSdp 579ce8eb11aSdp if (firstmn != mnode) 580ce8eb11aSdp continue; 581ce8eb11aSdp 5827c478bd9Sstevel@tonic-gate /* add in space for hpm_counters */ 5837c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 584ce8eb11aSdp r_base = physbase; 5857c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 586ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 587ce8eb11aSdp 5887c478bd9Sstevel@tonic-gate /* 5897c478bd9Sstevel@tonic-gate * Round up to always allocate on pointer sized 5907c478bd9Sstevel@tonic-gate * boundaries. 5917c478bd9Sstevel@tonic-gate */ 5927c478bd9Sstevel@tonic-gate ctrs_sz += P2ROUNDUP((r_pgcnt * sizeof (hpmctr_t)), 5937c478bd9Sstevel@tonic-gate sizeof (hpmctr_t *)); 5947c478bd9Sstevel@tonic-gate } 5957c478bd9Sstevel@tonic-gate } 5967c478bd9Sstevel@tonic-gate 5977c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 5987c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * sizeof (hw_page_map_t)); 5995d07b933Sdp } 6005d07b933Sdp 6015d07b933Sdp /* add in space for page_ctrs_cands and pcc_color_free */ 6025d07b933Sdp ctrs_sz += sizeof (pcc_info_t *) * max_mem_nodes * 6035d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6045d07b933Sdp 6055d07b933Sdp for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6065d07b933Sdp 6075d07b933Sdp if (mem_node_config[mnode].exists == 0) 6085d07b933Sdp continue; 6097c478bd9Sstevel@tonic-gate 6105d07b933Sdp nranges = mnode_nranges[mnode]; 6115d07b933Sdp ctrs_sz += sizeof (pcc_info_t) * nranges * 6125d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6135d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6145d07b933Sdp ctrs_sz += sizeof (pgcnt_t) * nranges * 6155d07b933Sdp colors_per_szc[r] * NPC_MUTEX; 6165d07b933Sdp } 6177c478bd9Sstevel@tonic-gate } 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6207c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * NPC_MUTEX * sizeof (kmutex_t)); 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate /* size for page list counts */ 6237c478bd9Sstevel@tonic-gate PLCNT_SZ(ctrs_sz); 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate /* 6267c478bd9Sstevel@tonic-gate * add some slop for roundups. page_ctrs_alloc will roundup the start 6277c478bd9Sstevel@tonic-gate * address of the counters to ecache_alignsize boundary for every 6287c478bd9Sstevel@tonic-gate * memory node. 6297c478bd9Sstevel@tonic-gate */ 6307c478bd9Sstevel@tonic-gate return (ctrs_sz + max_mem_nodes * L2CACHE_ALIGN); 6317c478bd9Sstevel@tonic-gate } 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate caddr_t 6347c478bd9Sstevel@tonic-gate page_ctrs_alloc(caddr_t alloc_base) 6357c478bd9Sstevel@tonic-gate { 6367c478bd9Sstevel@tonic-gate int mnode; 6375d07b933Sdp int mrange, nranges; 6387c478bd9Sstevel@tonic-gate int r; /* region size */ 6397c478bd9Sstevel@tonic-gate int i; 640ce8eb11aSdp int firstmn; /* first mnode that exists */ 641ce8eb11aSdp pfn_t physbase; 642ce8eb11aSdp pfn_t physmax; 6437c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 6447c478bd9Sstevel@tonic-gate 6457c478bd9Sstevel@tonic-gate /* 6467c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 6477c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 6487c478bd9Sstevel@tonic-gate * arrays. 6497c478bd9Sstevel@tonic-gate */ 6505d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 6515d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 6527c478bd9Sstevel@tonic-gate } 6537c478bd9Sstevel@tonic-gate 6547c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 6557c478bd9Sstevel@tonic-gate page_counters[r] = (hw_page_map_t *)alloc_base; 6567c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (hw_page_map_t)); 6577c478bd9Sstevel@tonic-gate } 6587c478bd9Sstevel@tonic-gate 6595d07b933Sdp /* page_ctrs_cands and pcc_color_free array */ 6605d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 6615d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6627c478bd9Sstevel@tonic-gate 6635d07b933Sdp page_ctrs_cands[i][r] = (pcc_info_t **)alloc_base; 6645d07b933Sdp alloc_base += sizeof (pcc_info_t *) * max_mem_nodes; 6657c478bd9Sstevel@tonic-gate 6667c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6675d07b933Sdp pcc_info_t *pi; 6685d07b933Sdp 6695d07b933Sdp if (mem_node_config[mnode].exists == 0) 6705d07b933Sdp continue; 6715d07b933Sdp 6725d07b933Sdp nranges = mnode_nranges[mnode]; 6735d07b933Sdp 6745d07b933Sdp pi = (pcc_info_t *)alloc_base; 6755d07b933Sdp alloc_base += sizeof (pcc_info_t) * nranges; 6765d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 6775d07b933Sdp 6785d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 6795d07b933Sdp pi->pcc_color_free = 6805d07b933Sdp (pgcnt_t *)alloc_base; 6815d07b933Sdp alloc_base += sizeof (pgcnt_t) * 6825d07b933Sdp colors_per_szc[r]; 6835d07b933Sdp pi++; 6845d07b933Sdp } 6857c478bd9Sstevel@tonic-gate } 6867c478bd9Sstevel@tonic-gate } 6877c478bd9Sstevel@tonic-gate } 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6907c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 6917c478bd9Sstevel@tonic-gate ctr_mutex[i] = (kmutex_t *)alloc_base; 6927c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (kmutex_t)); 6937c478bd9Sstevel@tonic-gate } 6947c478bd9Sstevel@tonic-gate 6957c478bd9Sstevel@tonic-gate /* initialize page list counts */ 6967c478bd9Sstevel@tonic-gate PLCNT_INIT(alloc_base); 6977c478bd9Sstevel@tonic-gate 698ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 6997c478bd9Sstevel@tonic-gate 7007c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 7017c478bd9Sstevel@tonic-gate pfn_t r_base; 7027c478bd9Sstevel@tonic-gate pgcnt_t r_align; 7037c478bd9Sstevel@tonic-gate int r_shift; 7045d07b933Sdp int nranges = mnode_nranges[mnode]; 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 7077c478bd9Sstevel@tonic-gate continue; 7087c478bd9Sstevel@tonic-gate 709ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 710ce8eb11aSdp 7117c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 7127c478bd9Sstevel@tonic-gate /* 7137c478bd9Sstevel@tonic-gate * the page_counters base has to be aligned to the 7147c478bd9Sstevel@tonic-gate * page count of page size code r otherwise the counts 7157c478bd9Sstevel@tonic-gate * will cross large page boundaries. 7167c478bd9Sstevel@tonic-gate */ 7177c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 718ce8eb11aSdp r_base = physbase; 7197c478bd9Sstevel@tonic-gate /* base needs to be aligned - lower to aligned value */ 7207c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 721ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 7227c478bd9Sstevel@tonic-gate r_shift = PAGE_BSZS_SHIFT(r); 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = r_shift; 7257c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = r_pgcnt; 7267c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = r_base; 7275d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7285d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 7295d07b933Sdp r, mrange) = (size_t *)alloc_base; 7305d07b933Sdp alloc_base += sizeof (size_t) * 7315d07b933Sdp colors_per_szc[r]; 7325d07b933Sdp } 7337c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 7345d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 7355d07b933Sdp pfn_t pfnum = r_base; 7365d07b933Sdp size_t idx; 7375d07b933Sdp int mrange; 738ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 7395d07b933Sdp 740b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, r, &it); 741b779d3e0Sdp if (pfnum == (pfn_t)-1) { 742b779d3e0Sdp idx = 0; 743b779d3e0Sdp } else { 744b779d3e0Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 745b779d3e0Sdp color_mask, color_mask, &it); 746b779d3e0Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 747b779d3e0Sdp idx = (idx >= r_pgcnt) ? 0 : idx; 748b779d3e0Sdp } 7495d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7505d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, 7515d07b933Sdp r, i, mrange) = idx; 7525d07b933Sdp } 7537c478bd9Sstevel@tonic-gate } 754ce8eb11aSdp 755ce8eb11aSdp /* hpm_counters may be shared by all mnodes */ 756ce8eb11aSdp if (firstmn == mnode) { 757ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 758ce8eb11aSdp (hpmctr_t *)alloc_base; 759ce8eb11aSdp alloc_base += 760ce8eb11aSdp P2ROUNDUP((sizeof (hpmctr_t) * r_pgcnt), 761ce8eb11aSdp sizeof (hpmctr_t *)); 762ce8eb11aSdp } else { 763ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 764ce8eb11aSdp PAGE_COUNTERS_COUNTERS(firstmn, r); 765ce8eb11aSdp } 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate /* 7687c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 7697c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 7707c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 7717c478bd9Sstevel@tonic-gate * and get consistent values. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 7747c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 7757c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 7767c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, r_base))) == r_base); 7777c478bd9Sstevel@tonic-gate } 7787c478bd9Sstevel@tonic-gate /* 7797c478bd9Sstevel@tonic-gate * Roundup the start address of the page_counters to 7807c478bd9Sstevel@tonic-gate * cache aligned boundary for every memory node. 7817c478bd9Sstevel@tonic-gate * page_ctrs_sz() has added some slop for these roundups. 7827c478bd9Sstevel@tonic-gate */ 7837c478bd9Sstevel@tonic-gate alloc_base = (caddr_t)P2ROUNDUP((uintptr_t)alloc_base, 784ce8eb11aSdp L2CACHE_ALIGN); 7857c478bd9Sstevel@tonic-gate } 7867c478bd9Sstevel@tonic-gate 7877c478bd9Sstevel@tonic-gate /* Initialize other page counter specific data structures. */ 7887c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < MAX_MEM_NODES; mnode++) { 7897c478bd9Sstevel@tonic-gate rw_init(&page_ctrs_rwlock[mnode], NULL, RW_DEFAULT, NULL); 7907c478bd9Sstevel@tonic-gate } 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate return (alloc_base); 7937c478bd9Sstevel@tonic-gate } 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate /* 7967c478bd9Sstevel@tonic-gate * Functions to adjust region counters for each size free list. 7977c478bd9Sstevel@tonic-gate * Caller is responsible to acquire the ctr_mutex lock if necessary and 7987c478bd9Sstevel@tonic-gate * thus can be called during startup without locks. 7997c478bd9Sstevel@tonic-gate */ 8007c478bd9Sstevel@tonic-gate /* ARGSUSED */ 8017c478bd9Sstevel@tonic-gate void 802affbd3ccSkchow page_ctr_add_internal(int mnode, int mtype, page_t *pp, int flags) 8037c478bd9Sstevel@tonic-gate { 8047c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8057c478bd9Sstevel@tonic-gate ssize_t idx; 8067c478bd9Sstevel@tonic-gate pfn_t pfnum; 8077c478bd9Sstevel@tonic-gate int lckidx; 8087c478bd9Sstevel@tonic-gate 809affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 810affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 811affbd3ccSkchow 8127c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8137c478bd9Sstevel@tonic-gate 814affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 8157c478bd9Sstevel@tonic-gate 8167c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8177c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8187c478bd9Sstevel@tonic-gate return; 8197c478bd9Sstevel@tonic-gate } 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8227c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8237c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8247c478bd9Sstevel@tonic-gate 8257c478bd9Sstevel@tonic-gate /* 8267c478bd9Sstevel@tonic-gate * Increment the count of free pages for the current 8277c478bd9Sstevel@tonic-gate * region. Continue looping up in region size incrementing 8287c478bd9Sstevel@tonic-gate * count if the preceeding region is full. 8297c478bd9Sstevel@tonic-gate */ 8307c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8317c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8327c478bd9Sstevel@tonic-gate 8337c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8347c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) < FULL_REGION_CNT(r)); 8357c478bd9Sstevel@tonic-gate 8365d07b933Sdp if (++PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r)) { 8377c478bd9Sstevel@tonic-gate break; 8385d07b933Sdp } else { 8395d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 8405d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 8415d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 8427c478bd9Sstevel@tonic-gate 8435d07b933Sdp cand->pcc_pages_free++; 8445d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]++; 8455d07b933Sdp } 8467c478bd9Sstevel@tonic-gate r++; 8477c478bd9Sstevel@tonic-gate } 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate void 851affbd3ccSkchow page_ctr_add(int mnode, int mtype, page_t *pp, int flags) 8527c478bd9Sstevel@tonic-gate { 8537c478bd9Sstevel@tonic-gate int lckidx = PP_CTR_LOCK_INDX(pp); 8547c478bd9Sstevel@tonic-gate kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate mutex_enter(lock); 857affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 8587c478bd9Sstevel@tonic-gate mutex_exit(lock); 8597c478bd9Sstevel@tonic-gate } 8607c478bd9Sstevel@tonic-gate 8617c478bd9Sstevel@tonic-gate void 8625d07b933Sdp page_ctr_sub_internal(int mnode, int mtype, page_t *pp, int flags) 8637c478bd9Sstevel@tonic-gate { 8647c478bd9Sstevel@tonic-gate int lckidx; 8657c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8667c478bd9Sstevel@tonic-gate ssize_t idx; 8677c478bd9Sstevel@tonic-gate pfn_t pfnum; 8687c478bd9Sstevel@tonic-gate 869affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 870affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 871affbd3ccSkchow 8727c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8737c478bd9Sstevel@tonic-gate 874affbd3ccSkchow PLCNT_DECR(pp, mnode, mtype, pp->p_szc, flags); 8757c478bd9Sstevel@tonic-gate 8767c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8777c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8787c478bd9Sstevel@tonic-gate return; 8797c478bd9Sstevel@tonic-gate } 8807c478bd9Sstevel@tonic-gate 8817c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8827c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8837c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate /* 8867c478bd9Sstevel@tonic-gate * Decrement the count of free pages for the current 8877c478bd9Sstevel@tonic-gate * region. Continue looping up in region size decrementing 8887c478bd9Sstevel@tonic-gate * count if the preceeding region was full. 8897c478bd9Sstevel@tonic-gate */ 8907c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8917c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8927c478bd9Sstevel@tonic-gate 8937c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8947c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) > 0); 8957c478bd9Sstevel@tonic-gate 8967c478bd9Sstevel@tonic-gate if (--PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r) - 1) { 8977c478bd9Sstevel@tonic-gate break; 8985d07b933Sdp } else { 8995d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 9005d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 9015d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 9027c478bd9Sstevel@tonic-gate 9035d07b933Sdp ASSERT(cand->pcc_pages_free != 0); 9045d07b933Sdp ASSERT(cand->pcc_color_free[PP_2_BIN_SZC(pp, r)] != 0); 9055d07b933Sdp 9065d07b933Sdp cand->pcc_pages_free--; 9075d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]--; 9085d07b933Sdp } 9097c478bd9Sstevel@tonic-gate r++; 9107c478bd9Sstevel@tonic-gate } 9115d07b933Sdp } 9125d07b933Sdp 9135d07b933Sdp void 9145d07b933Sdp page_ctr_sub(int mnode, int mtype, page_t *pp, int flags) 9155d07b933Sdp { 9165d07b933Sdp int lckidx = PP_CTR_LOCK_INDX(pp); 9175d07b933Sdp kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 9185d07b933Sdp 9195d07b933Sdp mutex_enter(lock); 9205d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 9217c478bd9Sstevel@tonic-gate mutex_exit(lock); 9227c478bd9Sstevel@tonic-gate } 9237c478bd9Sstevel@tonic-gate 9247c478bd9Sstevel@tonic-gate /* 9257c478bd9Sstevel@tonic-gate * Adjust page counters following a memory attach, since typically the 9267c478bd9Sstevel@tonic-gate * size of the array needs to change, and the PFN to counter index 9277c478bd9Sstevel@tonic-gate * mapping needs to change. 9285d07b933Sdp * 9295d07b933Sdp * It is possible this mnode did not exist at startup. In that case 9305d07b933Sdp * allocate pcc_info_t and pcc_color_free arrays. Also, allow for nranges 9315d07b933Sdp * to change (a theoretical possibility on x86), which means pcc_color_free 9325d07b933Sdp * arrays must be extended. 9337c478bd9Sstevel@tonic-gate */ 9347c478bd9Sstevel@tonic-gate uint_t 9357c478bd9Sstevel@tonic-gate page_ctrs_adjust(int mnode) 9367c478bd9Sstevel@tonic-gate { 9377c478bd9Sstevel@tonic-gate pgcnt_t npgs; 9387c478bd9Sstevel@tonic-gate int r; /* region size */ 9397c478bd9Sstevel@tonic-gate int i; 9407c478bd9Sstevel@tonic-gate size_t pcsz, old_csz; 9417c478bd9Sstevel@tonic-gate hpmctr_t *new_ctr, *old_ctr; 9427c478bd9Sstevel@tonic-gate pfn_t oldbase, newbase; 943ce8eb11aSdp pfn_t physbase, physmax; 9447c478bd9Sstevel@tonic-gate size_t old_npgs; 9457c478bd9Sstevel@tonic-gate hpmctr_t *ctr_cache[MMU_PAGE_SIZES]; 9467c478bd9Sstevel@tonic-gate size_t size_cache[MMU_PAGE_SIZES]; 9475d07b933Sdp size_t *color_cache[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 9485d07b933Sdp size_t *old_color_array[MAX_MNODE_MRANGES]; 9497c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 9505d07b933Sdp pcc_info_t **cands_cache; 9515d07b933Sdp pcc_info_t *old_pi, *pi; 9525d07b933Sdp pgcnt_t *pgcntp; 9535d07b933Sdp int nr, old_nranges, mrange, nranges = MNODE_RANGE_CNT(mnode); 9545d07b933Sdp int cands_cache_nranges; 9555d07b933Sdp int old_maxmrange, new_maxmrange; 9565d07b933Sdp int rc = 0; 957*9853d9e8SJason Beloro int oldmnode; 9587c478bd9Sstevel@tonic-gate 9595d07b933Sdp cands_cache = kmem_zalloc(sizeof (pcc_info_t *) * NPC_MUTEX * 9605d07b933Sdp MMU_PAGE_SIZES, KM_NOSLEEP); 9615d07b933Sdp if (cands_cache == NULL) 9625d07b933Sdp return (ENOMEM); 9635d07b933Sdp 964ce8eb11aSdp i = -1; 965ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, i); 966ce8eb11aSdp 967ce8eb11aSdp newbase = physbase & ~PC_BASE_ALIGN_MASK; 968ce8eb11aSdp npgs = roundup(physmax, PC_BASE_ALIGN) - newbase; 969ce8eb11aSdp 9705d07b933Sdp /* prepare to free non-null pointers on the way out */ 9715d07b933Sdp cands_cache_nranges = nranges; 9725d07b933Sdp bzero(ctr_cache, sizeof (ctr_cache)); 9735d07b933Sdp bzero(color_cache, sizeof (color_cache)); 9745d07b933Sdp 9757c478bd9Sstevel@tonic-gate /* 9767c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 9777c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 9787c478bd9Sstevel@tonic-gate * arrays. 9797c478bd9Sstevel@tonic-gate */ 9805d07b933Sdp for (r = 0; r < mmu_page_sizes; r++) { 9815d07b933Sdp colors_per_szc[r] = PAGE_GET_PAGECOLORS(r); 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate 9847c478bd9Sstevel@tonic-gate /* 9857c478bd9Sstevel@tonic-gate * Preallocate all of the new hpm_counters arrays as we can't 9867c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 9877c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 9887c478bd9Sstevel@tonic-gate * and return failure. 9897c478bd9Sstevel@tonic-gate */ 9907c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 9917c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_BSZS_SHIFT(r); 9925d07b933Sdp size_cache[r] = pcsz; 9937c478bd9Sstevel@tonic-gate ctr_cache[r] = kmem_zalloc(pcsz * 9947c478bd9Sstevel@tonic-gate sizeof (hpmctr_t), KM_NOSLEEP); 9957c478bd9Sstevel@tonic-gate if (ctr_cache[r] == NULL) { 9965d07b933Sdp rc = ENOMEM; 9975d07b933Sdp goto cleanup; 9987c478bd9Sstevel@tonic-gate } 9997c478bd9Sstevel@tonic-gate } 10005d07b933Sdp 10017c478bd9Sstevel@tonic-gate /* 10027c478bd9Sstevel@tonic-gate * Preallocate all of the new color current arrays as we can't 10037c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 10047c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 10057c478bd9Sstevel@tonic-gate * and return failure. 10067c478bd9Sstevel@tonic-gate */ 10077c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10085d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 10095d07b933Sdp color_cache[r][mrange] = kmem_zalloc(sizeof (size_t) * 10105d07b933Sdp colors_per_szc[r], KM_NOSLEEP); 10115d07b933Sdp if (color_cache[r][mrange] == NULL) { 10125d07b933Sdp rc = ENOMEM; 10135d07b933Sdp goto cleanup; 10145d07b933Sdp } 10155d07b933Sdp } 10165d07b933Sdp } 10175d07b933Sdp 10185d07b933Sdp /* 10195d07b933Sdp * Preallocate all of the new pcc_info_t arrays as we can't 10205d07b933Sdp * hold the page_ctrs_rwlock as a writer and allocate memory. 10215d07b933Sdp * If we can't allocate all of the arrays, undo our work so far 10225d07b933Sdp * and return failure. 10235d07b933Sdp */ 10245d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 10255d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 10265d07b933Sdp pi = kmem_zalloc(nranges * sizeof (pcc_info_t), 10275d07b933Sdp KM_NOSLEEP); 10285d07b933Sdp if (pi == NULL) { 10295d07b933Sdp rc = ENOMEM; 10305d07b933Sdp goto cleanup; 10317c478bd9Sstevel@tonic-gate } 10325d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = pi; 10335d07b933Sdp 10345d07b933Sdp for (mrange = 0; mrange < nranges; mrange++, pi++) { 10355d07b933Sdp pgcntp = kmem_zalloc(colors_per_szc[r] * 10365d07b933Sdp sizeof (pgcnt_t), KM_NOSLEEP); 10375d07b933Sdp if (pgcntp == NULL) { 10385d07b933Sdp rc = ENOMEM; 10395d07b933Sdp goto cleanup; 10405d07b933Sdp } 10415d07b933Sdp pi->pcc_color_free = pgcntp; 10427c478bd9Sstevel@tonic-gate } 10437c478bd9Sstevel@tonic-gate } 10447c478bd9Sstevel@tonic-gate } 10457c478bd9Sstevel@tonic-gate 10467c478bd9Sstevel@tonic-gate /* 10477c478bd9Sstevel@tonic-gate * Grab the write lock to prevent others from walking these arrays 10487c478bd9Sstevel@tonic-gate * while we are modifying them. 10497c478bd9Sstevel@tonic-gate */ 1050ce8eb11aSdp PAGE_CTRS_WRITE_LOCK(mnode); 10515d07b933Sdp 1052*9853d9e8SJason Beloro /* 1053*9853d9e8SJason Beloro * For interleaved mnodes, find the first mnode 1054*9853d9e8SJason Beloro * with valid page counters since the current 1055*9853d9e8SJason Beloro * mnode may have just been added and not have 1056*9853d9e8SJason Beloro * valid page counters. 1057*9853d9e8SJason Beloro */ 1058*9853d9e8SJason Beloro if (interleaved_mnodes) { 1059*9853d9e8SJason Beloro for (i = 0; i < max_mem_nodes; i++) 1060*9853d9e8SJason Beloro if (PAGE_COUNTERS_COUNTERS(i, 1) != NULL) 1061*9853d9e8SJason Beloro break; 1062*9853d9e8SJason Beloro ASSERT(i < max_mem_nodes); 1063*9853d9e8SJason Beloro oldmnode = i; 1064*9853d9e8SJason Beloro } else 1065*9853d9e8SJason Beloro oldmnode = mnode; 1066*9853d9e8SJason Beloro 10675d07b933Sdp old_nranges = mnode_nranges[mnode]; 10685d07b933Sdp cands_cache_nranges = old_nranges; 10695d07b933Sdp mnode_nranges[mnode] = nranges; 10705d07b933Sdp old_maxmrange = mnode_maxmrange[mnode]; 10715d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 10725d07b933Sdp new_maxmrange = mnode_maxmrange[mnode]; 10735d07b933Sdp 10747c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10757c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = PAGE_BSZS_SHIFT(r); 1076*9853d9e8SJason Beloro old_ctr = PAGE_COUNTERS_COUNTERS(oldmnode, r); 1077*9853d9e8SJason Beloro old_csz = PAGE_COUNTERS_ENTRIES(oldmnode, r); 1078*9853d9e8SJason Beloro oldbase = PAGE_COUNTERS_BASE(oldmnode, r); 1079*9853d9e8SJason Beloro old_npgs = old_csz << PAGE_COUNTERS_SHIFT(oldmnode, r); 10805d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 10815d07b933Sdp old_color_array[mrange] = 10825d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 1083ce8eb11aSdp r, mrange); 10845d07b933Sdp } 10857c478bd9Sstevel@tonic-gate 10867c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_COUNTERS_SHIFT(mnode, r); 10877c478bd9Sstevel@tonic-gate new_ctr = ctr_cache[r]; 10887c478bd9Sstevel@tonic-gate ctr_cache[r] = NULL; 10897c478bd9Sstevel@tonic-gate if (old_ctr != NULL && 10907c478bd9Sstevel@tonic-gate (oldbase + old_npgs > newbase) && 10917c478bd9Sstevel@tonic-gate (newbase + npgs > oldbase)) { 10927c478bd9Sstevel@tonic-gate /* 10937c478bd9Sstevel@tonic-gate * Map the intersection of the old and new 10947c478bd9Sstevel@tonic-gate * counters into the new array. 10957c478bd9Sstevel@tonic-gate */ 10967c478bd9Sstevel@tonic-gate size_t offset; 10977c478bd9Sstevel@tonic-gate if (newbase > oldbase) { 10987c478bd9Sstevel@tonic-gate offset = (newbase - oldbase) >> 10997c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 11007c478bd9Sstevel@tonic-gate bcopy(old_ctr + offset, new_ctr, 11017c478bd9Sstevel@tonic-gate MIN(pcsz, (old_csz - offset)) * 11027c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 11037c478bd9Sstevel@tonic-gate } else { 11047c478bd9Sstevel@tonic-gate offset = (oldbase - newbase) >> 11057c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 11067c478bd9Sstevel@tonic-gate bcopy(old_ctr, new_ctr + offset, 11077c478bd9Sstevel@tonic-gate MIN(pcsz - offset, old_csz) * 11087c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 11097c478bd9Sstevel@tonic-gate } 11107c478bd9Sstevel@tonic-gate } 11117c478bd9Sstevel@tonic-gate 11127c478bd9Sstevel@tonic-gate PAGE_COUNTERS_COUNTERS(mnode, r) = new_ctr; 11137c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = pcsz; 11147c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = newbase; 1115ce8eb11aSdp 1116ce8eb11aSdp /* update shared hpm_counters in other mnodes */ 1117ce8eb11aSdp if (interleaved_mnodes) { 1118ce8eb11aSdp for (i = 0; i < max_mem_nodes; i++) { 1119ce8eb11aSdp if (i == mnode) 1120ce8eb11aSdp continue; 1121*9853d9e8SJason Beloro ASSERT( 1122*9853d9e8SJason Beloro PAGE_COUNTERS_COUNTERS(i, r) == old_ctr || 1123*9853d9e8SJason Beloro PAGE_COUNTERS_COUNTERS(i, r) == NULL); 1124ce8eb11aSdp if (mem_node_config[i].exists == 0) 1125ce8eb11aSdp continue; 1126ce8eb11aSdp PAGE_COUNTERS_COUNTERS(i, r) = new_ctr; 1127ce8eb11aSdp PAGE_COUNTERS_ENTRIES(i, r) = pcsz; 1128ce8eb11aSdp PAGE_COUNTERS_BASE(i, r) = newbase; 1129ce8eb11aSdp } 1130ce8eb11aSdp } 1131ce8eb11aSdp 11325d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11335d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, r, mrange) = 11345d07b933Sdp color_cache[r][mrange]; 11355d07b933Sdp color_cache[r][mrange] = NULL; 11365d07b933Sdp } 11377c478bd9Sstevel@tonic-gate /* 11387c478bd9Sstevel@tonic-gate * for now, just reset on these events as it's probably 11397c478bd9Sstevel@tonic-gate * not worthwhile to try and optimize this. 11407c478bd9Sstevel@tonic-gate */ 11417c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 11425d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 1143ce8eb11aSdp int mlo = interleaved_mnodes ? 0 : mnode; 1144ce8eb11aSdp int mhi = interleaved_mnodes ? max_mem_nodes : 1145ce8eb11aSdp (mnode + 1); 1146ce8eb11aSdp int m; 1147*9853d9e8SJason Beloro pfn_t pfnum; 11485d07b933Sdp size_t idx; 1149ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 11505d07b933Sdp 1151ce8eb11aSdp for (m = mlo; m < mhi; m++) { 1152ce8eb11aSdp if (mem_node_config[m].exists == 0) 1153ce8eb11aSdp continue; 1154*9853d9e8SJason Beloro pfnum = newbase; 1155b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, m, r, &it); 1156b779d3e0Sdp if (pfnum == (pfn_t)-1) { 1157b779d3e0Sdp idx = 0; 1158b779d3e0Sdp } else { 1159b779d3e0Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 1160b779d3e0Sdp color_mask, color_mask, &it); 1161b779d3e0Sdp idx = PNUM_TO_IDX(m, r, pfnum); 1162b779d3e0Sdp idx = (idx < pcsz) ? idx : 0; 1163b779d3e0Sdp } 1164ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 1165*9853d9e8SJason Beloro if (PAGE_COUNTERS_CURRENT_COLOR_ARRAY(m, 1166*9853d9e8SJason Beloro r, mrange) != NULL) 1167*9853d9e8SJason Beloro PAGE_COUNTERS_CURRENT_COLOR(m, 1168*9853d9e8SJason Beloro r, i, mrange) = idx; 1169ce8eb11aSdp } 11705d07b933Sdp } 11717c478bd9Sstevel@tonic-gate } 11727c478bd9Sstevel@tonic-gate 11737c478bd9Sstevel@tonic-gate /* cache info for freeing out of the critical path */ 11747c478bd9Sstevel@tonic-gate if ((caddr_t)old_ctr >= kernelheap && 11757c478bd9Sstevel@tonic-gate (caddr_t)old_ctr < ekernelheap) { 11767c478bd9Sstevel@tonic-gate ctr_cache[r] = old_ctr; 11777c478bd9Sstevel@tonic-gate size_cache[r] = old_csz; 11787c478bd9Sstevel@tonic-gate } 11795d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11805d07b933Sdp size_t *tmp = old_color_array[mrange]; 11815d07b933Sdp if ((caddr_t)tmp >= kernelheap && 11825d07b933Sdp (caddr_t)tmp < ekernelheap) { 11835d07b933Sdp color_cache[r][mrange] = tmp; 11845d07b933Sdp } 11857c478bd9Sstevel@tonic-gate } 11867c478bd9Sstevel@tonic-gate /* 11877c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 11887c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 11897c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 11907c478bd9Sstevel@tonic-gate * and get consistent values. 11917c478bd9Sstevel@tonic-gate */ 11927c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 11937c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 11947c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 11957c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, newbase))) == newbase); 11965d07b933Sdp 11975d07b933Sdp /* pcc_info_t and pcc_color_free */ 11985d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 11995d07b933Sdp pcc_info_t *epi; 12005d07b933Sdp pcc_info_t *eold_pi; 12015d07b933Sdp 12025d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12035d07b933Sdp old_pi = page_ctrs_cands[i][r][mnode]; 12045d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 12055d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = old_pi; 12065d07b933Sdp 12075d07b933Sdp /* preserve old pcc_color_free values, if any */ 12085d07b933Sdp if (old_pi == NULL) 12095d07b933Sdp continue; 12105d07b933Sdp 12115d07b933Sdp /* 12125d07b933Sdp * when/if x86 does DR, must account for 12135d07b933Sdp * possible change in range index when 12145d07b933Sdp * preserving pcc_info 12155d07b933Sdp */ 12165d07b933Sdp epi = &pi[nranges]; 12175d07b933Sdp eold_pi = &old_pi[old_nranges]; 12185d07b933Sdp if (new_maxmrange > old_maxmrange) { 12195d07b933Sdp pi += new_maxmrange - old_maxmrange; 12205d07b933Sdp } else if (new_maxmrange < old_maxmrange) { 12215d07b933Sdp old_pi += old_maxmrange - new_maxmrange; 12225d07b933Sdp } 12235d07b933Sdp for (; pi < epi && old_pi < eold_pi; pi++, old_pi++) { 12245d07b933Sdp pcc_info_t tmp = *pi; 12255d07b933Sdp *pi = *old_pi; 12265d07b933Sdp *old_pi = tmp; 12275d07b933Sdp } 12285d07b933Sdp } 12297c478bd9Sstevel@tonic-gate } 1230ce8eb11aSdp PAGE_CTRS_WRITE_UNLOCK(mnode); 12317c478bd9Sstevel@tonic-gate 12327c478bd9Sstevel@tonic-gate /* 12337c478bd9Sstevel@tonic-gate * Now that we have dropped the write lock, it is safe to free all 12347c478bd9Sstevel@tonic-gate * of the memory we have cached above. 12355d07b933Sdp * We come thru here to free memory when pre-alloc fails, and also to 12365d07b933Sdp * free old pointers which were recorded while locked. 12377c478bd9Sstevel@tonic-gate */ 12385d07b933Sdp cleanup: 12397c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 12407c478bd9Sstevel@tonic-gate if (ctr_cache[r] != NULL) { 12417c478bd9Sstevel@tonic-gate kmem_free(ctr_cache[r], 12427c478bd9Sstevel@tonic-gate size_cache[r] * sizeof (hpmctr_t)); 12437c478bd9Sstevel@tonic-gate } 12445d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 12455d07b933Sdp if (color_cache[r][mrange] != NULL) { 12465d07b933Sdp kmem_free(color_cache[r][mrange], 12475d07b933Sdp colors_per_szc[r] * sizeof (size_t)); 12485d07b933Sdp } 12495d07b933Sdp } 12505d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 12515d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12525d07b933Sdp if (pi == NULL) 12535d07b933Sdp continue; 12545d07b933Sdp nr = cands_cache_nranges; 12555d07b933Sdp for (mrange = 0; mrange < nr; mrange++, pi++) { 12565d07b933Sdp pgcntp = pi->pcc_color_free; 12575d07b933Sdp if (pgcntp == NULL) 12585d07b933Sdp continue; 12595d07b933Sdp if ((caddr_t)pgcntp >= kernelheap && 12605d07b933Sdp (caddr_t)pgcntp < ekernelheap) { 12615d07b933Sdp kmem_free(pgcntp, 12625d07b933Sdp colors_per_szc[r] * 12635d07b933Sdp sizeof (pgcnt_t)); 12645d07b933Sdp } 12655d07b933Sdp } 12665d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12675d07b933Sdp if ((caddr_t)pi >= kernelheap && 12685d07b933Sdp (caddr_t)pi < ekernelheap) { 12695d07b933Sdp kmem_free(pi, nr * sizeof (pcc_info_t)); 12705d07b933Sdp } 12717c478bd9Sstevel@tonic-gate } 12727c478bd9Sstevel@tonic-gate } 12737c478bd9Sstevel@tonic-gate 12745d07b933Sdp kmem_free(cands_cache, 12755d07b933Sdp sizeof (pcc_info_t *) * NPC_MUTEX * MMU_PAGE_SIZES); 12765d07b933Sdp return (rc); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate 12795d07b933Sdp 12807c478bd9Sstevel@tonic-gate #ifdef DEBUG 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate /* 12837c478bd9Sstevel@tonic-gate * confirm pp is a large page corresponding to szc 12847c478bd9Sstevel@tonic-gate */ 12857c478bd9Sstevel@tonic-gate void 12867c478bd9Sstevel@tonic-gate chk_lpg(page_t *pp, uchar_t szc) 12877c478bd9Sstevel@tonic-gate { 12887c478bd9Sstevel@tonic-gate spgcnt_t npgs = page_get_pagecnt(pp->p_szc); 12897c478bd9Sstevel@tonic-gate uint_t noreloc; 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate if (npgs == 1) { 12927c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 12937c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == pp); 12947c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == pp); 12957c478bd9Sstevel@tonic-gate return; 12967c478bd9Sstevel@tonic-gate } 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12997c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 13007c478bd9Sstevel@tonic-gate 13017c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pp->p_pagenum, npgs)); 13027c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == (pp->p_next->p_pagenum - 1)); 13037c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev->p_pagenum == (pp->p_pagenum + (npgs - 1))); 13047c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == (pp + (npgs - 1))); 13057c478bd9Sstevel@tonic-gate 13067c478bd9Sstevel@tonic-gate /* 13077c478bd9Sstevel@tonic-gate * Check list of pages. 13087c478bd9Sstevel@tonic-gate */ 13097c478bd9Sstevel@tonic-gate noreloc = PP_ISNORELOC(pp); 13107c478bd9Sstevel@tonic-gate while (npgs--) { 13117c478bd9Sstevel@tonic-gate if (npgs != 0) { 13127c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == pp->p_next->p_pagenum - 1); 13137c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == (pp + 1)); 13147c478bd9Sstevel@tonic-gate } 13157c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 13167c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 13177c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 13187c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 13197c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 13207c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 13217c478bd9Sstevel@tonic-gate ASSERT(PP_ISNORELOC(pp) == noreloc); 13227c478bd9Sstevel@tonic-gate 13237c478bd9Sstevel@tonic-gate pp = pp->p_next; 13247c478bd9Sstevel@tonic-gate } 13257c478bd9Sstevel@tonic-gate } 13267c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 13277c478bd9Sstevel@tonic-gate 13287c478bd9Sstevel@tonic-gate void 13297c478bd9Sstevel@tonic-gate page_freelist_lock(int mnode) 13307c478bd9Sstevel@tonic-gate { 13317c478bd9Sstevel@tonic-gate int i; 13327c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13337c478bd9Sstevel@tonic-gate mutex_enter(FPC_MUTEX(mnode, i)); 13347c478bd9Sstevel@tonic-gate mutex_enter(CPC_MUTEX(mnode, i)); 13357c478bd9Sstevel@tonic-gate } 13367c478bd9Sstevel@tonic-gate } 13377c478bd9Sstevel@tonic-gate 13387c478bd9Sstevel@tonic-gate void 13397c478bd9Sstevel@tonic-gate page_freelist_unlock(int mnode) 13407c478bd9Sstevel@tonic-gate { 13417c478bd9Sstevel@tonic-gate int i; 13427c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13437c478bd9Sstevel@tonic-gate mutex_exit(FPC_MUTEX(mnode, i)); 13447c478bd9Sstevel@tonic-gate mutex_exit(CPC_MUTEX(mnode, i)); 13457c478bd9Sstevel@tonic-gate } 13467c478bd9Sstevel@tonic-gate } 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate /* 13497c478bd9Sstevel@tonic-gate * add pp to the specified page list. Defaults to head of the page list 13507c478bd9Sstevel@tonic-gate * unless PG_LIST_TAIL is specified. 13517c478bd9Sstevel@tonic-gate */ 13527c478bd9Sstevel@tonic-gate void 13537c478bd9Sstevel@tonic-gate page_list_add(page_t *pp, int flags) 13547c478bd9Sstevel@tonic-gate { 13557c478bd9Sstevel@tonic-gate page_t **ppp; 13567c478bd9Sstevel@tonic-gate kmutex_t *pcm; 13577c478bd9Sstevel@tonic-gate uint_t bin, mtype; 13587c478bd9Sstevel@tonic-gate int mnode; 13597c478bd9Sstevel@tonic-gate 13607c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 13617c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 13627c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 13637c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(pp) == 0); 13647c478bd9Sstevel@tonic-gate 13657c478bd9Sstevel@tonic-gate /* 13667c478bd9Sstevel@tonic-gate * Large pages should be freed via page_list_add_pages(). 13677c478bd9Sstevel@tonic-gate */ 13687c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 13697c478bd9Sstevel@tonic-gate 13707c478bd9Sstevel@tonic-gate /* 13717c478bd9Sstevel@tonic-gate * Don't need to lock the freelist first here 13727c478bd9Sstevel@tonic-gate * because the page isn't on the freelist yet. 13737c478bd9Sstevel@tonic-gate * This means p_szc can't change on us. 13747c478bd9Sstevel@tonic-gate */ 13757c478bd9Sstevel@tonic-gate 13767c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 13777c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 13787c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 13797c478bd9Sstevel@tonic-gate 13807c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 13817c478bd9Sstevel@tonic-gate /* 13827c478bd9Sstevel@tonic-gate * PG_LIST_ISINIT is set during system startup (ie. single 13837c478bd9Sstevel@tonic-gate * threaded), add a page to the free list and add to the 13847c478bd9Sstevel@tonic-gate * the free region counters w/o any locking 13857c478bd9Sstevel@tonic-gate */ 13867c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13877c478bd9Sstevel@tonic-gate 13887c478bd9Sstevel@tonic-gate /* inline version of page_add() */ 13897c478bd9Sstevel@tonic-gate if (*ppp != NULL) { 13907c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 13917c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 13927c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 13937c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 13947c478bd9Sstevel@tonic-gate } else 13957c478bd9Sstevel@tonic-gate *ppp = pp; 13967c478bd9Sstevel@tonic-gate 1397affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 1398affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13997c478bd9Sstevel@tonic-gate } else { 14007c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 14017c478bd9Sstevel@tonic-gate 14027c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1403affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 14047c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 14057c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14067c478bd9Sstevel@tonic-gate 14077c478bd9Sstevel@tonic-gate } else { 1408affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_cache); 14097c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 14107c478bd9Sstevel@tonic-gate ASSERT((pp->p_offset & PAGEOFFSET) == 0); 14117c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14127c478bd9Sstevel@tonic-gate } 14137c478bd9Sstevel@tonic-gate mutex_enter(pcm); 14147c478bd9Sstevel@tonic-gate page_add(ppp, pp); 14157c478bd9Sstevel@tonic-gate 14167c478bd9Sstevel@tonic-gate if (flags & PG_LIST_TAIL) 14177c478bd9Sstevel@tonic-gate *ppp = (*ppp)->p_next; 14187c478bd9Sstevel@tonic-gate /* 14197c478bd9Sstevel@tonic-gate * Add counters before releasing pcm mutex to avoid a race with 14205d07b933Sdp * page_freelist_coalesce and page_freelist_split. 14217c478bd9Sstevel@tonic-gate */ 1422affbd3ccSkchow page_ctr_add(mnode, mtype, pp, flags); 14237c478bd9Sstevel@tonic-gate mutex_exit(pcm); 14247c478bd9Sstevel@tonic-gate } 14257c478bd9Sstevel@tonic-gate 14267c478bd9Sstevel@tonic-gate 14277c478bd9Sstevel@tonic-gate #if defined(__sparc) 14287c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 14297c478bd9Sstevel@tonic-gate kcage_freemem_add(1); 14307c478bd9Sstevel@tonic-gate } 14317c478bd9Sstevel@tonic-gate #endif 14327c478bd9Sstevel@tonic-gate /* 14337c478bd9Sstevel@tonic-gate * It is up to the caller to unlock the page! 14347c478bd9Sstevel@tonic-gate */ 14357c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 14367c478bd9Sstevel@tonic-gate } 14377c478bd9Sstevel@tonic-gate 14387c478bd9Sstevel@tonic-gate 14397c478bd9Sstevel@tonic-gate #ifdef __sparc 14407c478bd9Sstevel@tonic-gate /* 14417c478bd9Sstevel@tonic-gate * This routine is only used by kcage_init during system startup. 14427c478bd9Sstevel@tonic-gate * It performs the function of page_list_sub/PP_SETNORELOC/page_list_add 14437c478bd9Sstevel@tonic-gate * without the overhead of taking locks and updating counters. 14447c478bd9Sstevel@tonic-gate */ 14457c478bd9Sstevel@tonic-gate void 14467c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 14477c478bd9Sstevel@tonic-gate { 14487c478bd9Sstevel@tonic-gate page_t **ppp; 14497c478bd9Sstevel@tonic-gate uint_t bin; 14507c478bd9Sstevel@tonic-gate int mnode; 14517c478bd9Sstevel@tonic-gate int mtype; 1452e21bae1bSkchow int flags = 0; 14537c478bd9Sstevel@tonic-gate 14547c478bd9Sstevel@tonic-gate /* 14557c478bd9Sstevel@tonic-gate * If this is a large page on the freelist then 14567c478bd9Sstevel@tonic-gate * break it up into smaller pages. 14577c478bd9Sstevel@tonic-gate */ 14587c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 14597c478bd9Sstevel@tonic-gate page_boot_demote(pp); 14607c478bd9Sstevel@tonic-gate 14617c478bd9Sstevel@tonic-gate /* 14627c478bd9Sstevel@tonic-gate * Get list page is currently on. 14637c478bd9Sstevel@tonic-gate */ 14647c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 14657c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 14667c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14677c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_RELOC); 14687c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 14697c478bd9Sstevel@tonic-gate 14707c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14717c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14727c478bd9Sstevel@tonic-gate flags |= PG_FREE_LIST; 14737c478bd9Sstevel@tonic-gate } else { 14747c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14757c478bd9Sstevel@tonic-gate flags |= PG_CACHE_LIST; 14767c478bd9Sstevel@tonic-gate } 14777c478bd9Sstevel@tonic-gate 14787c478bd9Sstevel@tonic-gate ASSERT(*ppp != NULL); 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate /* 14817c478bd9Sstevel@tonic-gate * Delete page from current list. 14827c478bd9Sstevel@tonic-gate */ 14837c478bd9Sstevel@tonic-gate if (*ppp == pp) 14847c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 14857c478bd9Sstevel@tonic-gate if (*ppp == pp) { 14867c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 14877c478bd9Sstevel@tonic-gate } else { 14887c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 14897c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 14907c478bd9Sstevel@tonic-gate } 14917c478bd9Sstevel@tonic-gate 14925d07b933Sdp /* 14935d07b933Sdp * Decrement page counters 14945d07b933Sdp */ 14955d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 14967c478bd9Sstevel@tonic-gate 14977c478bd9Sstevel@tonic-gate /* 14987c478bd9Sstevel@tonic-gate * Set no reloc for cage initted pages. 14997c478bd9Sstevel@tonic-gate */ 15007c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 15037c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_NORELOC); 15047c478bd9Sstevel@tonic-gate 15057c478bd9Sstevel@tonic-gate /* 15067c478bd9Sstevel@tonic-gate * Get new list for page. 15077c478bd9Sstevel@tonic-gate */ 15087c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 15097c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 15107c478bd9Sstevel@tonic-gate } else { 15117c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 15127c478bd9Sstevel@tonic-gate } 15137c478bd9Sstevel@tonic-gate 15147c478bd9Sstevel@tonic-gate /* 15157c478bd9Sstevel@tonic-gate * Insert page on new list. 15167c478bd9Sstevel@tonic-gate */ 15177c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 15187c478bd9Sstevel@tonic-gate *ppp = pp; 15197c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 15207c478bd9Sstevel@tonic-gate } else { 15217c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 15227c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 15237c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 15247c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 15257c478bd9Sstevel@tonic-gate } 15267c478bd9Sstevel@tonic-gate 15275d07b933Sdp /* 15285d07b933Sdp * Increment page counters 15295d07b933Sdp */ 15305d07b933Sdp page_ctr_add_internal(mnode, mtype, pp, flags); 15317c478bd9Sstevel@tonic-gate 15327c478bd9Sstevel@tonic-gate /* 15337c478bd9Sstevel@tonic-gate * Update cage freemem counter 15347c478bd9Sstevel@tonic-gate */ 15357c478bd9Sstevel@tonic-gate atomic_add_long(&kcage_freemem, 1); 15367c478bd9Sstevel@tonic-gate } 15377c478bd9Sstevel@tonic-gate #else /* __sparc */ 15387c478bd9Sstevel@tonic-gate 15397c478bd9Sstevel@tonic-gate /* ARGSUSED */ 15407c478bd9Sstevel@tonic-gate void 15417c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 15427c478bd9Sstevel@tonic-gate { 15437c478bd9Sstevel@tonic-gate panic("page_list_noreloc_startup: should be here only for sparc"); 15447c478bd9Sstevel@tonic-gate } 15457c478bd9Sstevel@tonic-gate #endif 15467c478bd9Sstevel@tonic-gate 15477c478bd9Sstevel@tonic-gate void 15487c478bd9Sstevel@tonic-gate page_list_add_pages(page_t *pp, int flags) 15497c478bd9Sstevel@tonic-gate { 15507c478bd9Sstevel@tonic-gate kmutex_t *pcm; 15517c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 15527c478bd9Sstevel@tonic-gate uint_t bin, mtype, i; 15537c478bd9Sstevel@tonic-gate int mnode; 15547c478bd9Sstevel@tonic-gate 15557c478bd9Sstevel@tonic-gate /* default to freelist/head */ 15567c478bd9Sstevel@tonic-gate ASSERT((flags & (PG_CACHE_LIST | PG_LIST_TAIL)) == 0); 15577c478bd9Sstevel@tonic-gate 15587c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 1559affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[pp->p_szc]); 15607c478bd9Sstevel@tonic-gate 15617c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 15627c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 15637c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 15667c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == mmu_page_sizes - 1); 15677c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 15687c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 1569affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 15707c478bd9Sstevel@tonic-gate } else { 15717c478bd9Sstevel@tonic-gate 15727c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 15737c478bd9Sstevel@tonic-gate 15747c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 15757c478bd9Sstevel@tonic-gate 15767c478bd9Sstevel@tonic-gate mutex_enter(pcm); 15777c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 1578affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 15797c478bd9Sstevel@tonic-gate mutex_exit(pcm); 15807c478bd9Sstevel@tonic-gate 15817c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 15827c478bd9Sstevel@tonic-gate #if defined(__sparc) 15837c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) 15847c478bd9Sstevel@tonic-gate kcage_freemem_add(pgcnt); 15857c478bd9Sstevel@tonic-gate #endif 15867c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++, pp++) 15878b464eb8Smec page_unlock_nocapture(pp); 15887c478bd9Sstevel@tonic-gate } 15897c478bd9Sstevel@tonic-gate } 15907c478bd9Sstevel@tonic-gate 15917c478bd9Sstevel@tonic-gate /* 15927c478bd9Sstevel@tonic-gate * During boot, need to demote a large page to base 15937c478bd9Sstevel@tonic-gate * pagesize pages for seg_kmem for use in boot_alloc() 15947c478bd9Sstevel@tonic-gate */ 15957c478bd9Sstevel@tonic-gate void 15967c478bd9Sstevel@tonic-gate page_boot_demote(page_t *pp) 15977c478bd9Sstevel@tonic-gate { 15987c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0); 15997c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16007c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate (void) page_demote(PP_2_MEM_NODE(pp), 160319397407SSherry Moore PFN_BASE(pp->p_pagenum, pp->p_szc), 0, pp->p_szc, 0, PC_NO_COLOR, 16047c478bd9Sstevel@tonic-gate PC_FREE); 16057c478bd9Sstevel@tonic-gate 16067c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16077c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16087c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 16097c478bd9Sstevel@tonic-gate } 16107c478bd9Sstevel@tonic-gate 16117c478bd9Sstevel@tonic-gate /* 16127c478bd9Sstevel@tonic-gate * Take a particular page off of whatever freelist the page 16137c478bd9Sstevel@tonic-gate * is claimed to be on. 16147c478bd9Sstevel@tonic-gate * 16157c478bd9Sstevel@tonic-gate * NOTE: Only used for PAGESIZE pages. 16167c478bd9Sstevel@tonic-gate */ 16177c478bd9Sstevel@tonic-gate void 16187c478bd9Sstevel@tonic-gate page_list_sub(page_t *pp, int flags) 16197c478bd9Sstevel@tonic-gate { 16207c478bd9Sstevel@tonic-gate int bin; 16217c478bd9Sstevel@tonic-gate uint_t mtype; 16227c478bd9Sstevel@tonic-gate int mnode; 16237c478bd9Sstevel@tonic-gate kmutex_t *pcm; 16247c478bd9Sstevel@tonic-gate page_t **ppp; 16257c478bd9Sstevel@tonic-gate 16267c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 16277c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16287c478bd9Sstevel@tonic-gate 16297c478bd9Sstevel@tonic-gate /* 16307c478bd9Sstevel@tonic-gate * The p_szc field can only be changed by page_promote() 16317c478bd9Sstevel@tonic-gate * and page_demote(). Only free pages can be promoted and 16327c478bd9Sstevel@tonic-gate * demoted and the free list MUST be locked during these 16337c478bd9Sstevel@tonic-gate * operations. So to prevent a race in page_list_sub() 16347c478bd9Sstevel@tonic-gate * between computing which bin of the freelist lock to 16357c478bd9Sstevel@tonic-gate * grab and actually grabing the lock we check again that 16367c478bd9Sstevel@tonic-gate * the bin we locked is still the correct one. Notice that 16377c478bd9Sstevel@tonic-gate * the p_szc field could have actually changed on us but 16387c478bd9Sstevel@tonic-gate * if the bin happens to still be the same we are safe. 16397c478bd9Sstevel@tonic-gate */ 16407c478bd9Sstevel@tonic-gate try_again: 16417c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 16427c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 16437c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 16447c478bd9Sstevel@tonic-gate mutex_enter(pcm); 16457c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 16467c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16477c478bd9Sstevel@tonic-gate goto try_again; 16487c478bd9Sstevel@tonic-gate } 16497c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 16507c478bd9Sstevel@tonic-gate 16517c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1652affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[0]); 16537c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16547c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 16557c478bd9Sstevel@tonic-gate } else { 1656affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_cache); 16577c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 16587c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate 16617c478bd9Sstevel@tonic-gate /* 16627c478bd9Sstevel@tonic-gate * Common PAGESIZE case. 16637c478bd9Sstevel@tonic-gate * 16647c478bd9Sstevel@tonic-gate * Note that we locked the freelist. This prevents 16657c478bd9Sstevel@tonic-gate * any page promotion/demotion operations. Therefore 16667c478bd9Sstevel@tonic-gate * the p_szc will not change until we drop pcm mutex. 16677c478bd9Sstevel@tonic-gate */ 16687c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 16697c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 16707c478bd9Sstevel@tonic-gate /* 16717c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16727c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16737c478bd9Sstevel@tonic-gate */ 1674affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 16757c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16767c478bd9Sstevel@tonic-gate 16777c478bd9Sstevel@tonic-gate #if defined(__sparc) 16787c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 16797c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 16807c478bd9Sstevel@tonic-gate } 16817c478bd9Sstevel@tonic-gate #endif 16827c478bd9Sstevel@tonic-gate return; 16837c478bd9Sstevel@tonic-gate } 16847c478bd9Sstevel@tonic-gate 16857c478bd9Sstevel@tonic-gate /* 16867c478bd9Sstevel@tonic-gate * Large pages on the cache list are not supported. 16877c478bd9Sstevel@tonic-gate */ 16887c478bd9Sstevel@tonic-gate if (flags & PG_CACHE_LIST) 16897c478bd9Sstevel@tonic-gate panic("page_list_sub: large page on cachelist"); 16907c478bd9Sstevel@tonic-gate 16917c478bd9Sstevel@tonic-gate /* 16927c478bd9Sstevel@tonic-gate * Slow but rare. 16937c478bd9Sstevel@tonic-gate * 16947c478bd9Sstevel@tonic-gate * Somebody wants this particular page which is part 16957c478bd9Sstevel@tonic-gate * of a large page. In this case we just demote the page 16967c478bd9Sstevel@tonic-gate * if it's on the freelist. 16977c478bd9Sstevel@tonic-gate * 16987c478bd9Sstevel@tonic-gate * We have to drop pcm before locking the entire freelist. 16997c478bd9Sstevel@tonic-gate * Once we have re-locked the freelist check to make sure 17007c478bd9Sstevel@tonic-gate * the page hasn't already been demoted or completely 17017c478bd9Sstevel@tonic-gate * freed. 17027c478bd9Sstevel@tonic-gate */ 17037c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17047c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 17057c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 17067c478bd9Sstevel@tonic-gate /* 17077c478bd9Sstevel@tonic-gate * Large page is on freelist. 17087c478bd9Sstevel@tonic-gate */ 17097c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, pp->p_szc), 171019397407SSherry Moore 0, pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 17117c478bd9Sstevel@tonic-gate } 17127c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17137c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17147c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 17157c478bd9Sstevel@tonic-gate 17167c478bd9Sstevel@tonic-gate /* 17177c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 17187c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 17197c478bd9Sstevel@tonic-gate */ 17207c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17217c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 17227c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 17237c478bd9Sstevel@tonic-gate 17247c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 1725affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 17267c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17277c478bd9Sstevel@tonic-gate 17287c478bd9Sstevel@tonic-gate #if defined(__sparc) 17297c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17307c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 17317c478bd9Sstevel@tonic-gate } 17327c478bd9Sstevel@tonic-gate #endif 17337c478bd9Sstevel@tonic-gate } 17347c478bd9Sstevel@tonic-gate 17357c478bd9Sstevel@tonic-gate void 17367c478bd9Sstevel@tonic-gate page_list_sub_pages(page_t *pp, uint_t szc) 17377c478bd9Sstevel@tonic-gate { 17387c478bd9Sstevel@tonic-gate kmutex_t *pcm; 17397c478bd9Sstevel@tonic-gate uint_t bin, mtype; 17407c478bd9Sstevel@tonic-gate int mnode; 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 17437c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17447c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate /* 17477c478bd9Sstevel@tonic-gate * See comment in page_list_sub(). 17487c478bd9Sstevel@tonic-gate */ 17497c478bd9Sstevel@tonic-gate try_again: 17507c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17517c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 17527c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 17537c478bd9Sstevel@tonic-gate mutex_enter(pcm); 17547c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 17557c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17567c478bd9Sstevel@tonic-gate goto try_again; 17577c478bd9Sstevel@tonic-gate } 17587c478bd9Sstevel@tonic-gate 17597c478bd9Sstevel@tonic-gate /* 17607c478bd9Sstevel@tonic-gate * If we're called with a page larger than szc or it got 17617c478bd9Sstevel@tonic-gate * promoted above szc before we locked the freelist then 17627c478bd9Sstevel@tonic-gate * drop pcm and re-lock entire freelist. If page still larger 17637c478bd9Sstevel@tonic-gate * than szc then demote it. 17647c478bd9Sstevel@tonic-gate */ 17657c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 17667c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17677c478bd9Sstevel@tonic-gate pcm = NULL; 17687c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 17697c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 1770affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szcbig); 17717c478bd9Sstevel@tonic-gate (void) page_demote(mnode, 177219397407SSherry Moore PFN_BASE(pp->p_pagenum, pp->p_szc), 0, 17737c478bd9Sstevel@tonic-gate pp->p_szc, szc, PC_NO_COLOR, PC_FREE); 17747c478bd9Sstevel@tonic-gate } 17757c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17767c478bd9Sstevel@tonic-gate } 17777c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17787c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17797c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc <= szc); 17807c478bd9Sstevel@tonic-gate ASSERT(pp == PP_PAGEROOT(pp)); 17817c478bd9Sstevel@tonic-gate 1782affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[pp->p_szc]); 1783affbd3ccSkchow 17847c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 17857c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 17867c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17877c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 17887c478bd9Sstevel@tonic-gate } else { 1789affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szc0); 17907c478bd9Sstevel@tonic-gate page_sub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17917c478bd9Sstevel@tonic-gate } 1792affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 17937c478bd9Sstevel@tonic-gate 17947c478bd9Sstevel@tonic-gate if (pcm != NULL) { 17957c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17967c478bd9Sstevel@tonic-gate } else { 17977c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17987c478bd9Sstevel@tonic-gate } 17997c478bd9Sstevel@tonic-gate 18007c478bd9Sstevel@tonic-gate #if defined(__sparc) 18017c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 18027c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 18037c478bd9Sstevel@tonic-gate 18047c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 18057c478bd9Sstevel@tonic-gate kcage_freemem_sub(pgcnt); 18067c478bd9Sstevel@tonic-gate } 18077c478bd9Sstevel@tonic-gate #endif 18087c478bd9Sstevel@tonic-gate } 18097c478bd9Sstevel@tonic-gate 18107c478bd9Sstevel@tonic-gate /* 18117c478bd9Sstevel@tonic-gate * Add the page to the front of a linked list of pages 18127c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 18137c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 18147c478bd9Sstevel@tonic-gate */ 18157c478bd9Sstevel@tonic-gate void 18167c478bd9Sstevel@tonic-gate mach_page_add(page_t **ppp, page_t *pp) 18177c478bd9Sstevel@tonic-gate { 18187c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 18197c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 18207c478bd9Sstevel@tonic-gate } else { 18217c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 18227c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 18237c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 18247c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 18257c478bd9Sstevel@tonic-gate } 18267c478bd9Sstevel@tonic-gate *ppp = pp; 18277c478bd9Sstevel@tonic-gate } 18287c478bd9Sstevel@tonic-gate 18297c478bd9Sstevel@tonic-gate /* 18307c478bd9Sstevel@tonic-gate * Remove this page from a linked list of pages 18317c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 18327c478bd9Sstevel@tonic-gate * 18337c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 18347c478bd9Sstevel@tonic-gate */ 18357c478bd9Sstevel@tonic-gate void 18367c478bd9Sstevel@tonic-gate mach_page_sub(page_t **ppp, page_t *pp) 18377c478bd9Sstevel@tonic-gate { 18387c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 18397c478bd9Sstevel@tonic-gate 18407c478bd9Sstevel@tonic-gate if (*ppp == NULL || pp == NULL) 18417c478bd9Sstevel@tonic-gate panic("mach_page_sub"); 18427c478bd9Sstevel@tonic-gate 18437c478bd9Sstevel@tonic-gate if (*ppp == pp) 18447c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate if (*ppp == pp) 18477c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 18487c478bd9Sstevel@tonic-gate else { 18497c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 18507c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 18517c478bd9Sstevel@tonic-gate } 18527c478bd9Sstevel@tonic-gate pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 18537c478bd9Sstevel@tonic-gate } 18547c478bd9Sstevel@tonic-gate 18557c478bd9Sstevel@tonic-gate /* 18567c478bd9Sstevel@tonic-gate * Routine fsflush uses to gradually coalesce the free list into larger pages. 18577c478bd9Sstevel@tonic-gate */ 18587c478bd9Sstevel@tonic-gate void 18597c478bd9Sstevel@tonic-gate page_promote_size(page_t *pp, uint_t cur_szc) 18607c478bd9Sstevel@tonic-gate { 18617c478bd9Sstevel@tonic-gate pfn_t pfn; 18627c478bd9Sstevel@tonic-gate int mnode; 18637c478bd9Sstevel@tonic-gate int idx; 18647c478bd9Sstevel@tonic-gate int new_szc = cur_szc + 1; 18657c478bd9Sstevel@tonic-gate int full = FULL_REGION_CNT(new_szc); 18667c478bd9Sstevel@tonic-gate 18677c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 18687c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(pfn); 18697c478bd9Sstevel@tonic-gate 18707c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, new_szc, pfn); 18737c478bd9Sstevel@tonic-gate if (PAGE_COUNTERS(mnode, new_szc, idx) == full) 18745d07b933Sdp (void) page_promote(mnode, pfn, new_szc, PC_FREE, PC_MTYPE_ANY); 18757c478bd9Sstevel@tonic-gate 18767c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 18777c478bd9Sstevel@tonic-gate } 18787c478bd9Sstevel@tonic-gate 18797c478bd9Sstevel@tonic-gate static uint_t page_promote_err; 18807c478bd9Sstevel@tonic-gate static uint_t page_promote_noreloc_err; 18817c478bd9Sstevel@tonic-gate 18827c478bd9Sstevel@tonic-gate /* 18837c478bd9Sstevel@tonic-gate * Create a single larger page (of szc new_szc) from smaller contiguous pages 18847c478bd9Sstevel@tonic-gate * for the given mnode starting at pfnum. Pages involved are on the freelist 18857c478bd9Sstevel@tonic-gate * before the call and may be returned to the caller if requested, otherwise 18867c478bd9Sstevel@tonic-gate * they will be placed back on the freelist. 18877c478bd9Sstevel@tonic-gate * If flags is PC_ALLOC, then the large page will be returned to the user in 18887c478bd9Sstevel@tonic-gate * a state which is consistent with a page being taken off the freelist. If 18897c478bd9Sstevel@tonic-gate * we failed to lock the new large page, then we will return NULL to the 18907c478bd9Sstevel@tonic-gate * caller and put the large page on the freelist instead. 18917c478bd9Sstevel@tonic-gate * If flags is PC_FREE, then the large page will be placed on the freelist, 18927c478bd9Sstevel@tonic-gate * and NULL will be returned. 18937c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 18947c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 18957c478bd9Sstevel@tonic-gate * 18967c478bd9Sstevel@tonic-gate * RFE: For performance pass in pp instead of pfnum so 18977c478bd9Sstevel@tonic-gate * we can avoid excessive calls to page_numtopp_nolock(). 18987c478bd9Sstevel@tonic-gate * This would depend on an assumption that all contiguous 18997c478bd9Sstevel@tonic-gate * pages are in the same memseg so we can just add/dec 19007c478bd9Sstevel@tonic-gate * our pp. 19017c478bd9Sstevel@tonic-gate * 19027c478bd9Sstevel@tonic-gate * Lock ordering: 19037c478bd9Sstevel@tonic-gate * 19047c478bd9Sstevel@tonic-gate * There is a potential but rare deadlock situation 19057c478bd9Sstevel@tonic-gate * for page promotion and demotion operations. The problem 19067c478bd9Sstevel@tonic-gate * is there are two paths into the freelist manager and 19077c478bd9Sstevel@tonic-gate * they have different lock orders: 19087c478bd9Sstevel@tonic-gate * 19097c478bd9Sstevel@tonic-gate * page_create() 19107c478bd9Sstevel@tonic-gate * lock freelist 19117c478bd9Sstevel@tonic-gate * page_lock(EXCL) 19127c478bd9Sstevel@tonic-gate * unlock freelist 19137c478bd9Sstevel@tonic-gate * return 19147c478bd9Sstevel@tonic-gate * caller drops page_lock 19157c478bd9Sstevel@tonic-gate * 19167c478bd9Sstevel@tonic-gate * page_free() and page_reclaim() 19177c478bd9Sstevel@tonic-gate * caller grabs page_lock(EXCL) 19187c478bd9Sstevel@tonic-gate * 19197c478bd9Sstevel@tonic-gate * lock freelist 19207c478bd9Sstevel@tonic-gate * unlock freelist 19217c478bd9Sstevel@tonic-gate * drop page_lock 19227c478bd9Sstevel@tonic-gate * 19237c478bd9Sstevel@tonic-gate * What prevents a thread in page_create() from deadlocking 19247c478bd9Sstevel@tonic-gate * with a thread freeing or reclaiming the same page is the 19257c478bd9Sstevel@tonic-gate * page_trylock() in page_get_freelist(). If the trylock fails 19267c478bd9Sstevel@tonic-gate * it skips the page. 19277c478bd9Sstevel@tonic-gate * 19287c478bd9Sstevel@tonic-gate * The lock ordering for promotion and demotion is the same as 19297c478bd9Sstevel@tonic-gate * for page_create(). Since the same deadlock could occur during 19307c478bd9Sstevel@tonic-gate * page promotion and freeing or reclaiming of a page on the 19317c478bd9Sstevel@tonic-gate * cache list we might have to fail the operation and undo what 19327c478bd9Sstevel@tonic-gate * have done so far. Again this is rare. 19337c478bd9Sstevel@tonic-gate */ 19347c478bd9Sstevel@tonic-gate page_t * 19355d07b933Sdp page_promote(int mnode, pfn_t pfnum, uchar_t new_szc, int flags, int mtype) 19367c478bd9Sstevel@tonic-gate { 19377c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *tpp, *start_pp; 19387c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, npgs; 19397c478bd9Sstevel@tonic-gate uint_t bin; 19407c478bd9Sstevel@tonic-gate pgcnt_t tmpnpgs, pages_left; 19417c478bd9Sstevel@tonic-gate uint_t noreloc; 19427c478bd9Sstevel@tonic-gate int which_list; 19437c478bd9Sstevel@tonic-gate ulong_t index; 19447c478bd9Sstevel@tonic-gate kmutex_t *phm; 19457c478bd9Sstevel@tonic-gate 19467c478bd9Sstevel@tonic-gate /* 19477c478bd9Sstevel@tonic-gate * General algorithm: 19487c478bd9Sstevel@tonic-gate * Find the starting page 19497c478bd9Sstevel@tonic-gate * Walk each page struct removing it from the freelist, 19507c478bd9Sstevel@tonic-gate * and linking it to all the other pages removed. 19517c478bd9Sstevel@tonic-gate * Once all pages are off the freelist, 19527c478bd9Sstevel@tonic-gate * walk the list, modifying p_szc to new_szc and what 19537c478bd9Sstevel@tonic-gate * ever other info needs to be done to create a large free page. 19547c478bd9Sstevel@tonic-gate * According to the flags, either return the page or put it 19557c478bd9Sstevel@tonic-gate * on the freelist. 19567c478bd9Sstevel@tonic-gate */ 19577c478bd9Sstevel@tonic-gate 19587c478bd9Sstevel@tonic-gate start_pp = page_numtopp_nolock(pfnum); 19597c478bd9Sstevel@tonic-gate ASSERT(start_pp && (start_pp->p_pagenum == pfnum)); 19607c478bd9Sstevel@tonic-gate new_npgs = page_get_pagecnt(new_szc); 19617c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pfnum, new_npgs)); 19627c478bd9Sstevel@tonic-gate 19635d07b933Sdp /* don't return page of the wrong mtype */ 19645d07b933Sdp if (mtype != PC_MTYPE_ANY && mtype != PP_2_MTYPE(start_pp)) 19655d07b933Sdp return (NULL); 19665d07b933Sdp 19677c478bd9Sstevel@tonic-gate /* 19687c478bd9Sstevel@tonic-gate * Loop through smaller pages to confirm that all pages 19697c478bd9Sstevel@tonic-gate * give the same result for PP_ISNORELOC(). 19707c478bd9Sstevel@tonic-gate * We can check this reliably here as the protocol for setting 19717c478bd9Sstevel@tonic-gate * P_NORELOC requires pages to be taken off the free list first. 19727c478bd9Sstevel@tonic-gate */ 19735d07b933Sdp noreloc = PP_ISNORELOC(start_pp); 19745d07b933Sdp for (pp = start_pp + new_npgs; --pp > start_pp; ) { 19755d07b933Sdp if (noreloc != PP_ISNORELOC(pp)) { 19767c478bd9Sstevel@tonic-gate page_promote_noreloc_err++; 19777c478bd9Sstevel@tonic-gate page_promote_err++; 19787c478bd9Sstevel@tonic-gate return (NULL); 19797c478bd9Sstevel@tonic-gate } 19807c478bd9Sstevel@tonic-gate } 19817c478bd9Sstevel@tonic-gate 19827c478bd9Sstevel@tonic-gate pages_left = new_npgs; 19837c478bd9Sstevel@tonic-gate pplist = NULL; 19847c478bd9Sstevel@tonic-gate pp = start_pp; 19857c478bd9Sstevel@tonic-gate 19867c478bd9Sstevel@tonic-gate /* Loop around coalescing the smaller pages into a big page. */ 19877c478bd9Sstevel@tonic-gate while (pages_left) { 19887c478bd9Sstevel@tonic-gate /* 19897c478bd9Sstevel@tonic-gate * Remove from the freelist. 19907c478bd9Sstevel@tonic-gate */ 19917c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 19927c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 19937c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 19947c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 19957c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 19967c478bd9Sstevel@tonic-gate 19977c478bd9Sstevel@tonic-gate /* 19987c478bd9Sstevel@tonic-gate * PG_FREE_LIST 19997c478bd9Sstevel@tonic-gate */ 20007c478bd9Sstevel@tonic-gate if (pp->p_szc) { 20017c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, 20027c478bd9Sstevel@tonic-gate pp->p_szc, bin, mtype), pp); 20037c478bd9Sstevel@tonic-gate } else { 20047c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_FREELISTS(mnode, 0, 20057c478bd9Sstevel@tonic-gate bin, mtype), pp); 20067c478bd9Sstevel@tonic-gate } 20077c478bd9Sstevel@tonic-gate which_list = PG_FREE_LIST; 20087c478bd9Sstevel@tonic-gate } else { 20097c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 20107c478bd9Sstevel@tonic-gate 20117c478bd9Sstevel@tonic-gate /* 20127c478bd9Sstevel@tonic-gate * PG_CACHE_LIST 20137c478bd9Sstevel@tonic-gate * 20147c478bd9Sstevel@tonic-gate * Since this page comes from the 20157c478bd9Sstevel@tonic-gate * cachelist, we must destroy the 20167c478bd9Sstevel@tonic-gate * vnode association. 20177c478bd9Sstevel@tonic-gate */ 20187c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 20197c478bd9Sstevel@tonic-gate goto fail_promote; 20207c478bd9Sstevel@tonic-gate } 20217c478bd9Sstevel@tonic-gate 20227c478bd9Sstevel@tonic-gate /* 20237c478bd9Sstevel@tonic-gate * We need to be careful not to deadlock 20247c478bd9Sstevel@tonic-gate * with another thread in page_lookup(). 20257c478bd9Sstevel@tonic-gate * The page_lookup() thread could be holding 20267c478bd9Sstevel@tonic-gate * the same phm that we need if the two 20277c478bd9Sstevel@tonic-gate * pages happen to hash to the same phm lock. 20287c478bd9Sstevel@tonic-gate * At this point we have locked the entire 20297c478bd9Sstevel@tonic-gate * freelist and page_lookup() could be trying 20307c478bd9Sstevel@tonic-gate * to grab a freelist lock. 20317c478bd9Sstevel@tonic-gate */ 20327c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(pp->p_vnode, pp->p_offset); 20337c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 20347c478bd9Sstevel@tonic-gate if (!mutex_tryenter(phm)) { 20358b464eb8Smec page_unlock_nocapture(pp); 20367c478bd9Sstevel@tonic-gate goto fail_promote; 20377c478bd9Sstevel@tonic-gate } 20387c478bd9Sstevel@tonic-gate 20397c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_CACHELISTS(mnode, bin, mtype), pp); 20407c478bd9Sstevel@tonic-gate page_hashout(pp, phm); 20417c478bd9Sstevel@tonic-gate mutex_exit(phm); 20427c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 20438b464eb8Smec page_unlock_nocapture(pp); 20447c478bd9Sstevel@tonic-gate which_list = PG_CACHE_LIST; 20457c478bd9Sstevel@tonic-gate } 2046affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, which_list); 20477c478bd9Sstevel@tonic-gate 20487c478bd9Sstevel@tonic-gate /* 20497c478bd9Sstevel@tonic-gate * Concatenate the smaller page(s) onto 20507c478bd9Sstevel@tonic-gate * the large page list. 20517c478bd9Sstevel@tonic-gate */ 20527c478bd9Sstevel@tonic-gate tmpnpgs = npgs = page_get_pagecnt(pp->p_szc); 20537c478bd9Sstevel@tonic-gate pages_left -= npgs; 20547c478bd9Sstevel@tonic-gate tpp = pp; 20557c478bd9Sstevel@tonic-gate while (npgs--) { 20567c478bd9Sstevel@tonic-gate tpp->p_szc = new_szc; 20577c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 20587c478bd9Sstevel@tonic-gate } 20597c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 20607c478bd9Sstevel@tonic-gate pp += tmpnpgs; 20617c478bd9Sstevel@tonic-gate } 20627c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 20637c478bd9Sstevel@tonic-gate 20647c478bd9Sstevel@tonic-gate /* 20657c478bd9Sstevel@tonic-gate * return the page to the user if requested 20667c478bd9Sstevel@tonic-gate * in the properly locked state. 20677c478bd9Sstevel@tonic-gate */ 20687c478bd9Sstevel@tonic-gate if (flags == PC_ALLOC && (page_trylock_cons(pplist, SE_EXCL))) { 20697c478bd9Sstevel@tonic-gate return (pplist); 20707c478bd9Sstevel@tonic-gate } 20717c478bd9Sstevel@tonic-gate 20727c478bd9Sstevel@tonic-gate /* 20737c478bd9Sstevel@tonic-gate * Otherwise place the new large page on the freelist 20747c478bd9Sstevel@tonic-gate */ 20757c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 20767c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pplist); 20777c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 20787c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, bin, mtype), pplist); 20797c478bd9Sstevel@tonic-gate 2080affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, PG_FREE_LIST); 20817c478bd9Sstevel@tonic-gate return (NULL); 20827c478bd9Sstevel@tonic-gate 20837c478bd9Sstevel@tonic-gate fail_promote: 20847c478bd9Sstevel@tonic-gate /* 20857c478bd9Sstevel@tonic-gate * A thread must have still been freeing or 20867c478bd9Sstevel@tonic-gate * reclaiming the page on the cachelist. 20877c478bd9Sstevel@tonic-gate * To prevent a deadlock undo what we have 20887c478bd9Sstevel@tonic-gate * done sofar and return failure. This 20897c478bd9Sstevel@tonic-gate * situation can only happen while promoting 20907c478bd9Sstevel@tonic-gate * PAGESIZE pages. 20917c478bd9Sstevel@tonic-gate */ 20927c478bd9Sstevel@tonic-gate page_promote_err++; 20937c478bd9Sstevel@tonic-gate while (pplist) { 20947c478bd9Sstevel@tonic-gate pp = pplist; 20957c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 20967c478bd9Sstevel@tonic-gate pp->p_szc = 0; 20977c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 20987c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 20997c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, mtype), pp); 2100affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 21017c478bd9Sstevel@tonic-gate } 21027c478bd9Sstevel@tonic-gate return (NULL); 21037c478bd9Sstevel@tonic-gate 21047c478bd9Sstevel@tonic-gate } 21057c478bd9Sstevel@tonic-gate 21067c478bd9Sstevel@tonic-gate /* 21077c478bd9Sstevel@tonic-gate * Break up a large page into smaller size pages. 21087c478bd9Sstevel@tonic-gate * Pages involved are on the freelist before the call and may 21097c478bd9Sstevel@tonic-gate * be returned to the caller if requested, otherwise they will 21107c478bd9Sstevel@tonic-gate * be placed back on the freelist. 21117c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 21127c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 21137c478bd9Sstevel@tonic-gate * If flags is not PC_ALLOC, the color argument is ignored, and thus 21147c478bd9Sstevel@tonic-gate * technically, any value may be passed in but PC_NO_COLOR is the standard 21157c478bd9Sstevel@tonic-gate * which should be followed for clarity's sake. 211619397407SSherry Moore * Returns a page whose pfn is < pfnmax 21177c478bd9Sstevel@tonic-gate */ 21187c478bd9Sstevel@tonic-gate page_t * 211919397407SSherry Moore page_demote(int mnode, pfn_t pfnum, pfn_t pfnmax, uchar_t cur_szc, 212019397407SSherry Moore uchar_t new_szc, int color, int flags) 21217c478bd9Sstevel@tonic-gate { 21227c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *npplist; 21237c478bd9Sstevel@tonic-gate pgcnt_t npgs, n; 21247c478bd9Sstevel@tonic-gate uint_t bin; 21257c478bd9Sstevel@tonic-gate uint_t mtype; 21267c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 21277c478bd9Sstevel@tonic-gate 21287c478bd9Sstevel@tonic-gate ASSERT(cur_szc != 0); 21297c478bd9Sstevel@tonic-gate ASSERT(new_szc < cur_szc); 21307c478bd9Sstevel@tonic-gate 21317c478bd9Sstevel@tonic-gate pplist = page_numtopp_nolock(pfnum); 21327c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 21337c478bd9Sstevel@tonic-gate 21347c478bd9Sstevel@tonic-gate ASSERT(pplist->p_szc == cur_szc); 21357c478bd9Sstevel@tonic-gate 21367c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 21377c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pplist)); 21387c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 21397c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, cur_szc, bin, mtype), pplist); 21407c478bd9Sstevel@tonic-gate 21417c478bd9Sstevel@tonic-gate CHK_LPG(pplist, cur_szc); 2142affbd3ccSkchow page_ctr_sub(mnode, mtype, pplist, PG_FREE_LIST); 21437c478bd9Sstevel@tonic-gate 21447c478bd9Sstevel@tonic-gate /* 21457c478bd9Sstevel@tonic-gate * Number of PAGESIZE pages for smaller new_szc 21467c478bd9Sstevel@tonic-gate * page. 21477c478bd9Sstevel@tonic-gate */ 21487c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(new_szc); 21497c478bd9Sstevel@tonic-gate 21507c478bd9Sstevel@tonic-gate while (pplist) { 21517c478bd9Sstevel@tonic-gate pp = pplist; 21527c478bd9Sstevel@tonic-gate 21537c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21547c478bd9Sstevel@tonic-gate 21557c478bd9Sstevel@tonic-gate /* 21567c478bd9Sstevel@tonic-gate * We either break it up into PAGESIZE pages or larger. 21577c478bd9Sstevel@tonic-gate */ 21587c478bd9Sstevel@tonic-gate if (npgs == 1) { /* PAGESIZE case */ 21597c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 21607c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21617c478bd9Sstevel@tonic-gate ASSERT(new_szc == 0); 21627c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 21637c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21647c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 21657c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 216619397407SSherry Moore (ret_pp == NULL) && (pfnmax == 0 || 216719397407SSherry Moore pp->p_pagenum < pfnmax) && 21687c478bd9Sstevel@tonic-gate page_trylock_cons(pp, SE_EXCL)) { 21697c478bd9Sstevel@tonic-gate ret_pp = pp; 21707c478bd9Sstevel@tonic-gate } else { 21717c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21727c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, 21737c478bd9Sstevel@tonic-gate mtype), pp); 2174affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 21757c478bd9Sstevel@tonic-gate } 21767c478bd9Sstevel@tonic-gate } else { 217719397407SSherry Moore page_t *try_to_return_this_page = NULL; 217819397407SSherry Moore int count = 0; 21797c478bd9Sstevel@tonic-gate 21807c478bd9Sstevel@tonic-gate /* 21817c478bd9Sstevel@tonic-gate * Break down into smaller lists of pages. 21827c478bd9Sstevel@tonic-gate */ 21837c478bd9Sstevel@tonic-gate page_list_break(&pplist, &npplist, npgs); 21847c478bd9Sstevel@tonic-gate 21857c478bd9Sstevel@tonic-gate pp = pplist; 21867c478bd9Sstevel@tonic-gate n = npgs; 21877c478bd9Sstevel@tonic-gate while (n--) { 21887c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 218919397407SSherry Moore /* 219019397407SSherry Moore * Check whether all the pages in this list 219119397407SSherry Moore * fit the request criteria. 219219397407SSherry Moore */ 219319397407SSherry Moore if (pfnmax == 0 || pp->p_pagenum < pfnmax) { 219419397407SSherry Moore count++; 219519397407SSherry Moore } 21967c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21977c478bd9Sstevel@tonic-gate pp = pp->p_next; 21987c478bd9Sstevel@tonic-gate } 21997c478bd9Sstevel@tonic-gate 220019397407SSherry Moore if (count == npgs && 220119397407SSherry Moore (pfnmax == 0 || pp->p_pagenum < pfnmax)) { 220219397407SSherry Moore try_to_return_this_page = pp; 220319397407SSherry Moore } 220419397407SSherry Moore 22057c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 22067c478bd9Sstevel@tonic-gate 22077c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 220819397407SSherry Moore if (try_to_return_this_page) 220919397407SSherry Moore ASSERT(mnode == 221019397407SSherry Moore PP_2_MEM_NODE(try_to_return_this_page)); 22117c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 221219397407SSherry Moore (ret_pp == NULL) && try_to_return_this_page && 221319397407SSherry Moore page_trylock_cons(try_to_return_this_page, 221419397407SSherry Moore SE_EXCL)) { 221519397407SSherry Moore ret_pp = try_to_return_this_page; 22167c478bd9Sstevel@tonic-gate } else { 22177c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 22187c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, 22197c478bd9Sstevel@tonic-gate bin, mtype), pplist); 22207c478bd9Sstevel@tonic-gate 2221affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, 2222affbd3ccSkchow PG_FREE_LIST); 22237c478bd9Sstevel@tonic-gate } 22247c478bd9Sstevel@tonic-gate pplist = npplist; 22257c478bd9Sstevel@tonic-gate } 22267c478bd9Sstevel@tonic-gate } 22277c478bd9Sstevel@tonic-gate return (ret_pp); 22287c478bd9Sstevel@tonic-gate } 22297c478bd9Sstevel@tonic-gate 22307c478bd9Sstevel@tonic-gate int mpss_coalesce_disable = 0; 22317c478bd9Sstevel@tonic-gate 22327c478bd9Sstevel@tonic-gate /* 22337c478bd9Sstevel@tonic-gate * Coalesce free pages into a page of the given szc and color if possible. 22347c478bd9Sstevel@tonic-gate * Return the pointer to the page created, otherwise, return NULL. 22355d07b933Sdp * 22365d07b933Sdp * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 22377c478bd9Sstevel@tonic-gate */ 22385d07b933Sdp page_t * 22395d07b933Sdp page_freelist_coalesce(int mnode, uchar_t szc, uint_t color, uint_t ceq_mask, 22405d07b933Sdp int mtype, pfn_t pfnhi) 22417c478bd9Sstevel@tonic-gate { 22425d07b933Sdp int r = szc; /* region size */ 22435d07b933Sdp int mrange; 22445d07b933Sdp uint_t full, bin, color_mask, wrap = 0; 22455d07b933Sdp pfn_t pfnum, lo, hi; 22465d07b933Sdp size_t len, idx, idx0; 22475d07b933Sdp pgcnt_t cands = 0, szcpgcnt = page_get_pagecnt(szc); 22487c478bd9Sstevel@tonic-gate page_t *ret_pp; 2249ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 22505d07b933Sdp #if defined(__sparc) 22515d07b933Sdp pfn_t pfnum0, nlo, nhi; 22525d07b933Sdp #endif 22537c478bd9Sstevel@tonic-gate 22547c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 22555d07b933Sdp ASSERT(szc < MMU_PAGE_SIZES); 22565d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[szc][0]); 22577c478bd9Sstevel@tonic-gate return (NULL); 22587c478bd9Sstevel@tonic-gate } 22597c478bd9Sstevel@tonic-gate 22605d07b933Sdp ASSERT(szc < mmu_page_sizes); 22615d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(szc) - 1; 22625d07b933Sdp ASSERT(ceq_mask <= color_mask); 22635d07b933Sdp ASSERT(color <= color_mask); 22645d07b933Sdp color &= ceq_mask; 22655d07b933Sdp 22665d07b933Sdp /* Prevent page_counters dynamic memory from being freed */ 22675d07b933Sdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 22685d07b933Sdp 22695d07b933Sdp mrange = MTYPE_2_MRANGE(mnode, mtype); 22705d07b933Sdp ASSERT(mrange < mnode_nranges[mnode]); 22715d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[r][mrange]); 22725d07b933Sdp 22735d07b933Sdp /* get pfn range for mtype */ 22745d07b933Sdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 22755d07b933Sdp MNODETYPE_2_PFN(mnode, mtype, lo, hi); 22765d07b933Sdp hi++; 22775d07b933Sdp 22785d07b933Sdp /* use lower limit if given */ 22795d07b933Sdp if (pfnhi != PFNNULL && pfnhi < hi) 22805d07b933Sdp hi = pfnhi; 22815d07b933Sdp 22825d07b933Sdp /* round to szcpgcnt boundaries */ 22835d07b933Sdp lo = P2ROUNDUP(lo, szcpgcnt); 2284b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 2285b779d3e0Sdp if (lo == (pfn_t)-1) { 2286b779d3e0Sdp rw_exit(&page_ctrs_rwlock[mnode]); 2287b779d3e0Sdp return (NULL); 2288b779d3e0Sdp } 22895d07b933Sdp hi = hi & ~(szcpgcnt - 1); 22905d07b933Sdp 22915d07b933Sdp /* set lo to the closest pfn of the right color */ 2292ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ color) & ceq_mask) || 2293ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(lo) != mnode)) { 2294ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, color, ceq_mask, color_mask, 2295ce8eb11aSdp &it); 22965d07b933Sdp } 22975d07b933Sdp 22985d07b933Sdp if (hi <= lo) { 22995d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 23007c478bd9Sstevel@tonic-gate return (NULL); 23017c478bd9Sstevel@tonic-gate } 23025d07b933Sdp 23037c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 23047c478bd9Sstevel@tonic-gate 23055d07b933Sdp /* calculate the number of page candidates and initial search index */ 23065d07b933Sdp bin = color; 23075d07b933Sdp idx0 = (size_t)(-1); 23085d07b933Sdp do { 23095d07b933Sdp pgcnt_t acand; 23105d07b933Sdp 23115d07b933Sdp PGCTRS_CANDS_GETVALUECOLOR(mnode, mrange, r, bin, acand); 23125d07b933Sdp if (acand) { 23135d07b933Sdp idx = PAGE_COUNTERS_CURRENT_COLOR(mnode, 23145d07b933Sdp r, bin, mrange); 23155d07b933Sdp idx0 = MIN(idx0, idx); 23165d07b933Sdp cands += acand; 23175d07b933Sdp } 23185d07b933Sdp bin = ADD_MASKED(bin, 1, ceq_mask, color_mask); 23195d07b933Sdp } while (bin != color); 23205d07b933Sdp 23215d07b933Sdp if (cands == 0) { 23225d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_cands_skip[r][mrange]); 23235d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 23245d07b933Sdp return (NULL); 23255d07b933Sdp } 23265d07b933Sdp 23275d07b933Sdp pfnum = IDX_TO_PNUM(mnode, r, idx0); 23285d07b933Sdp if (pfnum < lo || pfnum >= hi) { 23295d07b933Sdp pfnum = lo; 2330ce8eb11aSdp } else { 2331b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2332ce8eb11aSdp if (pfnum == (pfn_t)-1) { 2333ce8eb11aSdp pfnum = lo; 2334b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2335ce8eb11aSdp ASSERT(pfnum != (pfn_t)-1); 2336ce8eb11aSdp } else if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & ceq_mask || 2337ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(pfnum) != mnode)) { 2338ce8eb11aSdp /* invalid color, get the closest correct pfn */ 2339ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2340ce8eb11aSdp color_mask, &it); 2341ce8eb11aSdp if (pfnum >= hi) { 2342ce8eb11aSdp pfnum = lo; 2343b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2344ce8eb11aSdp } 2345ce8eb11aSdp } 23465d07b933Sdp } 23475d07b933Sdp 23485d07b933Sdp /* set starting index */ 23495d07b933Sdp idx0 = PNUM_TO_IDX(mnode, r, pfnum); 23505d07b933Sdp ASSERT(idx0 < len); 23515d07b933Sdp 23525d07b933Sdp #if defined(__sparc) 23535d07b933Sdp pfnum0 = pfnum; /* page corresponding to idx0 */ 23545d07b933Sdp nhi = 0; /* search kcage ranges */ 23555d07b933Sdp #endif 23565d07b933Sdp 23575d07b933Sdp for (idx = idx0; wrap == 0 || (idx < idx0 && wrap < 2); ) { 23585d07b933Sdp 23595d07b933Sdp #if defined(__sparc) 23605d07b933Sdp /* 23615d07b933Sdp * Find lowest intersection of kcage ranges and mnode. 23625d07b933Sdp * MTYPE_NORELOC means look in the cage, otherwise outside. 23635d07b933Sdp */ 23645d07b933Sdp if (nhi <= pfnum) { 23655d07b933Sdp if (kcage_next_range(mtype == MTYPE_NORELOC, pfnum, 23665d07b933Sdp (wrap == 0 ? hi : pfnum0), &nlo, &nhi)) 23675d07b933Sdp goto wrapit; 23685d07b933Sdp 23695d07b933Sdp /* jump to the next page in the range */ 23705d07b933Sdp if (pfnum < nlo) { 23715d07b933Sdp pfnum = P2ROUNDUP(nlo, szcpgcnt); 2372b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 23735d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 23745d07b933Sdp if (idx >= len || pfnum >= hi) 23755d07b933Sdp goto wrapit; 2376ce8eb11aSdp if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & 23775d07b933Sdp ceq_mask) 23785d07b933Sdp goto next; 2379ce8eb11aSdp if (interleaved_mnodes && 2380ce8eb11aSdp PFN_2_MEM_NODE(pfnum) != mnode) 2381ce8eb11aSdp goto next; 23827c478bd9Sstevel@tonic-gate } 23835d07b933Sdp } 23845d07b933Sdp #endif 23855d07b933Sdp 23865d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) != full) 23875d07b933Sdp goto next; 23885d07b933Sdp 23895d07b933Sdp /* 23905d07b933Sdp * RFE: For performance maybe we can do something less 23915d07b933Sdp * brutal than locking the entire freelist. So far 23925d07b933Sdp * this doesn't seem to be a performance problem? 23935d07b933Sdp */ 23945d07b933Sdp page_freelist_lock(mnode); 23955d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 23965d07b933Sdp ret_pp = 23975d07b933Sdp page_promote(mnode, pfnum, r, PC_ALLOC, mtype); 23987c478bd9Sstevel@tonic-gate if (ret_pp != NULL) { 23995d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfc_coalok[r][mrange]); 24005d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, 2401ce8eb11aSdp PFN_2_COLOR(pfnum, szc, &it), mrange) = idx; 24027c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 24037c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 24047c478bd9Sstevel@tonic-gate #if defined(__sparc) 24057c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 24067c478bd9Sstevel@tonic-gate pgcnt_t npgs; 24077c478bd9Sstevel@tonic-gate 24087c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(ret_pp->p_szc); 24097c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 24107c478bd9Sstevel@tonic-gate } 24117c478bd9Sstevel@tonic-gate #endif 24127c478bd9Sstevel@tonic-gate return (ret_pp); 24137c478bd9Sstevel@tonic-gate } 24145d07b933Sdp } else { 24155d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_changed[r][mrange]); 24165d07b933Sdp } 24175d07b933Sdp 24185d07b933Sdp page_freelist_unlock(mnode); 24195d07b933Sdp /* 24205d07b933Sdp * No point looking for another page if we've 24215d07b933Sdp * already tried all of the ones that 24225d07b933Sdp * page_ctr_cands indicated. Stash off where we left 24235d07b933Sdp * off. 24245d07b933Sdp * Note: this is not exact since we don't hold the 24255d07b933Sdp * page_freelist_locks before we initially get the 24265d07b933Sdp * value of cands for performance reasons, but should 24275d07b933Sdp * be a decent approximation. 24285d07b933Sdp */ 24295d07b933Sdp if (--cands == 0) { 24305d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, color, mrange) = 24315d07b933Sdp idx; 24325d07b933Sdp break; 24335d07b933Sdp } 24345d07b933Sdp next: 24355d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2436ce8eb11aSdp color_mask, &it); 24375d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 24385d07b933Sdp if (idx >= len || pfnum >= hi) { 24395d07b933Sdp wrapit: 24405d07b933Sdp pfnum = lo; 2441b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 24425d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 24435d07b933Sdp wrap++; 24445d07b933Sdp #if defined(__sparc) 24455d07b933Sdp nhi = 0; /* search kcage ranges */ 24465d07b933Sdp #endif 24477c478bd9Sstevel@tonic-gate } 24487c478bd9Sstevel@tonic-gate } 24495d07b933Sdp 24507c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 24515d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_failed[r][mrange]); 24527c478bd9Sstevel@tonic-gate return (NULL); 24537c478bd9Sstevel@tonic-gate } 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate /* 24567c478bd9Sstevel@tonic-gate * For the given mnode, promote as many small pages to large pages as possible. 2457ce8eb11aSdp * mnode can be -1, which means do them all 24587c478bd9Sstevel@tonic-gate */ 24597c478bd9Sstevel@tonic-gate void 24607c478bd9Sstevel@tonic-gate page_freelist_coalesce_all(int mnode) 24617c478bd9Sstevel@tonic-gate { 24627c478bd9Sstevel@tonic-gate int r; /* region size */ 24637c478bd9Sstevel@tonic-gate int idx, full; 24647c478bd9Sstevel@tonic-gate size_t len; 2465ce8eb11aSdp int doall = interleaved_mnodes || mnode < 0; 2466ce8eb11aSdp int mlo = doall ? 0 : mnode; 2467ce8eb11aSdp int mhi = doall ? max_mem_nodes : (mnode + 1); 24687c478bd9Sstevel@tonic-gate 24697c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce_all); 24707c478bd9Sstevel@tonic-gate 24717c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 24727c478bd9Sstevel@tonic-gate return; 24737c478bd9Sstevel@tonic-gate } 24747c478bd9Sstevel@tonic-gate 24757c478bd9Sstevel@tonic-gate /* 24767c478bd9Sstevel@tonic-gate * Lock the entire freelist and coalesce what we can. 24777c478bd9Sstevel@tonic-gate * 24787c478bd9Sstevel@tonic-gate * Always promote to the largest page possible 24797c478bd9Sstevel@tonic-gate * first to reduce the number of page promotions. 24807c478bd9Sstevel@tonic-gate */ 2481ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2482ce8eb11aSdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 2483ce8eb11aSdp page_freelist_lock(mnode); 2484ce8eb11aSdp } 24857c478bd9Sstevel@tonic-gate for (r = mmu_page_sizes - 1; r > 0; r--) { 2486ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2487ce8eb11aSdp pgcnt_t cands = 0; 2488ce8eb11aSdp int mrange, nranges = mnode_nranges[mnode]; 24897c478bd9Sstevel@tonic-gate 2490ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 2491ce8eb11aSdp PGCTRS_CANDS_GETVALUE(mnode, mrange, r, cands); 2492ce8eb11aSdp if (cands != 0) 2493ce8eb11aSdp break; 2494ce8eb11aSdp } 2495ce8eb11aSdp if (cands == 0) { 2496ce8eb11aSdp VM_STAT_ADD(vmm_vmstats. 2497ce8eb11aSdp page_ctrs_cands_skip_all); 2498ce8eb11aSdp continue; 2499ce8eb11aSdp } 25007c478bd9Sstevel@tonic-gate 2501ce8eb11aSdp full = FULL_REGION_CNT(r); 2502ce8eb11aSdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 2503ce8eb11aSdp 2504ce8eb11aSdp for (idx = 0; idx < len; idx++) { 2505ce8eb11aSdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 2506ce8eb11aSdp pfn_t pfnum = 2507ce8eb11aSdp IDX_TO_PNUM(mnode, r, idx); 2508ce8eb11aSdp int tmnode = interleaved_mnodes ? 2509ce8eb11aSdp PFN_2_MEM_NODE(pfnum) : mnode; 2510ce8eb11aSdp 2511ce8eb11aSdp ASSERT(pfnum >= 2512ce8eb11aSdp mem_node_config[tmnode].physbase && 2513ce8eb11aSdp pfnum < 2514ce8eb11aSdp mem_node_config[tmnode].physmax); 2515ce8eb11aSdp 2516ce8eb11aSdp (void) page_promote(tmnode, 2517ce8eb11aSdp pfnum, r, PC_FREE, PC_MTYPE_ANY); 2518ce8eb11aSdp } 25197c478bd9Sstevel@tonic-gate } 2520ce8eb11aSdp /* shared hpm_counters covers all mnodes, so we quit */ 2521ce8eb11aSdp if (interleaved_mnodes) 2522ce8eb11aSdp break; 25237c478bd9Sstevel@tonic-gate } 25247c478bd9Sstevel@tonic-gate } 2525ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2526ce8eb11aSdp page_freelist_unlock(mnode); 2527ce8eb11aSdp rw_exit(&page_ctrs_rwlock[mnode]); 2528ce8eb11aSdp } 25297c478bd9Sstevel@tonic-gate } 25307c478bd9Sstevel@tonic-gate 25317c478bd9Sstevel@tonic-gate /* 25327c478bd9Sstevel@tonic-gate * This is where all polices for moving pages around 25337c478bd9Sstevel@tonic-gate * to different page size free lists is implemented. 25347c478bd9Sstevel@tonic-gate * Returns 1 on success, 0 on failure. 25357c478bd9Sstevel@tonic-gate * 25367c478bd9Sstevel@tonic-gate * So far these are the priorities for this algorithm in descending 25377c478bd9Sstevel@tonic-gate * order: 25387c478bd9Sstevel@tonic-gate * 25397c478bd9Sstevel@tonic-gate * 1) When servicing a request try to do so with a free page 25407c478bd9Sstevel@tonic-gate * from next size up. Helps defer fragmentation as long 25417c478bd9Sstevel@tonic-gate * as possible. 25427c478bd9Sstevel@tonic-gate * 25437c478bd9Sstevel@tonic-gate * 2) Page coalesce on demand. Only when a freelist 25447c478bd9Sstevel@tonic-gate * larger than PAGESIZE is empty and step 1 25457c478bd9Sstevel@tonic-gate * will not work since all larger size lists are 25467c478bd9Sstevel@tonic-gate * also empty. 25477c478bd9Sstevel@tonic-gate * 25487c478bd9Sstevel@tonic-gate * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 25497c478bd9Sstevel@tonic-gate */ 25505d07b933Sdp 25517c478bd9Sstevel@tonic-gate page_t * 25525d07b933Sdp page_freelist_split(uchar_t szc, uint_t color, int mnode, int mtype, 255319397407SSherry Moore pfn_t pfnlo, pfn_t pfnhi, page_list_walker_t *plw) 25547c478bd9Sstevel@tonic-gate { 25557c478bd9Sstevel@tonic-gate uchar_t nszc = szc + 1; 25565d07b933Sdp uint_t bin, sbin, bin_prev; 25577c478bd9Sstevel@tonic-gate page_t *pp, *firstpp; 25587c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 25595d07b933Sdp uint_t color_mask; 25607c478bd9Sstevel@tonic-gate 25615d07b933Sdp if (nszc == mmu_page_sizes) 25625d07b933Sdp return (NULL); 25637c478bd9Sstevel@tonic-gate 25645d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25655d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nszc) - 1; 25665d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(szc, color); 25675d07b933Sdp bin_prev = (plw->plw_bin_split_prev == color) ? INVALID_COLOR : 25685d07b933Sdp PAGE_GET_NSZ_COLOR(szc, plw->plw_bin_split_prev); 25695d07b933Sdp 25705d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_req[szc]); 25717c478bd9Sstevel@tonic-gate /* 25725d07b933Sdp * First try to break up a larger page to fill current size freelist. 25737c478bd9Sstevel@tonic-gate */ 25745d07b933Sdp while (plw->plw_bins[nszc] != 0) { 25755d07b933Sdp 25765d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25775d07b933Sdp 25787c478bd9Sstevel@tonic-gate /* 25797c478bd9Sstevel@tonic-gate * If page found then demote it. 25807c478bd9Sstevel@tonic-gate */ 25817c478bd9Sstevel@tonic-gate if (PAGE_FREELISTS(mnode, nszc, bin, mtype)) { 25827c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 25837c478bd9Sstevel@tonic-gate firstpp = pp = PAGE_FREELISTS(mnode, nszc, bin, mtype); 25847c478bd9Sstevel@tonic-gate 25857c478bd9Sstevel@tonic-gate /* 25867c478bd9Sstevel@tonic-gate * If pfnhi is not PFNNULL, look for large page below 25877c478bd9Sstevel@tonic-gate * pfnhi. PFNNULL signifies no pfn requirement. 25887c478bd9Sstevel@tonic-gate */ 25897196569bSSherry Moore if (pp && 25907196569bSSherry Moore ((pfnhi != PFNNULL && pp->p_pagenum >= pfnhi) || 25917196569bSSherry Moore (pfnlo != PFNNULL && pp->p_pagenum < pfnlo))) { 25927c478bd9Sstevel@tonic-gate do { 25937c478bd9Sstevel@tonic-gate pp = pp->p_vpnext; 25947c478bd9Sstevel@tonic-gate if (pp == firstpp) { 25957c478bd9Sstevel@tonic-gate pp = NULL; 25967c478bd9Sstevel@tonic-gate break; 25977c478bd9Sstevel@tonic-gate } 259819397407SSherry Moore } while ((pfnhi != PFNNULL && 259919397407SSherry Moore pp->p_pagenum >= pfnhi) || 260019397407SSherry Moore (pfnlo != PFNNULL && 260119397407SSherry Moore pp->p_pagenum < pfnlo)); 260219397407SSherry Moore 260319397407SSherry Moore if (pfnhi != PFNNULL && pp != NULL) 260419397407SSherry Moore ASSERT(pp->p_pagenum < pfnhi); 260519397407SSherry Moore 260619397407SSherry Moore if (pfnlo != PFNNULL && pp != NULL) 260719397407SSherry Moore ASSERT(pp->p_pagenum >= pfnlo); 26087c478bd9Sstevel@tonic-gate } 26097c478bd9Sstevel@tonic-gate if (pp) { 26105d07b933Sdp uint_t ccolor = page_correct_color(szc, nszc, 26115d07b933Sdp color, bin, plw->plw_ceq_mask[szc]); 26125d07b933Sdp 26137c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == nszc); 26145d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_demote[nszc]); 26157c478bd9Sstevel@tonic-gate ret_pp = page_demote(mnode, pp->p_pagenum, 261619397407SSherry Moore pfnhi, pp->p_szc, szc, ccolor, PC_ALLOC); 26177c478bd9Sstevel@tonic-gate if (ret_pp) { 26187c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 26197c478bd9Sstevel@tonic-gate #if defined(__sparc) 26207c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 26217c478bd9Sstevel@tonic-gate pgcnt_t npgs; 26227c478bd9Sstevel@tonic-gate 26237c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt( 26247c478bd9Sstevel@tonic-gate ret_pp->p_szc); 26257c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 26267c478bd9Sstevel@tonic-gate } 26277c478bd9Sstevel@tonic-gate #endif 26287c478bd9Sstevel@tonic-gate return (ret_pp); 26297c478bd9Sstevel@tonic-gate } 26307c478bd9Sstevel@tonic-gate } 26317c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 26327c478bd9Sstevel@tonic-gate } 26337c478bd9Sstevel@tonic-gate 26345d07b933Sdp /* loop through next size bins */ 26355d07b933Sdp bin = ADD_MASKED(bin, 1, plw->plw_ceq_mask[nszc], color_mask); 26365d07b933Sdp plw->plw_bins[nszc]--; 26375d07b933Sdp 26385d07b933Sdp if (bin == sbin) { 26395d07b933Sdp uchar_t nnszc = nszc + 1; 26405d07b933Sdp 26415d07b933Sdp /* we are done with this page size - check next */ 26425d07b933Sdp if (plw->plw_bins[nnszc] == 0) 26435d07b933Sdp /* we have already checked next size bins */ 26445d07b933Sdp break; 26455d07b933Sdp 26465d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(nszc, bin); 26475d07b933Sdp if (bin_prev != INVALID_COLOR) { 26485d07b933Sdp bin_prev = PAGE_GET_NSZ_COLOR(nszc, bin_prev); 26495d07b933Sdp if (!((bin ^ bin_prev) & 26505d07b933Sdp plw->plw_ceq_mask[nnszc])) 26515d07b933Sdp break; 26525d07b933Sdp } 26535d07b933Sdp ASSERT(nnszc < mmu_page_sizes); 26545d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nnszc) - 1; 26555d07b933Sdp nszc = nnszc; 26565d07b933Sdp ASSERT(nszc < mmu_page_sizes); 26575d07b933Sdp } 26587c478bd9Sstevel@tonic-gate } 26597c478bd9Sstevel@tonic-gate 26607c478bd9Sstevel@tonic-gate return (ret_pp); 26617c478bd9Sstevel@tonic-gate } 26627c478bd9Sstevel@tonic-gate 26637c478bd9Sstevel@tonic-gate /* 26647c478bd9Sstevel@tonic-gate * Helper routine used only by the freelist code to lock 26657c478bd9Sstevel@tonic-gate * a page. If the page is a large page then it succeeds in 26667c478bd9Sstevel@tonic-gate * locking all the constituent pages or none at all. 26677c478bd9Sstevel@tonic-gate * Returns 1 on sucess, 0 on failure. 26687c478bd9Sstevel@tonic-gate */ 26697c478bd9Sstevel@tonic-gate static int 26707c478bd9Sstevel@tonic-gate page_trylock_cons(page_t *pp, se_t se) 26717c478bd9Sstevel@tonic-gate { 26727c478bd9Sstevel@tonic-gate page_t *tpp, *first_pp = pp; 26737c478bd9Sstevel@tonic-gate 26747c478bd9Sstevel@tonic-gate /* 26757c478bd9Sstevel@tonic-gate * Fail if can't lock first or only page. 26767c478bd9Sstevel@tonic-gate */ 26777c478bd9Sstevel@tonic-gate if (!page_trylock(pp, se)) { 26787c478bd9Sstevel@tonic-gate return (0); 26797c478bd9Sstevel@tonic-gate } 26807c478bd9Sstevel@tonic-gate 26817c478bd9Sstevel@tonic-gate /* 26827c478bd9Sstevel@tonic-gate * PAGESIZE: common case. 26837c478bd9Sstevel@tonic-gate */ 26847c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 26857c478bd9Sstevel@tonic-gate return (1); 26867c478bd9Sstevel@tonic-gate } 26877c478bd9Sstevel@tonic-gate 26887c478bd9Sstevel@tonic-gate /* 26897c478bd9Sstevel@tonic-gate * Large page case. 26907c478bd9Sstevel@tonic-gate */ 26917c478bd9Sstevel@tonic-gate tpp = pp->p_next; 26927c478bd9Sstevel@tonic-gate while (tpp != pp) { 26937c478bd9Sstevel@tonic-gate if (!page_trylock(tpp, se)) { 26947c478bd9Sstevel@tonic-gate /* 26958b464eb8Smec * On failure unlock what we have locked so far. 26968b464eb8Smec * We want to avoid attempting to capture these 26978b464eb8Smec * pages as the pcm mutex may be held which could 26988b464eb8Smec * lead to a recursive mutex panic. 26997c478bd9Sstevel@tonic-gate */ 27007c478bd9Sstevel@tonic-gate while (first_pp != tpp) { 27018b464eb8Smec page_unlock_nocapture(first_pp); 27027c478bd9Sstevel@tonic-gate first_pp = first_pp->p_next; 27037c478bd9Sstevel@tonic-gate } 27047c478bd9Sstevel@tonic-gate return (0); 27057c478bd9Sstevel@tonic-gate } 27067c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 27077c478bd9Sstevel@tonic-gate } 27087c478bd9Sstevel@tonic-gate return (1); 27097c478bd9Sstevel@tonic-gate } 27107c478bd9Sstevel@tonic-gate 27115d07b933Sdp /* 27125d07b933Sdp * init context for walking page lists 27135d07b933Sdp * Called when a page of the given szc in unavailable. Sets markers 27145d07b933Sdp * for the beginning of the search to detect when search has 27155d07b933Sdp * completed a full cycle. Sets flags for splitting larger pages 27165d07b933Sdp * and coalescing smaller pages. Page walking procedes until a page 27175d07b933Sdp * of the desired equivalent color is found. 27185d07b933Sdp */ 27195d07b933Sdp void 27205d07b933Sdp page_list_walk_init(uchar_t szc, uint_t flags, uint_t bin, int can_split, 27215d07b933Sdp int use_ceq, page_list_walker_t *plw) 27227c478bd9Sstevel@tonic-gate { 27235d07b933Sdp uint_t nszc, ceq_mask, colors; 27245d07b933Sdp uchar_t ceq = use_ceq ? colorequivszc[szc] : 0; 27257c478bd9Sstevel@tonic-gate 27267c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 27275d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 27287c478bd9Sstevel@tonic-gate 27295d07b933Sdp plw->plw_colors = colors; 27305d07b933Sdp plw->plw_color_mask = colors - 1; 27315d07b933Sdp plw->plw_bin_marker = plw->plw_bin0 = bin; 27325d07b933Sdp plw->plw_bin_split_prev = bin; 27335d07b933Sdp plw->plw_bin_step = (szc == 0) ? vac_colors : 1; 27347c478bd9Sstevel@tonic-gate 27355d07b933Sdp /* 27365d07b933Sdp * if vac aliasing is possible make sure lower order color 27375d07b933Sdp * bits are never ignored 27385d07b933Sdp */ 27395d07b933Sdp if (vac_colors > 1) 27405d07b933Sdp ceq &= 0xf0; 27417c478bd9Sstevel@tonic-gate 27427c478bd9Sstevel@tonic-gate /* 27435d07b933Sdp * calculate the number of non-equivalent colors and 27445d07b933Sdp * color equivalency mask 27457c478bd9Sstevel@tonic-gate */ 27465d07b933Sdp plw->plw_ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 27475d07b933Sdp ASSERT(szc > 0 || plw->plw_ceq_dif >= vac_colors); 27485d07b933Sdp ASSERT(plw->plw_ceq_dif > 0); 27495d07b933Sdp plw->plw_ceq_mask[szc] = (plw->plw_ceq_dif - 1) << (ceq & 0xf); 27507c478bd9Sstevel@tonic-gate 27515d07b933Sdp if (flags & PG_MATCH_COLOR) { 27525d07b933Sdp if (cpu_page_colors < 0) { 27535d07b933Sdp /* 27545d07b933Sdp * this is a heterogeneous machine with different CPUs 27555d07b933Sdp * having different size e$ (not supported for ni2/rock 27565d07b933Sdp */ 27575d07b933Sdp uint_t cpucolors = CPUSETSIZE() >> PAGE_GET_SHIFT(szc); 27585d07b933Sdp cpucolors = MAX(cpucolors, 1); 27595d07b933Sdp ceq_mask = plw->plw_color_mask & (cpucolors - 1); 27605d07b933Sdp plw->plw_ceq_mask[szc] = 27615d07b933Sdp MIN(ceq_mask, plw->plw_ceq_mask[szc]); 27625d07b933Sdp } 27635d07b933Sdp plw->plw_ceq_dif = 1; 27645d07b933Sdp } 27657c478bd9Sstevel@tonic-gate 27665d07b933Sdp /* we can split pages in the freelist, but not the cachelist */ 27675d07b933Sdp if (can_split) { 2768ce8eb11aSdp plw->plw_do_split = (szc + 1 < mmu_page_sizes) ? 1 : 0; 27697c478bd9Sstevel@tonic-gate 2770ce8eb11aSdp /* set next szc color masks and number of free list bins */ 2771ce8eb11aSdp for (nszc = szc + 1; nszc < mmu_page_sizes; nszc++, szc++) { 2772ce8eb11aSdp plw->plw_ceq_mask[nszc] = PAGE_GET_NSZ_MASK(szc, 2773ce8eb11aSdp plw->plw_ceq_mask[szc]); 2774ce8eb11aSdp plw->plw_bins[nszc] = PAGE_GET_PAGECOLORS(nszc); 2775ce8eb11aSdp } 2776ce8eb11aSdp plw->plw_ceq_mask[nszc] = INVALID_MASK; 2777ce8eb11aSdp plw->plw_bins[nszc] = 0; 27787c478bd9Sstevel@tonic-gate 27795d07b933Sdp } else { 2780ce8eb11aSdp ASSERT(szc == 0); 2781ce8eb11aSdp plw->plw_do_split = 0; 2782ce8eb11aSdp plw->plw_bins[1] = 0; 2783ce8eb11aSdp plw->plw_ceq_mask[1] = INVALID_MASK; 27847c478bd9Sstevel@tonic-gate } 27855d07b933Sdp } 27867c478bd9Sstevel@tonic-gate 27875d07b933Sdp /* 27885d07b933Sdp * set mark to flag where next split should occur 27895d07b933Sdp */ 27905d07b933Sdp #define PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw) { \ 27915d07b933Sdp uint_t bin_nsz = PAGE_GET_NSZ_COLOR(szc, bin); \ 27925d07b933Sdp uint_t bin0_nsz = PAGE_GET_NSZ_COLOR(szc, plw->plw_bin0); \ 27935d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[nszc] & plw->plw_color_mask; \ 27945d07b933Sdp plw->plw_split_next = \ 27955d07b933Sdp INC_MASKED(bin_nsz, neq_mask, plw->plw_color_mask); \ 27965d07b933Sdp if (!((plw->plw_split_next ^ bin0_nsz) & plw->plw_ceq_mask[nszc])) { \ 27975d07b933Sdp plw->plw_split_next = \ 27985d07b933Sdp INC_MASKED(plw->plw_split_next, \ 27995d07b933Sdp neq_mask, plw->plw_color_mask); \ 28005d07b933Sdp } \ 28015d07b933Sdp } 28027c478bd9Sstevel@tonic-gate 28035d07b933Sdp uint_t 28045d07b933Sdp page_list_walk_next_bin(uchar_t szc, uint_t bin, page_list_walker_t *plw) 28055d07b933Sdp { 28065d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[szc] & plw->plw_color_mask; 28075d07b933Sdp uint_t bin0_nsz, nbin_nsz, nbin0, nbin; 28085d07b933Sdp uchar_t nszc = szc + 1; 28095d07b933Sdp 28105d07b933Sdp nbin = ADD_MASKED(bin, 28115d07b933Sdp plw->plw_bin_step, neq_mask, plw->plw_color_mask); 28125d07b933Sdp 28135d07b933Sdp if (plw->plw_do_split) { 28145d07b933Sdp plw->plw_bin_split_prev = bin; 28155d07b933Sdp PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw); 28165d07b933Sdp plw->plw_do_split = 0; 28175d07b933Sdp } 28185d07b933Sdp 28195d07b933Sdp if (szc == 0) { 28205d07b933Sdp if (plw->plw_count != 0 || plw->plw_ceq_dif == vac_colors) { 28215d07b933Sdp if (nbin == plw->plw_bin0 && 28225d07b933Sdp (vac_colors == 1 || nbin != plw->plw_bin_marker)) { 28235d07b933Sdp nbin = ADD_MASKED(nbin, plw->plw_bin_step, 28245d07b933Sdp neq_mask, plw->plw_color_mask); 28255d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 28265d07b933Sdp } 28275d07b933Sdp 28285d07b933Sdp if (vac_colors > 1 && nbin == plw->plw_bin_marker) { 28295d07b933Sdp plw->plw_bin_marker = 28305d07b933Sdp nbin = INC_MASKED(nbin, neq_mask, 2831ce8eb11aSdp plw->plw_color_mask); 28325d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 28335d07b933Sdp /* 28345d07b933Sdp * large pages all have the same vac color 28355d07b933Sdp * so by now we should be done with next 28365d07b933Sdp * size page splitting process 28375d07b933Sdp */ 28385d07b933Sdp ASSERT(plw->plw_bins[1] == 0); 28395d07b933Sdp plw->plw_do_split = 0; 28405d07b933Sdp return (nbin); 28415d07b933Sdp } 28425d07b933Sdp 28435d07b933Sdp } else { 28445d07b933Sdp uint_t bin_jump = (vac_colors == 1) ? 28455d07b933Sdp (BIN_STEP & ~3) - (plw->plw_bin0 & 3) : BIN_STEP; 28465d07b933Sdp 28475d07b933Sdp bin_jump &= ~(vac_colors - 1); 28485d07b933Sdp 28495d07b933Sdp nbin0 = ADD_MASKED(plw->plw_bin0, bin_jump, neq_mask, 28505d07b933Sdp plw->plw_color_mask); 28515d07b933Sdp 28525d07b933Sdp if ((nbin0 ^ plw->plw_bin0) & plw->plw_ceq_mask[szc]) { 28535d07b933Sdp 28545d07b933Sdp plw->plw_bin_marker = nbin = nbin0; 28555d07b933Sdp 28565d07b933Sdp if (plw->plw_bins[nszc] != 0) { 28575d07b933Sdp /* 28585d07b933Sdp * check if next page size bin is the 28595d07b933Sdp * same as the next page size bin for 28605d07b933Sdp * bin0 28615d07b933Sdp */ 28625d07b933Sdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, 28635d07b933Sdp nbin); 28645d07b933Sdp bin0_nsz = PAGE_GET_NSZ_COLOR(szc, 28655d07b933Sdp plw->plw_bin0); 28665d07b933Sdp 28675d07b933Sdp if ((bin0_nsz ^ nbin_nsz) & 28685d07b933Sdp plw->plw_ceq_mask[nszc]) 28695d07b933Sdp plw->plw_do_split = 1; 28705d07b933Sdp } 28715d07b933Sdp return (nbin); 28725d07b933Sdp } 28735d07b933Sdp } 28745d07b933Sdp } 28755d07b933Sdp 28765d07b933Sdp if (plw->plw_bins[nszc] != 0) { 2877ce8eb11aSdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, nbin); 2878ce8eb11aSdp if (!((plw->plw_split_next ^ nbin_nsz) & 2879ce8eb11aSdp plw->plw_ceq_mask[nszc])) 2880ce8eb11aSdp plw->plw_do_split = 1; 28815d07b933Sdp } 28825d07b933Sdp 28835d07b933Sdp return (nbin); 28845d07b933Sdp } 28855d07b933Sdp 28865d07b933Sdp page_t * 28875d07b933Sdp page_get_mnode_freelist(int mnode, uint_t bin, int mtype, uchar_t szc, 28885d07b933Sdp uint_t flags) 28895d07b933Sdp { 28905d07b933Sdp kmutex_t *pcm; 28915d07b933Sdp page_t *pp, *first_pp; 28925d07b933Sdp uint_t sbin; 28935d07b933Sdp int plw_initialized; 28945d07b933Sdp page_list_walker_t plw; 28955d07b933Sdp 28965d07b933Sdp ASSERT(szc < mmu_page_sizes); 28975d07b933Sdp 28985d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_alloc[szc]); 28995d07b933Sdp 29005d07b933Sdp MTYPE_START(mnode, mtype, flags); 29015d07b933Sdp if (mtype < 0) { /* mnode does not have memory in mtype range */ 29025d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocempty[szc]); 29035d07b933Sdp return (NULL); 29045d07b933Sdp } 29055d07b933Sdp try_again: 29065d07b933Sdp 29075d07b933Sdp plw_initialized = 0; 29085d07b933Sdp plw.plw_ceq_dif = 1; 29097c478bd9Sstevel@tonic-gate 29107c478bd9Sstevel@tonic-gate /* 29117c478bd9Sstevel@tonic-gate * Only hold one freelist lock at a time, that way we 29127c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 29137c478bd9Sstevel@tonic-gate * ordering. 29147c478bd9Sstevel@tonic-gate */ 29155d07b933Sdp for (plw.plw_count = 0; 29165d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 29175d07b933Sdp sbin = bin; 29185d07b933Sdp do { 29195d07b933Sdp if (!PAGE_FREELISTS(mnode, szc, bin, mtype)) 29205d07b933Sdp goto bin_empty_1; 29215d07b933Sdp 29227c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 29237c478bd9Sstevel@tonic-gate mutex_enter(pcm); 29247c478bd9Sstevel@tonic-gate pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 29255d07b933Sdp if (pp == NULL) 29265d07b933Sdp goto bin_empty_0; 29275d07b933Sdp 29285d07b933Sdp /* 29295d07b933Sdp * These were set before the page 29305d07b933Sdp * was put on the free list, 29315d07b933Sdp * they must still be set. 29325d07b933Sdp */ 29335d07b933Sdp ASSERT(PP_ISFREE(pp)); 29345d07b933Sdp ASSERT(PP_ISAGED(pp)); 29355d07b933Sdp ASSERT(pp->p_vnode == NULL); 29365d07b933Sdp ASSERT(pp->p_hash == NULL); 29375d07b933Sdp ASSERT(pp->p_offset == (u_offset_t)-1); 29385d07b933Sdp ASSERT(pp->p_szc == szc); 29395d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 29405d07b933Sdp 29415d07b933Sdp /* 29425d07b933Sdp * Walk down the hash chain. 29435d07b933Sdp * 8k pages are linked on p_next 29445d07b933Sdp * and p_prev fields. Large pages 29455d07b933Sdp * are a contiguous group of 29465d07b933Sdp * constituent pages linked together 29475d07b933Sdp * on their p_next and p_prev fields. 29485d07b933Sdp * The large pages are linked together 29495d07b933Sdp * on the hash chain using p_vpnext 29505d07b933Sdp * p_vpprev of the base constituent 29515d07b933Sdp * page of each large page. 29525d07b933Sdp */ 29535d07b933Sdp first_pp = pp; 29545d07b933Sdp while (!page_trylock_cons(pp, SE_EXCL)) { 29555d07b933Sdp if (szc == 0) { 29565d07b933Sdp pp = pp->p_next; 29575d07b933Sdp } else { 29585d07b933Sdp pp = pp->p_vpnext; 29595d07b933Sdp } 29605d07b933Sdp 29617c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 29627c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 29637c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 29647c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL); 29657c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1); 29667c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 29677c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 29687c478bd9Sstevel@tonic-gate 29695d07b933Sdp if (pp == first_pp) 29705d07b933Sdp goto bin_empty_0; 29715d07b933Sdp } 29727c478bd9Sstevel@tonic-gate 29735d07b933Sdp ASSERT(pp != NULL); 29745d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 29755d07b933Sdp ASSERT(pp->p_szc == szc); 29765d07b933Sdp if (szc == 0) { 29775d07b933Sdp page_sub(&PAGE_FREELISTS(mnode, 29785d07b933Sdp szc, bin, mtype), pp); 29795d07b933Sdp } else { 29805d07b933Sdp page_vpsub(&PAGE_FREELISTS(mnode, 29815d07b933Sdp szc, bin, mtype), pp); 29825d07b933Sdp CHK_LPG(pp, szc); 29835d07b933Sdp } 29845d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 29857c478bd9Sstevel@tonic-gate 29865d07b933Sdp if ((PP_ISFREE(pp) == 0) || (PP_ISAGED(pp) == 0)) 29875d07b933Sdp panic("free page is not. pp %p", (void *)pp); 29885d07b933Sdp mutex_exit(pcm); 29897c478bd9Sstevel@tonic-gate 29907c478bd9Sstevel@tonic-gate #if defined(__sparc) 29915d07b933Sdp ASSERT(!kcage_on || PP_ISNORELOC(pp) || 29925d07b933Sdp (flags & PG_NORELOC) == 0); 29937c478bd9Sstevel@tonic-gate 29945d07b933Sdp if (PP_ISNORELOC(pp)) 29955d07b933Sdp kcage_freemem_sub(page_get_pagecnt(szc)); 29967c478bd9Sstevel@tonic-gate #endif 29975d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocok[szc]); 29985d07b933Sdp return (pp); 29997c478bd9Sstevel@tonic-gate 30005d07b933Sdp bin_empty_0: 30015d07b933Sdp mutex_exit(pcm); 30025d07b933Sdp bin_empty_1: 30035d07b933Sdp if (plw_initialized == 0) { 30045d07b933Sdp page_list_walk_init(szc, flags, bin, 1, 1, 30055d07b933Sdp &plw); 30065d07b933Sdp plw_initialized = 1; 30075d07b933Sdp ASSERT(plw.plw_colors <= 30085d07b933Sdp PAGE_GET_PAGECOLORS(szc)); 30095d07b933Sdp ASSERT(plw.plw_colors > 0); 30105d07b933Sdp ASSERT((plw.plw_colors & 30115d07b933Sdp (plw.plw_colors - 1)) == 0); 30125d07b933Sdp ASSERT(bin < plw.plw_colors); 30135d07b933Sdp ASSERT(plw.plw_ceq_mask[szc] < plw.plw_colors); 30147c478bd9Sstevel@tonic-gate } 30155d07b933Sdp /* calculate the next bin with equivalent color */ 30165d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 30175d07b933Sdp plw.plw_ceq_mask[szc], plw.plw_color_mask); 30185d07b933Sdp } while (sbin != bin); 30197c478bd9Sstevel@tonic-gate 30207c478bd9Sstevel@tonic-gate /* 30215d07b933Sdp * color bins are all empty if color match. Try and 30225d07b933Sdp * satisfy the request by breaking up or coalescing 30235d07b933Sdp * pages from a different size freelist of the correct 30245d07b933Sdp * color that satisfies the ORIGINAL color requested. 30255d07b933Sdp * If that fails then try pages of the same size but 30265d07b933Sdp * different colors assuming we are not called with 30277c478bd9Sstevel@tonic-gate * PG_MATCH_COLOR. 30287c478bd9Sstevel@tonic-gate */ 30295d07b933Sdp if (plw.plw_do_split && 30305d07b933Sdp (pp = page_freelist_split(szc, bin, mnode, 303119397407SSherry Moore mtype, PFNNULL, PFNNULL, &plw)) != NULL) 3032ce8eb11aSdp return (pp); 30337c478bd9Sstevel@tonic-gate 30345d07b933Sdp if (szc > 0 && (pp = page_freelist_coalesce(mnode, szc, 30355d07b933Sdp bin, plw.plw_ceq_mask[szc], mtype, PFNNULL)) != NULL) 30365d07b933Sdp return (pp); 30377c478bd9Sstevel@tonic-gate 30385d07b933Sdp if (plw.plw_ceq_dif > 1) 30395d07b933Sdp bin = page_list_walk_next_bin(szc, bin, &plw); 30407c478bd9Sstevel@tonic-gate } 30417c478bd9Sstevel@tonic-gate 3042affbd3ccSkchow /* if allowed, cycle through additional mtypes */ 3043affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3044affbd3ccSkchow if (mtype >= 0) 30455d07b933Sdp goto try_again; 3046affbd3ccSkchow 30477c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmf_allocfailed[szc]); 30487c478bd9Sstevel@tonic-gate 30497c478bd9Sstevel@tonic-gate return (NULL); 30507c478bd9Sstevel@tonic-gate } 30517c478bd9Sstevel@tonic-gate 30527c478bd9Sstevel@tonic-gate /* 30537c478bd9Sstevel@tonic-gate * Returns the count of free pages for 'pp' with size code 'szc'. 30547c478bd9Sstevel@tonic-gate * Note: This function does not return an exact value as the page freelist 30557c478bd9Sstevel@tonic-gate * locks are not held and thus the values in the page_counters may be 30567c478bd9Sstevel@tonic-gate * changing as we walk through the data. 30577c478bd9Sstevel@tonic-gate */ 30587c478bd9Sstevel@tonic-gate static int 30597c478bd9Sstevel@tonic-gate page_freecnt(int mnode, page_t *pp, uchar_t szc) 30607c478bd9Sstevel@tonic-gate { 30617c478bd9Sstevel@tonic-gate pgcnt_t pgfree; 30627c478bd9Sstevel@tonic-gate pgcnt_t cnt; 30637c478bd9Sstevel@tonic-gate ssize_t r = szc; /* region size */ 30647c478bd9Sstevel@tonic-gate ssize_t idx; 30657c478bd9Sstevel@tonic-gate int i; 30667c478bd9Sstevel@tonic-gate int full, range; 30677c478bd9Sstevel@tonic-gate 30687c478bd9Sstevel@tonic-gate /* Make sure pagenum passed in is aligned properly */ 30697c478bd9Sstevel@tonic-gate ASSERT((pp->p_pagenum & (PNUM_SIZE(szc) - 1)) == 0); 30707c478bd9Sstevel@tonic-gate ASSERT(szc > 0); 30717c478bd9Sstevel@tonic-gate 30727c478bd9Sstevel@tonic-gate /* Prevent page_counters dynamic memory from being freed */ 30737c478bd9Sstevel@tonic-gate rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 30747c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30757c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30767c478bd9Sstevel@tonic-gate pgfree = cnt << PNUM_SHIFT(r - 1); 30777c478bd9Sstevel@tonic-gate range = FULL_REGION_CNT(szc); 30787c478bd9Sstevel@tonic-gate 30797c478bd9Sstevel@tonic-gate /* Check for completely full region */ 30807c478bd9Sstevel@tonic-gate if (cnt == range) { 30817c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30827c478bd9Sstevel@tonic-gate return (pgfree); 30837c478bd9Sstevel@tonic-gate } 30847c478bd9Sstevel@tonic-gate 30857c478bd9Sstevel@tonic-gate while (--r > 0) { 30867c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30877c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 30887c478bd9Sstevel@tonic-gate for (i = 0; i < range; i++, idx++) { 30897c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30907c478bd9Sstevel@tonic-gate /* 30917c478bd9Sstevel@tonic-gate * If cnt here is full, that means we have already 30927c478bd9Sstevel@tonic-gate * accounted for these pages earlier. 30937c478bd9Sstevel@tonic-gate */ 30947c478bd9Sstevel@tonic-gate if (cnt != full) { 30957c478bd9Sstevel@tonic-gate pgfree += (cnt << PNUM_SHIFT(r - 1)); 30967c478bd9Sstevel@tonic-gate } 30977c478bd9Sstevel@tonic-gate } 30987c478bd9Sstevel@tonic-gate range *= full; 30997c478bd9Sstevel@tonic-gate } 31007c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 31017c478bd9Sstevel@tonic-gate return (pgfree); 31027c478bd9Sstevel@tonic-gate } 31037c478bd9Sstevel@tonic-gate 31047c478bd9Sstevel@tonic-gate /* 31057c478bd9Sstevel@tonic-gate * Called from page_geti_contig_pages to exclusively lock constituent pages 31067c478bd9Sstevel@tonic-gate * starting from 'spp' for page size code 'szc'. 31077c478bd9Sstevel@tonic-gate * 31087c478bd9Sstevel@tonic-gate * If 'ptcpthreshold' is set, the number of free pages needed in the 'szc' 31097c478bd9Sstevel@tonic-gate * region needs to be greater than or equal to the threshold. 31107c478bd9Sstevel@tonic-gate */ 31117c478bd9Sstevel@tonic-gate static int 31127c478bd9Sstevel@tonic-gate page_trylock_contig_pages(int mnode, page_t *spp, uchar_t szc, int flags) 31137c478bd9Sstevel@tonic-gate { 31147c478bd9Sstevel@tonic-gate pgcnt_t pgcnt = PNUM_SIZE(szc); 31157c478bd9Sstevel@tonic-gate pgcnt_t pgfree, i; 31167c478bd9Sstevel@tonic-gate page_t *pp; 31177c478bd9Sstevel@tonic-gate 31187c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcp[szc]); 31197c478bd9Sstevel@tonic-gate 31207c478bd9Sstevel@tonic-gate 31217c478bd9Sstevel@tonic-gate if ((ptcpthreshold == 0) || (flags & PGI_PGCPHIPRI)) 31227c478bd9Sstevel@tonic-gate goto skipptcpcheck; 31237c478bd9Sstevel@tonic-gate /* 31247c478bd9Sstevel@tonic-gate * check if there are sufficient free pages available before attempting 31257c478bd9Sstevel@tonic-gate * to trylock. Count is approximate as page counters can change. 31267c478bd9Sstevel@tonic-gate */ 31277c478bd9Sstevel@tonic-gate pgfree = page_freecnt(mnode, spp, szc); 31287c478bd9Sstevel@tonic-gate 31297c478bd9Sstevel@tonic-gate /* attempt to trylock if there are sufficient already free pages */ 31307c478bd9Sstevel@tonic-gate if (pgfree < pgcnt/ptcpthreshold) { 31317c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfreethresh[szc]); 31327c478bd9Sstevel@tonic-gate return (0); 31337c478bd9Sstevel@tonic-gate } 31347c478bd9Sstevel@tonic-gate 31357c478bd9Sstevel@tonic-gate skipptcpcheck: 31367c478bd9Sstevel@tonic-gate 31377c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++) { 31387c478bd9Sstevel@tonic-gate pp = &spp[i]; 31397c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 31407c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailexcl[szc]); 31417c478bd9Sstevel@tonic-gate while (--i != (pgcnt_t)-1) { 31427c478bd9Sstevel@tonic-gate pp = &spp[i]; 31437c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31448b464eb8Smec page_unlock_nocapture(pp); 31457c478bd9Sstevel@tonic-gate } 31467c478bd9Sstevel@tonic-gate return (0); 31477c478bd9Sstevel@tonic-gate } 31487c478bd9Sstevel@tonic-gate ASSERT(spp[i].p_pagenum == spp->p_pagenum + i); 31497c478bd9Sstevel@tonic-gate if ((pp->p_szc > szc || (szc && pp->p_szc == szc)) && 31507c478bd9Sstevel@tonic-gate !PP_ISFREE(pp)) { 31517c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailszc[szc]); 31527c478bd9Sstevel@tonic-gate ASSERT(i == 0); 31538b464eb8Smec page_unlock_nocapture(pp); 31547c478bd9Sstevel@tonic-gate return (0); 31557c478bd9Sstevel@tonic-gate } 31567c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 31577c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailcage[szc]); 31587c478bd9Sstevel@tonic-gate while (i != (pgcnt_t)-1) { 31597c478bd9Sstevel@tonic-gate pp = &spp[i]; 31607c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31618b464eb8Smec page_unlock_nocapture(pp); 31627c478bd9Sstevel@tonic-gate i--; 31637c478bd9Sstevel@tonic-gate } 31647c478bd9Sstevel@tonic-gate return (0); 31657c478bd9Sstevel@tonic-gate } 31667c478bd9Sstevel@tonic-gate } 31677c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpok[szc]); 31687c478bd9Sstevel@tonic-gate return (1); 31697c478bd9Sstevel@tonic-gate } 31707c478bd9Sstevel@tonic-gate 31717c478bd9Sstevel@tonic-gate /* 31727c478bd9Sstevel@tonic-gate * Claim large page pointed to by 'pp'. 'pp' is the starting set 31737c478bd9Sstevel@tonic-gate * of 'szc' constituent pages that had been locked exclusively previously. 31747c478bd9Sstevel@tonic-gate * Will attempt to relocate constituent pages in use. 31757c478bd9Sstevel@tonic-gate */ 31767c478bd9Sstevel@tonic-gate static page_t * 31777c478bd9Sstevel@tonic-gate page_claim_contig_pages(page_t *pp, uchar_t szc, int flags) 31787c478bd9Sstevel@tonic-gate { 31797c478bd9Sstevel@tonic-gate spgcnt_t pgcnt, npgs, i; 31807c478bd9Sstevel@tonic-gate page_t *targpp, *rpp, *hpp; 31817c478bd9Sstevel@tonic-gate page_t *replpp = NULL; 31827c478bd9Sstevel@tonic-gate page_t *pplist = NULL; 31837c478bd9Sstevel@tonic-gate 31847c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 31857c478bd9Sstevel@tonic-gate 31867c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(szc); 31877c478bd9Sstevel@tonic-gate while (pgcnt) { 31887c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31897c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 31907c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 31917c478bd9Sstevel@tonic-gate /* 31927c478bd9Sstevel@tonic-gate * If this is a PG_FREE_LIST page then its 31937c478bd9Sstevel@tonic-gate * size code can change underneath us due to 31947c478bd9Sstevel@tonic-gate * page promotion or demotion. As an optimzation 31957c478bd9Sstevel@tonic-gate * use page_list_sub_pages() instead of 31967c478bd9Sstevel@tonic-gate * page_list_sub(). 31977c478bd9Sstevel@tonic-gate */ 31987c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 31997c478bd9Sstevel@tonic-gate page_list_sub_pages(pp, szc); 32007c478bd9Sstevel@tonic-gate if (pp->p_szc == szc) { 32017c478bd9Sstevel@tonic-gate return (pp); 32027c478bd9Sstevel@tonic-gate } 32037c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < szc); 32047c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 32057c478bd9Sstevel@tonic-gate hpp = pp; 32067c478bd9Sstevel@tonic-gate for (i = 0; i < npgs; i++, pp++) { 32077c478bd9Sstevel@tonic-gate pp->p_szc = szc; 32087c478bd9Sstevel@tonic-gate } 32097c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &hpp); 32107c478bd9Sstevel@tonic-gate pgcnt -= npgs; 32117c478bd9Sstevel@tonic-gate continue; 32127c478bd9Sstevel@tonic-gate } 32137c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 32147c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 32157c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 32167c478bd9Sstevel@tonic-gate page_hashout(pp, NULL); 32177c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 32187c478bd9Sstevel@tonic-gate pp->p_szc = szc; 32197c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 32207c478bd9Sstevel@tonic-gate pp++; 32217c478bd9Sstevel@tonic-gate pgcnt--; 32227c478bd9Sstevel@tonic-gate continue; 32237c478bd9Sstevel@tonic-gate } 32247c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 32257c478bd9Sstevel@tonic-gate 32267c478bd9Sstevel@tonic-gate /* 32277c478bd9Sstevel@tonic-gate * page_create_wait freemem accounting done by caller of 32287c478bd9Sstevel@tonic-gate * page_get_freelist and not necessary to call it prior to 32297c478bd9Sstevel@tonic-gate * calling page_get_replacement_page. 32307c478bd9Sstevel@tonic-gate * 32317c478bd9Sstevel@tonic-gate * page_get_replacement_page can call page_get_contig_pages 32327c478bd9Sstevel@tonic-gate * to acquire a large page (szc > 0); the replacement must be 32337c478bd9Sstevel@tonic-gate * smaller than the contig page size to avoid looping or 32347c478bd9Sstevel@tonic-gate * szc == 0 and PGI_PGCPSZC0 is set. 32357c478bd9Sstevel@tonic-gate */ 32367c478bd9Sstevel@tonic-gate if (pp->p_szc < szc || (szc == 0 && (flags & PGI_PGCPSZC0))) { 32377c478bd9Sstevel@tonic-gate replpp = page_get_replacement_page(pp, NULL, 0); 32387c478bd9Sstevel@tonic-gate if (replpp) { 32397c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 32407c478bd9Sstevel@tonic-gate ASSERT(npgs <= pgcnt); 32417c478bd9Sstevel@tonic-gate targpp = pp; 32427c478bd9Sstevel@tonic-gate } 32437c478bd9Sstevel@tonic-gate } 32447c478bd9Sstevel@tonic-gate 32457c478bd9Sstevel@tonic-gate /* 32467c478bd9Sstevel@tonic-gate * If replacement is NULL or do_page_relocate fails, fail 32477c478bd9Sstevel@tonic-gate * coalescing of pages. 32487c478bd9Sstevel@tonic-gate */ 32497c478bd9Sstevel@tonic-gate if (replpp == NULL || (do_page_relocate(&targpp, &replpp, 0, 32507c478bd9Sstevel@tonic-gate &npgs, NULL) != 0)) { 32517c478bd9Sstevel@tonic-gate /* 32527c478bd9Sstevel@tonic-gate * Unlock un-processed target list 32537c478bd9Sstevel@tonic-gate */ 32547c478bd9Sstevel@tonic-gate while (pgcnt--) { 32557c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32568b464eb8Smec page_unlock_nocapture(pp); 32577c478bd9Sstevel@tonic-gate pp++; 32587c478bd9Sstevel@tonic-gate } 32597c478bd9Sstevel@tonic-gate /* 32607c478bd9Sstevel@tonic-gate * Free the processed target list. 32617c478bd9Sstevel@tonic-gate */ 32627c478bd9Sstevel@tonic-gate while (pplist) { 32637c478bd9Sstevel@tonic-gate pp = pplist; 32647c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 32657c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32667c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 32677c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 32687c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 32697c478bd9Sstevel@tonic-gate pp->p_szc = 0; 32707c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 32718b464eb8Smec page_unlock_nocapture(pp); 32727c478bd9Sstevel@tonic-gate } 32737c478bd9Sstevel@tonic-gate 32747c478bd9Sstevel@tonic-gate if (replpp != NULL) 32757c478bd9Sstevel@tonic-gate page_free_replacement_page(replpp); 32767c478bd9Sstevel@tonic-gate 32777c478bd9Sstevel@tonic-gate return (NULL); 32787c478bd9Sstevel@tonic-gate } 32797c478bd9Sstevel@tonic-gate ASSERT(pp == targpp); 32807c478bd9Sstevel@tonic-gate 32817c478bd9Sstevel@tonic-gate /* LINTED */ 32827c478bd9Sstevel@tonic-gate ASSERT(hpp = pp); /* That's right, it's an assignment */ 32837c478bd9Sstevel@tonic-gate 32847c478bd9Sstevel@tonic-gate pp += npgs; 32857c478bd9Sstevel@tonic-gate pgcnt -= npgs; 32867c478bd9Sstevel@tonic-gate 32877c478bd9Sstevel@tonic-gate while (npgs--) { 32887c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(targpp)); 32897c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(targpp)); 32907c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(targpp)); 32917c478bd9Sstevel@tonic-gate PP_SETFREE(targpp); 32927c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(targpp)); 32937c478bd9Sstevel@tonic-gate ASSERT(targpp->p_szc < szc || (szc == 0 && 32947c478bd9Sstevel@tonic-gate (flags & PGI_PGCPSZC0))); 32957c478bd9Sstevel@tonic-gate targpp->p_szc = szc; 32967c478bd9Sstevel@tonic-gate targpp = targpp->p_next; 32977c478bd9Sstevel@tonic-gate 32987c478bd9Sstevel@tonic-gate rpp = replpp; 32997c478bd9Sstevel@tonic-gate ASSERT(rpp != NULL); 33007c478bd9Sstevel@tonic-gate page_sub(&replpp, rpp); 33017c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(rpp)); 33027c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(rpp)); 33038b464eb8Smec page_unlock_nocapture(rpp); 33047c478bd9Sstevel@tonic-gate } 33057c478bd9Sstevel@tonic-gate ASSERT(targpp == hpp); 33067c478bd9Sstevel@tonic-gate ASSERT(replpp == NULL); 33077c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &targpp); 33087c478bd9Sstevel@tonic-gate } 33097c478bd9Sstevel@tonic-gate CHK_LPG(pplist, szc); 33107c478bd9Sstevel@tonic-gate return (pplist); 33117c478bd9Sstevel@tonic-gate } 33127c478bd9Sstevel@tonic-gate 33137c478bd9Sstevel@tonic-gate /* 33147c478bd9Sstevel@tonic-gate * Trim kernel cage from pfnlo-pfnhi and store result in lo-hi. Return code 33157c478bd9Sstevel@tonic-gate * of 0 means nothing left after trim. 33167c478bd9Sstevel@tonic-gate */ 33177c478bd9Sstevel@tonic-gate int 33187c478bd9Sstevel@tonic-gate trimkcage(struct memseg *mseg, pfn_t *lo, pfn_t *hi, pfn_t pfnlo, pfn_t pfnhi) 33197c478bd9Sstevel@tonic-gate { 33207c478bd9Sstevel@tonic-gate pfn_t kcagepfn; 33217c478bd9Sstevel@tonic-gate int decr; 33227c478bd9Sstevel@tonic-gate int rc = 0; 33237c478bd9Sstevel@tonic-gate 33247c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->pages)) { 33257c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1) == 0) { 33267c478bd9Sstevel@tonic-gate 33277c478bd9Sstevel@tonic-gate /* lower part of this mseg inside kernel cage */ 33287c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 33297c478bd9Sstevel@tonic-gate 33307c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 33317c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 33327c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 33337c478bd9Sstevel@tonic-gate ASSERT(decr == 0); 333478b03d3aSkchow *lo = MAX(kcagepfn, pfnlo); 333578b03d3aSkchow *hi = MIN(pfnhi, (mseg->pages_end - 1)); 33367c478bd9Sstevel@tonic-gate rc = 1; 33377c478bd9Sstevel@tonic-gate } 33387c478bd9Sstevel@tonic-gate } 33397c478bd9Sstevel@tonic-gate /* else entire mseg in the cage */ 33407c478bd9Sstevel@tonic-gate } else { 33417c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1)) { 33427c478bd9Sstevel@tonic-gate 33437c478bd9Sstevel@tonic-gate /* upper part of this mseg inside kernel cage */ 33447c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 33457c478bd9Sstevel@tonic-gate 33467c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 33477c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 33487c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 33497c478bd9Sstevel@tonic-gate ASSERT(decr); 335078b03d3aSkchow *hi = MIN(kcagepfn, pfnhi); 33517c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 33527c478bd9Sstevel@tonic-gate rc = 1; 33537c478bd9Sstevel@tonic-gate } 33547c478bd9Sstevel@tonic-gate } else { 33557c478bd9Sstevel@tonic-gate /* entire mseg outside of kernel cage */ 33567c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 33577c478bd9Sstevel@tonic-gate *hi = MIN(pfnhi, (mseg->pages_end - 1)); 33587c478bd9Sstevel@tonic-gate rc = 1; 33597c478bd9Sstevel@tonic-gate } 33607c478bd9Sstevel@tonic-gate } 33617c478bd9Sstevel@tonic-gate return (rc); 33627c478bd9Sstevel@tonic-gate } 33637c478bd9Sstevel@tonic-gate 33647c478bd9Sstevel@tonic-gate /* 33655d07b933Sdp * called from page_get_contig_pages to search 'pfnlo' thru 'pfnhi' to claim a 33667c478bd9Sstevel@tonic-gate * page with size code 'szc'. Claiming such a page requires acquiring 33677c478bd9Sstevel@tonic-gate * exclusive locks on all constituent pages (page_trylock_contig_pages), 33687c478bd9Sstevel@tonic-gate * relocating pages in use and concatenating these constituent pages into a 33697c478bd9Sstevel@tonic-gate * large page. 33707c478bd9Sstevel@tonic-gate * 33715d07b933Sdp * The page lists do not have such a large page and page_freelist_split has 33727c478bd9Sstevel@tonic-gate * already failed to demote larger pages and/or coalesce smaller free pages. 33737c478bd9Sstevel@tonic-gate * 33747c478bd9Sstevel@tonic-gate * 'flags' may specify PG_COLOR_MATCH which would limit the search of large 33757c478bd9Sstevel@tonic-gate * pages with the same color as 'bin'. 33767c478bd9Sstevel@tonic-gate * 33777c478bd9Sstevel@tonic-gate * 'pfnflag' specifies the subset of the pfn range to search. 33787c478bd9Sstevel@tonic-gate */ 33797c478bd9Sstevel@tonic-gate 33807c478bd9Sstevel@tonic-gate static page_t * 33817c478bd9Sstevel@tonic-gate page_geti_contig_pages(int mnode, uint_t bin, uchar_t szc, int flags, 338283f9b804Skchow pfn_t pfnlo, pfn_t pfnhi, pgcnt_t pfnflag) 33837c478bd9Sstevel@tonic-gate { 33847c478bd9Sstevel@tonic-gate struct memseg *mseg; 33857c478bd9Sstevel@tonic-gate pgcnt_t szcpgcnt = page_get_pagecnt(szc); 33867c478bd9Sstevel@tonic-gate pgcnt_t szcpgmask = szcpgcnt - 1; 33877c478bd9Sstevel@tonic-gate pfn_t randpfn; 33887c478bd9Sstevel@tonic-gate page_t *pp, *randpp, *endpp; 33895d07b933Sdp uint_t colors, ceq_mask; 33905d07b933Sdp /* LINTED : set but not used in function */ 33915d07b933Sdp uint_t color_mask; 33927c478bd9Sstevel@tonic-gate pfn_t hi, lo; 33937c478bd9Sstevel@tonic-gate uint_t skip; 3394ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 33957c478bd9Sstevel@tonic-gate 33967c478bd9Sstevel@tonic-gate ASSERT(szc != 0 || (flags & PGI_PGCPSZC0)); 33977c478bd9Sstevel@tonic-gate 339802bc52beSkchow pfnlo = P2ROUNDUP(pfnlo, szcpgcnt); 339902bc52beSkchow 340002bc52beSkchow if ((pfnhi - pfnlo) + 1 < szcpgcnt || pfnlo >= pfnhi) 34017c478bd9Sstevel@tonic-gate return (NULL); 34027c478bd9Sstevel@tonic-gate 34037c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 34047c478bd9Sstevel@tonic-gate 34055d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 34065d07b933Sdp color_mask = colors - 1; 34075d07b933Sdp if ((colors > 1) && (flags & PG_MATCH_COLOR)) { 34085d07b933Sdp uchar_t ceq = colorequivszc[szc]; 34095d07b933Sdp uint_t ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 34105d07b933Sdp 34115d07b933Sdp ASSERT(ceq_dif > 0); 34125d07b933Sdp ceq_mask = (ceq_dif - 1) << (ceq & 0xf); 34135d07b933Sdp } else { 34145d07b933Sdp ceq_mask = 0; 34155d07b933Sdp } 34167c478bd9Sstevel@tonic-gate 34177c478bd9Sstevel@tonic-gate ASSERT(bin < colors); 34187c478bd9Sstevel@tonic-gate 34195d07b933Sdp /* clear "non-significant" color bits */ 34205d07b933Sdp bin &= ceq_mask; 34215d07b933Sdp 34227c478bd9Sstevel@tonic-gate /* 34237c478bd9Sstevel@tonic-gate * trim the pfn range to search based on pfnflag. pfnflag is set 34247c478bd9Sstevel@tonic-gate * when there have been previous page_get_contig_page failures to 34257c478bd9Sstevel@tonic-gate * limit the search. 34267c478bd9Sstevel@tonic-gate * 34277c478bd9Sstevel@tonic-gate * The high bit in pfnflag specifies the number of 'slots' in the 34287c478bd9Sstevel@tonic-gate * pfn range and the remainder of pfnflag specifies which slot. 34297c478bd9Sstevel@tonic-gate * For example, a value of 1010b would mean the second slot of 34307c478bd9Sstevel@tonic-gate * the pfn range that has been divided into 8 slots. 34317c478bd9Sstevel@tonic-gate */ 34327c478bd9Sstevel@tonic-gate if (pfnflag > 1) { 34337c478bd9Sstevel@tonic-gate int slots = 1 << (highbit(pfnflag) - 1); 34347c478bd9Sstevel@tonic-gate int slotid = pfnflag & (slots - 1); 34357c478bd9Sstevel@tonic-gate pgcnt_t szcpages; 34367c478bd9Sstevel@tonic-gate int slotlen; 34377c478bd9Sstevel@tonic-gate 343802bc52beSkchow pfnhi = P2ALIGN((pfnhi + 1), szcpgcnt) - 1; 34397c478bd9Sstevel@tonic-gate szcpages = ((pfnhi - pfnlo) + 1) / szcpgcnt; 34407c478bd9Sstevel@tonic-gate slotlen = howmany(szcpages, slots); 344102bc52beSkchow /* skip if 'slotid' slot is empty */ 344202bc52beSkchow if (slotid * slotlen >= szcpages) 344302bc52beSkchow return (NULL); 34447c478bd9Sstevel@tonic-gate pfnlo = pfnlo + (((slotid * slotlen) % szcpages) * szcpgcnt); 34457c478bd9Sstevel@tonic-gate ASSERT(pfnlo < pfnhi); 34467c478bd9Sstevel@tonic-gate if (pfnhi > pfnlo + (slotlen * szcpgcnt)) 344702bc52beSkchow pfnhi = pfnlo + (slotlen * szcpgcnt) - 1; 34487c478bd9Sstevel@tonic-gate } 34497c478bd9Sstevel@tonic-gate 34507c478bd9Sstevel@tonic-gate memsegs_lock(0); 34517c478bd9Sstevel@tonic-gate 34527c478bd9Sstevel@tonic-gate /* 34537c478bd9Sstevel@tonic-gate * loop through memsegs to look for contig page candidates 34547c478bd9Sstevel@tonic-gate */ 34557c478bd9Sstevel@tonic-gate 34567c478bd9Sstevel@tonic-gate for (mseg = memsegs; mseg != NULL; mseg = mseg->next) { 34577c478bd9Sstevel@tonic-gate if (pfnhi < mseg->pages_base || pfnlo >= mseg->pages_end) { 34587c478bd9Sstevel@tonic-gate /* no overlap */ 34597c478bd9Sstevel@tonic-gate continue; 34607c478bd9Sstevel@tonic-gate } 34617c478bd9Sstevel@tonic-gate 34627c478bd9Sstevel@tonic-gate if (mseg->pages_end - mseg->pages_base < szcpgcnt) 34637c478bd9Sstevel@tonic-gate /* mseg too small */ 34647c478bd9Sstevel@tonic-gate continue; 34657c478bd9Sstevel@tonic-gate 346678b03d3aSkchow /* 346778b03d3aSkchow * trim off kernel cage pages from pfn range and check for 346878b03d3aSkchow * a trimmed pfn range returned that does not span the 346978b03d3aSkchow * desired large page size. 347078b03d3aSkchow */ 34717c478bd9Sstevel@tonic-gate if (kcage_on) { 347278b03d3aSkchow if (trimkcage(mseg, &lo, &hi, pfnlo, pfnhi) == 0 || 3473d94d1888Skchow lo >= hi || ((hi - lo) + 1) < szcpgcnt) 34747c478bd9Sstevel@tonic-gate continue; 34757c478bd9Sstevel@tonic-gate } else { 34767c478bd9Sstevel@tonic-gate lo = MAX(pfnlo, mseg->pages_base); 34777c478bd9Sstevel@tonic-gate hi = MIN(pfnhi, (mseg->pages_end - 1)); 34787c478bd9Sstevel@tonic-gate } 34797c478bd9Sstevel@tonic-gate 34807c478bd9Sstevel@tonic-gate /* round to szcpgcnt boundaries */ 34817c478bd9Sstevel@tonic-gate lo = P2ROUNDUP(lo, szcpgcnt); 348202bc52beSkchow 3483b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 348402bc52beSkchow hi = P2ALIGN((hi + 1), szcpgcnt) - 1; 34857c478bd9Sstevel@tonic-gate 34867c478bd9Sstevel@tonic-gate if (hi <= lo) 34877c478bd9Sstevel@tonic-gate continue; 34887c478bd9Sstevel@tonic-gate 34897c478bd9Sstevel@tonic-gate /* 34907c478bd9Sstevel@tonic-gate * set lo to point to the pfn for the desired bin. Large 34917c478bd9Sstevel@tonic-gate * page sizes may only have a single page color 34927c478bd9Sstevel@tonic-gate */ 34935d07b933Sdp skip = szcpgcnt; 3494ce8eb11aSdp if (ceq_mask > 0 || interleaved_mnodes) { 34955d07b933Sdp /* set lo to point at appropriate color */ 3496ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ bin) & ceq_mask) || 3497ce8eb11aSdp (interleaved_mnodes && 3498ce8eb11aSdp PFN_2_MEM_NODE(lo) != mnode)) { 3499ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, bin, ceq_mask, 3500ce8eb11aSdp color_mask, &it); 3501ce8eb11aSdp } 35025d07b933Sdp if (hi <= lo) 35035d07b933Sdp /* mseg cannot satisfy color request */ 35045d07b933Sdp continue; 35057c478bd9Sstevel@tonic-gate } 35067c478bd9Sstevel@tonic-gate 35077c478bd9Sstevel@tonic-gate /* randomly choose a point between lo and hi to begin search */ 35087c478bd9Sstevel@tonic-gate 35097c478bd9Sstevel@tonic-gate randpfn = (pfn_t)GETTICK(); 35107c478bd9Sstevel@tonic-gate randpfn = ((randpfn % (hi - lo)) + lo) & ~(skip - 1); 3511b779d3e0Sdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, szc, &it); 3512b779d3e0Sdp if (ceq_mask || interleaved_mnodes || randpfn == (pfn_t)-1) { 3513a7c3ca36Sdp if (randpfn != (pfn_t)-1) { 3514ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(randpfn, szc, bin, 3515ce8eb11aSdp ceq_mask, color_mask, &it); 3516a7c3ca36Sdp } 3517ce8eb11aSdp if (randpfn >= hi) { 3518ce8eb11aSdp randpfn = lo; 3519b779d3e0Sdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, szc, 3520b779d3e0Sdp &it); 3521ce8eb11aSdp } 35225d07b933Sdp } 35237c478bd9Sstevel@tonic-gate randpp = mseg->pages + (randpfn - mseg->pages_base); 35247c478bd9Sstevel@tonic-gate 35257c478bd9Sstevel@tonic-gate ASSERT(randpp->p_pagenum == randpfn); 35267c478bd9Sstevel@tonic-gate 35277c478bd9Sstevel@tonic-gate pp = randpp; 352802bc52beSkchow endpp = mseg->pages + (hi - mseg->pages_base) + 1; 35297c478bd9Sstevel@tonic-gate 35307c478bd9Sstevel@tonic-gate ASSERT(randpp + szcpgcnt <= endpp); 35317c478bd9Sstevel@tonic-gate 35327c478bd9Sstevel@tonic-gate do { 35337c478bd9Sstevel@tonic-gate ASSERT(!(pp->p_pagenum & szcpgmask)); 35345d07b933Sdp ASSERT(((PP_2_BIN(pp) ^ bin) & ceq_mask) == 0); 35355d07b933Sdp 35367c478bd9Sstevel@tonic-gate if (page_trylock_contig_pages(mnode, pp, szc, flags)) { 35377c478bd9Sstevel@tonic-gate /* pages unlocked by page_claim on failure */ 35387c478bd9Sstevel@tonic-gate if (page_claim_contig_pages(pp, szc, flags)) { 35397c478bd9Sstevel@tonic-gate memsegs_unlock(0); 35407c478bd9Sstevel@tonic-gate return (pp); 35417c478bd9Sstevel@tonic-gate } 35427c478bd9Sstevel@tonic-gate } 35437c478bd9Sstevel@tonic-gate 3544ce8eb11aSdp if (ceq_mask == 0 && !interleaved_mnodes) { 35455d07b933Sdp pp += skip; 35465d07b933Sdp } else { 35475d07b933Sdp pfn_t pfn = pp->p_pagenum; 35485d07b933Sdp 35495d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfn, szc, bin, 3550ce8eb11aSdp ceq_mask, color_mask, &it); 3551ce8eb11aSdp if (pfn == (pfn_t)-1) { 3552ce8eb11aSdp pp = endpp; 3553ce8eb11aSdp } else { 3554ce8eb11aSdp pp = mseg->pages + 3555ce8eb11aSdp (pfn - mseg->pages_base); 3556ce8eb11aSdp } 35575d07b933Sdp } 35587c478bd9Sstevel@tonic-gate if (pp >= endpp) { 35597c478bd9Sstevel@tonic-gate /* start from the beginning */ 3560b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 35617c478bd9Sstevel@tonic-gate pp = mseg->pages + (lo - mseg->pages_base); 35627c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == lo); 35637c478bd9Sstevel@tonic-gate ASSERT(pp + szcpgcnt <= endpp); 35647c478bd9Sstevel@tonic-gate } 35657c478bd9Sstevel@tonic-gate } while (pp != randpp); 35667c478bd9Sstevel@tonic-gate } 35677c478bd9Sstevel@tonic-gate memsegs_unlock(0); 35687c478bd9Sstevel@tonic-gate return (NULL); 35697c478bd9Sstevel@tonic-gate } 35707c478bd9Sstevel@tonic-gate 35717c478bd9Sstevel@tonic-gate 35727c478bd9Sstevel@tonic-gate /* 35737c478bd9Sstevel@tonic-gate * controlling routine that searches through physical memory in an attempt to 35747c478bd9Sstevel@tonic-gate * claim a large page based on the input parameters. 35757c478bd9Sstevel@tonic-gate * on the page free lists. 35767c478bd9Sstevel@tonic-gate * 35777c478bd9Sstevel@tonic-gate * calls page_geti_contig_pages with an initial pfn range from the mnode 35787c478bd9Sstevel@tonic-gate * and mtype. page_geti_contig_pages will trim off the parts of the pfn range 35797c478bd9Sstevel@tonic-gate * that overlaps with the kernel cage or does not match the requested page 35807c478bd9Sstevel@tonic-gate * color if PG_MATCH_COLOR is set. Since this search is very expensive, 35817c478bd9Sstevel@tonic-gate * page_geti_contig_pages may further limit the search range based on 35827c478bd9Sstevel@tonic-gate * previous failure counts (pgcpfailcnt[]). 35837c478bd9Sstevel@tonic-gate * 35847c478bd9Sstevel@tonic-gate * for PGI_PGCPSZC0 requests, page_get_contig_pages will relocate a base 35857c478bd9Sstevel@tonic-gate * pagesize page that satisfies mtype. 35867c478bd9Sstevel@tonic-gate */ 35877c478bd9Sstevel@tonic-gate page_t * 35887c478bd9Sstevel@tonic-gate page_get_contig_pages(int mnode, uint_t bin, int mtype, uchar_t szc, 35897c478bd9Sstevel@tonic-gate uint_t flags) 35907c478bd9Sstevel@tonic-gate { 35917c478bd9Sstevel@tonic-gate pfn_t pfnlo, pfnhi; /* contig pages pfn range */ 35927c478bd9Sstevel@tonic-gate page_t *pp; 359383f9b804Skchow pgcnt_t pfnflag = 0; /* no limit on search if 0 */ 35947c478bd9Sstevel@tonic-gate 35957c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_alloc[szc]); 35967c478bd9Sstevel@tonic-gate 35970b5aa17bSmec /* no allocations from cage */ 35980b5aa17bSmec flags |= PGI_NOCAGE; 35990b5aa17bSmec 36007c478bd9Sstevel@tonic-gate /* LINTED */ 36017c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 36027c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 36037c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocempty[szc]); 36047c478bd9Sstevel@tonic-gate return (NULL); 36057c478bd9Sstevel@tonic-gate } 36067c478bd9Sstevel@tonic-gate 36077c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 36087c478bd9Sstevel@tonic-gate 36097c478bd9Sstevel@tonic-gate /* do not limit search and ignore color if hi pri */ 36107c478bd9Sstevel@tonic-gate 36117c478bd9Sstevel@tonic-gate if (pgcplimitsearch && ((flags & PGI_PGCPHIPRI) == 0)) 36127c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 36137c478bd9Sstevel@tonic-gate 36147c478bd9Sstevel@tonic-gate /* remove color match to improve chances */ 36157c478bd9Sstevel@tonic-gate 36167c478bd9Sstevel@tonic-gate if (flags & PGI_PGCPHIPRI || pfnflag) 36177c478bd9Sstevel@tonic-gate flags &= ~PG_MATCH_COLOR; 36187c478bd9Sstevel@tonic-gate 36197c478bd9Sstevel@tonic-gate do { 36207c478bd9Sstevel@tonic-gate /* get pfn range based on mnode and mtype */ 36217c478bd9Sstevel@tonic-gate MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi); 36227c478bd9Sstevel@tonic-gate 36237c478bd9Sstevel@tonic-gate ASSERT(pfnhi >= pfnlo); 36247c478bd9Sstevel@tonic-gate 36257c478bd9Sstevel@tonic-gate pp = page_geti_contig_pages(mnode, bin, szc, flags, 36267c478bd9Sstevel@tonic-gate pfnlo, pfnhi, pfnflag); 36277c478bd9Sstevel@tonic-gate 36287c478bd9Sstevel@tonic-gate if (pp != NULL) { 36297c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 36307c478bd9Sstevel@tonic-gate if (pfnflag) { 36317c478bd9Sstevel@tonic-gate /* double the search size */ 36327c478bd9Sstevel@tonic-gate pgcpfailcnt[szc] = pfnflag >> 1; 36337c478bd9Sstevel@tonic-gate } 36347c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocok[szc]); 36357c478bd9Sstevel@tonic-gate return (pp); 36367c478bd9Sstevel@tonic-gate } 3637affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3638affbd3ccSkchow } while (mtype >= 0); 36397c478bd9Sstevel@tonic-gate 36407c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocfailed[szc]); 36417c478bd9Sstevel@tonic-gate return (NULL); 36427c478bd9Sstevel@tonic-gate } 36437c478bd9Sstevel@tonic-gate 364478b03d3aSkchow #if defined(__i386) || defined(__amd64) 364578b03d3aSkchow /* 364678b03d3aSkchow * Determine the likelihood of finding/coalescing a szc page. 364778b03d3aSkchow * Return 0 if the likelihood is small otherwise return 1. 364878b03d3aSkchow * 364978b03d3aSkchow * For now, be conservative and check only 1g pages and return 0 365078b03d3aSkchow * if there had been previous coalescing failures and the szc pages 365178b03d3aSkchow * needed to satisfy request would exhaust most of freemem. 365278b03d3aSkchow */ 365378b03d3aSkchow int 365478b03d3aSkchow page_chk_freelist(uint_t szc) 365578b03d3aSkchow { 365678b03d3aSkchow pgcnt_t pgcnt; 365778b03d3aSkchow 365878b03d3aSkchow if (szc <= 1) 365978b03d3aSkchow return (1); 366078b03d3aSkchow 366178b03d3aSkchow pgcnt = page_get_pagecnt(szc); 366278b03d3aSkchow if (pgcpfailcnt[szc] && pgcnt + throttlefree >= freemem) { 366378b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_deny[szc]); 366478b03d3aSkchow return (0); 366578b03d3aSkchow } 366678b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_allow[szc]); 366778b03d3aSkchow return (1); 366878b03d3aSkchow } 366978b03d3aSkchow #endif 36707c478bd9Sstevel@tonic-gate 36717c478bd9Sstevel@tonic-gate /* 36727c478bd9Sstevel@tonic-gate * Find the `best' page on the freelist for this (vp,off) (as,vaddr) pair. 36737c478bd9Sstevel@tonic-gate * 36747c478bd9Sstevel@tonic-gate * Does its own locking and accounting. 36757c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 36767c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 36777c478bd9Sstevel@tonic-gate * 36787c478bd9Sstevel@tonic-gate * Finds a page, removes it, THEN locks it. 36797c478bd9Sstevel@tonic-gate */ 36807c478bd9Sstevel@tonic-gate 36817c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 36827c478bd9Sstevel@tonic-gate page_t * 36837c478bd9Sstevel@tonic-gate page_get_freelist(struct vnode *vp, u_offset_t off, struct seg *seg, 36847c478bd9Sstevel@tonic-gate caddr_t vaddr, size_t size, uint_t flags, struct lgrp *lgrp) 36857c478bd9Sstevel@tonic-gate { 36867c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 36877c478bd9Sstevel@tonic-gate page_t *pp = NULL; 36887c478bd9Sstevel@tonic-gate ulong_t bin; 36897c478bd9Sstevel@tonic-gate uchar_t szc; 36907c478bd9Sstevel@tonic-gate int mnode; 36917c478bd9Sstevel@tonic-gate int mtype; 36927c478bd9Sstevel@tonic-gate page_t *(*page_get_func)(int, uint_t, int, uchar_t, uint_t); 36937c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 36947c478bd9Sstevel@tonic-gate 36957c478bd9Sstevel@tonic-gate page_get_func = page_get_mnode_freelist; 36967c478bd9Sstevel@tonic-gate 36977c478bd9Sstevel@tonic-gate /* 36987c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or passed a freed lgrp 36997c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 37007c478bd9Sstevel@tonic-gate */ 37017c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 37027c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 37037c478bd9Sstevel@tonic-gate 37047c478bd9Sstevel@tonic-gate if (kcage_on) { 37057c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC)) == PG_NORELOC && 37067c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + btop(size) && 37077c478bd9Sstevel@tonic-gate curthread != kcage_cageout_thread) { 37087c478bd9Sstevel@tonic-gate /* 37097c478bd9Sstevel@tonic-gate * Set a "reserve" of kcage_throttlefree pages for 37107c478bd9Sstevel@tonic-gate * PG_PANIC and cageout thread allocations. 37117c478bd9Sstevel@tonic-gate * 37127c478bd9Sstevel@tonic-gate * Everybody else has to serialize in 37137c478bd9Sstevel@tonic-gate * page_create_get_something() to get a cage page, so 37147c478bd9Sstevel@tonic-gate * that we don't deadlock cageout! 37157c478bd9Sstevel@tonic-gate */ 37167c478bd9Sstevel@tonic-gate return (NULL); 37177c478bd9Sstevel@tonic-gate } 37187c478bd9Sstevel@tonic-gate } else { 37197c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 37207c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 37217c478bd9Sstevel@tonic-gate } 37227c478bd9Sstevel@tonic-gate 37237c478bd9Sstevel@tonic-gate /* LINTED */ 372407ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, size); 37257c478bd9Sstevel@tonic-gate 37267c478bd9Sstevel@tonic-gate /* 37277c478bd9Sstevel@tonic-gate * Convert size to page size code. 37287c478bd9Sstevel@tonic-gate */ 37297c478bd9Sstevel@tonic-gate if ((szc = page_szc(size)) == (uchar_t)-1) 37307c478bd9Sstevel@tonic-gate panic("page_get_freelist: illegal page size request"); 37317c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 37327c478bd9Sstevel@tonic-gate 37337c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_alloc[szc]); 37347c478bd9Sstevel@tonic-gate 37357c478bd9Sstevel@tonic-gate /* LINTED */ 37365d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, szc); 37377c478bd9Sstevel@tonic-gate 37385d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(szc)); 37397c478bd9Sstevel@tonic-gate 37407c478bd9Sstevel@tonic-gate /* 37417c478bd9Sstevel@tonic-gate * Try to get a local page first, but try remote if we can't 37427c478bd9Sstevel@tonic-gate * get a page of the right color. 37437c478bd9Sstevel@tonic-gate */ 37447c478bd9Sstevel@tonic-gate pgretry: 37457c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 37467c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 37477c478bd9Sstevel@tonic-gate pp = page_get_func(mnode, bin, mtype, szc, flags); 37487c478bd9Sstevel@tonic-gate if (pp != NULL) { 37497c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocok[szc]); 37507c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 37517c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 37527c478bd9Sstevel@tonic-gate int, mnode, 37537c478bd9Sstevel@tonic-gate ulong_t, bin, 37547c478bd9Sstevel@tonic-gate uint_t, flags); 37557c478bd9Sstevel@tonic-gate return (pp); 37567c478bd9Sstevel@tonic-gate } 37577c478bd9Sstevel@tonic-gate } 37587c478bd9Sstevel@tonic-gate ASSERT(pp == NULL); 37597c478bd9Sstevel@tonic-gate 37607c478bd9Sstevel@tonic-gate /* 37617c478bd9Sstevel@tonic-gate * for non-SZC0 PAGESIZE requests, check cachelist before checking 37627c478bd9Sstevel@tonic-gate * remote free lists. Caller expected to call page_get_cachelist which 37637c478bd9Sstevel@tonic-gate * will check local cache lists and remote free lists. 37647c478bd9Sstevel@tonic-gate */ 37657c478bd9Sstevel@tonic-gate if (szc == 0 && ((flags & PGI_PGCPSZC0) == 0)) { 37667c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocdeferred); 37677c478bd9Sstevel@tonic-gate return (NULL); 37687c478bd9Sstevel@tonic-gate } 37697c478bd9Sstevel@tonic-gate 37707c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 37717c478bd9Sstevel@tonic-gate 37727c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 37737c478bd9Sstevel@tonic-gate 37742cb27123Saguzovsk if (!(flags & PG_LOCAL)) { 37752cb27123Saguzovsk /* 37762cb27123Saguzovsk * Try to get a non-local freelist page. 37772cb27123Saguzovsk */ 37782cb27123Saguzovsk LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 37792cb27123Saguzovsk while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 37802cb27123Saguzovsk pp = page_get_func(mnode, bin, mtype, szc, flags); 37812cb27123Saguzovsk if (pp != NULL) { 37822cb27123Saguzovsk DTRACE_PROBE4(page__get, 37832cb27123Saguzovsk lgrp_t *, lgrp, 37842cb27123Saguzovsk int, mnode, 37852cb27123Saguzovsk ulong_t, bin, 37862cb27123Saguzovsk uint_t, flags); 37872cb27123Saguzovsk VM_STAT_ADD(vmm_vmstats.pgf_allocokrem[szc]); 37882cb27123Saguzovsk return (pp); 37892cb27123Saguzovsk } 37907c478bd9Sstevel@tonic-gate } 37912cb27123Saguzovsk ASSERT(pp == NULL); 37927c478bd9Sstevel@tonic-gate } 37937c478bd9Sstevel@tonic-gate 37947c478bd9Sstevel@tonic-gate /* 37957c478bd9Sstevel@tonic-gate * when the cage is off chances are page_get_contig_pages() will fail 37967c478bd9Sstevel@tonic-gate * to lock a large page chunk therefore when the cage is off it's not 37977c478bd9Sstevel@tonic-gate * called by default. this can be changed via /etc/system. 37987c478bd9Sstevel@tonic-gate * 37997c478bd9Sstevel@tonic-gate * page_get_contig_pages() also called to acquire a base pagesize page 38007c478bd9Sstevel@tonic-gate * for page_create_get_something(). 38017c478bd9Sstevel@tonic-gate */ 38027c478bd9Sstevel@tonic-gate if (!(flags & PG_NORELOC) && (pg_contig_disable == 0) && 38037c478bd9Sstevel@tonic-gate (kcage_on || pg_lpgcreate_nocage || szc == 0) && 38047c478bd9Sstevel@tonic-gate (page_get_func != page_get_contig_pages)) { 38057c478bd9Sstevel@tonic-gate 38067c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocretry[szc]); 38077c478bd9Sstevel@tonic-gate page_get_func = page_get_contig_pages; 38087c478bd9Sstevel@tonic-gate goto pgretry; 38097c478bd9Sstevel@tonic-gate } 38107c478bd9Sstevel@tonic-gate 38112cb27123Saguzovsk if (!(flags & PG_LOCAL) && pgcplimitsearch && 38122cb27123Saguzovsk page_get_func == page_get_contig_pages) 381383f9b804Skchow SETPGCPFAILCNT(szc); 38147c478bd9Sstevel@tonic-gate 38157c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocfailed[szc]); 38167c478bd9Sstevel@tonic-gate return (NULL); 38177c478bd9Sstevel@tonic-gate } 38187c478bd9Sstevel@tonic-gate 38197c478bd9Sstevel@tonic-gate /* 38207c478bd9Sstevel@tonic-gate * Find the `best' page on the cachelist for this (vp,off) (as,vaddr) pair. 38217c478bd9Sstevel@tonic-gate * 38227c478bd9Sstevel@tonic-gate * Does its own locking. 38237c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 38247c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 38257c478bd9Sstevel@tonic-gate * Otherwise, scan the bins for ones with pages. For each bin with pages, 38267c478bd9Sstevel@tonic-gate * try to lock one of them. If no page can be locked, try the 38277c478bd9Sstevel@tonic-gate * next bin. Return NULL if a page can not be found and locked. 38287c478bd9Sstevel@tonic-gate * 38297c478bd9Sstevel@tonic-gate * Finds a pages, trys to lock it, then removes it. 38307c478bd9Sstevel@tonic-gate */ 38317c478bd9Sstevel@tonic-gate 38327c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 38337c478bd9Sstevel@tonic-gate page_t * 38347c478bd9Sstevel@tonic-gate page_get_cachelist(struct vnode *vp, u_offset_t off, struct seg *seg, 38357c478bd9Sstevel@tonic-gate caddr_t vaddr, uint_t flags, struct lgrp *lgrp) 38367c478bd9Sstevel@tonic-gate { 38377c478bd9Sstevel@tonic-gate page_t *pp; 38387c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 38397c478bd9Sstevel@tonic-gate ulong_t bin; 38407c478bd9Sstevel@tonic-gate /*LINTED*/ 38417c478bd9Sstevel@tonic-gate int mnode; 38427c478bd9Sstevel@tonic-gate int mtype; 38437c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 38447c478bd9Sstevel@tonic-gate 38457c478bd9Sstevel@tonic-gate /* 38467c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or pasased a freed lgrp 38477c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 38487c478bd9Sstevel@tonic-gate */ 38497c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 38507c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 38517c478bd9Sstevel@tonic-gate 38527c478bd9Sstevel@tonic-gate if (!kcage_on) { 38537c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 38547c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 38557c478bd9Sstevel@tonic-gate } 38567c478bd9Sstevel@tonic-gate 38577c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == PG_NORELOC && 38587c478bd9Sstevel@tonic-gate kcage_freemem <= kcage_throttlefree) { 38597c478bd9Sstevel@tonic-gate /* 38607c478bd9Sstevel@tonic-gate * Reserve kcage_throttlefree pages for critical kernel 38617c478bd9Sstevel@tonic-gate * threads. 38627c478bd9Sstevel@tonic-gate * 38637c478bd9Sstevel@tonic-gate * Everybody else has to go to page_create_get_something() 38647c478bd9Sstevel@tonic-gate * to get a cage page, so we don't deadlock cageout. 38657c478bd9Sstevel@tonic-gate */ 38667c478bd9Sstevel@tonic-gate return (NULL); 38677c478bd9Sstevel@tonic-gate } 38687c478bd9Sstevel@tonic-gate 38697c478bd9Sstevel@tonic-gate /* LINTED */ 38705d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, 0); 38717c478bd9Sstevel@tonic-gate 38725d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(0)); 38737c478bd9Sstevel@tonic-gate 38747c478bd9Sstevel@tonic-gate /* LINTED */ 387507ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, MMU_PAGESIZE); 38767c478bd9Sstevel@tonic-gate 38777c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_alloc); 38787c478bd9Sstevel@tonic-gate 38797c478bd9Sstevel@tonic-gate /* 38807c478bd9Sstevel@tonic-gate * Try local cachelists first 38817c478bd9Sstevel@tonic-gate */ 38827c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 38837c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38847c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 38857c478bd9Sstevel@tonic-gate if (pp != NULL) { 38867c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocok); 38877c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38887c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38897c478bd9Sstevel@tonic-gate int, mnode, 38907c478bd9Sstevel@tonic-gate ulong_t, bin, 38917c478bd9Sstevel@tonic-gate uint_t, flags); 38927c478bd9Sstevel@tonic-gate return (pp); 38937c478bd9Sstevel@tonic-gate } 38947c478bd9Sstevel@tonic-gate } 38957c478bd9Sstevel@tonic-gate 38967c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 38977c478bd9Sstevel@tonic-gate 38987c478bd9Sstevel@tonic-gate /* 38997c478bd9Sstevel@tonic-gate * Try freelists/cachelists that are farther away 39007c478bd9Sstevel@tonic-gate * This is our only chance to allocate remote pages for PAGESIZE 39017c478bd9Sstevel@tonic-gate * requests. 39027c478bd9Sstevel@tonic-gate */ 39037c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 39047c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 39057c478bd9Sstevel@tonic-gate pp = page_get_mnode_freelist(mnode, bin, mtype, 39067c478bd9Sstevel@tonic-gate 0, flags); 39077c478bd9Sstevel@tonic-gate if (pp != NULL) { 39087c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokdeferred); 39097c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 39107c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 39117c478bd9Sstevel@tonic-gate int, mnode, 39127c478bd9Sstevel@tonic-gate ulong_t, bin, 39137c478bd9Sstevel@tonic-gate uint_t, flags); 39147c478bd9Sstevel@tonic-gate return (pp); 39157c478bd9Sstevel@tonic-gate } 39167c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 39177c478bd9Sstevel@tonic-gate if (pp != NULL) { 39187c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokrem); 39197c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 39207c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 39217c478bd9Sstevel@tonic-gate int, mnode, 39227c478bd9Sstevel@tonic-gate ulong_t, bin, 39237c478bd9Sstevel@tonic-gate uint_t, flags); 39247c478bd9Sstevel@tonic-gate return (pp); 39257c478bd9Sstevel@tonic-gate } 39267c478bd9Sstevel@tonic-gate } 39277c478bd9Sstevel@tonic-gate 39287c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocfailed); 39297c478bd9Sstevel@tonic-gate return (NULL); 39307c478bd9Sstevel@tonic-gate } 39317c478bd9Sstevel@tonic-gate 39327c478bd9Sstevel@tonic-gate page_t * 39337c478bd9Sstevel@tonic-gate page_get_mnode_cachelist(uint_t bin, uint_t flags, int mnode, int mtype) 39347c478bd9Sstevel@tonic-gate { 39355d07b933Sdp kmutex_t *pcm; 39365d07b933Sdp page_t *pp, *first_pp; 39375d07b933Sdp uint_t sbin; 39385d07b933Sdp int plw_initialized; 39395d07b933Sdp page_list_walker_t plw; 39407c478bd9Sstevel@tonic-gate 39417c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_alloc); 39427c478bd9Sstevel@tonic-gate 39437c478bd9Sstevel@tonic-gate /* LINTED */ 39447c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 39457c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 39467c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocempty); 39477c478bd9Sstevel@tonic-gate return (NULL); 39487c478bd9Sstevel@tonic-gate } 39497c478bd9Sstevel@tonic-gate 39505d07b933Sdp try_again: 39517c478bd9Sstevel@tonic-gate 39525d07b933Sdp plw_initialized = 0; 39535d07b933Sdp plw.plw_ceq_dif = 1; 39547c478bd9Sstevel@tonic-gate 39557c478bd9Sstevel@tonic-gate /* 39567c478bd9Sstevel@tonic-gate * Only hold one cachelist lock at a time, that way we 39577c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 39587c478bd9Sstevel@tonic-gate * ordering. 39597c478bd9Sstevel@tonic-gate */ 39607c478bd9Sstevel@tonic-gate 39615d07b933Sdp for (plw.plw_count = 0; 39625d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 39635d07b933Sdp sbin = bin; 39645d07b933Sdp do { 39655d07b933Sdp 39665d07b933Sdp if (!PAGE_CACHELISTS(mnode, bin, mtype)) 39675d07b933Sdp goto bin_empty_1; 39687c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 39697c478bd9Sstevel@tonic-gate mutex_enter(pcm); 39707c478bd9Sstevel@tonic-gate pp = PAGE_CACHELISTS(mnode, bin, mtype); 39715d07b933Sdp if (pp == NULL) 39725d07b933Sdp goto bin_empty_0; 39737c478bd9Sstevel@tonic-gate 39745d07b933Sdp first_pp = pp; 39755d07b933Sdp ASSERT(pp->p_vnode); 39765d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39775d07b933Sdp ASSERT(pp->p_szc == 0); 39785d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 39795d07b933Sdp while (!page_trylock(pp, SE_EXCL)) { 39805d07b933Sdp pp = pp->p_next; 39815d07b933Sdp ASSERT(pp->p_szc == 0); 39825d07b933Sdp if (pp == first_pp) { 39837c478bd9Sstevel@tonic-gate /* 39845d07b933Sdp * We have searched the complete list! 39855d07b933Sdp * And all of them (might only be one) 39865d07b933Sdp * are locked. This can happen since 39875d07b933Sdp * these pages can also be found via 39885d07b933Sdp * the hash list. When found via the 39895d07b933Sdp * hash list, they are locked first, 39905d07b933Sdp * then removed. We give up to let the 39915d07b933Sdp * other thread run. 39927c478bd9Sstevel@tonic-gate */ 39935d07b933Sdp pp = NULL; 39945d07b933Sdp break; 39957c478bd9Sstevel@tonic-gate } 39965d07b933Sdp ASSERT(pp->p_vnode); 39975d07b933Sdp ASSERT(PP_ISFREE(pp)); 39985d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39995d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == 40005d07b933Sdp mnode); 40017c478bd9Sstevel@tonic-gate } 40027c478bd9Sstevel@tonic-gate 40035d07b933Sdp if (pp) { 40045d07b933Sdp page_t **ppp; 40055d07b933Sdp /* 40065d07b933Sdp * Found and locked a page. 40075d07b933Sdp * Pull it off the list. 40085d07b933Sdp */ 40095d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 40105d07b933Sdp ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 40115d07b933Sdp page_sub(ppp, pp); 40125d07b933Sdp /* 40135d07b933Sdp * Subtract counters before releasing pcm mutex 40145d07b933Sdp * to avoid a race with page_freelist_coalesce 40155d07b933Sdp * and page_freelist_split. 40165d07b933Sdp */ 40175d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 40185d07b933Sdp mutex_exit(pcm); 40195d07b933Sdp ASSERT(pp->p_vnode); 40205d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 40215d07b933Sdp #if defined(__sparc) 40225d07b933Sdp ASSERT(!kcage_on || 40235d07b933Sdp (flags & PG_NORELOC) == 0 || 40245d07b933Sdp PP_ISNORELOC(pp)); 40255d07b933Sdp if (PP_ISNORELOC(pp)) { 40265d07b933Sdp kcage_freemem_sub(1); 40277c478bd9Sstevel@tonic-gate } 40285d07b933Sdp #endif 40295d07b933Sdp VM_STAT_ADD(vmm_vmstats. pgmc_allocok); 40305d07b933Sdp return (pp); 40317c478bd9Sstevel@tonic-gate } 40325d07b933Sdp bin_empty_0: 40335d07b933Sdp mutex_exit(pcm); 40345d07b933Sdp bin_empty_1: 40355d07b933Sdp if (plw_initialized == 0) { 40365d07b933Sdp page_list_walk_init(0, flags, bin, 0, 1, &plw); 40375d07b933Sdp plw_initialized = 1; 40387c478bd9Sstevel@tonic-gate } 40395d07b933Sdp /* calculate the next bin with equivalent color */ 40405d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 40415d07b933Sdp plw.plw_ceq_mask[0], plw.plw_color_mask); 40425d07b933Sdp } while (sbin != bin); 40437c478bd9Sstevel@tonic-gate 40445d07b933Sdp if (plw.plw_ceq_dif > 1) 40455d07b933Sdp bin = page_list_walk_next_bin(0, bin, &plw); 40467c478bd9Sstevel@tonic-gate } 40477c478bd9Sstevel@tonic-gate 4048affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 4049affbd3ccSkchow if (mtype >= 0) 40505d07b933Sdp goto try_again; 4051affbd3ccSkchow 40527c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocfailed); 40537c478bd9Sstevel@tonic-gate return (NULL); 40547c478bd9Sstevel@tonic-gate } 40557c478bd9Sstevel@tonic-gate 40567c478bd9Sstevel@tonic-gate #ifdef DEBUG 40577c478bd9Sstevel@tonic-gate #define REPL_PAGE_STATS 40587c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 40597c478bd9Sstevel@tonic-gate 40607c478bd9Sstevel@tonic-gate #ifdef REPL_PAGE_STATS 40617c478bd9Sstevel@tonic-gate struct repl_page_stats { 40627c478bd9Sstevel@tonic-gate uint_t ngets; 40637c478bd9Sstevel@tonic-gate uint_t ngets_noreloc; 40647c478bd9Sstevel@tonic-gate uint_t npgr_noreloc; 40657c478bd9Sstevel@tonic-gate uint_t nnopage_first; 40667c478bd9Sstevel@tonic-gate uint_t nnopage; 40677c478bd9Sstevel@tonic-gate uint_t nhashout; 40687c478bd9Sstevel@tonic-gate uint_t nnofree; 40697c478bd9Sstevel@tonic-gate uint_t nnext_pp; 40707c478bd9Sstevel@tonic-gate } repl_page_stats; 40717c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) atomic_add_32(&repl_page_stats.v, 1) 40727c478bd9Sstevel@tonic-gate #else /* REPL_PAGE_STATS */ 40737c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) 40747c478bd9Sstevel@tonic-gate #endif /* REPL_PAGE_STATS */ 40757c478bd9Sstevel@tonic-gate 40767c478bd9Sstevel@tonic-gate int pgrppgcp; 40777c478bd9Sstevel@tonic-gate 40787c478bd9Sstevel@tonic-gate /* 40797c478bd9Sstevel@tonic-gate * The freemem accounting must be done by the caller. 40807c478bd9Sstevel@tonic-gate * First we try to get a replacement page of the same size as like_pp, 40817c478bd9Sstevel@tonic-gate * if that is not possible, then we just get a set of discontiguous 40827c478bd9Sstevel@tonic-gate * PAGESIZE pages. 40837c478bd9Sstevel@tonic-gate */ 40847c478bd9Sstevel@tonic-gate page_t * 40852dae3fb5Sjjc page_get_replacement_page(page_t *orig_like_pp, struct lgrp *lgrp_target, 40867c478bd9Sstevel@tonic-gate uint_t pgrflags) 40877c478bd9Sstevel@tonic-gate { 40887c478bd9Sstevel@tonic-gate page_t *like_pp; 40897c478bd9Sstevel@tonic-gate page_t *pp, *pplist; 40907c478bd9Sstevel@tonic-gate page_t *pl = NULL; 40917c478bd9Sstevel@tonic-gate ulong_t bin; 40927c478bd9Sstevel@tonic-gate int mnode, page_mnode; 40937c478bd9Sstevel@tonic-gate int szc; 40947c478bd9Sstevel@tonic-gate spgcnt_t npgs, pg_cnt; 40957c478bd9Sstevel@tonic-gate pfn_t pfnum; 40967c478bd9Sstevel@tonic-gate int mtype; 40977c478bd9Sstevel@tonic-gate int flags = 0; 40987c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 40992dae3fb5Sjjc lgrp_t *lgrp; 41007c478bd9Sstevel@tonic-gate 41017c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets); 41027c478bd9Sstevel@tonic-gate like_pp = orig_like_pp; 41037c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(like_pp)); 41047c478bd9Sstevel@tonic-gate 41057c478bd9Sstevel@tonic-gate szc = like_pp->p_szc; 41067c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 41077c478bd9Sstevel@tonic-gate /* 41087c478bd9Sstevel@tonic-gate * Now we reset like_pp to the base page_t. 41097c478bd9Sstevel@tonic-gate * That way, we won't walk past the end of this 'szc' page. 41107c478bd9Sstevel@tonic-gate */ 41117c478bd9Sstevel@tonic-gate pfnum = PFN_BASE(like_pp->p_pagenum, szc); 41127c478bd9Sstevel@tonic-gate like_pp = page_numtopp_nolock(pfnum); 41137c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == szc); 41147c478bd9Sstevel@tonic-gate 41157c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(like_pp)) { 41167c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 41177c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets_noreloc); 41187c478bd9Sstevel@tonic-gate flags = PGI_RELOCONLY; 41197c478bd9Sstevel@tonic-gate } else if (pgrflags & PGR_NORELOC) { 41207c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 41217c478bd9Sstevel@tonic-gate REPL_STAT_INCR(npgr_noreloc); 41227c478bd9Sstevel@tonic-gate flags = PG_NORELOC; 41237c478bd9Sstevel@tonic-gate } 41247c478bd9Sstevel@tonic-gate 41257c478bd9Sstevel@tonic-gate /* 41267c478bd9Sstevel@tonic-gate * Kernel pages must always be replaced with the same size 41277c478bd9Sstevel@tonic-gate * pages, since we cannot properly handle demotion of kernel 41287c478bd9Sstevel@tonic-gate * pages. 41297c478bd9Sstevel@tonic-gate */ 4130ad23a2dbSjohansen if (PP_ISKAS(like_pp)) 41317c478bd9Sstevel@tonic-gate pgrflags |= PGR_SAMESZC; 41327c478bd9Sstevel@tonic-gate 41337c478bd9Sstevel@tonic-gate /* LINTED */ 413407ad560dSkchow MTYPE_PGR_INIT(mtype, flags, like_pp, page_mnode, npgs); 41357c478bd9Sstevel@tonic-gate 41367c478bd9Sstevel@tonic-gate while (npgs) { 41377c478bd9Sstevel@tonic-gate pplist = NULL; 41387c478bd9Sstevel@tonic-gate for (;;) { 41397c478bd9Sstevel@tonic-gate pg_cnt = page_get_pagecnt(szc); 41407c478bd9Sstevel@tonic-gate bin = PP_2_BIN(like_pp); 41417c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == orig_like_pp->p_szc); 41427c478bd9Sstevel@tonic-gate ASSERT(pg_cnt <= npgs); 41437c478bd9Sstevel@tonic-gate 41447c478bd9Sstevel@tonic-gate /* 41457c478bd9Sstevel@tonic-gate * If an lgroup was specified, try to get the 41467c478bd9Sstevel@tonic-gate * page from that lgroup. 41472dae3fb5Sjjc * NOTE: Must be careful with code below because 41482dae3fb5Sjjc * lgroup may disappear and reappear since there 41492dae3fb5Sjjc * is no locking for lgroup here. 41507c478bd9Sstevel@tonic-gate */ 41512dae3fb5Sjjc if (LGRP_EXISTS(lgrp_target)) { 41522dae3fb5Sjjc /* 41532dae3fb5Sjjc * Keep local variable for lgroup separate 41542dae3fb5Sjjc * from lgroup argument since this code should 41552dae3fb5Sjjc * only be exercised when lgroup argument 41562dae3fb5Sjjc * exists.... 41572dae3fb5Sjjc */ 41582dae3fb5Sjjc lgrp = lgrp_target; 41592dae3fb5Sjjc 41607c478bd9Sstevel@tonic-gate /* Try the lgroup's freelists first */ 41617c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41627c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41637c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41647c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41657c478bd9Sstevel@tonic-gate != -1) { 4166ce8eb11aSdp pplist = 4167ce8eb11aSdp page_get_mnode_freelist(mnode, bin, 4168ce8eb11aSdp mtype, szc, flags); 41697c478bd9Sstevel@tonic-gate } 41707c478bd9Sstevel@tonic-gate 41717c478bd9Sstevel@tonic-gate /* 41727c478bd9Sstevel@tonic-gate * Now try it's cachelists if this is a 41737c478bd9Sstevel@tonic-gate * small page. Don't need to do it for 41747c478bd9Sstevel@tonic-gate * larger ones since page_freelist_coalesce() 41757c478bd9Sstevel@tonic-gate * already failed. 41767c478bd9Sstevel@tonic-gate */ 41777c478bd9Sstevel@tonic-gate if (pplist != NULL || szc != 0) 41787c478bd9Sstevel@tonic-gate break; 41797c478bd9Sstevel@tonic-gate 41807c478bd9Sstevel@tonic-gate /* Now try it's cachelists */ 41817c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41827c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41837c478bd9Sstevel@tonic-gate 41847c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41857c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41867c478bd9Sstevel@tonic-gate != -1) { 4187ce8eb11aSdp pplist = 4188ce8eb11aSdp page_get_mnode_cachelist(bin, flags, 4189ce8eb11aSdp mnode, mtype); 41907c478bd9Sstevel@tonic-gate } 41917c478bd9Sstevel@tonic-gate if (pplist != NULL) { 41927c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 41937c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 41947c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 41957c478bd9Sstevel@tonic-gate break; 41967c478bd9Sstevel@tonic-gate } 41977c478bd9Sstevel@tonic-gate /* Done looking in this lgroup. Bail out. */ 41987c478bd9Sstevel@tonic-gate break; 41997c478bd9Sstevel@tonic-gate } 42007c478bd9Sstevel@tonic-gate 42017c478bd9Sstevel@tonic-gate /* 42022dae3fb5Sjjc * No lgroup was specified (or lgroup was removed by 42032dae3fb5Sjjc * DR, so just try to get the page as close to 42042dae3fb5Sjjc * like_pp's mnode as possible. 42057c478bd9Sstevel@tonic-gate * First try the local freelist... 42067c478bd9Sstevel@tonic-gate */ 42077c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(like_pp); 42087c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, bin, 42097c478bd9Sstevel@tonic-gate mtype, szc, flags); 42107c478bd9Sstevel@tonic-gate if (pplist != NULL) 42117c478bd9Sstevel@tonic-gate break; 42127c478bd9Sstevel@tonic-gate 42137c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnofree); 42147c478bd9Sstevel@tonic-gate 42157c478bd9Sstevel@tonic-gate /* 42167c478bd9Sstevel@tonic-gate * ...then the local cachelist. Don't need to do it for 42177c478bd9Sstevel@tonic-gate * larger pages cause page_freelist_coalesce() already 42187c478bd9Sstevel@tonic-gate * failed there anyway. 42197c478bd9Sstevel@tonic-gate */ 42207c478bd9Sstevel@tonic-gate if (szc == 0) { 42217c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, flags, 42227c478bd9Sstevel@tonic-gate mnode, mtype); 42237c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42247c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 42257c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 42267c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 42277c478bd9Sstevel@tonic-gate break; 42287c478bd9Sstevel@tonic-gate } 42297c478bd9Sstevel@tonic-gate } 42307c478bd9Sstevel@tonic-gate 42317c478bd9Sstevel@tonic-gate /* Now try remote freelists */ 42327c478bd9Sstevel@tonic-gate page_mnode = mnode; 42337c478bd9Sstevel@tonic-gate lgrp = 42347c478bd9Sstevel@tonic-gate lgrp_hand_to_lgrp(MEM_NODE_2_LGRPHAND(page_mnode)); 42357c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42367c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42377c478bd9Sstevel@tonic-gate while (pplist == NULL && 42387c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 42397c478bd9Sstevel@tonic-gate != -1) { 42407c478bd9Sstevel@tonic-gate /* 42417c478bd9Sstevel@tonic-gate * Skip local mnode. 42427c478bd9Sstevel@tonic-gate */ 42437c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 42447c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 42457c478bd9Sstevel@tonic-gate continue; 42467c478bd9Sstevel@tonic-gate 42477c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, 42487c478bd9Sstevel@tonic-gate bin, mtype, szc, flags); 42497c478bd9Sstevel@tonic-gate } 42507c478bd9Sstevel@tonic-gate 42517c478bd9Sstevel@tonic-gate if (pplist != NULL) 42527c478bd9Sstevel@tonic-gate break; 42537c478bd9Sstevel@tonic-gate 42547c478bd9Sstevel@tonic-gate 42557c478bd9Sstevel@tonic-gate /* Now try remote cachelists */ 42567c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42577c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42587c478bd9Sstevel@tonic-gate while (pplist == NULL && szc == 0) { 42597c478bd9Sstevel@tonic-gate mnode = lgrp_memnode_choose(&lgrp_cookie); 42607c478bd9Sstevel@tonic-gate if (mnode == -1) 42617c478bd9Sstevel@tonic-gate break; 42627c478bd9Sstevel@tonic-gate /* 42637c478bd9Sstevel@tonic-gate * Skip local mnode. 42647c478bd9Sstevel@tonic-gate */ 42657c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 42667c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 42677c478bd9Sstevel@tonic-gate continue; 42687c478bd9Sstevel@tonic-gate 42697c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, 42707c478bd9Sstevel@tonic-gate flags, mnode, mtype); 42717c478bd9Sstevel@tonic-gate 42727c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42737c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 42747c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 42757c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 42767c478bd9Sstevel@tonic-gate break; 42777c478bd9Sstevel@tonic-gate } 42787c478bd9Sstevel@tonic-gate } 42797c478bd9Sstevel@tonic-gate 42807c478bd9Sstevel@tonic-gate /* 42817c478bd9Sstevel@tonic-gate * Break out of while loop under the following cases: 42827c478bd9Sstevel@tonic-gate * - If we successfully got a page. 42837c478bd9Sstevel@tonic-gate * - If pgrflags specified only returning a specific 42847c478bd9Sstevel@tonic-gate * page size and we could not find that page size. 42857c478bd9Sstevel@tonic-gate * - If we could not satisfy the request with PAGESIZE 42867c478bd9Sstevel@tonic-gate * or larger pages. 42877c478bd9Sstevel@tonic-gate */ 42887c478bd9Sstevel@tonic-gate if (pplist != NULL || szc == 0) 42897c478bd9Sstevel@tonic-gate break; 42907c478bd9Sstevel@tonic-gate 42917c478bd9Sstevel@tonic-gate if ((pgrflags & PGR_SAMESZC) || pgrppgcp) { 42927c478bd9Sstevel@tonic-gate /* try to find contig page */ 42937c478bd9Sstevel@tonic-gate 42947c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42957c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42967c478bd9Sstevel@tonic-gate 42977c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 42987c478bd9Sstevel@tonic-gate (mnode = 4299ce8eb11aSdp lgrp_memnode_choose(&lgrp_cookie)) 43007c478bd9Sstevel@tonic-gate != -1) { 43017c478bd9Sstevel@tonic-gate pplist = page_get_contig_pages( 4302ce8eb11aSdp mnode, bin, mtype, szc, 4303ce8eb11aSdp flags | PGI_PGCPHIPRI); 43047c478bd9Sstevel@tonic-gate } 43057c478bd9Sstevel@tonic-gate break; 43067c478bd9Sstevel@tonic-gate } 43077c478bd9Sstevel@tonic-gate 43087c478bd9Sstevel@tonic-gate /* 43097c478bd9Sstevel@tonic-gate * The correct thing to do here is try the next 43107c478bd9Sstevel@tonic-gate * page size down using szc--. Due to a bug 43117c478bd9Sstevel@tonic-gate * with the processing of HAT_RELOAD_SHARE 43127c478bd9Sstevel@tonic-gate * where the sfmmu_ttecnt arrays of all 43137c478bd9Sstevel@tonic-gate * hats sharing an ISM segment don't get updated, 43147c478bd9Sstevel@tonic-gate * using intermediate size pages for relocation 43157c478bd9Sstevel@tonic-gate * can lead to continuous page faults. 43167c478bd9Sstevel@tonic-gate */ 43177c478bd9Sstevel@tonic-gate szc = 0; 43187c478bd9Sstevel@tonic-gate } 43197c478bd9Sstevel@tonic-gate 43207c478bd9Sstevel@tonic-gate if (pplist != NULL) { 43217c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 43227c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 43237c478bd9Sstevel@tonic-gate int, mnode, 43247c478bd9Sstevel@tonic-gate ulong_t, bin, 43257c478bd9Sstevel@tonic-gate uint_t, flags); 43267c478bd9Sstevel@tonic-gate 43277c478bd9Sstevel@tonic-gate while (pplist != NULL && pg_cnt--) { 43287c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 43297c478bd9Sstevel@tonic-gate pp = pplist; 43307c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 43317c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 43327c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 43337c478bd9Sstevel@tonic-gate page_list_concat(&pl, &pp); 43347c478bd9Sstevel@tonic-gate npgs--; 43357c478bd9Sstevel@tonic-gate like_pp = like_pp + 1; 43367c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnext_pp); 43377c478bd9Sstevel@tonic-gate } 43387c478bd9Sstevel@tonic-gate ASSERT(pg_cnt == 0); 43397c478bd9Sstevel@tonic-gate } else { 43407c478bd9Sstevel@tonic-gate break; 43417c478bd9Sstevel@tonic-gate } 43427c478bd9Sstevel@tonic-gate } 43437c478bd9Sstevel@tonic-gate 43447c478bd9Sstevel@tonic-gate if (npgs) { 43457c478bd9Sstevel@tonic-gate /* 43467c478bd9Sstevel@tonic-gate * We were unable to allocate the necessary number 43477c478bd9Sstevel@tonic-gate * of pages. 43487c478bd9Sstevel@tonic-gate * We need to free up any pl. 43497c478bd9Sstevel@tonic-gate */ 43507c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnopage); 43517c478bd9Sstevel@tonic-gate page_free_replacement_page(pl); 43527c478bd9Sstevel@tonic-gate return (NULL); 43537c478bd9Sstevel@tonic-gate } else { 43547c478bd9Sstevel@tonic-gate return (pl); 43557c478bd9Sstevel@tonic-gate } 43567c478bd9Sstevel@tonic-gate } 43577c478bd9Sstevel@tonic-gate 43587c478bd9Sstevel@tonic-gate /* 43597c478bd9Sstevel@tonic-gate * demote a free large page to it's constituent pages 43607c478bd9Sstevel@tonic-gate */ 43617c478bd9Sstevel@tonic-gate void 43627c478bd9Sstevel@tonic-gate page_demote_free_pages(page_t *pp) 43637c478bd9Sstevel@tonic-gate { 43647c478bd9Sstevel@tonic-gate 43657c478bd9Sstevel@tonic-gate int mnode; 43667c478bd9Sstevel@tonic-gate 43677c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 43687c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 43697c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 43707c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 43717c478bd9Sstevel@tonic-gate 43727c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 43737c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 43747c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 43757c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, 437619397407SSherry Moore pp->p_szc), 0, pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 43777c478bd9Sstevel@tonic-gate } 43787c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 43797c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 43807c478bd9Sstevel@tonic-gate } 4381932dc8e5Sdp 4382932dc8e5Sdp /* 4383932dc8e5Sdp * Factor in colorequiv to check additional 'equivalent' bins. 4384932dc8e5Sdp * colorequiv may be set in /etc/system 4385932dc8e5Sdp */ 4386932dc8e5Sdp void 4387932dc8e5Sdp page_set_colorequiv_arr(void) 4388932dc8e5Sdp { 4389932dc8e5Sdp if (colorequiv > 1) { 4390932dc8e5Sdp int i; 4391fe70c9cfSdp uint_t sv_a = lowbit(colorequiv) - 1; 4392932dc8e5Sdp 4393fe70c9cfSdp if (sv_a > 15) 4394fe70c9cfSdp sv_a = 15; 4395932dc8e5Sdp 4396932dc8e5Sdp for (i = 0; i < MMU_PAGE_SIZES; i++) { 4397fe70c9cfSdp uint_t colors; 4398fe70c9cfSdp uint_t a = sv_a; 4399932dc8e5Sdp 4400932dc8e5Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) { 4401932dc8e5Sdp continue; 4402932dc8e5Sdp } 4403932dc8e5Sdp while ((colors >> a) == 0) 4404932dc8e5Sdp a--; 4405932dc8e5Sdp if ((a << 4) > colorequivszc[i]) { 4406932dc8e5Sdp colorequivszc[i] = (a << 4); 4407932dc8e5Sdp } 4408932dc8e5Sdp } 4409932dc8e5Sdp } 4410932dc8e5Sdp } 4411