17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50b5aa17bSmec * Common Development and Distribution License (the "License"). 60b5aa17bSmec * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22a7c3ca36Sdp * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 317c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate 357c478bd9Sstevel@tonic-gate /* 367c478bd9Sstevel@tonic-gate * This file contains common functions to access and manage the page lists. 377c478bd9Sstevel@tonic-gate * Many of these routines originated from platform dependent modules 387c478bd9Sstevel@tonic-gate * (sun4/vm/vm_dep.c, i86pc/vm/vm_machdep.c) and modified to function in 397c478bd9Sstevel@tonic-gate * a platform independent manner. 407c478bd9Sstevel@tonic-gate * 417c478bd9Sstevel@tonic-gate * vm/vm_dep.h provides for platform specific support. 427c478bd9Sstevel@tonic-gate */ 437c478bd9Sstevel@tonic-gate 447c478bd9Sstevel@tonic-gate #include <sys/types.h> 457c478bd9Sstevel@tonic-gate #include <sys/debug.h> 467c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 477c478bd9Sstevel@tonic-gate #include <sys/systm.h> 487c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 497c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 507c478bd9Sstevel@tonic-gate #include <vm/as.h> 517c478bd9Sstevel@tonic-gate #include <vm/page.h> 527c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 537c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 5478b03d3aSkchow #include <sys/vmsystm.h> 557c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 567c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 577c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 587c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 597c478bd9Sstevel@tonic-gate #include <sys/callb.h> 607c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 617c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 627c478bd9Sstevel@tonic-gate 637c478bd9Sstevel@tonic-gate extern uint_t vac_colors; 647c478bd9Sstevel@tonic-gate 656061ce8aSkchow #define MAX_PRAGMA_ALIGN 128 666061ce8aSkchow 676061ce8aSkchow /* vm_cpu_data0 for the boot cpu before kmem is initialized */ 686061ce8aSkchow 696061ce8aSkchow #if L2CACHE_ALIGN_MAX <= MAX_PRAGMA_ALIGN 70affbd3ccSkchow #pragma align L2CACHE_ALIGN_MAX(vm_cpu_data0) 716061ce8aSkchow #else 726061ce8aSkchow #pragma align MAX_PRAGMA_ALIGN(vm_cpu_data0) 736061ce8aSkchow #endif 74affbd3ccSkchow char vm_cpu_data0[VM_CPU_DATA_PADSIZE]; 75affbd3ccSkchow 767c478bd9Sstevel@tonic-gate /* 777c478bd9Sstevel@tonic-gate * number of page colors equivalent to reqested color in page_get routines. 787c478bd9Sstevel@tonic-gate * If set, keeps large pages intact longer and keeps MPO allocation 797c478bd9Sstevel@tonic-gate * from the local mnode in favor of acquiring the 'correct' page color from 807c478bd9Sstevel@tonic-gate * a demoted large page or from a remote mnode. 817c478bd9Sstevel@tonic-gate */ 825d07b933Sdp uint_t colorequiv; 835d07b933Sdp 845d07b933Sdp /* 855d07b933Sdp * color equivalency mask for each page size. 865d07b933Sdp * Mask is computed based on cpu L2$ way sizes and colorequiv global. 875d07b933Sdp * High 4 bits determine the number of high order bits of the color to ignore. 885d07b933Sdp * Low 4 bits determines number of low order bits of color to ignore (it's only 895d07b933Sdp * relevant for hashed index based page coloring). 905d07b933Sdp */ 915d07b933Sdp uchar_t colorequivszc[MMU_PAGE_SIZES]; 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate /* 947c478bd9Sstevel@tonic-gate * if set, specifies the percentage of large pages that are free from within 957c478bd9Sstevel@tonic-gate * a large page region before attempting to lock those pages for 967c478bd9Sstevel@tonic-gate * page_get_contig_pages processing. 977c478bd9Sstevel@tonic-gate * 987c478bd9Sstevel@tonic-gate * Should be turned on when kpr is available when page_trylock_contig_pages 997c478bd9Sstevel@tonic-gate * can be more selective. 1007c478bd9Sstevel@tonic-gate */ 1017c478bd9Sstevel@tonic-gate 1027c478bd9Sstevel@tonic-gate int ptcpthreshold; 1037c478bd9Sstevel@tonic-gate 1047c478bd9Sstevel@tonic-gate /* 1057c478bd9Sstevel@tonic-gate * Limit page get contig page search based on failure cnts in pgcpfailcnt[]. 10683f9b804Skchow * Enabled by default via pgcplimitsearch. 10783f9b804Skchow * 10883f9b804Skchow * pgcpfailcnt[] is bounded by PGCPFAILMAX (>= 1/2 of installed 10983f9b804Skchow * memory). When reached, pgcpfailcnt[] is reset to 1/2 of this upper 11083f9b804Skchow * bound. This upper bound range guarantees: 11183f9b804Skchow * - all large page 'slots' will be searched over time 11283f9b804Skchow * - the minimum (1) large page candidates considered on each pgcp call 11383f9b804Skchow * - count doesn't wrap around to 0 1147c478bd9Sstevel@tonic-gate */ 11583f9b804Skchow pgcnt_t pgcpfailcnt[MMU_PAGE_SIZES]; 1167c478bd9Sstevel@tonic-gate int pgcplimitsearch = 1; 1177c478bd9Sstevel@tonic-gate 11883f9b804Skchow #define PGCPFAILMAX (1 << (highbit(physinstalled) - 1)) 11983f9b804Skchow #define SETPGCPFAILCNT(szc) \ 12083f9b804Skchow if (++pgcpfailcnt[szc] >= PGCPFAILMAX) \ 12183f9b804Skchow pgcpfailcnt[szc] = PGCPFAILMAX / 2; 12283f9b804Skchow 1237c478bd9Sstevel@tonic-gate #ifdef VM_STATS 1247c478bd9Sstevel@tonic-gate struct vmm_vmstats_str vmm_vmstats; 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 1277c478bd9Sstevel@tonic-gate 1287c478bd9Sstevel@tonic-gate #if defined(__sparc) 1297c478bd9Sstevel@tonic-gate #define LPGCREATE 0 1307c478bd9Sstevel@tonic-gate #else 1317c478bd9Sstevel@tonic-gate /* enable page_get_contig_pages */ 1327c478bd9Sstevel@tonic-gate #define LPGCREATE 1 1337c478bd9Sstevel@tonic-gate #endif 1347c478bd9Sstevel@tonic-gate 1357c478bd9Sstevel@tonic-gate int pg_contig_disable; 1367c478bd9Sstevel@tonic-gate int pg_lpgcreate_nocage = LPGCREATE; 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate /* 139*19397407SSherry Moore * page_freelist_split pfn flag to signify no lo or hi pfn requirement. 1407c478bd9Sstevel@tonic-gate */ 1417c478bd9Sstevel@tonic-gate #define PFNNULL 0 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate /* Flags involved in promotion and demotion routines */ 1447c478bd9Sstevel@tonic-gate #define PC_FREE 0x1 /* put page on freelist */ 1457c478bd9Sstevel@tonic-gate #define PC_ALLOC 0x2 /* return page for allocation */ 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate /* 1487c478bd9Sstevel@tonic-gate * Flag for page_demote to be used with PC_FREE to denote that we don't care 1497c478bd9Sstevel@tonic-gate * what the color is as the color parameter to the function is ignored. 1507c478bd9Sstevel@tonic-gate */ 1517c478bd9Sstevel@tonic-gate #define PC_NO_COLOR (-1) 1527c478bd9Sstevel@tonic-gate 1535d07b933Sdp /* mtype value for page_promote to use when mtype does not matter */ 1545d07b933Sdp #define PC_MTYPE_ANY (-1) 1555d07b933Sdp 1567c478bd9Sstevel@tonic-gate /* 1577c478bd9Sstevel@tonic-gate * page counters candidates info 1587c478bd9Sstevel@tonic-gate * See page_ctrs_cands comment below for more details. 1597c478bd9Sstevel@tonic-gate * fields are as follows: 1607c478bd9Sstevel@tonic-gate * pcc_pages_free: # pages which freelist coalesce can create 1617c478bd9Sstevel@tonic-gate * pcc_color_free: pointer to page free counts per color 1627c478bd9Sstevel@tonic-gate */ 1637c478bd9Sstevel@tonic-gate typedef struct pcc_info { 1647c478bd9Sstevel@tonic-gate pgcnt_t pcc_pages_free; 1657c478bd9Sstevel@tonic-gate pgcnt_t *pcc_color_free; 16606fb6a36Sdv uint_t pad[12]; 1677c478bd9Sstevel@tonic-gate } pcc_info_t; 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate /* 1707c478bd9Sstevel@tonic-gate * On big machines it can take a long time to check page_counters 1717c478bd9Sstevel@tonic-gate * arrays. page_ctrs_cands is a summary array whose elements are a dynamically 1727c478bd9Sstevel@tonic-gate * updated sum of all elements of the corresponding page_counters arrays. 1737c478bd9Sstevel@tonic-gate * page_freelist_coalesce() searches page_counters only if an appropriate 1747c478bd9Sstevel@tonic-gate * element of page_ctrs_cands array is greater than 0. 1757c478bd9Sstevel@tonic-gate * 1765d07b933Sdp * page_ctrs_cands is indexed by mutex (i), region (r), mnode (m), mrange (g) 1777c478bd9Sstevel@tonic-gate */ 1785d07b933Sdp pcc_info_t **page_ctrs_cands[NPC_MUTEX][MMU_PAGE_SIZES]; 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate /* 1817c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1825d07b933Sdp * for the given mnode (m), mrange (g), and region size (r) 1837c478bd9Sstevel@tonic-gate */ 1845d07b933Sdp #define PGCTRS_CANDS_GETVALUE(m, g, r, val) { \ 1857c478bd9Sstevel@tonic-gate int i; \ 1867c478bd9Sstevel@tonic-gate val = 0; \ 1877c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 1885d07b933Sdp val += page_ctrs_cands[i][(r)][(m)][(g)].pcc_pages_free; \ 1897c478bd9Sstevel@tonic-gate } \ 1907c478bd9Sstevel@tonic-gate } 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate /* 1937c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1945d07b933Sdp * for the given mnode (m), mrange (g), region size (r), and color (c) 1957c478bd9Sstevel@tonic-gate */ 1965d07b933Sdp #define PGCTRS_CANDS_GETVALUECOLOR(m, g, r, c, val) { \ 1977c478bd9Sstevel@tonic-gate int i; \ 1987c478bd9Sstevel@tonic-gate val = 0; \ 1995d07b933Sdp ASSERT((c) < PAGE_GET_PAGECOLORS(r)); \ 2007c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 2015d07b933Sdp val += \ 2025d07b933Sdp page_ctrs_cands[i][(r)][(m)][(g)].pcc_color_free[(c)]; \ 2037c478bd9Sstevel@tonic-gate } \ 2047c478bd9Sstevel@tonic-gate } 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate /* 2077c478bd9Sstevel@tonic-gate * We can only allow a single thread to update a counter within the physical 2087c478bd9Sstevel@tonic-gate * range of the largest supported page size. That is the finest granularity 2097c478bd9Sstevel@tonic-gate * possible since the counter values are dependent on each other 2107c478bd9Sstevel@tonic-gate * as you move accross region sizes. PP_CTR_LOCK_INDX is used to determine the 2117c478bd9Sstevel@tonic-gate * ctr_mutex lock index for a particular physical range. 2127c478bd9Sstevel@tonic-gate */ 2137c478bd9Sstevel@tonic-gate static kmutex_t *ctr_mutex[NPC_MUTEX]; 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate #define PP_CTR_LOCK_INDX(pp) \ 2165d07b933Sdp (((pp)->p_pagenum >> \ 2177c478bd9Sstevel@tonic-gate (PAGE_BSZS_SHIFT(mmu_page_sizes - 1))) & (NPC_MUTEX - 1)) 2187c478bd9Sstevel@tonic-gate 2195d07b933Sdp #define INVALID_COLOR 0xffffffff 2205d07b933Sdp #define INVALID_MASK 0xffffffff 2215d07b933Sdp 2227c478bd9Sstevel@tonic-gate /* 2237c478bd9Sstevel@tonic-gate * Local functions prototypes. 2247c478bd9Sstevel@tonic-gate */ 2257c478bd9Sstevel@tonic-gate 226affbd3ccSkchow void page_ctr_add(int, int, page_t *, int); 227affbd3ccSkchow void page_ctr_add_internal(int, int, page_t *, int); 228affbd3ccSkchow void page_ctr_sub(int, int, page_t *, int); 2295d07b933Sdp void page_ctr_sub_internal(int, int, page_t *, int); 2307c478bd9Sstevel@tonic-gate void page_freelist_lock(int); 2317c478bd9Sstevel@tonic-gate void page_freelist_unlock(int); 2325d07b933Sdp page_t *page_promote(int, pfn_t, uchar_t, int, int); 233*19397407SSherry Moore page_t *page_demote(int, pfn_t, pfn_t, uchar_t, uchar_t, int, int); 2345d07b933Sdp page_t *page_freelist_split(uchar_t, 235*19397407SSherry Moore uint_t, int, int, pfn_t, pfn_t, page_list_walker_t *); 2367c478bd9Sstevel@tonic-gate page_t *page_get_mnode_cachelist(uint_t, uint_t, int, int); 2377c478bd9Sstevel@tonic-gate static int page_trylock_cons(page_t *pp, se_t se); 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * The page_counters array below is used to keep track of free contiguous 2417c478bd9Sstevel@tonic-gate * physical memory. A hw_page_map_t will be allocated per mnode per szc. 2427c478bd9Sstevel@tonic-gate * This contains an array of counters, the size of the array, a shift value 2437c478bd9Sstevel@tonic-gate * used to convert a pagenum into a counter array index or vice versa, as 2447c478bd9Sstevel@tonic-gate * well as a cache of the last successful index to be promoted to a larger 2457c478bd9Sstevel@tonic-gate * page size. As an optimization, we keep track of the last successful index 2467c478bd9Sstevel@tonic-gate * to be promoted per page color for the given size region, and this is 2477c478bd9Sstevel@tonic-gate * allocated dynamically based upon the number of colors for a given 2487c478bd9Sstevel@tonic-gate * region size. 2497c478bd9Sstevel@tonic-gate * 2507c478bd9Sstevel@tonic-gate * Conceptually, the page counters are represented as: 2517c478bd9Sstevel@tonic-gate * 2527c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode] 2537c478bd9Sstevel@tonic-gate * 2547c478bd9Sstevel@tonic-gate * region_size: size code of a candidate larger page made up 2557c478bd9Sstevel@tonic-gate * of contiguous free smaller pages. 2567c478bd9Sstevel@tonic-gate * 2577c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode].hpm_counters[index]: 2587c478bd9Sstevel@tonic-gate * represents how many (region_size - 1) pages either 2597c478bd9Sstevel@tonic-gate * exist or can be created within the given index range. 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * Let's look at a sparc example: 2627c478bd9Sstevel@tonic-gate * If we want to create a free 512k page, we look at region_size 2 2637c478bd9Sstevel@tonic-gate * for the mnode we want. We calculate the index and look at a specific 2647c478bd9Sstevel@tonic-gate * hpm_counters location. If we see 8 (FULL_REGION_CNT on sparc) at 2657c478bd9Sstevel@tonic-gate * this location, it means that 8 64k pages either exist or can be created 2667c478bd9Sstevel@tonic-gate * from 8K pages in order to make a single free 512k page at the given 2677c478bd9Sstevel@tonic-gate * index. Note that when a region is full, it will contribute to the 2687c478bd9Sstevel@tonic-gate * counts in the region above it. Thus we will not know what page 2697c478bd9Sstevel@tonic-gate * size the free pages will be which can be promoted to this new free 2707c478bd9Sstevel@tonic-gate * page unless we look at all regions below the current region. 2717c478bd9Sstevel@tonic-gate */ 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate /* 2747c478bd9Sstevel@tonic-gate * Note: hpmctr_t is defined in platform vm_dep.h 2757c478bd9Sstevel@tonic-gate * hw_page_map_t contains all the information needed for the page_counters 2767c478bd9Sstevel@tonic-gate * logic. The fields are as follows: 2777c478bd9Sstevel@tonic-gate * 2787c478bd9Sstevel@tonic-gate * hpm_counters: dynamically allocated array to hold counter data 2797c478bd9Sstevel@tonic-gate * hpm_entries: entries in hpm_counters 2807c478bd9Sstevel@tonic-gate * hpm_shift: shift for pnum/array index conv 2817c478bd9Sstevel@tonic-gate * hpm_base: PFN mapped to counter index 0 2827c478bd9Sstevel@tonic-gate * hpm_color_current: last index in counter array for this color at 2837c478bd9Sstevel@tonic-gate * which we successfully created a large page 2847c478bd9Sstevel@tonic-gate */ 2857c478bd9Sstevel@tonic-gate typedef struct hw_page_map { 2867c478bd9Sstevel@tonic-gate hpmctr_t *hpm_counters; 2877c478bd9Sstevel@tonic-gate size_t hpm_entries; 2887c478bd9Sstevel@tonic-gate int hpm_shift; 2897c478bd9Sstevel@tonic-gate pfn_t hpm_base; 2905d07b933Sdp size_t *hpm_color_current[MAX_MNODE_MRANGES]; 29106fb6a36Sdv #if defined(__sparc) 29206fb6a36Sdv uint_t pad[4]; 29306fb6a36Sdv #endif 2947c478bd9Sstevel@tonic-gate } hw_page_map_t; 2957c478bd9Sstevel@tonic-gate 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * Element zero is not used, but is allocated for convenience. 2987c478bd9Sstevel@tonic-gate */ 2997c478bd9Sstevel@tonic-gate static hw_page_map_t *page_counters[MMU_PAGE_SIZES]; 3007c478bd9Sstevel@tonic-gate 3015d07b933Sdp /* 3025d07b933Sdp * Cached value of MNODE_RANGE_CNT(mnode). 3035d07b933Sdp * This is a function call in x86. 3045d07b933Sdp */ 3055d07b933Sdp static int mnode_nranges[MAX_MEM_NODES]; 3065d07b933Sdp static int mnode_maxmrange[MAX_MEM_NODES]; 3075d07b933Sdp 3087c478bd9Sstevel@tonic-gate /* 3097c478bd9Sstevel@tonic-gate * The following macros are convenient ways to get access to the individual 3107c478bd9Sstevel@tonic-gate * elements of the page_counters arrays. They can be used on both 3117c478bd9Sstevel@tonic-gate * the left side and right side of equations. 3127c478bd9Sstevel@tonic-gate */ 3137c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS(mnode, rg_szc, idx) \ 3147c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters[(idx)]) 3157c478bd9Sstevel@tonic-gate 3167c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_COUNTERS(mnode, rg_szc) \ 3177c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters) 3187c478bd9Sstevel@tonic-gate 3197c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_SHIFT(mnode, rg_szc) \ 3207c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_shift) 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_ENTRIES(mnode, rg_szc) \ 3237c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_entries) 3247c478bd9Sstevel@tonic-gate 3257c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_BASE(mnode, rg_szc) \ 3267c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_base) 3277c478bd9Sstevel@tonic-gate 3285d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, rg_szc, g) \ 3295d07b933Sdp (page_counters[(rg_szc)][(mnode)].hpm_color_current[(g)]) 3307c478bd9Sstevel@tonic-gate 3315d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR(mnode, rg_szc, color, mrange) \ 3325d07b933Sdp (page_counters[(rg_szc)][(mnode)]. \ 3335d07b933Sdp hpm_color_current[(mrange)][(color)]) 3347c478bd9Sstevel@tonic-gate 3357c478bd9Sstevel@tonic-gate #define PNUM_TO_IDX(mnode, rg_szc, pnum) \ 3367c478bd9Sstevel@tonic-gate (((pnum) - PAGE_COUNTERS_BASE((mnode), (rg_szc))) >> \ 3377c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT((mnode), (rg_szc))) 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate #define IDX_TO_PNUM(mnode, rg_szc, index) \ 3407c478bd9Sstevel@tonic-gate (PAGE_COUNTERS_BASE((mnode), (rg_szc)) + \ 3417c478bd9Sstevel@tonic-gate ((index) << PAGE_COUNTERS_SHIFT((mnode), (rg_szc)))) 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate /* 3447c478bd9Sstevel@tonic-gate * Protects the hpm_counters and hpm_color_current memory from changing while 3457c478bd9Sstevel@tonic-gate * looking at page counters information. 3467c478bd9Sstevel@tonic-gate * Grab the write lock to modify what these fields point at. 3477c478bd9Sstevel@tonic-gate * Grab the read lock to prevent any pointers from changing. 3487c478bd9Sstevel@tonic-gate * The write lock can not be held during memory allocation due to a possible 3497c478bd9Sstevel@tonic-gate * recursion deadlock with trying to grab the read lock while the 3507c478bd9Sstevel@tonic-gate * write lock is already held. 3517c478bd9Sstevel@tonic-gate */ 3527c478bd9Sstevel@tonic-gate krwlock_t page_ctrs_rwlock[MAX_MEM_NODES]; 3537c478bd9Sstevel@tonic-gate 354affbd3ccSkchow 355affbd3ccSkchow /* 356affbd3ccSkchow * initialize cpu_vm_data to point at cache aligned vm_cpu_data_t. 357affbd3ccSkchow */ 358affbd3ccSkchow void 359affbd3ccSkchow cpu_vm_data_init(struct cpu *cp) 360affbd3ccSkchow { 361affbd3ccSkchow if (cp == CPU0) { 362affbd3ccSkchow cp->cpu_vm_data = (void *)&vm_cpu_data0; 363affbd3ccSkchow } else { 364affbd3ccSkchow void *kmptr; 3656061ce8aSkchow int align; 3666061ce8aSkchow size_t sz; 367affbd3ccSkchow 3686061ce8aSkchow align = (L2CACHE_ALIGN) ? L2CACHE_ALIGN : L2CACHE_ALIGN_MAX; 3696061ce8aSkchow sz = P2ROUNDUP(sizeof (vm_cpu_data_t), align) + align; 3706061ce8aSkchow kmptr = kmem_zalloc(sz, KM_SLEEP); 371affbd3ccSkchow cp->cpu_vm_data = (void *) P2ROUNDUP((uintptr_t)kmptr, align); 372affbd3ccSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr = kmptr; 3736061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize = sz; 374affbd3ccSkchow } 375affbd3ccSkchow } 376affbd3ccSkchow 377affbd3ccSkchow /* 378affbd3ccSkchow * free cpu_vm_data 379affbd3ccSkchow */ 380affbd3ccSkchow void 381affbd3ccSkchow cpu_vm_data_destroy(struct cpu *cp) 382affbd3ccSkchow { 383affbd3ccSkchow if (cp->cpu_seqid && cp->cpu_vm_data) { 384affbd3ccSkchow ASSERT(cp != CPU0); 385affbd3ccSkchow kmem_free(((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr, 3866061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize); 387affbd3ccSkchow } 388affbd3ccSkchow cp->cpu_vm_data = NULL; 389affbd3ccSkchow } 390affbd3ccSkchow 391affbd3ccSkchow 3927c478bd9Sstevel@tonic-gate /* 3937c478bd9Sstevel@tonic-gate * page size to page size code 3947c478bd9Sstevel@tonic-gate */ 3957c478bd9Sstevel@tonic-gate int 3967c478bd9Sstevel@tonic-gate page_szc(size_t pagesize) 3977c478bd9Sstevel@tonic-gate { 3987c478bd9Sstevel@tonic-gate int i = 0; 3997c478bd9Sstevel@tonic-gate 4007c478bd9Sstevel@tonic-gate while (hw_page_array[i].hp_size) { 4017c478bd9Sstevel@tonic-gate if (pagesize == hw_page_array[i].hp_size) 4027c478bd9Sstevel@tonic-gate return (i); 4037c478bd9Sstevel@tonic-gate i++; 4047c478bd9Sstevel@tonic-gate } 4057c478bd9Sstevel@tonic-gate return (-1); 4067c478bd9Sstevel@tonic-gate } 4077c478bd9Sstevel@tonic-gate 4087c478bd9Sstevel@tonic-gate /* 4094abce959Smec * page size to page size code with the restriction that it be a supported 4104abce959Smec * user page size. If it's not a supported user page size, -1 will be returned. 4117c478bd9Sstevel@tonic-gate */ 4127c478bd9Sstevel@tonic-gate int 4134abce959Smec page_szc_user_filtered(size_t pagesize) 4147c478bd9Sstevel@tonic-gate { 4157c478bd9Sstevel@tonic-gate int szc = page_szc(pagesize); 4164abce959Smec if ((szc != -1) && (SZC_2_USERSZC(szc) != -1)) { 4174abce959Smec return (szc); 4184abce959Smec } 4197c478bd9Sstevel@tonic-gate return (-1); 4207c478bd9Sstevel@tonic-gate } 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate /* 4237c478bd9Sstevel@tonic-gate * Return how many page sizes are available for the user to use. This is 4247c478bd9Sstevel@tonic-gate * what the hardware supports and not based upon how the OS implements the 4257c478bd9Sstevel@tonic-gate * support of different page sizes. 42602bc52beSkchow * 42702bc52beSkchow * If legacy is non-zero, return the number of pagesizes available to legacy 42802bc52beSkchow * applications. The number of legacy page sizes might be less than the 42902bc52beSkchow * exported user page sizes. This is to prevent legacy applications that 43002bc52beSkchow * use the largest page size returned from getpagesizes(3c) from inadvertantly 43102bc52beSkchow * using the 'new' large pagesizes. 4327c478bd9Sstevel@tonic-gate */ 4337c478bd9Sstevel@tonic-gate uint_t 43402bc52beSkchow page_num_user_pagesizes(int legacy) 4357c478bd9Sstevel@tonic-gate { 43602bc52beSkchow if (legacy) 43702bc52beSkchow return (mmu_legacy_page_sizes); 4387c478bd9Sstevel@tonic-gate return (mmu_exported_page_sizes); 4397c478bd9Sstevel@tonic-gate } 4407c478bd9Sstevel@tonic-gate 4417c478bd9Sstevel@tonic-gate uint_t 4427c478bd9Sstevel@tonic-gate page_num_pagesizes(void) 4437c478bd9Sstevel@tonic-gate { 4447c478bd9Sstevel@tonic-gate return (mmu_page_sizes); 4457c478bd9Sstevel@tonic-gate } 4467c478bd9Sstevel@tonic-gate 4477c478bd9Sstevel@tonic-gate /* 4487c478bd9Sstevel@tonic-gate * returns the count of the number of base pagesize pages associated with szc 4497c478bd9Sstevel@tonic-gate */ 4507c478bd9Sstevel@tonic-gate pgcnt_t 4517c478bd9Sstevel@tonic-gate page_get_pagecnt(uint_t szc) 4527c478bd9Sstevel@tonic-gate { 4537c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4547c478bd9Sstevel@tonic-gate panic("page_get_pagecnt: out of range %d", szc); 4557c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_pgcnt); 4567c478bd9Sstevel@tonic-gate } 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate size_t 4597c478bd9Sstevel@tonic-gate page_get_pagesize(uint_t szc) 4607c478bd9Sstevel@tonic-gate { 4617c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4627c478bd9Sstevel@tonic-gate panic("page_get_pagesize: out of range %d", szc); 4637c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4647c478bd9Sstevel@tonic-gate } 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate /* 4677c478bd9Sstevel@tonic-gate * Return the size of a page based upon the index passed in. An index of 4687c478bd9Sstevel@tonic-gate * zero refers to the smallest page size in the system, and as index increases 4697c478bd9Sstevel@tonic-gate * it refers to the next larger supported page size in the system. 4707c478bd9Sstevel@tonic-gate * Note that szc and userszc may not be the same due to unsupported szc's on 4717c478bd9Sstevel@tonic-gate * some systems. 4727c478bd9Sstevel@tonic-gate */ 4737c478bd9Sstevel@tonic-gate size_t 4747c478bd9Sstevel@tonic-gate page_get_user_pagesize(uint_t userszc) 4757c478bd9Sstevel@tonic-gate { 4767c478bd9Sstevel@tonic-gate uint_t szc = USERSZC_2_SZC(userszc); 4777c478bd9Sstevel@tonic-gate 4787c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4797c478bd9Sstevel@tonic-gate panic("page_get_user_pagesize: out of range %d", szc); 4807c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4817c478bd9Sstevel@tonic-gate } 4827c478bd9Sstevel@tonic-gate 4837c478bd9Sstevel@tonic-gate uint_t 4847c478bd9Sstevel@tonic-gate page_get_shift(uint_t szc) 4857c478bd9Sstevel@tonic-gate { 4867c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4877c478bd9Sstevel@tonic-gate panic("page_get_shift: out of range %d", szc); 4885d07b933Sdp return (PAGE_GET_SHIFT(szc)); 4897c478bd9Sstevel@tonic-gate } 4907c478bd9Sstevel@tonic-gate 4917c478bd9Sstevel@tonic-gate uint_t 4927c478bd9Sstevel@tonic-gate page_get_pagecolors(uint_t szc) 4937c478bd9Sstevel@tonic-gate { 4945d07b933Sdp if (szc >= mmu_page_sizes) 4955d07b933Sdp panic("page_get_pagecolors: out of range %d", szc); 4965d07b933Sdp return (PAGE_GET_PAGECOLORS(szc)); 4975d07b933Sdp } 4985d07b933Sdp 4995d07b933Sdp /* 5005d07b933Sdp * this assigns the desired equivalent color after a split 5015d07b933Sdp */ 5025d07b933Sdp uint_t 5035d07b933Sdp page_correct_color(uchar_t szc, uchar_t nszc, uint_t color, 5045d07b933Sdp uint_t ncolor, uint_t ceq_mask) 5055d07b933Sdp { 5065d07b933Sdp ASSERT(nszc > szc); 5075d07b933Sdp ASSERT(szc < mmu_page_sizes); 5085d07b933Sdp ASSERT(color < PAGE_GET_PAGECOLORS(szc)); 5095d07b933Sdp ASSERT(ncolor < PAGE_GET_PAGECOLORS(nszc)); 5105d07b933Sdp 5115d07b933Sdp color &= ceq_mask; 512ce8eb11aSdp ncolor = PAGE_CONVERT_COLOR(ncolor, szc, nszc); 5135d07b933Sdp return (color | (ncolor & ~ceq_mask)); 5147c478bd9Sstevel@tonic-gate } 5157c478bd9Sstevel@tonic-gate 516ce8eb11aSdp /* 517ce8eb11aSdp * The interleaved_mnodes flag is set when mnodes overlap in 518ce8eb11aSdp * the physbase..physmax range, but have disjoint slices. 519ce8eb11aSdp * In this case hpm_counters is shared by all mnodes. 520ce8eb11aSdp * This flag is set dynamically by the platform. 521ce8eb11aSdp */ 522ce8eb11aSdp int interleaved_mnodes = 0; 523ce8eb11aSdp 5247c478bd9Sstevel@tonic-gate /* 5257c478bd9Sstevel@tonic-gate * Called by startup(). 5267c478bd9Sstevel@tonic-gate * Size up the per page size free list counters based on physmax 5277c478bd9Sstevel@tonic-gate * of each node and max_mem_nodes. 528ce8eb11aSdp * 529ce8eb11aSdp * If interleaved_mnodes is set we need to find the first mnode that 530ce8eb11aSdp * exists. hpm_counters for the first mnode will then be shared by 531ce8eb11aSdp * all other mnodes. If interleaved_mnodes is not set, just set 532ce8eb11aSdp * first=mnode each time. That means there will be no sharing. 5337c478bd9Sstevel@tonic-gate */ 5347c478bd9Sstevel@tonic-gate size_t 5357c478bd9Sstevel@tonic-gate page_ctrs_sz(void) 5367c478bd9Sstevel@tonic-gate { 5377c478bd9Sstevel@tonic-gate int r; /* region size */ 5387c478bd9Sstevel@tonic-gate int mnode; 539ce8eb11aSdp int firstmn; /* first mnode that exists */ 5405d07b933Sdp int nranges; 541ce8eb11aSdp pfn_t physbase; 542ce8eb11aSdp pfn_t physmax; 5437c478bd9Sstevel@tonic-gate uint_t ctrs_sz = 0; 5447c478bd9Sstevel@tonic-gate int i; 5457c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 5467c478bd9Sstevel@tonic-gate 5477c478bd9Sstevel@tonic-gate /* 5487c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 5497c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 5507c478bd9Sstevel@tonic-gate * arrays. 5517c478bd9Sstevel@tonic-gate */ 5525d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 5535d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 5547c478bd9Sstevel@tonic-gate } 5557c478bd9Sstevel@tonic-gate 556ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 5577c478bd9Sstevel@tonic-gate 5587c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 5597c478bd9Sstevel@tonic-gate pfn_t r_base; 5607c478bd9Sstevel@tonic-gate pgcnt_t r_align; 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 5637c478bd9Sstevel@tonic-gate continue; 5647c478bd9Sstevel@tonic-gate 565ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 5665d07b933Sdp nranges = MNODE_RANGE_CNT(mnode); 5675d07b933Sdp mnode_nranges[mnode] = nranges; 5685d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 5695d07b933Sdp 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * determine size needed for page counter arrays with 5727c478bd9Sstevel@tonic-gate * base aligned to large page size. 5737c478bd9Sstevel@tonic-gate */ 5747c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 575ce8eb11aSdp /* add in space for hpm_color_current */ 576ce8eb11aSdp ctrs_sz += sizeof (size_t) * 577ce8eb11aSdp colors_per_szc[r] * nranges; 578ce8eb11aSdp 579ce8eb11aSdp if (firstmn != mnode) 580ce8eb11aSdp continue; 581ce8eb11aSdp 5827c478bd9Sstevel@tonic-gate /* add in space for hpm_counters */ 5837c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 584ce8eb11aSdp r_base = physbase; 5857c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 586ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 587ce8eb11aSdp 5887c478bd9Sstevel@tonic-gate /* 5897c478bd9Sstevel@tonic-gate * Round up to always allocate on pointer sized 5907c478bd9Sstevel@tonic-gate * boundaries. 5917c478bd9Sstevel@tonic-gate */ 5927c478bd9Sstevel@tonic-gate ctrs_sz += P2ROUNDUP((r_pgcnt * sizeof (hpmctr_t)), 5937c478bd9Sstevel@tonic-gate sizeof (hpmctr_t *)); 5947c478bd9Sstevel@tonic-gate } 5957c478bd9Sstevel@tonic-gate } 5967c478bd9Sstevel@tonic-gate 5977c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 5987c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * sizeof (hw_page_map_t)); 5995d07b933Sdp } 6005d07b933Sdp 6015d07b933Sdp /* add in space for page_ctrs_cands and pcc_color_free */ 6025d07b933Sdp ctrs_sz += sizeof (pcc_info_t *) * max_mem_nodes * 6035d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6045d07b933Sdp 6055d07b933Sdp for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6065d07b933Sdp 6075d07b933Sdp if (mem_node_config[mnode].exists == 0) 6085d07b933Sdp continue; 6097c478bd9Sstevel@tonic-gate 6105d07b933Sdp nranges = mnode_nranges[mnode]; 6115d07b933Sdp ctrs_sz += sizeof (pcc_info_t) * nranges * 6125d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6135d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6145d07b933Sdp ctrs_sz += sizeof (pgcnt_t) * nranges * 6155d07b933Sdp colors_per_szc[r] * NPC_MUTEX; 6165d07b933Sdp } 6177c478bd9Sstevel@tonic-gate } 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6207c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * NPC_MUTEX * sizeof (kmutex_t)); 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate /* size for page list counts */ 6237c478bd9Sstevel@tonic-gate PLCNT_SZ(ctrs_sz); 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate /* 6267c478bd9Sstevel@tonic-gate * add some slop for roundups. page_ctrs_alloc will roundup the start 6277c478bd9Sstevel@tonic-gate * address of the counters to ecache_alignsize boundary for every 6287c478bd9Sstevel@tonic-gate * memory node. 6297c478bd9Sstevel@tonic-gate */ 6307c478bd9Sstevel@tonic-gate return (ctrs_sz + max_mem_nodes * L2CACHE_ALIGN); 6317c478bd9Sstevel@tonic-gate } 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate caddr_t 6347c478bd9Sstevel@tonic-gate page_ctrs_alloc(caddr_t alloc_base) 6357c478bd9Sstevel@tonic-gate { 6367c478bd9Sstevel@tonic-gate int mnode; 6375d07b933Sdp int mrange, nranges; 6387c478bd9Sstevel@tonic-gate int r; /* region size */ 6397c478bd9Sstevel@tonic-gate int i; 640ce8eb11aSdp int firstmn; /* first mnode that exists */ 641ce8eb11aSdp pfn_t physbase; 642ce8eb11aSdp pfn_t physmax; 6437c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 6447c478bd9Sstevel@tonic-gate 6457c478bd9Sstevel@tonic-gate /* 6467c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 6477c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 6487c478bd9Sstevel@tonic-gate * arrays. 6497c478bd9Sstevel@tonic-gate */ 6505d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 6515d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 6527c478bd9Sstevel@tonic-gate } 6537c478bd9Sstevel@tonic-gate 6547c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 6557c478bd9Sstevel@tonic-gate page_counters[r] = (hw_page_map_t *)alloc_base; 6567c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (hw_page_map_t)); 6577c478bd9Sstevel@tonic-gate } 6587c478bd9Sstevel@tonic-gate 6595d07b933Sdp /* page_ctrs_cands and pcc_color_free array */ 6605d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 6615d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6627c478bd9Sstevel@tonic-gate 6635d07b933Sdp page_ctrs_cands[i][r] = (pcc_info_t **)alloc_base; 6645d07b933Sdp alloc_base += sizeof (pcc_info_t *) * max_mem_nodes; 6657c478bd9Sstevel@tonic-gate 6667c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6675d07b933Sdp pcc_info_t *pi; 6685d07b933Sdp 6695d07b933Sdp if (mem_node_config[mnode].exists == 0) 6705d07b933Sdp continue; 6715d07b933Sdp 6725d07b933Sdp nranges = mnode_nranges[mnode]; 6735d07b933Sdp 6745d07b933Sdp pi = (pcc_info_t *)alloc_base; 6755d07b933Sdp alloc_base += sizeof (pcc_info_t) * nranges; 6765d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 6775d07b933Sdp 6785d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 6795d07b933Sdp pi->pcc_color_free = 6805d07b933Sdp (pgcnt_t *)alloc_base; 6815d07b933Sdp alloc_base += sizeof (pgcnt_t) * 6825d07b933Sdp colors_per_szc[r]; 6835d07b933Sdp pi++; 6845d07b933Sdp } 6857c478bd9Sstevel@tonic-gate } 6867c478bd9Sstevel@tonic-gate } 6877c478bd9Sstevel@tonic-gate } 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6907c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 6917c478bd9Sstevel@tonic-gate ctr_mutex[i] = (kmutex_t *)alloc_base; 6927c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (kmutex_t)); 6937c478bd9Sstevel@tonic-gate } 6947c478bd9Sstevel@tonic-gate 6957c478bd9Sstevel@tonic-gate /* initialize page list counts */ 6967c478bd9Sstevel@tonic-gate PLCNT_INIT(alloc_base); 6977c478bd9Sstevel@tonic-gate 698ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 6997c478bd9Sstevel@tonic-gate 7007c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 7017c478bd9Sstevel@tonic-gate pfn_t r_base; 7027c478bd9Sstevel@tonic-gate pgcnt_t r_align; 7037c478bd9Sstevel@tonic-gate int r_shift; 7045d07b933Sdp int nranges = mnode_nranges[mnode]; 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 7077c478bd9Sstevel@tonic-gate continue; 7087c478bd9Sstevel@tonic-gate 709ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 710ce8eb11aSdp 7117c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 7127c478bd9Sstevel@tonic-gate /* 7137c478bd9Sstevel@tonic-gate * the page_counters base has to be aligned to the 7147c478bd9Sstevel@tonic-gate * page count of page size code r otherwise the counts 7157c478bd9Sstevel@tonic-gate * will cross large page boundaries. 7167c478bd9Sstevel@tonic-gate */ 7177c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 718ce8eb11aSdp r_base = physbase; 7197c478bd9Sstevel@tonic-gate /* base needs to be aligned - lower to aligned value */ 7207c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 721ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 7227c478bd9Sstevel@tonic-gate r_shift = PAGE_BSZS_SHIFT(r); 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = r_shift; 7257c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = r_pgcnt; 7267c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = r_base; 7275d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7285d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 7295d07b933Sdp r, mrange) = (size_t *)alloc_base; 7305d07b933Sdp alloc_base += sizeof (size_t) * 7315d07b933Sdp colors_per_szc[r]; 7325d07b933Sdp } 7337c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 7345d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 7355d07b933Sdp pfn_t pfnum = r_base; 7365d07b933Sdp size_t idx; 7375d07b933Sdp int mrange; 738ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 7395d07b933Sdp 740b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, r, &it); 741b779d3e0Sdp if (pfnum == (pfn_t)-1) { 742b779d3e0Sdp idx = 0; 743b779d3e0Sdp } else { 744b779d3e0Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 745b779d3e0Sdp color_mask, color_mask, &it); 746b779d3e0Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 747b779d3e0Sdp idx = (idx >= r_pgcnt) ? 0 : idx; 748b779d3e0Sdp } 7495d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7505d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, 7515d07b933Sdp r, i, mrange) = idx; 7525d07b933Sdp } 7537c478bd9Sstevel@tonic-gate } 754ce8eb11aSdp 755ce8eb11aSdp /* hpm_counters may be shared by all mnodes */ 756ce8eb11aSdp if (firstmn == mnode) { 757ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 758ce8eb11aSdp (hpmctr_t *)alloc_base; 759ce8eb11aSdp alloc_base += 760ce8eb11aSdp P2ROUNDUP((sizeof (hpmctr_t) * r_pgcnt), 761ce8eb11aSdp sizeof (hpmctr_t *)); 762ce8eb11aSdp } else { 763ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 764ce8eb11aSdp PAGE_COUNTERS_COUNTERS(firstmn, r); 765ce8eb11aSdp } 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate /* 7687c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 7697c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 7707c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 7717c478bd9Sstevel@tonic-gate * and get consistent values. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 7747c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 7757c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 7767c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, r_base))) == r_base); 7777c478bd9Sstevel@tonic-gate } 7787c478bd9Sstevel@tonic-gate /* 7797c478bd9Sstevel@tonic-gate * Roundup the start address of the page_counters to 7807c478bd9Sstevel@tonic-gate * cache aligned boundary for every memory node. 7817c478bd9Sstevel@tonic-gate * page_ctrs_sz() has added some slop for these roundups. 7827c478bd9Sstevel@tonic-gate */ 7837c478bd9Sstevel@tonic-gate alloc_base = (caddr_t)P2ROUNDUP((uintptr_t)alloc_base, 784ce8eb11aSdp L2CACHE_ALIGN); 7857c478bd9Sstevel@tonic-gate } 7867c478bd9Sstevel@tonic-gate 7877c478bd9Sstevel@tonic-gate /* Initialize other page counter specific data structures. */ 7887c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < MAX_MEM_NODES; mnode++) { 7897c478bd9Sstevel@tonic-gate rw_init(&page_ctrs_rwlock[mnode], NULL, RW_DEFAULT, NULL); 7907c478bd9Sstevel@tonic-gate } 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate return (alloc_base); 7937c478bd9Sstevel@tonic-gate } 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate /* 7967c478bd9Sstevel@tonic-gate * Functions to adjust region counters for each size free list. 7977c478bd9Sstevel@tonic-gate * Caller is responsible to acquire the ctr_mutex lock if necessary and 7987c478bd9Sstevel@tonic-gate * thus can be called during startup without locks. 7997c478bd9Sstevel@tonic-gate */ 8007c478bd9Sstevel@tonic-gate /* ARGSUSED */ 8017c478bd9Sstevel@tonic-gate void 802affbd3ccSkchow page_ctr_add_internal(int mnode, int mtype, page_t *pp, int flags) 8037c478bd9Sstevel@tonic-gate { 8047c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8057c478bd9Sstevel@tonic-gate ssize_t idx; 8067c478bd9Sstevel@tonic-gate pfn_t pfnum; 8077c478bd9Sstevel@tonic-gate int lckidx; 8087c478bd9Sstevel@tonic-gate 809affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 810affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 811affbd3ccSkchow 8127c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8137c478bd9Sstevel@tonic-gate 814affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 8157c478bd9Sstevel@tonic-gate 8167c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8177c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8187c478bd9Sstevel@tonic-gate return; 8197c478bd9Sstevel@tonic-gate } 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8227c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8237c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8247c478bd9Sstevel@tonic-gate 8257c478bd9Sstevel@tonic-gate /* 8267c478bd9Sstevel@tonic-gate * Increment the count of free pages for the current 8277c478bd9Sstevel@tonic-gate * region. Continue looping up in region size incrementing 8287c478bd9Sstevel@tonic-gate * count if the preceeding region is full. 8297c478bd9Sstevel@tonic-gate */ 8307c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8317c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8327c478bd9Sstevel@tonic-gate 8337c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8347c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) < FULL_REGION_CNT(r)); 8357c478bd9Sstevel@tonic-gate 8365d07b933Sdp if (++PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r)) { 8377c478bd9Sstevel@tonic-gate break; 8385d07b933Sdp } else { 8395d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 8405d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 8415d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 8427c478bd9Sstevel@tonic-gate 8435d07b933Sdp cand->pcc_pages_free++; 8445d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]++; 8455d07b933Sdp } 8467c478bd9Sstevel@tonic-gate r++; 8477c478bd9Sstevel@tonic-gate } 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate void 851affbd3ccSkchow page_ctr_add(int mnode, int mtype, page_t *pp, int flags) 8527c478bd9Sstevel@tonic-gate { 8537c478bd9Sstevel@tonic-gate int lckidx = PP_CTR_LOCK_INDX(pp); 8547c478bd9Sstevel@tonic-gate kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate mutex_enter(lock); 857affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 8587c478bd9Sstevel@tonic-gate mutex_exit(lock); 8597c478bd9Sstevel@tonic-gate } 8607c478bd9Sstevel@tonic-gate 8617c478bd9Sstevel@tonic-gate void 8625d07b933Sdp page_ctr_sub_internal(int mnode, int mtype, page_t *pp, int flags) 8637c478bd9Sstevel@tonic-gate { 8647c478bd9Sstevel@tonic-gate int lckidx; 8657c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8667c478bd9Sstevel@tonic-gate ssize_t idx; 8677c478bd9Sstevel@tonic-gate pfn_t pfnum; 8687c478bd9Sstevel@tonic-gate 869affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 870affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 871affbd3ccSkchow 8727c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8737c478bd9Sstevel@tonic-gate 874affbd3ccSkchow PLCNT_DECR(pp, mnode, mtype, pp->p_szc, flags); 8757c478bd9Sstevel@tonic-gate 8767c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8777c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8787c478bd9Sstevel@tonic-gate return; 8797c478bd9Sstevel@tonic-gate } 8807c478bd9Sstevel@tonic-gate 8817c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8827c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8837c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate /* 8867c478bd9Sstevel@tonic-gate * Decrement the count of free pages for the current 8877c478bd9Sstevel@tonic-gate * region. Continue looping up in region size decrementing 8887c478bd9Sstevel@tonic-gate * count if the preceeding region was full. 8897c478bd9Sstevel@tonic-gate */ 8907c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8917c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8927c478bd9Sstevel@tonic-gate 8937c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8947c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) > 0); 8957c478bd9Sstevel@tonic-gate 8967c478bd9Sstevel@tonic-gate if (--PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r) - 1) { 8977c478bd9Sstevel@tonic-gate break; 8985d07b933Sdp } else { 8995d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 9005d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 9015d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 9027c478bd9Sstevel@tonic-gate 9035d07b933Sdp ASSERT(cand->pcc_pages_free != 0); 9045d07b933Sdp ASSERT(cand->pcc_color_free[PP_2_BIN_SZC(pp, r)] != 0); 9055d07b933Sdp 9065d07b933Sdp cand->pcc_pages_free--; 9075d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]--; 9085d07b933Sdp } 9097c478bd9Sstevel@tonic-gate r++; 9107c478bd9Sstevel@tonic-gate } 9115d07b933Sdp } 9125d07b933Sdp 9135d07b933Sdp void 9145d07b933Sdp page_ctr_sub(int mnode, int mtype, page_t *pp, int flags) 9155d07b933Sdp { 9165d07b933Sdp int lckidx = PP_CTR_LOCK_INDX(pp); 9175d07b933Sdp kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 9185d07b933Sdp 9195d07b933Sdp mutex_enter(lock); 9205d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 9217c478bd9Sstevel@tonic-gate mutex_exit(lock); 9227c478bd9Sstevel@tonic-gate } 9237c478bd9Sstevel@tonic-gate 9247c478bd9Sstevel@tonic-gate /* 9257c478bd9Sstevel@tonic-gate * Adjust page counters following a memory attach, since typically the 9267c478bd9Sstevel@tonic-gate * size of the array needs to change, and the PFN to counter index 9277c478bd9Sstevel@tonic-gate * mapping needs to change. 9285d07b933Sdp * 9295d07b933Sdp * It is possible this mnode did not exist at startup. In that case 9305d07b933Sdp * allocate pcc_info_t and pcc_color_free arrays. Also, allow for nranges 9315d07b933Sdp * to change (a theoretical possibility on x86), which means pcc_color_free 9325d07b933Sdp * arrays must be extended. 9337c478bd9Sstevel@tonic-gate */ 9347c478bd9Sstevel@tonic-gate uint_t 9357c478bd9Sstevel@tonic-gate page_ctrs_adjust(int mnode) 9367c478bd9Sstevel@tonic-gate { 9377c478bd9Sstevel@tonic-gate pgcnt_t npgs; 9387c478bd9Sstevel@tonic-gate int r; /* region size */ 9397c478bd9Sstevel@tonic-gate int i; 9407c478bd9Sstevel@tonic-gate size_t pcsz, old_csz; 9417c478bd9Sstevel@tonic-gate hpmctr_t *new_ctr, *old_ctr; 9427c478bd9Sstevel@tonic-gate pfn_t oldbase, newbase; 943ce8eb11aSdp pfn_t physbase, physmax; 9447c478bd9Sstevel@tonic-gate size_t old_npgs; 9457c478bd9Sstevel@tonic-gate hpmctr_t *ctr_cache[MMU_PAGE_SIZES]; 9467c478bd9Sstevel@tonic-gate size_t size_cache[MMU_PAGE_SIZES]; 9475d07b933Sdp size_t *color_cache[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 9485d07b933Sdp size_t *old_color_array[MAX_MNODE_MRANGES]; 9497c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 9505d07b933Sdp pcc_info_t **cands_cache; 9515d07b933Sdp pcc_info_t *old_pi, *pi; 9525d07b933Sdp pgcnt_t *pgcntp; 9535d07b933Sdp int nr, old_nranges, mrange, nranges = MNODE_RANGE_CNT(mnode); 9545d07b933Sdp int cands_cache_nranges; 9555d07b933Sdp int old_maxmrange, new_maxmrange; 9565d07b933Sdp int rc = 0; 9577c478bd9Sstevel@tonic-gate 9585d07b933Sdp cands_cache = kmem_zalloc(sizeof (pcc_info_t *) * NPC_MUTEX * 9595d07b933Sdp MMU_PAGE_SIZES, KM_NOSLEEP); 9605d07b933Sdp if (cands_cache == NULL) 9615d07b933Sdp return (ENOMEM); 9625d07b933Sdp 963ce8eb11aSdp i = -1; 964ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, i); 965ce8eb11aSdp 966ce8eb11aSdp newbase = physbase & ~PC_BASE_ALIGN_MASK; 967ce8eb11aSdp npgs = roundup(physmax, PC_BASE_ALIGN) - newbase; 968ce8eb11aSdp 9695d07b933Sdp /* prepare to free non-null pointers on the way out */ 9705d07b933Sdp cands_cache_nranges = nranges; 9715d07b933Sdp bzero(ctr_cache, sizeof (ctr_cache)); 9725d07b933Sdp bzero(color_cache, sizeof (color_cache)); 9735d07b933Sdp 9747c478bd9Sstevel@tonic-gate /* 9757c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 9767c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 9777c478bd9Sstevel@tonic-gate * arrays. 9787c478bd9Sstevel@tonic-gate */ 9795d07b933Sdp for (r = 0; r < mmu_page_sizes; r++) { 9805d07b933Sdp colors_per_szc[r] = PAGE_GET_PAGECOLORS(r); 9817c478bd9Sstevel@tonic-gate } 9827c478bd9Sstevel@tonic-gate 9837c478bd9Sstevel@tonic-gate /* 9847c478bd9Sstevel@tonic-gate * Preallocate all of the new hpm_counters arrays as we can't 9857c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 9867c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 9877c478bd9Sstevel@tonic-gate * and return failure. 9887c478bd9Sstevel@tonic-gate */ 9897c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 9907c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_BSZS_SHIFT(r); 9915d07b933Sdp size_cache[r] = pcsz; 9927c478bd9Sstevel@tonic-gate ctr_cache[r] = kmem_zalloc(pcsz * 9937c478bd9Sstevel@tonic-gate sizeof (hpmctr_t), KM_NOSLEEP); 9947c478bd9Sstevel@tonic-gate if (ctr_cache[r] == NULL) { 9955d07b933Sdp rc = ENOMEM; 9965d07b933Sdp goto cleanup; 9977c478bd9Sstevel@tonic-gate } 9987c478bd9Sstevel@tonic-gate } 9995d07b933Sdp 10007c478bd9Sstevel@tonic-gate /* 10017c478bd9Sstevel@tonic-gate * Preallocate all of the new color current arrays as we can't 10027c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 10037c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 10047c478bd9Sstevel@tonic-gate * and return failure. 10057c478bd9Sstevel@tonic-gate */ 10067c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10075d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 10085d07b933Sdp color_cache[r][mrange] = kmem_zalloc(sizeof (size_t) * 10095d07b933Sdp colors_per_szc[r], KM_NOSLEEP); 10105d07b933Sdp if (color_cache[r][mrange] == NULL) { 10115d07b933Sdp rc = ENOMEM; 10125d07b933Sdp goto cleanup; 10135d07b933Sdp } 10145d07b933Sdp } 10155d07b933Sdp } 10165d07b933Sdp 10175d07b933Sdp /* 10185d07b933Sdp * Preallocate all of the new pcc_info_t arrays as we can't 10195d07b933Sdp * hold the page_ctrs_rwlock as a writer and allocate memory. 10205d07b933Sdp * If we can't allocate all of the arrays, undo our work so far 10215d07b933Sdp * and return failure. 10225d07b933Sdp */ 10235d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 10245d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 10255d07b933Sdp pi = kmem_zalloc(nranges * sizeof (pcc_info_t), 10265d07b933Sdp KM_NOSLEEP); 10275d07b933Sdp if (pi == NULL) { 10285d07b933Sdp rc = ENOMEM; 10295d07b933Sdp goto cleanup; 10307c478bd9Sstevel@tonic-gate } 10315d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = pi; 10325d07b933Sdp 10335d07b933Sdp for (mrange = 0; mrange < nranges; mrange++, pi++) { 10345d07b933Sdp pgcntp = kmem_zalloc(colors_per_szc[r] * 10355d07b933Sdp sizeof (pgcnt_t), KM_NOSLEEP); 10365d07b933Sdp if (pgcntp == NULL) { 10375d07b933Sdp rc = ENOMEM; 10385d07b933Sdp goto cleanup; 10395d07b933Sdp } 10405d07b933Sdp pi->pcc_color_free = pgcntp; 10417c478bd9Sstevel@tonic-gate } 10427c478bd9Sstevel@tonic-gate } 10437c478bd9Sstevel@tonic-gate } 10447c478bd9Sstevel@tonic-gate 10457c478bd9Sstevel@tonic-gate /* 10467c478bd9Sstevel@tonic-gate * Grab the write lock to prevent others from walking these arrays 10477c478bd9Sstevel@tonic-gate * while we are modifying them. 10487c478bd9Sstevel@tonic-gate */ 1049ce8eb11aSdp PAGE_CTRS_WRITE_LOCK(mnode); 10505d07b933Sdp 10515d07b933Sdp old_nranges = mnode_nranges[mnode]; 10525d07b933Sdp cands_cache_nranges = old_nranges; 10535d07b933Sdp mnode_nranges[mnode] = nranges; 10545d07b933Sdp old_maxmrange = mnode_maxmrange[mnode]; 10555d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 10565d07b933Sdp new_maxmrange = mnode_maxmrange[mnode]; 10575d07b933Sdp 10587c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10597c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = PAGE_BSZS_SHIFT(r); 10607c478bd9Sstevel@tonic-gate old_ctr = PAGE_COUNTERS_COUNTERS(mnode, r); 10617c478bd9Sstevel@tonic-gate old_csz = PAGE_COUNTERS_ENTRIES(mnode, r); 10627c478bd9Sstevel@tonic-gate oldbase = PAGE_COUNTERS_BASE(mnode, r); 10637c478bd9Sstevel@tonic-gate old_npgs = old_csz << PAGE_COUNTERS_SHIFT(mnode, r); 10645d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 10655d07b933Sdp old_color_array[mrange] = 10665d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 1067ce8eb11aSdp r, mrange); 10685d07b933Sdp } 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_COUNTERS_SHIFT(mnode, r); 10717c478bd9Sstevel@tonic-gate new_ctr = ctr_cache[r]; 10727c478bd9Sstevel@tonic-gate ctr_cache[r] = NULL; 10737c478bd9Sstevel@tonic-gate if (old_ctr != NULL && 10747c478bd9Sstevel@tonic-gate (oldbase + old_npgs > newbase) && 10757c478bd9Sstevel@tonic-gate (newbase + npgs > oldbase)) { 10767c478bd9Sstevel@tonic-gate /* 10777c478bd9Sstevel@tonic-gate * Map the intersection of the old and new 10787c478bd9Sstevel@tonic-gate * counters into the new array. 10797c478bd9Sstevel@tonic-gate */ 10807c478bd9Sstevel@tonic-gate size_t offset; 10817c478bd9Sstevel@tonic-gate if (newbase > oldbase) { 10827c478bd9Sstevel@tonic-gate offset = (newbase - oldbase) >> 10837c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 10847c478bd9Sstevel@tonic-gate bcopy(old_ctr + offset, new_ctr, 10857c478bd9Sstevel@tonic-gate MIN(pcsz, (old_csz - offset)) * 10867c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 10877c478bd9Sstevel@tonic-gate } else { 10887c478bd9Sstevel@tonic-gate offset = (oldbase - newbase) >> 10897c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 10907c478bd9Sstevel@tonic-gate bcopy(old_ctr, new_ctr + offset, 10917c478bd9Sstevel@tonic-gate MIN(pcsz - offset, old_csz) * 10927c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 10937c478bd9Sstevel@tonic-gate } 10947c478bd9Sstevel@tonic-gate } 10957c478bd9Sstevel@tonic-gate 10967c478bd9Sstevel@tonic-gate PAGE_COUNTERS_COUNTERS(mnode, r) = new_ctr; 10977c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = pcsz; 10987c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = newbase; 1099ce8eb11aSdp 1100ce8eb11aSdp /* update shared hpm_counters in other mnodes */ 1101ce8eb11aSdp if (interleaved_mnodes) { 1102ce8eb11aSdp for (i = 0; i < max_mem_nodes; i++) { 1103ce8eb11aSdp if (i == mnode) 1104ce8eb11aSdp continue; 1105ce8eb11aSdp if (mem_node_config[i].exists == 0) 1106ce8eb11aSdp continue; 1107ce8eb11aSdp ASSERT(PAGE_COUNTERS_COUNTERS(i, r) == old_ctr); 1108ce8eb11aSdp PAGE_COUNTERS_COUNTERS(i, r) = new_ctr; 1109ce8eb11aSdp PAGE_COUNTERS_ENTRIES(i, r) = pcsz; 1110ce8eb11aSdp PAGE_COUNTERS_BASE(i, r) = newbase; 1111ce8eb11aSdp } 1112ce8eb11aSdp } 1113ce8eb11aSdp 11145d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11155d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, r, mrange) = 11165d07b933Sdp color_cache[r][mrange]; 11175d07b933Sdp color_cache[r][mrange] = NULL; 11185d07b933Sdp } 11197c478bd9Sstevel@tonic-gate /* 11207c478bd9Sstevel@tonic-gate * for now, just reset on these events as it's probably 11217c478bd9Sstevel@tonic-gate * not worthwhile to try and optimize this. 11227c478bd9Sstevel@tonic-gate */ 11237c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 11245d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 1125ce8eb11aSdp int mlo = interleaved_mnodes ? 0 : mnode; 1126ce8eb11aSdp int mhi = interleaved_mnodes ? max_mem_nodes : 1127ce8eb11aSdp (mnode + 1); 1128ce8eb11aSdp int m; 11295d07b933Sdp pfn_t pfnum = newbase; 11305d07b933Sdp size_t idx; 1131ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 11325d07b933Sdp 1133ce8eb11aSdp for (m = mlo; m < mhi; m++) { 1134ce8eb11aSdp if (mem_node_config[m].exists == 0) 1135ce8eb11aSdp continue; 1136b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, m, r, &it); 1137b779d3e0Sdp if (pfnum == (pfn_t)-1) { 1138b779d3e0Sdp idx = 0; 1139b779d3e0Sdp } else { 1140b779d3e0Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 1141b779d3e0Sdp color_mask, color_mask, &it); 1142b779d3e0Sdp idx = PNUM_TO_IDX(m, r, pfnum); 1143b779d3e0Sdp idx = (idx < pcsz) ? idx : 0; 1144b779d3e0Sdp } 1145ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 1146ce8eb11aSdp PAGE_COUNTERS_CURRENT_COLOR(m, 1147ce8eb11aSdp r, i, mrange) = idx; 1148ce8eb11aSdp } 11495d07b933Sdp } 11507c478bd9Sstevel@tonic-gate } 11517c478bd9Sstevel@tonic-gate 11527c478bd9Sstevel@tonic-gate /* cache info for freeing out of the critical path */ 11537c478bd9Sstevel@tonic-gate if ((caddr_t)old_ctr >= kernelheap && 11547c478bd9Sstevel@tonic-gate (caddr_t)old_ctr < ekernelheap) { 11557c478bd9Sstevel@tonic-gate ctr_cache[r] = old_ctr; 11567c478bd9Sstevel@tonic-gate size_cache[r] = old_csz; 11577c478bd9Sstevel@tonic-gate } 11585d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11595d07b933Sdp size_t *tmp = old_color_array[mrange]; 11605d07b933Sdp if ((caddr_t)tmp >= kernelheap && 11615d07b933Sdp (caddr_t)tmp < ekernelheap) { 11625d07b933Sdp color_cache[r][mrange] = tmp; 11635d07b933Sdp } 11647c478bd9Sstevel@tonic-gate } 11657c478bd9Sstevel@tonic-gate /* 11667c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 11677c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 11687c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 11697c478bd9Sstevel@tonic-gate * and get consistent values. 11707c478bd9Sstevel@tonic-gate */ 11717c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 11727c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 11737c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 11747c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, newbase))) == newbase); 11755d07b933Sdp 11765d07b933Sdp /* pcc_info_t and pcc_color_free */ 11775d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 11785d07b933Sdp pcc_info_t *epi; 11795d07b933Sdp pcc_info_t *eold_pi; 11805d07b933Sdp 11815d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 11825d07b933Sdp old_pi = page_ctrs_cands[i][r][mnode]; 11835d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 11845d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = old_pi; 11855d07b933Sdp 11865d07b933Sdp /* preserve old pcc_color_free values, if any */ 11875d07b933Sdp if (old_pi == NULL) 11885d07b933Sdp continue; 11895d07b933Sdp 11905d07b933Sdp /* 11915d07b933Sdp * when/if x86 does DR, must account for 11925d07b933Sdp * possible change in range index when 11935d07b933Sdp * preserving pcc_info 11945d07b933Sdp */ 11955d07b933Sdp epi = &pi[nranges]; 11965d07b933Sdp eold_pi = &old_pi[old_nranges]; 11975d07b933Sdp if (new_maxmrange > old_maxmrange) { 11985d07b933Sdp pi += new_maxmrange - old_maxmrange; 11995d07b933Sdp } else if (new_maxmrange < old_maxmrange) { 12005d07b933Sdp old_pi += old_maxmrange - new_maxmrange; 12015d07b933Sdp } 12025d07b933Sdp for (; pi < epi && old_pi < eold_pi; pi++, old_pi++) { 12035d07b933Sdp pcc_info_t tmp = *pi; 12045d07b933Sdp *pi = *old_pi; 12055d07b933Sdp *old_pi = tmp; 12065d07b933Sdp } 12075d07b933Sdp } 12087c478bd9Sstevel@tonic-gate } 1209ce8eb11aSdp PAGE_CTRS_WRITE_UNLOCK(mnode); 12107c478bd9Sstevel@tonic-gate 12117c478bd9Sstevel@tonic-gate /* 12127c478bd9Sstevel@tonic-gate * Now that we have dropped the write lock, it is safe to free all 12137c478bd9Sstevel@tonic-gate * of the memory we have cached above. 12145d07b933Sdp * We come thru here to free memory when pre-alloc fails, and also to 12155d07b933Sdp * free old pointers which were recorded while locked. 12167c478bd9Sstevel@tonic-gate */ 12175d07b933Sdp cleanup: 12187c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 12197c478bd9Sstevel@tonic-gate if (ctr_cache[r] != NULL) { 12207c478bd9Sstevel@tonic-gate kmem_free(ctr_cache[r], 12217c478bd9Sstevel@tonic-gate size_cache[r] * sizeof (hpmctr_t)); 12227c478bd9Sstevel@tonic-gate } 12235d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 12245d07b933Sdp if (color_cache[r][mrange] != NULL) { 12255d07b933Sdp kmem_free(color_cache[r][mrange], 12265d07b933Sdp colors_per_szc[r] * sizeof (size_t)); 12275d07b933Sdp } 12285d07b933Sdp } 12295d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 12305d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12315d07b933Sdp if (pi == NULL) 12325d07b933Sdp continue; 12335d07b933Sdp nr = cands_cache_nranges; 12345d07b933Sdp for (mrange = 0; mrange < nr; mrange++, pi++) { 12355d07b933Sdp pgcntp = pi->pcc_color_free; 12365d07b933Sdp if (pgcntp == NULL) 12375d07b933Sdp continue; 12385d07b933Sdp if ((caddr_t)pgcntp >= kernelheap && 12395d07b933Sdp (caddr_t)pgcntp < ekernelheap) { 12405d07b933Sdp kmem_free(pgcntp, 12415d07b933Sdp colors_per_szc[r] * 12425d07b933Sdp sizeof (pgcnt_t)); 12435d07b933Sdp } 12445d07b933Sdp } 12455d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12465d07b933Sdp if ((caddr_t)pi >= kernelheap && 12475d07b933Sdp (caddr_t)pi < ekernelheap) { 12485d07b933Sdp kmem_free(pi, nr * sizeof (pcc_info_t)); 12495d07b933Sdp } 12507c478bd9Sstevel@tonic-gate } 12517c478bd9Sstevel@tonic-gate } 12527c478bd9Sstevel@tonic-gate 12535d07b933Sdp kmem_free(cands_cache, 12545d07b933Sdp sizeof (pcc_info_t *) * NPC_MUTEX * MMU_PAGE_SIZES); 12555d07b933Sdp return (rc); 12567c478bd9Sstevel@tonic-gate } 12577c478bd9Sstevel@tonic-gate 12585d07b933Sdp 12597c478bd9Sstevel@tonic-gate #ifdef DEBUG 12607c478bd9Sstevel@tonic-gate 12617c478bd9Sstevel@tonic-gate /* 12627c478bd9Sstevel@tonic-gate * confirm pp is a large page corresponding to szc 12637c478bd9Sstevel@tonic-gate */ 12647c478bd9Sstevel@tonic-gate void 12657c478bd9Sstevel@tonic-gate chk_lpg(page_t *pp, uchar_t szc) 12667c478bd9Sstevel@tonic-gate { 12677c478bd9Sstevel@tonic-gate spgcnt_t npgs = page_get_pagecnt(pp->p_szc); 12687c478bd9Sstevel@tonic-gate uint_t noreloc; 12697c478bd9Sstevel@tonic-gate 12707c478bd9Sstevel@tonic-gate if (npgs == 1) { 12717c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 12727c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == pp); 12737c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == pp); 12747c478bd9Sstevel@tonic-gate return; 12757c478bd9Sstevel@tonic-gate } 12767c478bd9Sstevel@tonic-gate 12777c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12787c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 12797c478bd9Sstevel@tonic-gate 12807c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pp->p_pagenum, npgs)); 12817c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == (pp->p_next->p_pagenum - 1)); 12827c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev->p_pagenum == (pp->p_pagenum + (npgs - 1))); 12837c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == (pp + (npgs - 1))); 12847c478bd9Sstevel@tonic-gate 12857c478bd9Sstevel@tonic-gate /* 12867c478bd9Sstevel@tonic-gate * Check list of pages. 12877c478bd9Sstevel@tonic-gate */ 12887c478bd9Sstevel@tonic-gate noreloc = PP_ISNORELOC(pp); 12897c478bd9Sstevel@tonic-gate while (npgs--) { 12907c478bd9Sstevel@tonic-gate if (npgs != 0) { 12917c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == pp->p_next->p_pagenum - 1); 12927c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == (pp + 1)); 12937c478bd9Sstevel@tonic-gate } 12947c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 12957c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 12967c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 12977c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12987c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 12997c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 13007c478bd9Sstevel@tonic-gate ASSERT(PP_ISNORELOC(pp) == noreloc); 13017c478bd9Sstevel@tonic-gate 13027c478bd9Sstevel@tonic-gate pp = pp->p_next; 13037c478bd9Sstevel@tonic-gate } 13047c478bd9Sstevel@tonic-gate } 13057c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 13067c478bd9Sstevel@tonic-gate 13077c478bd9Sstevel@tonic-gate void 13087c478bd9Sstevel@tonic-gate page_freelist_lock(int mnode) 13097c478bd9Sstevel@tonic-gate { 13107c478bd9Sstevel@tonic-gate int i; 13117c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13127c478bd9Sstevel@tonic-gate mutex_enter(FPC_MUTEX(mnode, i)); 13137c478bd9Sstevel@tonic-gate mutex_enter(CPC_MUTEX(mnode, i)); 13147c478bd9Sstevel@tonic-gate } 13157c478bd9Sstevel@tonic-gate } 13167c478bd9Sstevel@tonic-gate 13177c478bd9Sstevel@tonic-gate void 13187c478bd9Sstevel@tonic-gate page_freelist_unlock(int mnode) 13197c478bd9Sstevel@tonic-gate { 13207c478bd9Sstevel@tonic-gate int i; 13217c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13227c478bd9Sstevel@tonic-gate mutex_exit(FPC_MUTEX(mnode, i)); 13237c478bd9Sstevel@tonic-gate mutex_exit(CPC_MUTEX(mnode, i)); 13247c478bd9Sstevel@tonic-gate } 13257c478bd9Sstevel@tonic-gate } 13267c478bd9Sstevel@tonic-gate 13277c478bd9Sstevel@tonic-gate /* 13287c478bd9Sstevel@tonic-gate * add pp to the specified page list. Defaults to head of the page list 13297c478bd9Sstevel@tonic-gate * unless PG_LIST_TAIL is specified. 13307c478bd9Sstevel@tonic-gate */ 13317c478bd9Sstevel@tonic-gate void 13327c478bd9Sstevel@tonic-gate page_list_add(page_t *pp, int flags) 13337c478bd9Sstevel@tonic-gate { 13347c478bd9Sstevel@tonic-gate page_t **ppp; 13357c478bd9Sstevel@tonic-gate kmutex_t *pcm; 13367c478bd9Sstevel@tonic-gate uint_t bin, mtype; 13377c478bd9Sstevel@tonic-gate int mnode; 13387c478bd9Sstevel@tonic-gate 13397c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 13407c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 13417c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 13427c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(pp) == 0); 13437c478bd9Sstevel@tonic-gate 13447c478bd9Sstevel@tonic-gate /* 13457c478bd9Sstevel@tonic-gate * Large pages should be freed via page_list_add_pages(). 13467c478bd9Sstevel@tonic-gate */ 13477c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 13487c478bd9Sstevel@tonic-gate 13497c478bd9Sstevel@tonic-gate /* 13507c478bd9Sstevel@tonic-gate * Don't need to lock the freelist first here 13517c478bd9Sstevel@tonic-gate * because the page isn't on the freelist yet. 13527c478bd9Sstevel@tonic-gate * This means p_szc can't change on us. 13537c478bd9Sstevel@tonic-gate */ 13547c478bd9Sstevel@tonic-gate 13557c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 13567c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 13577c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 13587c478bd9Sstevel@tonic-gate 13597c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 13607c478bd9Sstevel@tonic-gate /* 13617c478bd9Sstevel@tonic-gate * PG_LIST_ISINIT is set during system startup (ie. single 13627c478bd9Sstevel@tonic-gate * threaded), add a page to the free list and add to the 13637c478bd9Sstevel@tonic-gate * the free region counters w/o any locking 13647c478bd9Sstevel@tonic-gate */ 13657c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate /* inline version of page_add() */ 13687c478bd9Sstevel@tonic-gate if (*ppp != NULL) { 13697c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 13707c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 13717c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 13727c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 13737c478bd9Sstevel@tonic-gate } else 13747c478bd9Sstevel@tonic-gate *ppp = pp; 13757c478bd9Sstevel@tonic-gate 1376affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 1377affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13787c478bd9Sstevel@tonic-gate } else { 13797c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1382affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13837c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 13847c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13857c478bd9Sstevel@tonic-gate 13867c478bd9Sstevel@tonic-gate } else { 1387affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_cache); 13887c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 13897c478bd9Sstevel@tonic-gate ASSERT((pp->p_offset & PAGEOFFSET) == 0); 13907c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 13917c478bd9Sstevel@tonic-gate } 13927c478bd9Sstevel@tonic-gate mutex_enter(pcm); 13937c478bd9Sstevel@tonic-gate page_add(ppp, pp); 13947c478bd9Sstevel@tonic-gate 13957c478bd9Sstevel@tonic-gate if (flags & PG_LIST_TAIL) 13967c478bd9Sstevel@tonic-gate *ppp = (*ppp)->p_next; 13977c478bd9Sstevel@tonic-gate /* 13987c478bd9Sstevel@tonic-gate * Add counters before releasing pcm mutex to avoid a race with 13995d07b933Sdp * page_freelist_coalesce and page_freelist_split. 14007c478bd9Sstevel@tonic-gate */ 1401affbd3ccSkchow page_ctr_add(mnode, mtype, pp, flags); 14027c478bd9Sstevel@tonic-gate mutex_exit(pcm); 14037c478bd9Sstevel@tonic-gate } 14047c478bd9Sstevel@tonic-gate 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate #if defined(__sparc) 14077c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 14087c478bd9Sstevel@tonic-gate kcage_freemem_add(1); 14097c478bd9Sstevel@tonic-gate } 14107c478bd9Sstevel@tonic-gate #endif 14117c478bd9Sstevel@tonic-gate /* 14127c478bd9Sstevel@tonic-gate * It is up to the caller to unlock the page! 14137c478bd9Sstevel@tonic-gate */ 14147c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 14157c478bd9Sstevel@tonic-gate } 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate 14187c478bd9Sstevel@tonic-gate #ifdef __sparc 14197c478bd9Sstevel@tonic-gate /* 14207c478bd9Sstevel@tonic-gate * This routine is only used by kcage_init during system startup. 14217c478bd9Sstevel@tonic-gate * It performs the function of page_list_sub/PP_SETNORELOC/page_list_add 14227c478bd9Sstevel@tonic-gate * without the overhead of taking locks and updating counters. 14237c478bd9Sstevel@tonic-gate */ 14247c478bd9Sstevel@tonic-gate void 14257c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 14267c478bd9Sstevel@tonic-gate { 14277c478bd9Sstevel@tonic-gate page_t **ppp; 14287c478bd9Sstevel@tonic-gate uint_t bin; 14297c478bd9Sstevel@tonic-gate int mnode; 14307c478bd9Sstevel@tonic-gate int mtype; 1431e21bae1bSkchow int flags = 0; 14327c478bd9Sstevel@tonic-gate 14337c478bd9Sstevel@tonic-gate /* 14347c478bd9Sstevel@tonic-gate * If this is a large page on the freelist then 14357c478bd9Sstevel@tonic-gate * break it up into smaller pages. 14367c478bd9Sstevel@tonic-gate */ 14377c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 14387c478bd9Sstevel@tonic-gate page_boot_demote(pp); 14397c478bd9Sstevel@tonic-gate 14407c478bd9Sstevel@tonic-gate /* 14417c478bd9Sstevel@tonic-gate * Get list page is currently on. 14427c478bd9Sstevel@tonic-gate */ 14437c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 14447c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 14457c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14467c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_RELOC); 14477c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 14487c478bd9Sstevel@tonic-gate 14497c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14507c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14517c478bd9Sstevel@tonic-gate flags |= PG_FREE_LIST; 14527c478bd9Sstevel@tonic-gate } else { 14537c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14547c478bd9Sstevel@tonic-gate flags |= PG_CACHE_LIST; 14557c478bd9Sstevel@tonic-gate } 14567c478bd9Sstevel@tonic-gate 14577c478bd9Sstevel@tonic-gate ASSERT(*ppp != NULL); 14587c478bd9Sstevel@tonic-gate 14597c478bd9Sstevel@tonic-gate /* 14607c478bd9Sstevel@tonic-gate * Delete page from current list. 14617c478bd9Sstevel@tonic-gate */ 14627c478bd9Sstevel@tonic-gate if (*ppp == pp) 14637c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 14647c478bd9Sstevel@tonic-gate if (*ppp == pp) { 14657c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 14667c478bd9Sstevel@tonic-gate } else { 14677c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 14687c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 14697c478bd9Sstevel@tonic-gate } 14707c478bd9Sstevel@tonic-gate 14715d07b933Sdp /* 14725d07b933Sdp * Decrement page counters 14735d07b933Sdp */ 14745d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 14757c478bd9Sstevel@tonic-gate 14767c478bd9Sstevel@tonic-gate /* 14777c478bd9Sstevel@tonic-gate * Set no reloc for cage initted pages. 14787c478bd9Sstevel@tonic-gate */ 14797c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 14807c478bd9Sstevel@tonic-gate 14817c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14827c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_NORELOC); 14837c478bd9Sstevel@tonic-gate 14847c478bd9Sstevel@tonic-gate /* 14857c478bd9Sstevel@tonic-gate * Get new list for page. 14867c478bd9Sstevel@tonic-gate */ 14877c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14887c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14897c478bd9Sstevel@tonic-gate } else { 14907c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14917c478bd9Sstevel@tonic-gate } 14927c478bd9Sstevel@tonic-gate 14937c478bd9Sstevel@tonic-gate /* 14947c478bd9Sstevel@tonic-gate * Insert page on new list. 14957c478bd9Sstevel@tonic-gate */ 14967c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 14977c478bd9Sstevel@tonic-gate *ppp = pp; 14987c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 14997c478bd9Sstevel@tonic-gate } else { 15007c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 15017c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 15027c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 15037c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 15047c478bd9Sstevel@tonic-gate } 15057c478bd9Sstevel@tonic-gate 15065d07b933Sdp /* 15075d07b933Sdp * Increment page counters 15085d07b933Sdp */ 15095d07b933Sdp page_ctr_add_internal(mnode, mtype, pp, flags); 15107c478bd9Sstevel@tonic-gate 15117c478bd9Sstevel@tonic-gate /* 15127c478bd9Sstevel@tonic-gate * Update cage freemem counter 15137c478bd9Sstevel@tonic-gate */ 15147c478bd9Sstevel@tonic-gate atomic_add_long(&kcage_freemem, 1); 15157c478bd9Sstevel@tonic-gate } 15167c478bd9Sstevel@tonic-gate #else /* __sparc */ 15177c478bd9Sstevel@tonic-gate 15187c478bd9Sstevel@tonic-gate /* ARGSUSED */ 15197c478bd9Sstevel@tonic-gate void 15207c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 15217c478bd9Sstevel@tonic-gate { 15227c478bd9Sstevel@tonic-gate panic("page_list_noreloc_startup: should be here only for sparc"); 15237c478bd9Sstevel@tonic-gate } 15247c478bd9Sstevel@tonic-gate #endif 15257c478bd9Sstevel@tonic-gate 15267c478bd9Sstevel@tonic-gate void 15277c478bd9Sstevel@tonic-gate page_list_add_pages(page_t *pp, int flags) 15287c478bd9Sstevel@tonic-gate { 15297c478bd9Sstevel@tonic-gate kmutex_t *pcm; 15307c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 15317c478bd9Sstevel@tonic-gate uint_t bin, mtype, i; 15327c478bd9Sstevel@tonic-gate int mnode; 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate /* default to freelist/head */ 15357c478bd9Sstevel@tonic-gate ASSERT((flags & (PG_CACHE_LIST | PG_LIST_TAIL)) == 0); 15367c478bd9Sstevel@tonic-gate 15377c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 1538affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[pp->p_szc]); 15397c478bd9Sstevel@tonic-gate 15407c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 15417c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 15427c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 15437c478bd9Sstevel@tonic-gate 15447c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 15457c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == mmu_page_sizes - 1); 15467c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 15477c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 1548affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 15497c478bd9Sstevel@tonic-gate } else { 15507c478bd9Sstevel@tonic-gate 15517c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 15527c478bd9Sstevel@tonic-gate 15537c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 15547c478bd9Sstevel@tonic-gate 15557c478bd9Sstevel@tonic-gate mutex_enter(pcm); 15567c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 1557affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 15587c478bd9Sstevel@tonic-gate mutex_exit(pcm); 15597c478bd9Sstevel@tonic-gate 15607c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 15617c478bd9Sstevel@tonic-gate #if defined(__sparc) 15627c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) 15637c478bd9Sstevel@tonic-gate kcage_freemem_add(pgcnt); 15647c478bd9Sstevel@tonic-gate #endif 15657c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++, pp++) 15668b464eb8Smec page_unlock_nocapture(pp); 15677c478bd9Sstevel@tonic-gate } 15687c478bd9Sstevel@tonic-gate } 15697c478bd9Sstevel@tonic-gate 15707c478bd9Sstevel@tonic-gate /* 15717c478bd9Sstevel@tonic-gate * During boot, need to demote a large page to base 15727c478bd9Sstevel@tonic-gate * pagesize pages for seg_kmem for use in boot_alloc() 15737c478bd9Sstevel@tonic-gate */ 15747c478bd9Sstevel@tonic-gate void 15757c478bd9Sstevel@tonic-gate page_boot_demote(page_t *pp) 15767c478bd9Sstevel@tonic-gate { 15777c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0); 15787c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15797c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 15807c478bd9Sstevel@tonic-gate 15817c478bd9Sstevel@tonic-gate (void) page_demote(PP_2_MEM_NODE(pp), 1582*19397407SSherry Moore PFN_BASE(pp->p_pagenum, pp->p_szc), 0, pp->p_szc, 0, PC_NO_COLOR, 15837c478bd9Sstevel@tonic-gate PC_FREE); 15847c478bd9Sstevel@tonic-gate 15857c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15867c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 15877c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 15887c478bd9Sstevel@tonic-gate } 15897c478bd9Sstevel@tonic-gate 15907c478bd9Sstevel@tonic-gate /* 15917c478bd9Sstevel@tonic-gate * Take a particular page off of whatever freelist the page 15927c478bd9Sstevel@tonic-gate * is claimed to be on. 15937c478bd9Sstevel@tonic-gate * 15947c478bd9Sstevel@tonic-gate * NOTE: Only used for PAGESIZE pages. 15957c478bd9Sstevel@tonic-gate */ 15967c478bd9Sstevel@tonic-gate void 15977c478bd9Sstevel@tonic-gate page_list_sub(page_t *pp, int flags) 15987c478bd9Sstevel@tonic-gate { 15997c478bd9Sstevel@tonic-gate int bin; 16007c478bd9Sstevel@tonic-gate uint_t mtype; 16017c478bd9Sstevel@tonic-gate int mnode; 16027c478bd9Sstevel@tonic-gate kmutex_t *pcm; 16037c478bd9Sstevel@tonic-gate page_t **ppp; 16047c478bd9Sstevel@tonic-gate 16057c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 16067c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16077c478bd9Sstevel@tonic-gate 16087c478bd9Sstevel@tonic-gate /* 16097c478bd9Sstevel@tonic-gate * The p_szc field can only be changed by page_promote() 16107c478bd9Sstevel@tonic-gate * and page_demote(). Only free pages can be promoted and 16117c478bd9Sstevel@tonic-gate * demoted and the free list MUST be locked during these 16127c478bd9Sstevel@tonic-gate * operations. So to prevent a race in page_list_sub() 16137c478bd9Sstevel@tonic-gate * between computing which bin of the freelist lock to 16147c478bd9Sstevel@tonic-gate * grab and actually grabing the lock we check again that 16157c478bd9Sstevel@tonic-gate * the bin we locked is still the correct one. Notice that 16167c478bd9Sstevel@tonic-gate * the p_szc field could have actually changed on us but 16177c478bd9Sstevel@tonic-gate * if the bin happens to still be the same we are safe. 16187c478bd9Sstevel@tonic-gate */ 16197c478bd9Sstevel@tonic-gate try_again: 16207c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 16217c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 16227c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 16237c478bd9Sstevel@tonic-gate mutex_enter(pcm); 16247c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 16257c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16267c478bd9Sstevel@tonic-gate goto try_again; 16277c478bd9Sstevel@tonic-gate } 16287c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 16297c478bd9Sstevel@tonic-gate 16307c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1631affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[0]); 16327c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16337c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 16347c478bd9Sstevel@tonic-gate } else { 1635affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_cache); 16367c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 16377c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 16387c478bd9Sstevel@tonic-gate } 16397c478bd9Sstevel@tonic-gate 16407c478bd9Sstevel@tonic-gate /* 16417c478bd9Sstevel@tonic-gate * Common PAGESIZE case. 16427c478bd9Sstevel@tonic-gate * 16437c478bd9Sstevel@tonic-gate * Note that we locked the freelist. This prevents 16447c478bd9Sstevel@tonic-gate * any page promotion/demotion operations. Therefore 16457c478bd9Sstevel@tonic-gate * the p_szc will not change until we drop pcm mutex. 16467c478bd9Sstevel@tonic-gate */ 16477c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 16487c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 16497c478bd9Sstevel@tonic-gate /* 16507c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16517c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16527c478bd9Sstevel@tonic-gate */ 1653affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 16547c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16557c478bd9Sstevel@tonic-gate 16567c478bd9Sstevel@tonic-gate #if defined(__sparc) 16577c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 16587c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate #endif 16617c478bd9Sstevel@tonic-gate return; 16627c478bd9Sstevel@tonic-gate } 16637c478bd9Sstevel@tonic-gate 16647c478bd9Sstevel@tonic-gate /* 16657c478bd9Sstevel@tonic-gate * Large pages on the cache list are not supported. 16667c478bd9Sstevel@tonic-gate */ 16677c478bd9Sstevel@tonic-gate if (flags & PG_CACHE_LIST) 16687c478bd9Sstevel@tonic-gate panic("page_list_sub: large page on cachelist"); 16697c478bd9Sstevel@tonic-gate 16707c478bd9Sstevel@tonic-gate /* 16717c478bd9Sstevel@tonic-gate * Slow but rare. 16727c478bd9Sstevel@tonic-gate * 16737c478bd9Sstevel@tonic-gate * Somebody wants this particular page which is part 16747c478bd9Sstevel@tonic-gate * of a large page. In this case we just demote the page 16757c478bd9Sstevel@tonic-gate * if it's on the freelist. 16767c478bd9Sstevel@tonic-gate * 16777c478bd9Sstevel@tonic-gate * We have to drop pcm before locking the entire freelist. 16787c478bd9Sstevel@tonic-gate * Once we have re-locked the freelist check to make sure 16797c478bd9Sstevel@tonic-gate * the page hasn't already been demoted or completely 16807c478bd9Sstevel@tonic-gate * freed. 16817c478bd9Sstevel@tonic-gate */ 16827c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16837c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 16847c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 16857c478bd9Sstevel@tonic-gate /* 16867c478bd9Sstevel@tonic-gate * Large page is on freelist. 16877c478bd9Sstevel@tonic-gate */ 16887c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, pp->p_szc), 1689*19397407SSherry Moore 0, pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 16907c478bd9Sstevel@tonic-gate } 16917c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16927c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16937c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 16947c478bd9Sstevel@tonic-gate 16957c478bd9Sstevel@tonic-gate /* 16967c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16977c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16987c478bd9Sstevel@tonic-gate */ 16997c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17007c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 17017c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 17027c478bd9Sstevel@tonic-gate 17037c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 1704affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 17057c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17067c478bd9Sstevel@tonic-gate 17077c478bd9Sstevel@tonic-gate #if defined(__sparc) 17087c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17097c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 17107c478bd9Sstevel@tonic-gate } 17117c478bd9Sstevel@tonic-gate #endif 17127c478bd9Sstevel@tonic-gate } 17137c478bd9Sstevel@tonic-gate 17147c478bd9Sstevel@tonic-gate void 17157c478bd9Sstevel@tonic-gate page_list_sub_pages(page_t *pp, uint_t szc) 17167c478bd9Sstevel@tonic-gate { 17177c478bd9Sstevel@tonic-gate kmutex_t *pcm; 17187c478bd9Sstevel@tonic-gate uint_t bin, mtype; 17197c478bd9Sstevel@tonic-gate int mnode; 17207c478bd9Sstevel@tonic-gate 17217c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 17227c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17237c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17247c478bd9Sstevel@tonic-gate 17257c478bd9Sstevel@tonic-gate /* 17267c478bd9Sstevel@tonic-gate * See comment in page_list_sub(). 17277c478bd9Sstevel@tonic-gate */ 17287c478bd9Sstevel@tonic-gate try_again: 17297c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17307c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 17317c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 17327c478bd9Sstevel@tonic-gate mutex_enter(pcm); 17337c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 17347c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17357c478bd9Sstevel@tonic-gate goto try_again; 17367c478bd9Sstevel@tonic-gate } 17377c478bd9Sstevel@tonic-gate 17387c478bd9Sstevel@tonic-gate /* 17397c478bd9Sstevel@tonic-gate * If we're called with a page larger than szc or it got 17407c478bd9Sstevel@tonic-gate * promoted above szc before we locked the freelist then 17417c478bd9Sstevel@tonic-gate * drop pcm and re-lock entire freelist. If page still larger 17427c478bd9Sstevel@tonic-gate * than szc then demote it. 17437c478bd9Sstevel@tonic-gate */ 17447c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 17457c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17467c478bd9Sstevel@tonic-gate pcm = NULL; 17477c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 17487c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 1749affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szcbig); 17507c478bd9Sstevel@tonic-gate (void) page_demote(mnode, 1751*19397407SSherry Moore PFN_BASE(pp->p_pagenum, pp->p_szc), 0, 17527c478bd9Sstevel@tonic-gate pp->p_szc, szc, PC_NO_COLOR, PC_FREE); 17537c478bd9Sstevel@tonic-gate } 17547c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17557c478bd9Sstevel@tonic-gate } 17567c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17577c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17587c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc <= szc); 17597c478bd9Sstevel@tonic-gate ASSERT(pp == PP_PAGEROOT(pp)); 17607c478bd9Sstevel@tonic-gate 1761affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[pp->p_szc]); 1762affbd3ccSkchow 17637c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 17647c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 17657c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17667c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 17677c478bd9Sstevel@tonic-gate } else { 1768affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szc0); 17697c478bd9Sstevel@tonic-gate page_sub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17707c478bd9Sstevel@tonic-gate } 1771affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 17727c478bd9Sstevel@tonic-gate 17737c478bd9Sstevel@tonic-gate if (pcm != NULL) { 17747c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17757c478bd9Sstevel@tonic-gate } else { 17767c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17777c478bd9Sstevel@tonic-gate } 17787c478bd9Sstevel@tonic-gate 17797c478bd9Sstevel@tonic-gate #if defined(__sparc) 17807c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17817c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 17827c478bd9Sstevel@tonic-gate 17837c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 17847c478bd9Sstevel@tonic-gate kcage_freemem_sub(pgcnt); 17857c478bd9Sstevel@tonic-gate } 17867c478bd9Sstevel@tonic-gate #endif 17877c478bd9Sstevel@tonic-gate } 17887c478bd9Sstevel@tonic-gate 17897c478bd9Sstevel@tonic-gate /* 17907c478bd9Sstevel@tonic-gate * Add the page to the front of a linked list of pages 17917c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 17927c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 17937c478bd9Sstevel@tonic-gate */ 17947c478bd9Sstevel@tonic-gate void 17957c478bd9Sstevel@tonic-gate mach_page_add(page_t **ppp, page_t *pp) 17967c478bd9Sstevel@tonic-gate { 17977c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 17987c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 17997c478bd9Sstevel@tonic-gate } else { 18007c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 18017c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 18027c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 18037c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 18047c478bd9Sstevel@tonic-gate } 18057c478bd9Sstevel@tonic-gate *ppp = pp; 18067c478bd9Sstevel@tonic-gate } 18077c478bd9Sstevel@tonic-gate 18087c478bd9Sstevel@tonic-gate /* 18097c478bd9Sstevel@tonic-gate * Remove this page from a linked list of pages 18107c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 18117c478bd9Sstevel@tonic-gate * 18127c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 18137c478bd9Sstevel@tonic-gate */ 18147c478bd9Sstevel@tonic-gate void 18157c478bd9Sstevel@tonic-gate mach_page_sub(page_t **ppp, page_t *pp) 18167c478bd9Sstevel@tonic-gate { 18177c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 18187c478bd9Sstevel@tonic-gate 18197c478bd9Sstevel@tonic-gate if (*ppp == NULL || pp == NULL) 18207c478bd9Sstevel@tonic-gate panic("mach_page_sub"); 18217c478bd9Sstevel@tonic-gate 18227c478bd9Sstevel@tonic-gate if (*ppp == pp) 18237c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 18247c478bd9Sstevel@tonic-gate 18257c478bd9Sstevel@tonic-gate if (*ppp == pp) 18267c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 18277c478bd9Sstevel@tonic-gate else { 18287c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 18297c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 18307c478bd9Sstevel@tonic-gate } 18317c478bd9Sstevel@tonic-gate pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 18327c478bd9Sstevel@tonic-gate } 18337c478bd9Sstevel@tonic-gate 18347c478bd9Sstevel@tonic-gate /* 18357c478bd9Sstevel@tonic-gate * Routine fsflush uses to gradually coalesce the free list into larger pages. 18367c478bd9Sstevel@tonic-gate */ 18377c478bd9Sstevel@tonic-gate void 18387c478bd9Sstevel@tonic-gate page_promote_size(page_t *pp, uint_t cur_szc) 18397c478bd9Sstevel@tonic-gate { 18407c478bd9Sstevel@tonic-gate pfn_t pfn; 18417c478bd9Sstevel@tonic-gate int mnode; 18427c478bd9Sstevel@tonic-gate int idx; 18437c478bd9Sstevel@tonic-gate int new_szc = cur_szc + 1; 18447c478bd9Sstevel@tonic-gate int full = FULL_REGION_CNT(new_szc); 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 18477c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(pfn); 18487c478bd9Sstevel@tonic-gate 18497c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 18507c478bd9Sstevel@tonic-gate 18517c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, new_szc, pfn); 18527c478bd9Sstevel@tonic-gate if (PAGE_COUNTERS(mnode, new_szc, idx) == full) 18535d07b933Sdp (void) page_promote(mnode, pfn, new_szc, PC_FREE, PC_MTYPE_ANY); 18547c478bd9Sstevel@tonic-gate 18557c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 18567c478bd9Sstevel@tonic-gate } 18577c478bd9Sstevel@tonic-gate 18587c478bd9Sstevel@tonic-gate static uint_t page_promote_err; 18597c478bd9Sstevel@tonic-gate static uint_t page_promote_noreloc_err; 18607c478bd9Sstevel@tonic-gate 18617c478bd9Sstevel@tonic-gate /* 18627c478bd9Sstevel@tonic-gate * Create a single larger page (of szc new_szc) from smaller contiguous pages 18637c478bd9Sstevel@tonic-gate * for the given mnode starting at pfnum. Pages involved are on the freelist 18647c478bd9Sstevel@tonic-gate * before the call and may be returned to the caller if requested, otherwise 18657c478bd9Sstevel@tonic-gate * they will be placed back on the freelist. 18667c478bd9Sstevel@tonic-gate * If flags is PC_ALLOC, then the large page will be returned to the user in 18677c478bd9Sstevel@tonic-gate * a state which is consistent with a page being taken off the freelist. If 18687c478bd9Sstevel@tonic-gate * we failed to lock the new large page, then we will return NULL to the 18697c478bd9Sstevel@tonic-gate * caller and put the large page on the freelist instead. 18707c478bd9Sstevel@tonic-gate * If flags is PC_FREE, then the large page will be placed on the freelist, 18717c478bd9Sstevel@tonic-gate * and NULL will be returned. 18727c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 18737c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 18747c478bd9Sstevel@tonic-gate * 18757c478bd9Sstevel@tonic-gate * RFE: For performance pass in pp instead of pfnum so 18767c478bd9Sstevel@tonic-gate * we can avoid excessive calls to page_numtopp_nolock(). 18777c478bd9Sstevel@tonic-gate * This would depend on an assumption that all contiguous 18787c478bd9Sstevel@tonic-gate * pages are in the same memseg so we can just add/dec 18797c478bd9Sstevel@tonic-gate * our pp. 18807c478bd9Sstevel@tonic-gate * 18817c478bd9Sstevel@tonic-gate * Lock ordering: 18827c478bd9Sstevel@tonic-gate * 18837c478bd9Sstevel@tonic-gate * There is a potential but rare deadlock situation 18847c478bd9Sstevel@tonic-gate * for page promotion and demotion operations. The problem 18857c478bd9Sstevel@tonic-gate * is there are two paths into the freelist manager and 18867c478bd9Sstevel@tonic-gate * they have different lock orders: 18877c478bd9Sstevel@tonic-gate * 18887c478bd9Sstevel@tonic-gate * page_create() 18897c478bd9Sstevel@tonic-gate * lock freelist 18907c478bd9Sstevel@tonic-gate * page_lock(EXCL) 18917c478bd9Sstevel@tonic-gate * unlock freelist 18927c478bd9Sstevel@tonic-gate * return 18937c478bd9Sstevel@tonic-gate * caller drops page_lock 18947c478bd9Sstevel@tonic-gate * 18957c478bd9Sstevel@tonic-gate * page_free() and page_reclaim() 18967c478bd9Sstevel@tonic-gate * caller grabs page_lock(EXCL) 18977c478bd9Sstevel@tonic-gate * 18987c478bd9Sstevel@tonic-gate * lock freelist 18997c478bd9Sstevel@tonic-gate * unlock freelist 19007c478bd9Sstevel@tonic-gate * drop page_lock 19017c478bd9Sstevel@tonic-gate * 19027c478bd9Sstevel@tonic-gate * What prevents a thread in page_create() from deadlocking 19037c478bd9Sstevel@tonic-gate * with a thread freeing or reclaiming the same page is the 19047c478bd9Sstevel@tonic-gate * page_trylock() in page_get_freelist(). If the trylock fails 19057c478bd9Sstevel@tonic-gate * it skips the page. 19067c478bd9Sstevel@tonic-gate * 19077c478bd9Sstevel@tonic-gate * The lock ordering for promotion and demotion is the same as 19087c478bd9Sstevel@tonic-gate * for page_create(). Since the same deadlock could occur during 19097c478bd9Sstevel@tonic-gate * page promotion and freeing or reclaiming of a page on the 19107c478bd9Sstevel@tonic-gate * cache list we might have to fail the operation and undo what 19117c478bd9Sstevel@tonic-gate * have done so far. Again this is rare. 19127c478bd9Sstevel@tonic-gate */ 19137c478bd9Sstevel@tonic-gate page_t * 19145d07b933Sdp page_promote(int mnode, pfn_t pfnum, uchar_t new_szc, int flags, int mtype) 19157c478bd9Sstevel@tonic-gate { 19167c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *tpp, *start_pp; 19177c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, npgs; 19187c478bd9Sstevel@tonic-gate uint_t bin; 19197c478bd9Sstevel@tonic-gate pgcnt_t tmpnpgs, pages_left; 19207c478bd9Sstevel@tonic-gate uint_t noreloc; 19217c478bd9Sstevel@tonic-gate int which_list; 19227c478bd9Sstevel@tonic-gate ulong_t index; 19237c478bd9Sstevel@tonic-gate kmutex_t *phm; 19247c478bd9Sstevel@tonic-gate 19257c478bd9Sstevel@tonic-gate /* 19267c478bd9Sstevel@tonic-gate * General algorithm: 19277c478bd9Sstevel@tonic-gate * Find the starting page 19287c478bd9Sstevel@tonic-gate * Walk each page struct removing it from the freelist, 19297c478bd9Sstevel@tonic-gate * and linking it to all the other pages removed. 19307c478bd9Sstevel@tonic-gate * Once all pages are off the freelist, 19317c478bd9Sstevel@tonic-gate * walk the list, modifying p_szc to new_szc and what 19327c478bd9Sstevel@tonic-gate * ever other info needs to be done to create a large free page. 19337c478bd9Sstevel@tonic-gate * According to the flags, either return the page or put it 19347c478bd9Sstevel@tonic-gate * on the freelist. 19357c478bd9Sstevel@tonic-gate */ 19367c478bd9Sstevel@tonic-gate 19377c478bd9Sstevel@tonic-gate start_pp = page_numtopp_nolock(pfnum); 19387c478bd9Sstevel@tonic-gate ASSERT(start_pp && (start_pp->p_pagenum == pfnum)); 19397c478bd9Sstevel@tonic-gate new_npgs = page_get_pagecnt(new_szc); 19407c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pfnum, new_npgs)); 19417c478bd9Sstevel@tonic-gate 19425d07b933Sdp /* don't return page of the wrong mtype */ 19435d07b933Sdp if (mtype != PC_MTYPE_ANY && mtype != PP_2_MTYPE(start_pp)) 19445d07b933Sdp return (NULL); 19455d07b933Sdp 19467c478bd9Sstevel@tonic-gate /* 19477c478bd9Sstevel@tonic-gate * Loop through smaller pages to confirm that all pages 19487c478bd9Sstevel@tonic-gate * give the same result for PP_ISNORELOC(). 19497c478bd9Sstevel@tonic-gate * We can check this reliably here as the protocol for setting 19507c478bd9Sstevel@tonic-gate * P_NORELOC requires pages to be taken off the free list first. 19517c478bd9Sstevel@tonic-gate */ 19525d07b933Sdp noreloc = PP_ISNORELOC(start_pp); 19535d07b933Sdp for (pp = start_pp + new_npgs; --pp > start_pp; ) { 19545d07b933Sdp if (noreloc != PP_ISNORELOC(pp)) { 19557c478bd9Sstevel@tonic-gate page_promote_noreloc_err++; 19567c478bd9Sstevel@tonic-gate page_promote_err++; 19577c478bd9Sstevel@tonic-gate return (NULL); 19587c478bd9Sstevel@tonic-gate } 19597c478bd9Sstevel@tonic-gate } 19607c478bd9Sstevel@tonic-gate 19617c478bd9Sstevel@tonic-gate pages_left = new_npgs; 19627c478bd9Sstevel@tonic-gate pplist = NULL; 19637c478bd9Sstevel@tonic-gate pp = start_pp; 19647c478bd9Sstevel@tonic-gate 19657c478bd9Sstevel@tonic-gate /* Loop around coalescing the smaller pages into a big page. */ 19667c478bd9Sstevel@tonic-gate while (pages_left) { 19677c478bd9Sstevel@tonic-gate /* 19687c478bd9Sstevel@tonic-gate * Remove from the freelist. 19697c478bd9Sstevel@tonic-gate */ 19707c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 19717c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 19727c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 19737c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 19747c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 19757c478bd9Sstevel@tonic-gate 19767c478bd9Sstevel@tonic-gate /* 19777c478bd9Sstevel@tonic-gate * PG_FREE_LIST 19787c478bd9Sstevel@tonic-gate */ 19797c478bd9Sstevel@tonic-gate if (pp->p_szc) { 19807c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, 19817c478bd9Sstevel@tonic-gate pp->p_szc, bin, mtype), pp); 19827c478bd9Sstevel@tonic-gate } else { 19837c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_FREELISTS(mnode, 0, 19847c478bd9Sstevel@tonic-gate bin, mtype), pp); 19857c478bd9Sstevel@tonic-gate } 19867c478bd9Sstevel@tonic-gate which_list = PG_FREE_LIST; 19877c478bd9Sstevel@tonic-gate } else { 19887c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 19897c478bd9Sstevel@tonic-gate 19907c478bd9Sstevel@tonic-gate /* 19917c478bd9Sstevel@tonic-gate * PG_CACHE_LIST 19927c478bd9Sstevel@tonic-gate * 19937c478bd9Sstevel@tonic-gate * Since this page comes from the 19947c478bd9Sstevel@tonic-gate * cachelist, we must destroy the 19957c478bd9Sstevel@tonic-gate * vnode association. 19967c478bd9Sstevel@tonic-gate */ 19977c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 19987c478bd9Sstevel@tonic-gate goto fail_promote; 19997c478bd9Sstevel@tonic-gate } 20007c478bd9Sstevel@tonic-gate 20017c478bd9Sstevel@tonic-gate /* 20027c478bd9Sstevel@tonic-gate * We need to be careful not to deadlock 20037c478bd9Sstevel@tonic-gate * with another thread in page_lookup(). 20047c478bd9Sstevel@tonic-gate * The page_lookup() thread could be holding 20057c478bd9Sstevel@tonic-gate * the same phm that we need if the two 20067c478bd9Sstevel@tonic-gate * pages happen to hash to the same phm lock. 20077c478bd9Sstevel@tonic-gate * At this point we have locked the entire 20087c478bd9Sstevel@tonic-gate * freelist and page_lookup() could be trying 20097c478bd9Sstevel@tonic-gate * to grab a freelist lock. 20107c478bd9Sstevel@tonic-gate */ 20117c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(pp->p_vnode, pp->p_offset); 20127c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 20137c478bd9Sstevel@tonic-gate if (!mutex_tryenter(phm)) { 20148b464eb8Smec page_unlock_nocapture(pp); 20157c478bd9Sstevel@tonic-gate goto fail_promote; 20167c478bd9Sstevel@tonic-gate } 20177c478bd9Sstevel@tonic-gate 20187c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_CACHELISTS(mnode, bin, mtype), pp); 20197c478bd9Sstevel@tonic-gate page_hashout(pp, phm); 20207c478bd9Sstevel@tonic-gate mutex_exit(phm); 20217c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 20228b464eb8Smec page_unlock_nocapture(pp); 20237c478bd9Sstevel@tonic-gate which_list = PG_CACHE_LIST; 20247c478bd9Sstevel@tonic-gate } 2025affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, which_list); 20267c478bd9Sstevel@tonic-gate 20277c478bd9Sstevel@tonic-gate /* 20287c478bd9Sstevel@tonic-gate * Concatenate the smaller page(s) onto 20297c478bd9Sstevel@tonic-gate * the large page list. 20307c478bd9Sstevel@tonic-gate */ 20317c478bd9Sstevel@tonic-gate tmpnpgs = npgs = page_get_pagecnt(pp->p_szc); 20327c478bd9Sstevel@tonic-gate pages_left -= npgs; 20337c478bd9Sstevel@tonic-gate tpp = pp; 20347c478bd9Sstevel@tonic-gate while (npgs--) { 20357c478bd9Sstevel@tonic-gate tpp->p_szc = new_szc; 20367c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 20377c478bd9Sstevel@tonic-gate } 20387c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 20397c478bd9Sstevel@tonic-gate pp += tmpnpgs; 20407c478bd9Sstevel@tonic-gate } 20417c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 20427c478bd9Sstevel@tonic-gate 20437c478bd9Sstevel@tonic-gate /* 20447c478bd9Sstevel@tonic-gate * return the page to the user if requested 20457c478bd9Sstevel@tonic-gate * in the properly locked state. 20467c478bd9Sstevel@tonic-gate */ 20477c478bd9Sstevel@tonic-gate if (flags == PC_ALLOC && (page_trylock_cons(pplist, SE_EXCL))) { 20487c478bd9Sstevel@tonic-gate return (pplist); 20497c478bd9Sstevel@tonic-gate } 20507c478bd9Sstevel@tonic-gate 20517c478bd9Sstevel@tonic-gate /* 20527c478bd9Sstevel@tonic-gate * Otherwise place the new large page on the freelist 20537c478bd9Sstevel@tonic-gate */ 20547c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 20557c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pplist); 20567c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 20577c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, bin, mtype), pplist); 20587c478bd9Sstevel@tonic-gate 2059affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, PG_FREE_LIST); 20607c478bd9Sstevel@tonic-gate return (NULL); 20617c478bd9Sstevel@tonic-gate 20627c478bd9Sstevel@tonic-gate fail_promote: 20637c478bd9Sstevel@tonic-gate /* 20647c478bd9Sstevel@tonic-gate * A thread must have still been freeing or 20657c478bd9Sstevel@tonic-gate * reclaiming the page on the cachelist. 20667c478bd9Sstevel@tonic-gate * To prevent a deadlock undo what we have 20677c478bd9Sstevel@tonic-gate * done sofar and return failure. This 20687c478bd9Sstevel@tonic-gate * situation can only happen while promoting 20697c478bd9Sstevel@tonic-gate * PAGESIZE pages. 20707c478bd9Sstevel@tonic-gate */ 20717c478bd9Sstevel@tonic-gate page_promote_err++; 20727c478bd9Sstevel@tonic-gate while (pplist) { 20737c478bd9Sstevel@tonic-gate pp = pplist; 20747c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 20757c478bd9Sstevel@tonic-gate pp->p_szc = 0; 20767c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 20777c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 20787c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, mtype), pp); 2079affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 20807c478bd9Sstevel@tonic-gate } 20817c478bd9Sstevel@tonic-gate return (NULL); 20827c478bd9Sstevel@tonic-gate 20837c478bd9Sstevel@tonic-gate } 20847c478bd9Sstevel@tonic-gate 20857c478bd9Sstevel@tonic-gate /* 20867c478bd9Sstevel@tonic-gate * Break up a large page into smaller size pages. 20877c478bd9Sstevel@tonic-gate * Pages involved are on the freelist before the call and may 20887c478bd9Sstevel@tonic-gate * be returned to the caller if requested, otherwise they will 20897c478bd9Sstevel@tonic-gate * be placed back on the freelist. 20907c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 20917c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 20927c478bd9Sstevel@tonic-gate * If flags is not PC_ALLOC, the color argument is ignored, and thus 20937c478bd9Sstevel@tonic-gate * technically, any value may be passed in but PC_NO_COLOR is the standard 20947c478bd9Sstevel@tonic-gate * which should be followed for clarity's sake. 2095*19397407SSherry Moore * Returns a page whose pfn is < pfnmax 20967c478bd9Sstevel@tonic-gate */ 20977c478bd9Sstevel@tonic-gate page_t * 2098*19397407SSherry Moore page_demote(int mnode, pfn_t pfnum, pfn_t pfnmax, uchar_t cur_szc, 2099*19397407SSherry Moore uchar_t new_szc, int color, int flags) 21007c478bd9Sstevel@tonic-gate { 21017c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *npplist; 21027c478bd9Sstevel@tonic-gate pgcnt_t npgs, n; 21037c478bd9Sstevel@tonic-gate uint_t bin; 21047c478bd9Sstevel@tonic-gate uint_t mtype; 21057c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 21067c478bd9Sstevel@tonic-gate 21077c478bd9Sstevel@tonic-gate ASSERT(cur_szc != 0); 21087c478bd9Sstevel@tonic-gate ASSERT(new_szc < cur_szc); 21097c478bd9Sstevel@tonic-gate 21107c478bd9Sstevel@tonic-gate pplist = page_numtopp_nolock(pfnum); 21117c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 21127c478bd9Sstevel@tonic-gate 21137c478bd9Sstevel@tonic-gate ASSERT(pplist->p_szc == cur_szc); 21147c478bd9Sstevel@tonic-gate 21157c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 21167c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pplist)); 21177c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 21187c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, cur_szc, bin, mtype), pplist); 21197c478bd9Sstevel@tonic-gate 21207c478bd9Sstevel@tonic-gate CHK_LPG(pplist, cur_szc); 2121affbd3ccSkchow page_ctr_sub(mnode, mtype, pplist, PG_FREE_LIST); 21227c478bd9Sstevel@tonic-gate 21237c478bd9Sstevel@tonic-gate /* 21247c478bd9Sstevel@tonic-gate * Number of PAGESIZE pages for smaller new_szc 21257c478bd9Sstevel@tonic-gate * page. 21267c478bd9Sstevel@tonic-gate */ 21277c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(new_szc); 21287c478bd9Sstevel@tonic-gate 21297c478bd9Sstevel@tonic-gate while (pplist) { 21307c478bd9Sstevel@tonic-gate pp = pplist; 21317c478bd9Sstevel@tonic-gate 21327c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21337c478bd9Sstevel@tonic-gate 21347c478bd9Sstevel@tonic-gate /* 21357c478bd9Sstevel@tonic-gate * We either break it up into PAGESIZE pages or larger. 21367c478bd9Sstevel@tonic-gate */ 21377c478bd9Sstevel@tonic-gate if (npgs == 1) { /* PAGESIZE case */ 21387c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 21397c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21407c478bd9Sstevel@tonic-gate ASSERT(new_szc == 0); 21417c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 21427c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21437c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 21447c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 2145*19397407SSherry Moore (ret_pp == NULL) && (pfnmax == 0 || 2146*19397407SSherry Moore pp->p_pagenum < pfnmax) && 21477c478bd9Sstevel@tonic-gate page_trylock_cons(pp, SE_EXCL)) { 21487c478bd9Sstevel@tonic-gate ret_pp = pp; 21497c478bd9Sstevel@tonic-gate } else { 21507c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21517c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, 21527c478bd9Sstevel@tonic-gate mtype), pp); 2153affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 21547c478bd9Sstevel@tonic-gate } 21557c478bd9Sstevel@tonic-gate } else { 2156*19397407SSherry Moore page_t *try_to_return_this_page = NULL; 2157*19397407SSherry Moore int count = 0; 21587c478bd9Sstevel@tonic-gate 21597c478bd9Sstevel@tonic-gate /* 21607c478bd9Sstevel@tonic-gate * Break down into smaller lists of pages. 21617c478bd9Sstevel@tonic-gate */ 21627c478bd9Sstevel@tonic-gate page_list_break(&pplist, &npplist, npgs); 21637c478bd9Sstevel@tonic-gate 21647c478bd9Sstevel@tonic-gate pp = pplist; 21657c478bd9Sstevel@tonic-gate n = npgs; 21667c478bd9Sstevel@tonic-gate while (n--) { 21677c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 2168*19397407SSherry Moore /* 2169*19397407SSherry Moore * Check whether all the pages in this list 2170*19397407SSherry Moore * fit the request criteria. 2171*19397407SSherry Moore */ 2172*19397407SSherry Moore if (pfnmax == 0 || pp->p_pagenum < pfnmax) { 2173*19397407SSherry Moore count++; 2174*19397407SSherry Moore } 21757c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21767c478bd9Sstevel@tonic-gate pp = pp->p_next; 21777c478bd9Sstevel@tonic-gate } 21787c478bd9Sstevel@tonic-gate 2179*19397407SSherry Moore if (count == npgs && 2180*19397407SSherry Moore (pfnmax == 0 || pp->p_pagenum < pfnmax)) { 2181*19397407SSherry Moore try_to_return_this_page = pp; 2182*19397407SSherry Moore } 2183*19397407SSherry Moore 21847c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 21857c478bd9Sstevel@tonic-gate 21867c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 2187*19397407SSherry Moore if (try_to_return_this_page) 2188*19397407SSherry Moore ASSERT(mnode == 2189*19397407SSherry Moore PP_2_MEM_NODE(try_to_return_this_page)); 21907c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 2191*19397407SSherry Moore (ret_pp == NULL) && try_to_return_this_page && 2192*19397407SSherry Moore page_trylock_cons(try_to_return_this_page, 2193*19397407SSherry Moore SE_EXCL)) { 2194*19397407SSherry Moore ret_pp = try_to_return_this_page; 21957c478bd9Sstevel@tonic-gate } else { 21967c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21977c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, 21987c478bd9Sstevel@tonic-gate bin, mtype), pplist); 21997c478bd9Sstevel@tonic-gate 2200affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, 2201affbd3ccSkchow PG_FREE_LIST); 22027c478bd9Sstevel@tonic-gate } 22037c478bd9Sstevel@tonic-gate pplist = npplist; 22047c478bd9Sstevel@tonic-gate } 22057c478bd9Sstevel@tonic-gate } 22067c478bd9Sstevel@tonic-gate return (ret_pp); 22077c478bd9Sstevel@tonic-gate } 22087c478bd9Sstevel@tonic-gate 22097c478bd9Sstevel@tonic-gate int mpss_coalesce_disable = 0; 22107c478bd9Sstevel@tonic-gate 22117c478bd9Sstevel@tonic-gate /* 22127c478bd9Sstevel@tonic-gate * Coalesce free pages into a page of the given szc and color if possible. 22137c478bd9Sstevel@tonic-gate * Return the pointer to the page created, otherwise, return NULL. 22145d07b933Sdp * 22155d07b933Sdp * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 22167c478bd9Sstevel@tonic-gate */ 22175d07b933Sdp page_t * 22185d07b933Sdp page_freelist_coalesce(int mnode, uchar_t szc, uint_t color, uint_t ceq_mask, 22195d07b933Sdp int mtype, pfn_t pfnhi) 22207c478bd9Sstevel@tonic-gate { 22215d07b933Sdp int r = szc; /* region size */ 22225d07b933Sdp int mrange; 22235d07b933Sdp uint_t full, bin, color_mask, wrap = 0; 22245d07b933Sdp pfn_t pfnum, lo, hi; 22255d07b933Sdp size_t len, idx, idx0; 22265d07b933Sdp pgcnt_t cands = 0, szcpgcnt = page_get_pagecnt(szc); 22277c478bd9Sstevel@tonic-gate page_t *ret_pp; 2228ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 22295d07b933Sdp #if defined(__sparc) 22305d07b933Sdp pfn_t pfnum0, nlo, nhi; 22315d07b933Sdp #endif 22327c478bd9Sstevel@tonic-gate 22337c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 22345d07b933Sdp ASSERT(szc < MMU_PAGE_SIZES); 22355d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[szc][0]); 22367c478bd9Sstevel@tonic-gate return (NULL); 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate 22395d07b933Sdp ASSERT(szc < mmu_page_sizes); 22405d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(szc) - 1; 22415d07b933Sdp ASSERT(ceq_mask <= color_mask); 22425d07b933Sdp ASSERT(color <= color_mask); 22435d07b933Sdp color &= ceq_mask; 22445d07b933Sdp 22455d07b933Sdp /* Prevent page_counters dynamic memory from being freed */ 22465d07b933Sdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 22475d07b933Sdp 22485d07b933Sdp mrange = MTYPE_2_MRANGE(mnode, mtype); 22495d07b933Sdp ASSERT(mrange < mnode_nranges[mnode]); 22505d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[r][mrange]); 22515d07b933Sdp 22525d07b933Sdp /* get pfn range for mtype */ 22535d07b933Sdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 22545d07b933Sdp #if defined(__sparc) 22555d07b933Sdp lo = PAGE_COUNTERS_BASE(mnode, r); 22565d07b933Sdp hi = IDX_TO_PNUM(mnode, r, len); 22575d07b933Sdp #else 22585d07b933Sdp MNODETYPE_2_PFN(mnode, mtype, lo, hi); 22595d07b933Sdp hi++; 22605d07b933Sdp #endif 22615d07b933Sdp 22625d07b933Sdp /* use lower limit if given */ 22635d07b933Sdp if (pfnhi != PFNNULL && pfnhi < hi) 22645d07b933Sdp hi = pfnhi; 22655d07b933Sdp 22665d07b933Sdp /* round to szcpgcnt boundaries */ 22675d07b933Sdp lo = P2ROUNDUP(lo, szcpgcnt); 2268b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 2269b779d3e0Sdp if (lo == (pfn_t)-1) { 2270b779d3e0Sdp rw_exit(&page_ctrs_rwlock[mnode]); 2271b779d3e0Sdp return (NULL); 2272b779d3e0Sdp } 22735d07b933Sdp hi = hi & ~(szcpgcnt - 1); 22745d07b933Sdp 22755d07b933Sdp /* set lo to the closest pfn of the right color */ 2276ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ color) & ceq_mask) || 2277ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(lo) != mnode)) { 2278ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, color, ceq_mask, color_mask, 2279ce8eb11aSdp &it); 22805d07b933Sdp } 22815d07b933Sdp 22825d07b933Sdp if (hi <= lo) { 22835d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 22847c478bd9Sstevel@tonic-gate return (NULL); 22857c478bd9Sstevel@tonic-gate } 22865d07b933Sdp 22877c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 22887c478bd9Sstevel@tonic-gate 22895d07b933Sdp /* calculate the number of page candidates and initial search index */ 22905d07b933Sdp bin = color; 22915d07b933Sdp idx0 = (size_t)(-1); 22925d07b933Sdp do { 22935d07b933Sdp pgcnt_t acand; 22945d07b933Sdp 22955d07b933Sdp PGCTRS_CANDS_GETVALUECOLOR(mnode, mrange, r, bin, acand); 22965d07b933Sdp if (acand) { 22975d07b933Sdp idx = PAGE_COUNTERS_CURRENT_COLOR(mnode, 22985d07b933Sdp r, bin, mrange); 22995d07b933Sdp idx0 = MIN(idx0, idx); 23005d07b933Sdp cands += acand; 23015d07b933Sdp } 23025d07b933Sdp bin = ADD_MASKED(bin, 1, ceq_mask, color_mask); 23035d07b933Sdp } while (bin != color); 23045d07b933Sdp 23055d07b933Sdp if (cands == 0) { 23065d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_cands_skip[r][mrange]); 23075d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 23085d07b933Sdp return (NULL); 23095d07b933Sdp } 23105d07b933Sdp 23115d07b933Sdp pfnum = IDX_TO_PNUM(mnode, r, idx0); 23125d07b933Sdp if (pfnum < lo || pfnum >= hi) { 23135d07b933Sdp pfnum = lo; 2314ce8eb11aSdp } else { 2315b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2316ce8eb11aSdp if (pfnum == (pfn_t)-1) { 2317ce8eb11aSdp pfnum = lo; 2318b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2319ce8eb11aSdp ASSERT(pfnum != (pfn_t)-1); 2320ce8eb11aSdp } else if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & ceq_mask || 2321ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(pfnum) != mnode)) { 2322ce8eb11aSdp /* invalid color, get the closest correct pfn */ 2323ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2324ce8eb11aSdp color_mask, &it); 2325ce8eb11aSdp if (pfnum >= hi) { 2326ce8eb11aSdp pfnum = lo; 2327b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2328ce8eb11aSdp } 2329ce8eb11aSdp } 23305d07b933Sdp } 23315d07b933Sdp 23325d07b933Sdp /* set starting index */ 23335d07b933Sdp idx0 = PNUM_TO_IDX(mnode, r, pfnum); 23345d07b933Sdp ASSERT(idx0 < len); 23355d07b933Sdp 23365d07b933Sdp #if defined(__sparc) 23375d07b933Sdp pfnum0 = pfnum; /* page corresponding to idx0 */ 23385d07b933Sdp nhi = 0; /* search kcage ranges */ 23395d07b933Sdp #endif 23405d07b933Sdp 23415d07b933Sdp for (idx = idx0; wrap == 0 || (idx < idx0 && wrap < 2); ) { 23425d07b933Sdp 23435d07b933Sdp #if defined(__sparc) 23445d07b933Sdp /* 23455d07b933Sdp * Find lowest intersection of kcage ranges and mnode. 23465d07b933Sdp * MTYPE_NORELOC means look in the cage, otherwise outside. 23475d07b933Sdp */ 23485d07b933Sdp if (nhi <= pfnum) { 23495d07b933Sdp if (kcage_next_range(mtype == MTYPE_NORELOC, pfnum, 23505d07b933Sdp (wrap == 0 ? hi : pfnum0), &nlo, &nhi)) 23515d07b933Sdp goto wrapit; 23525d07b933Sdp 23535d07b933Sdp /* jump to the next page in the range */ 23545d07b933Sdp if (pfnum < nlo) { 23555d07b933Sdp pfnum = P2ROUNDUP(nlo, szcpgcnt); 2356b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 23575d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 23585d07b933Sdp if (idx >= len || pfnum >= hi) 23595d07b933Sdp goto wrapit; 2360ce8eb11aSdp if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & 23615d07b933Sdp ceq_mask) 23625d07b933Sdp goto next; 2363ce8eb11aSdp if (interleaved_mnodes && 2364ce8eb11aSdp PFN_2_MEM_NODE(pfnum) != mnode) 2365ce8eb11aSdp goto next; 23667c478bd9Sstevel@tonic-gate } 23675d07b933Sdp } 23685d07b933Sdp #endif 23695d07b933Sdp 23705d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) != full) 23715d07b933Sdp goto next; 23725d07b933Sdp 23735d07b933Sdp /* 23745d07b933Sdp * RFE: For performance maybe we can do something less 23755d07b933Sdp * brutal than locking the entire freelist. So far 23765d07b933Sdp * this doesn't seem to be a performance problem? 23775d07b933Sdp */ 23785d07b933Sdp page_freelist_lock(mnode); 23795d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 23805d07b933Sdp ret_pp = 23815d07b933Sdp page_promote(mnode, pfnum, r, PC_ALLOC, mtype); 23827c478bd9Sstevel@tonic-gate if (ret_pp != NULL) { 23835d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfc_coalok[r][mrange]); 23845d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, 2385ce8eb11aSdp PFN_2_COLOR(pfnum, szc, &it), mrange) = idx; 23867c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 23877c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 23887c478bd9Sstevel@tonic-gate #if defined(__sparc) 23897c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 23907c478bd9Sstevel@tonic-gate pgcnt_t npgs; 23917c478bd9Sstevel@tonic-gate 23927c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(ret_pp->p_szc); 23937c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 23947c478bd9Sstevel@tonic-gate } 23957c478bd9Sstevel@tonic-gate #endif 23967c478bd9Sstevel@tonic-gate return (ret_pp); 23977c478bd9Sstevel@tonic-gate } 23985d07b933Sdp } else { 23995d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_changed[r][mrange]); 24005d07b933Sdp } 24015d07b933Sdp 24025d07b933Sdp page_freelist_unlock(mnode); 24035d07b933Sdp /* 24045d07b933Sdp * No point looking for another page if we've 24055d07b933Sdp * already tried all of the ones that 24065d07b933Sdp * page_ctr_cands indicated. Stash off where we left 24075d07b933Sdp * off. 24085d07b933Sdp * Note: this is not exact since we don't hold the 24095d07b933Sdp * page_freelist_locks before we initially get the 24105d07b933Sdp * value of cands for performance reasons, but should 24115d07b933Sdp * be a decent approximation. 24125d07b933Sdp */ 24135d07b933Sdp if (--cands == 0) { 24145d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, color, mrange) = 24155d07b933Sdp idx; 24165d07b933Sdp break; 24175d07b933Sdp } 24185d07b933Sdp next: 24195d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2420ce8eb11aSdp color_mask, &it); 24215d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 24225d07b933Sdp if (idx >= len || pfnum >= hi) { 24235d07b933Sdp wrapit: 24245d07b933Sdp pfnum = lo; 2425b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 24265d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 24275d07b933Sdp wrap++; 24285d07b933Sdp #if defined(__sparc) 24295d07b933Sdp nhi = 0; /* search kcage ranges */ 24305d07b933Sdp #endif 24317c478bd9Sstevel@tonic-gate } 24327c478bd9Sstevel@tonic-gate } 24335d07b933Sdp 24347c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 24355d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_failed[r][mrange]); 24367c478bd9Sstevel@tonic-gate return (NULL); 24377c478bd9Sstevel@tonic-gate } 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate /* 24407c478bd9Sstevel@tonic-gate * For the given mnode, promote as many small pages to large pages as possible. 2441ce8eb11aSdp * mnode can be -1, which means do them all 24427c478bd9Sstevel@tonic-gate */ 24437c478bd9Sstevel@tonic-gate void 24447c478bd9Sstevel@tonic-gate page_freelist_coalesce_all(int mnode) 24457c478bd9Sstevel@tonic-gate { 24467c478bd9Sstevel@tonic-gate int r; /* region size */ 24477c478bd9Sstevel@tonic-gate int idx, full; 24487c478bd9Sstevel@tonic-gate size_t len; 2449ce8eb11aSdp int doall = interleaved_mnodes || mnode < 0; 2450ce8eb11aSdp int mlo = doall ? 0 : mnode; 2451ce8eb11aSdp int mhi = doall ? max_mem_nodes : (mnode + 1); 24527c478bd9Sstevel@tonic-gate 24537c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce_all); 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 24567c478bd9Sstevel@tonic-gate return; 24577c478bd9Sstevel@tonic-gate } 24587c478bd9Sstevel@tonic-gate 24597c478bd9Sstevel@tonic-gate /* 24607c478bd9Sstevel@tonic-gate * Lock the entire freelist and coalesce what we can. 24617c478bd9Sstevel@tonic-gate * 24627c478bd9Sstevel@tonic-gate * Always promote to the largest page possible 24637c478bd9Sstevel@tonic-gate * first to reduce the number of page promotions. 24647c478bd9Sstevel@tonic-gate */ 2465ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2466ce8eb11aSdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 2467ce8eb11aSdp page_freelist_lock(mnode); 2468ce8eb11aSdp } 24697c478bd9Sstevel@tonic-gate for (r = mmu_page_sizes - 1; r > 0; r--) { 2470ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2471ce8eb11aSdp pgcnt_t cands = 0; 2472ce8eb11aSdp int mrange, nranges = mnode_nranges[mnode]; 24737c478bd9Sstevel@tonic-gate 2474ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 2475ce8eb11aSdp PGCTRS_CANDS_GETVALUE(mnode, mrange, r, cands); 2476ce8eb11aSdp if (cands != 0) 2477ce8eb11aSdp break; 2478ce8eb11aSdp } 2479ce8eb11aSdp if (cands == 0) { 2480ce8eb11aSdp VM_STAT_ADD(vmm_vmstats. 2481ce8eb11aSdp page_ctrs_cands_skip_all); 2482ce8eb11aSdp continue; 2483ce8eb11aSdp } 24847c478bd9Sstevel@tonic-gate 2485ce8eb11aSdp full = FULL_REGION_CNT(r); 2486ce8eb11aSdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 2487ce8eb11aSdp 2488ce8eb11aSdp for (idx = 0; idx < len; idx++) { 2489ce8eb11aSdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 2490ce8eb11aSdp pfn_t pfnum = 2491ce8eb11aSdp IDX_TO_PNUM(mnode, r, idx); 2492ce8eb11aSdp int tmnode = interleaved_mnodes ? 2493ce8eb11aSdp PFN_2_MEM_NODE(pfnum) : mnode; 2494ce8eb11aSdp 2495ce8eb11aSdp ASSERT(pfnum >= 2496ce8eb11aSdp mem_node_config[tmnode].physbase && 2497ce8eb11aSdp pfnum < 2498ce8eb11aSdp mem_node_config[tmnode].physmax); 2499ce8eb11aSdp 2500ce8eb11aSdp (void) page_promote(tmnode, 2501ce8eb11aSdp pfnum, r, PC_FREE, PC_MTYPE_ANY); 2502ce8eb11aSdp } 25037c478bd9Sstevel@tonic-gate } 2504ce8eb11aSdp /* shared hpm_counters covers all mnodes, so we quit */ 2505ce8eb11aSdp if (interleaved_mnodes) 2506ce8eb11aSdp break; 25077c478bd9Sstevel@tonic-gate } 25087c478bd9Sstevel@tonic-gate } 2509ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2510ce8eb11aSdp page_freelist_unlock(mnode); 2511ce8eb11aSdp rw_exit(&page_ctrs_rwlock[mnode]); 2512ce8eb11aSdp } 25137c478bd9Sstevel@tonic-gate } 25147c478bd9Sstevel@tonic-gate 25157c478bd9Sstevel@tonic-gate /* 25167c478bd9Sstevel@tonic-gate * This is where all polices for moving pages around 25177c478bd9Sstevel@tonic-gate * to different page size free lists is implemented. 25187c478bd9Sstevel@tonic-gate * Returns 1 on success, 0 on failure. 25197c478bd9Sstevel@tonic-gate * 25207c478bd9Sstevel@tonic-gate * So far these are the priorities for this algorithm in descending 25217c478bd9Sstevel@tonic-gate * order: 25227c478bd9Sstevel@tonic-gate * 25237c478bd9Sstevel@tonic-gate * 1) When servicing a request try to do so with a free page 25247c478bd9Sstevel@tonic-gate * from next size up. Helps defer fragmentation as long 25257c478bd9Sstevel@tonic-gate * as possible. 25267c478bd9Sstevel@tonic-gate * 25277c478bd9Sstevel@tonic-gate * 2) Page coalesce on demand. Only when a freelist 25287c478bd9Sstevel@tonic-gate * larger than PAGESIZE is empty and step 1 25297c478bd9Sstevel@tonic-gate * will not work since all larger size lists are 25307c478bd9Sstevel@tonic-gate * also empty. 25317c478bd9Sstevel@tonic-gate * 25327c478bd9Sstevel@tonic-gate * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 25337c478bd9Sstevel@tonic-gate */ 25345d07b933Sdp 25357c478bd9Sstevel@tonic-gate page_t * 25365d07b933Sdp page_freelist_split(uchar_t szc, uint_t color, int mnode, int mtype, 2537*19397407SSherry Moore pfn_t pfnlo, pfn_t pfnhi, page_list_walker_t *plw) 25387c478bd9Sstevel@tonic-gate { 25397c478bd9Sstevel@tonic-gate uchar_t nszc = szc + 1; 25405d07b933Sdp uint_t bin, sbin, bin_prev; 25417c478bd9Sstevel@tonic-gate page_t *pp, *firstpp; 25427c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 25435d07b933Sdp uint_t color_mask; 25447c478bd9Sstevel@tonic-gate 25455d07b933Sdp if (nszc == mmu_page_sizes) 25465d07b933Sdp return (NULL); 25477c478bd9Sstevel@tonic-gate 25485d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25495d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nszc) - 1; 25505d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(szc, color); 25515d07b933Sdp bin_prev = (plw->plw_bin_split_prev == color) ? INVALID_COLOR : 25525d07b933Sdp PAGE_GET_NSZ_COLOR(szc, plw->plw_bin_split_prev); 25535d07b933Sdp 25545d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_req[szc]); 25557c478bd9Sstevel@tonic-gate /* 25565d07b933Sdp * First try to break up a larger page to fill current size freelist. 25577c478bd9Sstevel@tonic-gate */ 25585d07b933Sdp while (plw->plw_bins[nszc] != 0) { 25595d07b933Sdp 25605d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25615d07b933Sdp 25627c478bd9Sstevel@tonic-gate /* 25637c478bd9Sstevel@tonic-gate * If page found then demote it. 25647c478bd9Sstevel@tonic-gate */ 25657c478bd9Sstevel@tonic-gate if (PAGE_FREELISTS(mnode, nszc, bin, mtype)) { 25667c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 25677c478bd9Sstevel@tonic-gate firstpp = pp = PAGE_FREELISTS(mnode, nszc, bin, mtype); 25687c478bd9Sstevel@tonic-gate 25697c478bd9Sstevel@tonic-gate /* 25707c478bd9Sstevel@tonic-gate * If pfnhi is not PFNNULL, look for large page below 25717c478bd9Sstevel@tonic-gate * pfnhi. PFNNULL signifies no pfn requirement. 25727c478bd9Sstevel@tonic-gate */ 2573*19397407SSherry Moore if ((pfnhi != PFNNULL && pp->p_pagenum >= pfnhi) || 2574*19397407SSherry Moore (pfnlo != PFNNULL && pp->p_pagenum < pfnlo)) { 25757c478bd9Sstevel@tonic-gate do { 25767c478bd9Sstevel@tonic-gate pp = pp->p_vpnext; 25777c478bd9Sstevel@tonic-gate if (pp == firstpp) { 25787c478bd9Sstevel@tonic-gate pp = NULL; 25797c478bd9Sstevel@tonic-gate break; 25807c478bd9Sstevel@tonic-gate } 2581*19397407SSherry Moore } while ((pfnhi != PFNNULL && 2582*19397407SSherry Moore pp->p_pagenum >= pfnhi) || 2583*19397407SSherry Moore (pfnlo != PFNNULL && 2584*19397407SSherry Moore pp->p_pagenum < pfnlo)); 2585*19397407SSherry Moore 2586*19397407SSherry Moore if (pfnhi != PFNNULL && pp != NULL) 2587*19397407SSherry Moore ASSERT(pp->p_pagenum < pfnhi); 2588*19397407SSherry Moore 2589*19397407SSherry Moore if (pfnlo != PFNNULL && pp != NULL) 2590*19397407SSherry Moore ASSERT(pp->p_pagenum >= pfnlo); 25917c478bd9Sstevel@tonic-gate } 25927c478bd9Sstevel@tonic-gate if (pp) { 25935d07b933Sdp uint_t ccolor = page_correct_color(szc, nszc, 25945d07b933Sdp color, bin, plw->plw_ceq_mask[szc]); 25955d07b933Sdp 25967c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == nszc); 25975d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_demote[nszc]); 25987c478bd9Sstevel@tonic-gate ret_pp = page_demote(mnode, pp->p_pagenum, 2599*19397407SSherry Moore pfnhi, pp->p_szc, szc, ccolor, PC_ALLOC); 26007c478bd9Sstevel@tonic-gate if (ret_pp) { 26017c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 26027c478bd9Sstevel@tonic-gate #if defined(__sparc) 26037c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 26047c478bd9Sstevel@tonic-gate pgcnt_t npgs; 26057c478bd9Sstevel@tonic-gate 26067c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt( 26077c478bd9Sstevel@tonic-gate ret_pp->p_szc); 26087c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 26097c478bd9Sstevel@tonic-gate } 26107c478bd9Sstevel@tonic-gate #endif 26117c478bd9Sstevel@tonic-gate return (ret_pp); 26127c478bd9Sstevel@tonic-gate } 26137c478bd9Sstevel@tonic-gate } 26147c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 26157c478bd9Sstevel@tonic-gate } 26167c478bd9Sstevel@tonic-gate 26175d07b933Sdp /* loop through next size bins */ 26185d07b933Sdp bin = ADD_MASKED(bin, 1, plw->plw_ceq_mask[nszc], color_mask); 26195d07b933Sdp plw->plw_bins[nszc]--; 26205d07b933Sdp 26215d07b933Sdp if (bin == sbin) { 26225d07b933Sdp uchar_t nnszc = nszc + 1; 26235d07b933Sdp 26245d07b933Sdp /* we are done with this page size - check next */ 26255d07b933Sdp if (plw->plw_bins[nnszc] == 0) 26265d07b933Sdp /* we have already checked next size bins */ 26275d07b933Sdp break; 26285d07b933Sdp 26295d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(nszc, bin); 26305d07b933Sdp if (bin_prev != INVALID_COLOR) { 26315d07b933Sdp bin_prev = PAGE_GET_NSZ_COLOR(nszc, bin_prev); 26325d07b933Sdp if (!((bin ^ bin_prev) & 26335d07b933Sdp plw->plw_ceq_mask[nnszc])) 26345d07b933Sdp break; 26355d07b933Sdp } 26365d07b933Sdp ASSERT(nnszc < mmu_page_sizes); 26375d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nnszc) - 1; 26385d07b933Sdp nszc = nnszc; 26395d07b933Sdp ASSERT(nszc < mmu_page_sizes); 26405d07b933Sdp } 26417c478bd9Sstevel@tonic-gate } 26427c478bd9Sstevel@tonic-gate 26437c478bd9Sstevel@tonic-gate return (ret_pp); 26447c478bd9Sstevel@tonic-gate } 26457c478bd9Sstevel@tonic-gate 26467c478bd9Sstevel@tonic-gate /* 26477c478bd9Sstevel@tonic-gate * Helper routine used only by the freelist code to lock 26487c478bd9Sstevel@tonic-gate * a page. If the page is a large page then it succeeds in 26497c478bd9Sstevel@tonic-gate * locking all the constituent pages or none at all. 26507c478bd9Sstevel@tonic-gate * Returns 1 on sucess, 0 on failure. 26517c478bd9Sstevel@tonic-gate */ 26527c478bd9Sstevel@tonic-gate static int 26537c478bd9Sstevel@tonic-gate page_trylock_cons(page_t *pp, se_t se) 26547c478bd9Sstevel@tonic-gate { 26557c478bd9Sstevel@tonic-gate page_t *tpp, *first_pp = pp; 26567c478bd9Sstevel@tonic-gate 26577c478bd9Sstevel@tonic-gate /* 26587c478bd9Sstevel@tonic-gate * Fail if can't lock first or only page. 26597c478bd9Sstevel@tonic-gate */ 26607c478bd9Sstevel@tonic-gate if (!page_trylock(pp, se)) { 26617c478bd9Sstevel@tonic-gate return (0); 26627c478bd9Sstevel@tonic-gate } 26637c478bd9Sstevel@tonic-gate 26647c478bd9Sstevel@tonic-gate /* 26657c478bd9Sstevel@tonic-gate * PAGESIZE: common case. 26667c478bd9Sstevel@tonic-gate */ 26677c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 26687c478bd9Sstevel@tonic-gate return (1); 26697c478bd9Sstevel@tonic-gate } 26707c478bd9Sstevel@tonic-gate 26717c478bd9Sstevel@tonic-gate /* 26727c478bd9Sstevel@tonic-gate * Large page case. 26737c478bd9Sstevel@tonic-gate */ 26747c478bd9Sstevel@tonic-gate tpp = pp->p_next; 26757c478bd9Sstevel@tonic-gate while (tpp != pp) { 26767c478bd9Sstevel@tonic-gate if (!page_trylock(tpp, se)) { 26777c478bd9Sstevel@tonic-gate /* 26788b464eb8Smec * On failure unlock what we have locked so far. 26798b464eb8Smec * We want to avoid attempting to capture these 26808b464eb8Smec * pages as the pcm mutex may be held which could 26818b464eb8Smec * lead to a recursive mutex panic. 26827c478bd9Sstevel@tonic-gate */ 26837c478bd9Sstevel@tonic-gate while (first_pp != tpp) { 26848b464eb8Smec page_unlock_nocapture(first_pp); 26857c478bd9Sstevel@tonic-gate first_pp = first_pp->p_next; 26867c478bd9Sstevel@tonic-gate } 26877c478bd9Sstevel@tonic-gate return (0); 26887c478bd9Sstevel@tonic-gate } 26897c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 26907c478bd9Sstevel@tonic-gate } 26917c478bd9Sstevel@tonic-gate return (1); 26927c478bd9Sstevel@tonic-gate } 26937c478bd9Sstevel@tonic-gate 26945d07b933Sdp /* 26955d07b933Sdp * init context for walking page lists 26965d07b933Sdp * Called when a page of the given szc in unavailable. Sets markers 26975d07b933Sdp * for the beginning of the search to detect when search has 26985d07b933Sdp * completed a full cycle. Sets flags for splitting larger pages 26995d07b933Sdp * and coalescing smaller pages. Page walking procedes until a page 27005d07b933Sdp * of the desired equivalent color is found. 27015d07b933Sdp */ 27025d07b933Sdp void 27035d07b933Sdp page_list_walk_init(uchar_t szc, uint_t flags, uint_t bin, int can_split, 27045d07b933Sdp int use_ceq, page_list_walker_t *plw) 27057c478bd9Sstevel@tonic-gate { 27065d07b933Sdp uint_t nszc, ceq_mask, colors; 27075d07b933Sdp uchar_t ceq = use_ceq ? colorequivszc[szc] : 0; 27087c478bd9Sstevel@tonic-gate 27097c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 27105d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 27117c478bd9Sstevel@tonic-gate 27125d07b933Sdp plw->plw_colors = colors; 27135d07b933Sdp plw->plw_color_mask = colors - 1; 27145d07b933Sdp plw->plw_bin_marker = plw->plw_bin0 = bin; 27155d07b933Sdp plw->plw_bin_split_prev = bin; 27165d07b933Sdp plw->plw_bin_step = (szc == 0) ? vac_colors : 1; 27177c478bd9Sstevel@tonic-gate 27185d07b933Sdp /* 27195d07b933Sdp * if vac aliasing is possible make sure lower order color 27205d07b933Sdp * bits are never ignored 27215d07b933Sdp */ 27225d07b933Sdp if (vac_colors > 1) 27235d07b933Sdp ceq &= 0xf0; 27247c478bd9Sstevel@tonic-gate 27257c478bd9Sstevel@tonic-gate /* 27265d07b933Sdp * calculate the number of non-equivalent colors and 27275d07b933Sdp * color equivalency mask 27287c478bd9Sstevel@tonic-gate */ 27295d07b933Sdp plw->plw_ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 27305d07b933Sdp ASSERT(szc > 0 || plw->plw_ceq_dif >= vac_colors); 27315d07b933Sdp ASSERT(plw->plw_ceq_dif > 0); 27325d07b933Sdp plw->plw_ceq_mask[szc] = (plw->plw_ceq_dif - 1) << (ceq & 0xf); 27337c478bd9Sstevel@tonic-gate 27345d07b933Sdp if (flags & PG_MATCH_COLOR) { 27355d07b933Sdp if (cpu_page_colors < 0) { 27365d07b933Sdp /* 27375d07b933Sdp * this is a heterogeneous machine with different CPUs 27385d07b933Sdp * having different size e$ (not supported for ni2/rock 27395d07b933Sdp */ 27405d07b933Sdp uint_t cpucolors = CPUSETSIZE() >> PAGE_GET_SHIFT(szc); 27415d07b933Sdp cpucolors = MAX(cpucolors, 1); 27425d07b933Sdp ceq_mask = plw->plw_color_mask & (cpucolors - 1); 27435d07b933Sdp plw->plw_ceq_mask[szc] = 27445d07b933Sdp MIN(ceq_mask, plw->plw_ceq_mask[szc]); 27455d07b933Sdp } 27465d07b933Sdp plw->plw_ceq_dif = 1; 27475d07b933Sdp } 27487c478bd9Sstevel@tonic-gate 27495d07b933Sdp /* we can split pages in the freelist, but not the cachelist */ 27505d07b933Sdp if (can_split) { 2751ce8eb11aSdp plw->plw_do_split = (szc + 1 < mmu_page_sizes) ? 1 : 0; 27527c478bd9Sstevel@tonic-gate 2753ce8eb11aSdp /* set next szc color masks and number of free list bins */ 2754ce8eb11aSdp for (nszc = szc + 1; nszc < mmu_page_sizes; nszc++, szc++) { 2755ce8eb11aSdp plw->plw_ceq_mask[nszc] = PAGE_GET_NSZ_MASK(szc, 2756ce8eb11aSdp plw->plw_ceq_mask[szc]); 2757ce8eb11aSdp plw->plw_bins[nszc] = PAGE_GET_PAGECOLORS(nszc); 2758ce8eb11aSdp } 2759ce8eb11aSdp plw->plw_ceq_mask[nszc] = INVALID_MASK; 2760ce8eb11aSdp plw->plw_bins[nszc] = 0; 27617c478bd9Sstevel@tonic-gate 27625d07b933Sdp } else { 2763ce8eb11aSdp ASSERT(szc == 0); 2764ce8eb11aSdp plw->plw_do_split = 0; 2765ce8eb11aSdp plw->plw_bins[1] = 0; 2766ce8eb11aSdp plw->plw_ceq_mask[1] = INVALID_MASK; 27677c478bd9Sstevel@tonic-gate } 27685d07b933Sdp } 27697c478bd9Sstevel@tonic-gate 27705d07b933Sdp /* 27715d07b933Sdp * set mark to flag where next split should occur 27725d07b933Sdp */ 27735d07b933Sdp #define PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw) { \ 27745d07b933Sdp uint_t bin_nsz = PAGE_GET_NSZ_COLOR(szc, bin); \ 27755d07b933Sdp uint_t bin0_nsz = PAGE_GET_NSZ_COLOR(szc, plw->plw_bin0); \ 27765d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[nszc] & plw->plw_color_mask; \ 27775d07b933Sdp plw->plw_split_next = \ 27785d07b933Sdp INC_MASKED(bin_nsz, neq_mask, plw->plw_color_mask); \ 27795d07b933Sdp if (!((plw->plw_split_next ^ bin0_nsz) & plw->plw_ceq_mask[nszc])) { \ 27805d07b933Sdp plw->plw_split_next = \ 27815d07b933Sdp INC_MASKED(plw->plw_split_next, \ 27825d07b933Sdp neq_mask, plw->plw_color_mask); \ 27835d07b933Sdp } \ 27845d07b933Sdp } 27857c478bd9Sstevel@tonic-gate 27865d07b933Sdp uint_t 27875d07b933Sdp page_list_walk_next_bin(uchar_t szc, uint_t bin, page_list_walker_t *plw) 27885d07b933Sdp { 27895d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[szc] & plw->plw_color_mask; 27905d07b933Sdp uint_t bin0_nsz, nbin_nsz, nbin0, nbin; 27915d07b933Sdp uchar_t nszc = szc + 1; 27925d07b933Sdp 27935d07b933Sdp nbin = ADD_MASKED(bin, 27945d07b933Sdp plw->plw_bin_step, neq_mask, plw->plw_color_mask); 27955d07b933Sdp 27965d07b933Sdp if (plw->plw_do_split) { 27975d07b933Sdp plw->plw_bin_split_prev = bin; 27985d07b933Sdp PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw); 27995d07b933Sdp plw->plw_do_split = 0; 28005d07b933Sdp } 28015d07b933Sdp 28025d07b933Sdp if (szc == 0) { 28035d07b933Sdp if (plw->plw_count != 0 || plw->plw_ceq_dif == vac_colors) { 28045d07b933Sdp if (nbin == plw->plw_bin0 && 28055d07b933Sdp (vac_colors == 1 || nbin != plw->plw_bin_marker)) { 28065d07b933Sdp nbin = ADD_MASKED(nbin, plw->plw_bin_step, 28075d07b933Sdp neq_mask, plw->plw_color_mask); 28085d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 28095d07b933Sdp } 28105d07b933Sdp 28115d07b933Sdp if (vac_colors > 1 && nbin == plw->plw_bin_marker) { 28125d07b933Sdp plw->plw_bin_marker = 28135d07b933Sdp nbin = INC_MASKED(nbin, neq_mask, 2814ce8eb11aSdp plw->plw_color_mask); 28155d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 28165d07b933Sdp /* 28175d07b933Sdp * large pages all have the same vac color 28185d07b933Sdp * so by now we should be done with next 28195d07b933Sdp * size page splitting process 28205d07b933Sdp */ 28215d07b933Sdp ASSERT(plw->plw_bins[1] == 0); 28225d07b933Sdp plw->plw_do_split = 0; 28235d07b933Sdp return (nbin); 28245d07b933Sdp } 28255d07b933Sdp 28265d07b933Sdp } else { 28275d07b933Sdp uint_t bin_jump = (vac_colors == 1) ? 28285d07b933Sdp (BIN_STEP & ~3) - (plw->plw_bin0 & 3) : BIN_STEP; 28295d07b933Sdp 28305d07b933Sdp bin_jump &= ~(vac_colors - 1); 28315d07b933Sdp 28325d07b933Sdp nbin0 = ADD_MASKED(plw->plw_bin0, bin_jump, neq_mask, 28335d07b933Sdp plw->plw_color_mask); 28345d07b933Sdp 28355d07b933Sdp if ((nbin0 ^ plw->plw_bin0) & plw->plw_ceq_mask[szc]) { 28365d07b933Sdp 28375d07b933Sdp plw->plw_bin_marker = nbin = nbin0; 28385d07b933Sdp 28395d07b933Sdp if (plw->plw_bins[nszc] != 0) { 28405d07b933Sdp /* 28415d07b933Sdp * check if next page size bin is the 28425d07b933Sdp * same as the next page size bin for 28435d07b933Sdp * bin0 28445d07b933Sdp */ 28455d07b933Sdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, 28465d07b933Sdp nbin); 28475d07b933Sdp bin0_nsz = PAGE_GET_NSZ_COLOR(szc, 28485d07b933Sdp plw->plw_bin0); 28495d07b933Sdp 28505d07b933Sdp if ((bin0_nsz ^ nbin_nsz) & 28515d07b933Sdp plw->plw_ceq_mask[nszc]) 28525d07b933Sdp plw->plw_do_split = 1; 28535d07b933Sdp } 28545d07b933Sdp return (nbin); 28555d07b933Sdp } 28565d07b933Sdp } 28575d07b933Sdp } 28585d07b933Sdp 28595d07b933Sdp if (plw->plw_bins[nszc] != 0) { 2860ce8eb11aSdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, nbin); 2861ce8eb11aSdp if (!((plw->plw_split_next ^ nbin_nsz) & 2862ce8eb11aSdp plw->plw_ceq_mask[nszc])) 2863ce8eb11aSdp plw->plw_do_split = 1; 28645d07b933Sdp } 28655d07b933Sdp 28665d07b933Sdp return (nbin); 28675d07b933Sdp } 28685d07b933Sdp 28695d07b933Sdp page_t * 28705d07b933Sdp page_get_mnode_freelist(int mnode, uint_t bin, int mtype, uchar_t szc, 28715d07b933Sdp uint_t flags) 28725d07b933Sdp { 28735d07b933Sdp kmutex_t *pcm; 28745d07b933Sdp page_t *pp, *first_pp; 28755d07b933Sdp uint_t sbin; 28765d07b933Sdp int plw_initialized; 28775d07b933Sdp page_list_walker_t plw; 28785d07b933Sdp 28795d07b933Sdp ASSERT(szc < mmu_page_sizes); 28805d07b933Sdp 28815d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_alloc[szc]); 28825d07b933Sdp 28835d07b933Sdp MTYPE_START(mnode, mtype, flags); 28845d07b933Sdp if (mtype < 0) { /* mnode does not have memory in mtype range */ 28855d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocempty[szc]); 28865d07b933Sdp return (NULL); 28875d07b933Sdp } 28885d07b933Sdp try_again: 28895d07b933Sdp 28905d07b933Sdp plw_initialized = 0; 28915d07b933Sdp plw.plw_ceq_dif = 1; 28927c478bd9Sstevel@tonic-gate 28937c478bd9Sstevel@tonic-gate /* 28947c478bd9Sstevel@tonic-gate * Only hold one freelist lock at a time, that way we 28957c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 28967c478bd9Sstevel@tonic-gate * ordering. 28977c478bd9Sstevel@tonic-gate */ 28985d07b933Sdp for (plw.plw_count = 0; 28995d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 29005d07b933Sdp sbin = bin; 29015d07b933Sdp do { 29025d07b933Sdp if (!PAGE_FREELISTS(mnode, szc, bin, mtype)) 29035d07b933Sdp goto bin_empty_1; 29045d07b933Sdp 29057c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 29067c478bd9Sstevel@tonic-gate mutex_enter(pcm); 29077c478bd9Sstevel@tonic-gate pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 29085d07b933Sdp if (pp == NULL) 29095d07b933Sdp goto bin_empty_0; 29105d07b933Sdp 29115d07b933Sdp /* 29125d07b933Sdp * These were set before the page 29135d07b933Sdp * was put on the free list, 29145d07b933Sdp * they must still be set. 29155d07b933Sdp */ 29165d07b933Sdp ASSERT(PP_ISFREE(pp)); 29175d07b933Sdp ASSERT(PP_ISAGED(pp)); 29185d07b933Sdp ASSERT(pp->p_vnode == NULL); 29195d07b933Sdp ASSERT(pp->p_hash == NULL); 29205d07b933Sdp ASSERT(pp->p_offset == (u_offset_t)-1); 29215d07b933Sdp ASSERT(pp->p_szc == szc); 29225d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 29235d07b933Sdp 29245d07b933Sdp /* 29255d07b933Sdp * Walk down the hash chain. 29265d07b933Sdp * 8k pages are linked on p_next 29275d07b933Sdp * and p_prev fields. Large pages 29285d07b933Sdp * are a contiguous group of 29295d07b933Sdp * constituent pages linked together 29305d07b933Sdp * on their p_next and p_prev fields. 29315d07b933Sdp * The large pages are linked together 29325d07b933Sdp * on the hash chain using p_vpnext 29335d07b933Sdp * p_vpprev of the base constituent 29345d07b933Sdp * page of each large page. 29355d07b933Sdp */ 29365d07b933Sdp first_pp = pp; 29375d07b933Sdp while (!page_trylock_cons(pp, SE_EXCL)) { 29385d07b933Sdp if (szc == 0) { 29395d07b933Sdp pp = pp->p_next; 29405d07b933Sdp } else { 29415d07b933Sdp pp = pp->p_vpnext; 29425d07b933Sdp } 29435d07b933Sdp 29447c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 29457c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 29467c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 29477c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL); 29487c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1); 29497c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 29507c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 29517c478bd9Sstevel@tonic-gate 29525d07b933Sdp if (pp == first_pp) 29535d07b933Sdp goto bin_empty_0; 29545d07b933Sdp } 29557c478bd9Sstevel@tonic-gate 29565d07b933Sdp ASSERT(pp != NULL); 29575d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 29585d07b933Sdp ASSERT(pp->p_szc == szc); 29595d07b933Sdp if (szc == 0) { 29605d07b933Sdp page_sub(&PAGE_FREELISTS(mnode, 29615d07b933Sdp szc, bin, mtype), pp); 29625d07b933Sdp } else { 29635d07b933Sdp page_vpsub(&PAGE_FREELISTS(mnode, 29645d07b933Sdp szc, bin, mtype), pp); 29655d07b933Sdp CHK_LPG(pp, szc); 29665d07b933Sdp } 29675d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 29687c478bd9Sstevel@tonic-gate 29695d07b933Sdp if ((PP_ISFREE(pp) == 0) || (PP_ISAGED(pp) == 0)) 29705d07b933Sdp panic("free page is not. pp %p", (void *)pp); 29715d07b933Sdp mutex_exit(pcm); 29727c478bd9Sstevel@tonic-gate 29737c478bd9Sstevel@tonic-gate #if defined(__sparc) 29745d07b933Sdp ASSERT(!kcage_on || PP_ISNORELOC(pp) || 29755d07b933Sdp (flags & PG_NORELOC) == 0); 29767c478bd9Sstevel@tonic-gate 29775d07b933Sdp if (PP_ISNORELOC(pp)) 29785d07b933Sdp kcage_freemem_sub(page_get_pagecnt(szc)); 29797c478bd9Sstevel@tonic-gate #endif 29805d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocok[szc]); 29815d07b933Sdp return (pp); 29827c478bd9Sstevel@tonic-gate 29835d07b933Sdp bin_empty_0: 29845d07b933Sdp mutex_exit(pcm); 29855d07b933Sdp bin_empty_1: 29865d07b933Sdp if (plw_initialized == 0) { 29875d07b933Sdp page_list_walk_init(szc, flags, bin, 1, 1, 29885d07b933Sdp &plw); 29895d07b933Sdp plw_initialized = 1; 29905d07b933Sdp ASSERT(plw.plw_colors <= 29915d07b933Sdp PAGE_GET_PAGECOLORS(szc)); 29925d07b933Sdp ASSERT(plw.plw_colors > 0); 29935d07b933Sdp ASSERT((plw.plw_colors & 29945d07b933Sdp (plw.plw_colors - 1)) == 0); 29955d07b933Sdp ASSERT(bin < plw.plw_colors); 29965d07b933Sdp ASSERT(plw.plw_ceq_mask[szc] < plw.plw_colors); 29977c478bd9Sstevel@tonic-gate } 29985d07b933Sdp /* calculate the next bin with equivalent color */ 29995d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 30005d07b933Sdp plw.plw_ceq_mask[szc], plw.plw_color_mask); 30015d07b933Sdp } while (sbin != bin); 30027c478bd9Sstevel@tonic-gate 30037c478bd9Sstevel@tonic-gate /* 30045d07b933Sdp * color bins are all empty if color match. Try and 30055d07b933Sdp * satisfy the request by breaking up or coalescing 30065d07b933Sdp * pages from a different size freelist of the correct 30075d07b933Sdp * color that satisfies the ORIGINAL color requested. 30085d07b933Sdp * If that fails then try pages of the same size but 30095d07b933Sdp * different colors assuming we are not called with 30107c478bd9Sstevel@tonic-gate * PG_MATCH_COLOR. 30117c478bd9Sstevel@tonic-gate */ 30125d07b933Sdp if (plw.plw_do_split && 30135d07b933Sdp (pp = page_freelist_split(szc, bin, mnode, 3014*19397407SSherry Moore mtype, PFNNULL, PFNNULL, &plw)) != NULL) 3015ce8eb11aSdp return (pp); 30167c478bd9Sstevel@tonic-gate 30175d07b933Sdp if (szc > 0 && (pp = page_freelist_coalesce(mnode, szc, 30185d07b933Sdp bin, plw.plw_ceq_mask[szc], mtype, PFNNULL)) != NULL) 30195d07b933Sdp return (pp); 30207c478bd9Sstevel@tonic-gate 30215d07b933Sdp if (plw.plw_ceq_dif > 1) 30225d07b933Sdp bin = page_list_walk_next_bin(szc, bin, &plw); 30237c478bd9Sstevel@tonic-gate } 30247c478bd9Sstevel@tonic-gate 3025affbd3ccSkchow /* if allowed, cycle through additional mtypes */ 3026affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3027affbd3ccSkchow if (mtype >= 0) 30285d07b933Sdp goto try_again; 3029affbd3ccSkchow 30307c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmf_allocfailed[szc]); 30317c478bd9Sstevel@tonic-gate 30327c478bd9Sstevel@tonic-gate return (NULL); 30337c478bd9Sstevel@tonic-gate } 30347c478bd9Sstevel@tonic-gate 30357c478bd9Sstevel@tonic-gate /* 30367c478bd9Sstevel@tonic-gate * Returns the count of free pages for 'pp' with size code 'szc'. 30377c478bd9Sstevel@tonic-gate * Note: This function does not return an exact value as the page freelist 30387c478bd9Sstevel@tonic-gate * locks are not held and thus the values in the page_counters may be 30397c478bd9Sstevel@tonic-gate * changing as we walk through the data. 30407c478bd9Sstevel@tonic-gate */ 30417c478bd9Sstevel@tonic-gate static int 30427c478bd9Sstevel@tonic-gate page_freecnt(int mnode, page_t *pp, uchar_t szc) 30437c478bd9Sstevel@tonic-gate { 30447c478bd9Sstevel@tonic-gate pgcnt_t pgfree; 30457c478bd9Sstevel@tonic-gate pgcnt_t cnt; 30467c478bd9Sstevel@tonic-gate ssize_t r = szc; /* region size */ 30477c478bd9Sstevel@tonic-gate ssize_t idx; 30487c478bd9Sstevel@tonic-gate int i; 30497c478bd9Sstevel@tonic-gate int full, range; 30507c478bd9Sstevel@tonic-gate 30517c478bd9Sstevel@tonic-gate /* Make sure pagenum passed in is aligned properly */ 30527c478bd9Sstevel@tonic-gate ASSERT((pp->p_pagenum & (PNUM_SIZE(szc) - 1)) == 0); 30537c478bd9Sstevel@tonic-gate ASSERT(szc > 0); 30547c478bd9Sstevel@tonic-gate 30557c478bd9Sstevel@tonic-gate /* Prevent page_counters dynamic memory from being freed */ 30567c478bd9Sstevel@tonic-gate rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 30577c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30587c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30597c478bd9Sstevel@tonic-gate pgfree = cnt << PNUM_SHIFT(r - 1); 30607c478bd9Sstevel@tonic-gate range = FULL_REGION_CNT(szc); 30617c478bd9Sstevel@tonic-gate 30627c478bd9Sstevel@tonic-gate /* Check for completely full region */ 30637c478bd9Sstevel@tonic-gate if (cnt == range) { 30647c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30657c478bd9Sstevel@tonic-gate return (pgfree); 30667c478bd9Sstevel@tonic-gate } 30677c478bd9Sstevel@tonic-gate 30687c478bd9Sstevel@tonic-gate while (--r > 0) { 30697c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30707c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 30717c478bd9Sstevel@tonic-gate for (i = 0; i < range; i++, idx++) { 30727c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30737c478bd9Sstevel@tonic-gate /* 30747c478bd9Sstevel@tonic-gate * If cnt here is full, that means we have already 30757c478bd9Sstevel@tonic-gate * accounted for these pages earlier. 30767c478bd9Sstevel@tonic-gate */ 30777c478bd9Sstevel@tonic-gate if (cnt != full) { 30787c478bd9Sstevel@tonic-gate pgfree += (cnt << PNUM_SHIFT(r - 1)); 30797c478bd9Sstevel@tonic-gate } 30807c478bd9Sstevel@tonic-gate } 30817c478bd9Sstevel@tonic-gate range *= full; 30827c478bd9Sstevel@tonic-gate } 30837c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30847c478bd9Sstevel@tonic-gate return (pgfree); 30857c478bd9Sstevel@tonic-gate } 30867c478bd9Sstevel@tonic-gate 30877c478bd9Sstevel@tonic-gate /* 30887c478bd9Sstevel@tonic-gate * Called from page_geti_contig_pages to exclusively lock constituent pages 30897c478bd9Sstevel@tonic-gate * starting from 'spp' for page size code 'szc'. 30907c478bd9Sstevel@tonic-gate * 30917c478bd9Sstevel@tonic-gate * If 'ptcpthreshold' is set, the number of free pages needed in the 'szc' 30927c478bd9Sstevel@tonic-gate * region needs to be greater than or equal to the threshold. 30937c478bd9Sstevel@tonic-gate */ 30947c478bd9Sstevel@tonic-gate static int 30957c478bd9Sstevel@tonic-gate page_trylock_contig_pages(int mnode, page_t *spp, uchar_t szc, int flags) 30967c478bd9Sstevel@tonic-gate { 30977c478bd9Sstevel@tonic-gate pgcnt_t pgcnt = PNUM_SIZE(szc); 30987c478bd9Sstevel@tonic-gate pgcnt_t pgfree, i; 30997c478bd9Sstevel@tonic-gate page_t *pp; 31007c478bd9Sstevel@tonic-gate 31017c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcp[szc]); 31027c478bd9Sstevel@tonic-gate 31037c478bd9Sstevel@tonic-gate 31047c478bd9Sstevel@tonic-gate if ((ptcpthreshold == 0) || (flags & PGI_PGCPHIPRI)) 31057c478bd9Sstevel@tonic-gate goto skipptcpcheck; 31067c478bd9Sstevel@tonic-gate /* 31077c478bd9Sstevel@tonic-gate * check if there are sufficient free pages available before attempting 31087c478bd9Sstevel@tonic-gate * to trylock. Count is approximate as page counters can change. 31097c478bd9Sstevel@tonic-gate */ 31107c478bd9Sstevel@tonic-gate pgfree = page_freecnt(mnode, spp, szc); 31117c478bd9Sstevel@tonic-gate 31127c478bd9Sstevel@tonic-gate /* attempt to trylock if there are sufficient already free pages */ 31137c478bd9Sstevel@tonic-gate if (pgfree < pgcnt/ptcpthreshold) { 31147c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfreethresh[szc]); 31157c478bd9Sstevel@tonic-gate return (0); 31167c478bd9Sstevel@tonic-gate } 31177c478bd9Sstevel@tonic-gate 31187c478bd9Sstevel@tonic-gate skipptcpcheck: 31197c478bd9Sstevel@tonic-gate 31207c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++) { 31217c478bd9Sstevel@tonic-gate pp = &spp[i]; 31227c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 31237c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailexcl[szc]); 31247c478bd9Sstevel@tonic-gate while (--i != (pgcnt_t)-1) { 31257c478bd9Sstevel@tonic-gate pp = &spp[i]; 31267c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31278b464eb8Smec page_unlock_nocapture(pp); 31287c478bd9Sstevel@tonic-gate } 31297c478bd9Sstevel@tonic-gate return (0); 31307c478bd9Sstevel@tonic-gate } 31317c478bd9Sstevel@tonic-gate ASSERT(spp[i].p_pagenum == spp->p_pagenum + i); 31327c478bd9Sstevel@tonic-gate if ((pp->p_szc > szc || (szc && pp->p_szc == szc)) && 31337c478bd9Sstevel@tonic-gate !PP_ISFREE(pp)) { 31347c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailszc[szc]); 31357c478bd9Sstevel@tonic-gate ASSERT(i == 0); 31368b464eb8Smec page_unlock_nocapture(pp); 31377c478bd9Sstevel@tonic-gate return (0); 31387c478bd9Sstevel@tonic-gate } 31397c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 31407c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailcage[szc]); 31417c478bd9Sstevel@tonic-gate while (i != (pgcnt_t)-1) { 31427c478bd9Sstevel@tonic-gate pp = &spp[i]; 31437c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31448b464eb8Smec page_unlock_nocapture(pp); 31457c478bd9Sstevel@tonic-gate i--; 31467c478bd9Sstevel@tonic-gate } 31477c478bd9Sstevel@tonic-gate return (0); 31487c478bd9Sstevel@tonic-gate } 31497c478bd9Sstevel@tonic-gate } 31507c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpok[szc]); 31517c478bd9Sstevel@tonic-gate return (1); 31527c478bd9Sstevel@tonic-gate } 31537c478bd9Sstevel@tonic-gate 31547c478bd9Sstevel@tonic-gate /* 31557c478bd9Sstevel@tonic-gate * Claim large page pointed to by 'pp'. 'pp' is the starting set 31567c478bd9Sstevel@tonic-gate * of 'szc' constituent pages that had been locked exclusively previously. 31577c478bd9Sstevel@tonic-gate * Will attempt to relocate constituent pages in use. 31587c478bd9Sstevel@tonic-gate */ 31597c478bd9Sstevel@tonic-gate static page_t * 31607c478bd9Sstevel@tonic-gate page_claim_contig_pages(page_t *pp, uchar_t szc, int flags) 31617c478bd9Sstevel@tonic-gate { 31627c478bd9Sstevel@tonic-gate spgcnt_t pgcnt, npgs, i; 31637c478bd9Sstevel@tonic-gate page_t *targpp, *rpp, *hpp; 31647c478bd9Sstevel@tonic-gate page_t *replpp = NULL; 31657c478bd9Sstevel@tonic-gate page_t *pplist = NULL; 31667c478bd9Sstevel@tonic-gate 31677c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 31687c478bd9Sstevel@tonic-gate 31697c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(szc); 31707c478bd9Sstevel@tonic-gate while (pgcnt) { 31717c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31727c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 31737c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 31747c478bd9Sstevel@tonic-gate /* 31757c478bd9Sstevel@tonic-gate * If this is a PG_FREE_LIST page then its 31767c478bd9Sstevel@tonic-gate * size code can change underneath us due to 31777c478bd9Sstevel@tonic-gate * page promotion or demotion. As an optimzation 31787c478bd9Sstevel@tonic-gate * use page_list_sub_pages() instead of 31797c478bd9Sstevel@tonic-gate * page_list_sub(). 31807c478bd9Sstevel@tonic-gate */ 31817c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 31827c478bd9Sstevel@tonic-gate page_list_sub_pages(pp, szc); 31837c478bd9Sstevel@tonic-gate if (pp->p_szc == szc) { 31847c478bd9Sstevel@tonic-gate return (pp); 31857c478bd9Sstevel@tonic-gate } 31867c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < szc); 31877c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31887c478bd9Sstevel@tonic-gate hpp = pp; 31897c478bd9Sstevel@tonic-gate for (i = 0; i < npgs; i++, pp++) { 31907c478bd9Sstevel@tonic-gate pp->p_szc = szc; 31917c478bd9Sstevel@tonic-gate } 31927c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &hpp); 31937c478bd9Sstevel@tonic-gate pgcnt -= npgs; 31947c478bd9Sstevel@tonic-gate continue; 31957c478bd9Sstevel@tonic-gate } 31967c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 31977c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 31987c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 31997c478bd9Sstevel@tonic-gate page_hashout(pp, NULL); 32007c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 32017c478bd9Sstevel@tonic-gate pp->p_szc = szc; 32027c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 32037c478bd9Sstevel@tonic-gate pp++; 32047c478bd9Sstevel@tonic-gate pgcnt--; 32057c478bd9Sstevel@tonic-gate continue; 32067c478bd9Sstevel@tonic-gate } 32077c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 32087c478bd9Sstevel@tonic-gate 32097c478bd9Sstevel@tonic-gate /* 32107c478bd9Sstevel@tonic-gate * page_create_wait freemem accounting done by caller of 32117c478bd9Sstevel@tonic-gate * page_get_freelist and not necessary to call it prior to 32127c478bd9Sstevel@tonic-gate * calling page_get_replacement_page. 32137c478bd9Sstevel@tonic-gate * 32147c478bd9Sstevel@tonic-gate * page_get_replacement_page can call page_get_contig_pages 32157c478bd9Sstevel@tonic-gate * to acquire a large page (szc > 0); the replacement must be 32167c478bd9Sstevel@tonic-gate * smaller than the contig page size to avoid looping or 32177c478bd9Sstevel@tonic-gate * szc == 0 and PGI_PGCPSZC0 is set. 32187c478bd9Sstevel@tonic-gate */ 32197c478bd9Sstevel@tonic-gate if (pp->p_szc < szc || (szc == 0 && (flags & PGI_PGCPSZC0))) { 32207c478bd9Sstevel@tonic-gate replpp = page_get_replacement_page(pp, NULL, 0); 32217c478bd9Sstevel@tonic-gate if (replpp) { 32227c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 32237c478bd9Sstevel@tonic-gate ASSERT(npgs <= pgcnt); 32247c478bd9Sstevel@tonic-gate targpp = pp; 32257c478bd9Sstevel@tonic-gate } 32267c478bd9Sstevel@tonic-gate } 32277c478bd9Sstevel@tonic-gate 32287c478bd9Sstevel@tonic-gate /* 32297c478bd9Sstevel@tonic-gate * If replacement is NULL or do_page_relocate fails, fail 32307c478bd9Sstevel@tonic-gate * coalescing of pages. 32317c478bd9Sstevel@tonic-gate */ 32327c478bd9Sstevel@tonic-gate if (replpp == NULL || (do_page_relocate(&targpp, &replpp, 0, 32337c478bd9Sstevel@tonic-gate &npgs, NULL) != 0)) { 32347c478bd9Sstevel@tonic-gate /* 32357c478bd9Sstevel@tonic-gate * Unlock un-processed target list 32367c478bd9Sstevel@tonic-gate */ 32377c478bd9Sstevel@tonic-gate while (pgcnt--) { 32387c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32398b464eb8Smec page_unlock_nocapture(pp); 32407c478bd9Sstevel@tonic-gate pp++; 32417c478bd9Sstevel@tonic-gate } 32427c478bd9Sstevel@tonic-gate /* 32437c478bd9Sstevel@tonic-gate * Free the processed target list. 32447c478bd9Sstevel@tonic-gate */ 32457c478bd9Sstevel@tonic-gate while (pplist) { 32467c478bd9Sstevel@tonic-gate pp = pplist; 32477c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 32487c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32497c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 32507c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 32517c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 32527c478bd9Sstevel@tonic-gate pp->p_szc = 0; 32537c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 32548b464eb8Smec page_unlock_nocapture(pp); 32557c478bd9Sstevel@tonic-gate } 32567c478bd9Sstevel@tonic-gate 32577c478bd9Sstevel@tonic-gate if (replpp != NULL) 32587c478bd9Sstevel@tonic-gate page_free_replacement_page(replpp); 32597c478bd9Sstevel@tonic-gate 32607c478bd9Sstevel@tonic-gate return (NULL); 32617c478bd9Sstevel@tonic-gate } 32627c478bd9Sstevel@tonic-gate ASSERT(pp == targpp); 32637c478bd9Sstevel@tonic-gate 32647c478bd9Sstevel@tonic-gate /* LINTED */ 32657c478bd9Sstevel@tonic-gate ASSERT(hpp = pp); /* That's right, it's an assignment */ 32667c478bd9Sstevel@tonic-gate 32677c478bd9Sstevel@tonic-gate pp += npgs; 32687c478bd9Sstevel@tonic-gate pgcnt -= npgs; 32697c478bd9Sstevel@tonic-gate 32707c478bd9Sstevel@tonic-gate while (npgs--) { 32717c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(targpp)); 32727c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(targpp)); 32737c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(targpp)); 32747c478bd9Sstevel@tonic-gate PP_SETFREE(targpp); 32757c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(targpp)); 32767c478bd9Sstevel@tonic-gate ASSERT(targpp->p_szc < szc || (szc == 0 && 32777c478bd9Sstevel@tonic-gate (flags & PGI_PGCPSZC0))); 32787c478bd9Sstevel@tonic-gate targpp->p_szc = szc; 32797c478bd9Sstevel@tonic-gate targpp = targpp->p_next; 32807c478bd9Sstevel@tonic-gate 32817c478bd9Sstevel@tonic-gate rpp = replpp; 32827c478bd9Sstevel@tonic-gate ASSERT(rpp != NULL); 32837c478bd9Sstevel@tonic-gate page_sub(&replpp, rpp); 32847c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(rpp)); 32857c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(rpp)); 32868b464eb8Smec page_unlock_nocapture(rpp); 32877c478bd9Sstevel@tonic-gate } 32887c478bd9Sstevel@tonic-gate ASSERT(targpp == hpp); 32897c478bd9Sstevel@tonic-gate ASSERT(replpp == NULL); 32907c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &targpp); 32917c478bd9Sstevel@tonic-gate } 32927c478bd9Sstevel@tonic-gate CHK_LPG(pplist, szc); 32937c478bd9Sstevel@tonic-gate return (pplist); 32947c478bd9Sstevel@tonic-gate } 32957c478bd9Sstevel@tonic-gate 32967c478bd9Sstevel@tonic-gate /* 32977c478bd9Sstevel@tonic-gate * Trim kernel cage from pfnlo-pfnhi and store result in lo-hi. Return code 32987c478bd9Sstevel@tonic-gate * of 0 means nothing left after trim. 32997c478bd9Sstevel@tonic-gate */ 33007c478bd9Sstevel@tonic-gate int 33017c478bd9Sstevel@tonic-gate trimkcage(struct memseg *mseg, pfn_t *lo, pfn_t *hi, pfn_t pfnlo, pfn_t pfnhi) 33027c478bd9Sstevel@tonic-gate { 33037c478bd9Sstevel@tonic-gate pfn_t kcagepfn; 33047c478bd9Sstevel@tonic-gate int decr; 33057c478bd9Sstevel@tonic-gate int rc = 0; 33067c478bd9Sstevel@tonic-gate 33077c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->pages)) { 33087c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1) == 0) { 33097c478bd9Sstevel@tonic-gate 33107c478bd9Sstevel@tonic-gate /* lower part of this mseg inside kernel cage */ 33117c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 33127c478bd9Sstevel@tonic-gate 33137c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 33147c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 33157c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 33167c478bd9Sstevel@tonic-gate ASSERT(decr == 0); 331778b03d3aSkchow *lo = MAX(kcagepfn, pfnlo); 331878b03d3aSkchow *hi = MIN(pfnhi, (mseg->pages_end - 1)); 33197c478bd9Sstevel@tonic-gate rc = 1; 33207c478bd9Sstevel@tonic-gate } 33217c478bd9Sstevel@tonic-gate } 33227c478bd9Sstevel@tonic-gate /* else entire mseg in the cage */ 33237c478bd9Sstevel@tonic-gate } else { 33247c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1)) { 33257c478bd9Sstevel@tonic-gate 33267c478bd9Sstevel@tonic-gate /* upper part of this mseg inside kernel cage */ 33277c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 33287c478bd9Sstevel@tonic-gate 33297c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 33307c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 33317c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 33327c478bd9Sstevel@tonic-gate ASSERT(decr); 333378b03d3aSkchow *hi = MIN(kcagepfn, pfnhi); 33347c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 33357c478bd9Sstevel@tonic-gate rc = 1; 33367c478bd9Sstevel@tonic-gate } 33377c478bd9Sstevel@tonic-gate } else { 33387c478bd9Sstevel@tonic-gate /* entire mseg outside of kernel cage */ 33397c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 33407c478bd9Sstevel@tonic-gate *hi = MIN(pfnhi, (mseg->pages_end - 1)); 33417c478bd9Sstevel@tonic-gate rc = 1; 33427c478bd9Sstevel@tonic-gate } 33437c478bd9Sstevel@tonic-gate } 33447c478bd9Sstevel@tonic-gate return (rc); 33457c478bd9Sstevel@tonic-gate } 33467c478bd9Sstevel@tonic-gate 33477c478bd9Sstevel@tonic-gate /* 33485d07b933Sdp * called from page_get_contig_pages to search 'pfnlo' thru 'pfnhi' to claim a 33497c478bd9Sstevel@tonic-gate * page with size code 'szc'. Claiming such a page requires acquiring 33507c478bd9Sstevel@tonic-gate * exclusive locks on all constituent pages (page_trylock_contig_pages), 33517c478bd9Sstevel@tonic-gate * relocating pages in use and concatenating these constituent pages into a 33527c478bd9Sstevel@tonic-gate * large page. 33537c478bd9Sstevel@tonic-gate * 33545d07b933Sdp * The page lists do not have such a large page and page_freelist_split has 33557c478bd9Sstevel@tonic-gate * already failed to demote larger pages and/or coalesce smaller free pages. 33567c478bd9Sstevel@tonic-gate * 33577c478bd9Sstevel@tonic-gate * 'flags' may specify PG_COLOR_MATCH which would limit the search of large 33587c478bd9Sstevel@tonic-gate * pages with the same color as 'bin'. 33597c478bd9Sstevel@tonic-gate * 33607c478bd9Sstevel@tonic-gate * 'pfnflag' specifies the subset of the pfn range to search. 33617c478bd9Sstevel@tonic-gate */ 33627c478bd9Sstevel@tonic-gate 33637c478bd9Sstevel@tonic-gate static page_t * 33647c478bd9Sstevel@tonic-gate page_geti_contig_pages(int mnode, uint_t bin, uchar_t szc, int flags, 336583f9b804Skchow pfn_t pfnlo, pfn_t pfnhi, pgcnt_t pfnflag) 33667c478bd9Sstevel@tonic-gate { 33677c478bd9Sstevel@tonic-gate struct memseg *mseg; 33687c478bd9Sstevel@tonic-gate pgcnt_t szcpgcnt = page_get_pagecnt(szc); 33697c478bd9Sstevel@tonic-gate pgcnt_t szcpgmask = szcpgcnt - 1; 33707c478bd9Sstevel@tonic-gate pfn_t randpfn; 33717c478bd9Sstevel@tonic-gate page_t *pp, *randpp, *endpp; 33725d07b933Sdp uint_t colors, ceq_mask; 33735d07b933Sdp /* LINTED : set but not used in function */ 33745d07b933Sdp uint_t color_mask; 33757c478bd9Sstevel@tonic-gate pfn_t hi, lo; 33767c478bd9Sstevel@tonic-gate uint_t skip; 3377ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 33787c478bd9Sstevel@tonic-gate 33797c478bd9Sstevel@tonic-gate ASSERT(szc != 0 || (flags & PGI_PGCPSZC0)); 33807c478bd9Sstevel@tonic-gate 338102bc52beSkchow pfnlo = P2ROUNDUP(pfnlo, szcpgcnt); 338202bc52beSkchow 338302bc52beSkchow if ((pfnhi - pfnlo) + 1 < szcpgcnt || pfnlo >= pfnhi) 33847c478bd9Sstevel@tonic-gate return (NULL); 33857c478bd9Sstevel@tonic-gate 33867c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 33877c478bd9Sstevel@tonic-gate 33885d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 33895d07b933Sdp color_mask = colors - 1; 33905d07b933Sdp if ((colors > 1) && (flags & PG_MATCH_COLOR)) { 33915d07b933Sdp uchar_t ceq = colorequivszc[szc]; 33925d07b933Sdp uint_t ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 33935d07b933Sdp 33945d07b933Sdp ASSERT(ceq_dif > 0); 33955d07b933Sdp ceq_mask = (ceq_dif - 1) << (ceq & 0xf); 33965d07b933Sdp } else { 33975d07b933Sdp ceq_mask = 0; 33985d07b933Sdp } 33997c478bd9Sstevel@tonic-gate 34007c478bd9Sstevel@tonic-gate ASSERT(bin < colors); 34017c478bd9Sstevel@tonic-gate 34025d07b933Sdp /* clear "non-significant" color bits */ 34035d07b933Sdp bin &= ceq_mask; 34045d07b933Sdp 34057c478bd9Sstevel@tonic-gate /* 34067c478bd9Sstevel@tonic-gate * trim the pfn range to search based on pfnflag. pfnflag is set 34077c478bd9Sstevel@tonic-gate * when there have been previous page_get_contig_page failures to 34087c478bd9Sstevel@tonic-gate * limit the search. 34097c478bd9Sstevel@tonic-gate * 34107c478bd9Sstevel@tonic-gate * The high bit in pfnflag specifies the number of 'slots' in the 34117c478bd9Sstevel@tonic-gate * pfn range and the remainder of pfnflag specifies which slot. 34127c478bd9Sstevel@tonic-gate * For example, a value of 1010b would mean the second slot of 34137c478bd9Sstevel@tonic-gate * the pfn range that has been divided into 8 slots. 34147c478bd9Sstevel@tonic-gate */ 34157c478bd9Sstevel@tonic-gate if (pfnflag > 1) { 34167c478bd9Sstevel@tonic-gate int slots = 1 << (highbit(pfnflag) - 1); 34177c478bd9Sstevel@tonic-gate int slotid = pfnflag & (slots - 1); 34187c478bd9Sstevel@tonic-gate pgcnt_t szcpages; 34197c478bd9Sstevel@tonic-gate int slotlen; 34207c478bd9Sstevel@tonic-gate 342102bc52beSkchow pfnhi = P2ALIGN((pfnhi + 1), szcpgcnt) - 1; 34227c478bd9Sstevel@tonic-gate szcpages = ((pfnhi - pfnlo) + 1) / szcpgcnt; 34237c478bd9Sstevel@tonic-gate slotlen = howmany(szcpages, slots); 342402bc52beSkchow /* skip if 'slotid' slot is empty */ 342502bc52beSkchow if (slotid * slotlen >= szcpages) 342602bc52beSkchow return (NULL); 34277c478bd9Sstevel@tonic-gate pfnlo = pfnlo + (((slotid * slotlen) % szcpages) * szcpgcnt); 34287c478bd9Sstevel@tonic-gate ASSERT(pfnlo < pfnhi); 34297c478bd9Sstevel@tonic-gate if (pfnhi > pfnlo + (slotlen * szcpgcnt)) 343002bc52beSkchow pfnhi = pfnlo + (slotlen * szcpgcnt) - 1; 34317c478bd9Sstevel@tonic-gate } 34327c478bd9Sstevel@tonic-gate 34337c478bd9Sstevel@tonic-gate memsegs_lock(0); 34347c478bd9Sstevel@tonic-gate 34357c478bd9Sstevel@tonic-gate /* 34367c478bd9Sstevel@tonic-gate * loop through memsegs to look for contig page candidates 34377c478bd9Sstevel@tonic-gate */ 34387c478bd9Sstevel@tonic-gate 34397c478bd9Sstevel@tonic-gate for (mseg = memsegs; mseg != NULL; mseg = mseg->next) { 34407c478bd9Sstevel@tonic-gate if (pfnhi < mseg->pages_base || pfnlo >= mseg->pages_end) { 34417c478bd9Sstevel@tonic-gate /* no overlap */ 34427c478bd9Sstevel@tonic-gate continue; 34437c478bd9Sstevel@tonic-gate } 34447c478bd9Sstevel@tonic-gate 34457c478bd9Sstevel@tonic-gate if (mseg->pages_end - mseg->pages_base < szcpgcnt) 34467c478bd9Sstevel@tonic-gate /* mseg too small */ 34477c478bd9Sstevel@tonic-gate continue; 34487c478bd9Sstevel@tonic-gate 344978b03d3aSkchow /* 345078b03d3aSkchow * trim off kernel cage pages from pfn range and check for 345178b03d3aSkchow * a trimmed pfn range returned that does not span the 345278b03d3aSkchow * desired large page size. 345378b03d3aSkchow */ 34547c478bd9Sstevel@tonic-gate if (kcage_on) { 345578b03d3aSkchow if (trimkcage(mseg, &lo, &hi, pfnlo, pfnhi) == 0 || 3456d94d1888Skchow lo >= hi || ((hi - lo) + 1) < szcpgcnt) 34577c478bd9Sstevel@tonic-gate continue; 34587c478bd9Sstevel@tonic-gate } else { 34597c478bd9Sstevel@tonic-gate lo = MAX(pfnlo, mseg->pages_base); 34607c478bd9Sstevel@tonic-gate hi = MIN(pfnhi, (mseg->pages_end - 1)); 34617c478bd9Sstevel@tonic-gate } 34627c478bd9Sstevel@tonic-gate 34637c478bd9Sstevel@tonic-gate /* round to szcpgcnt boundaries */ 34647c478bd9Sstevel@tonic-gate lo = P2ROUNDUP(lo, szcpgcnt); 346502bc52beSkchow 3466b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 346702bc52beSkchow hi = P2ALIGN((hi + 1), szcpgcnt) - 1; 34687c478bd9Sstevel@tonic-gate 34697c478bd9Sstevel@tonic-gate if (hi <= lo) 34707c478bd9Sstevel@tonic-gate continue; 34717c478bd9Sstevel@tonic-gate 34727c478bd9Sstevel@tonic-gate /* 34737c478bd9Sstevel@tonic-gate * set lo to point to the pfn for the desired bin. Large 34747c478bd9Sstevel@tonic-gate * page sizes may only have a single page color 34757c478bd9Sstevel@tonic-gate */ 34765d07b933Sdp skip = szcpgcnt; 3477ce8eb11aSdp if (ceq_mask > 0 || interleaved_mnodes) { 34785d07b933Sdp /* set lo to point at appropriate color */ 3479ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ bin) & ceq_mask) || 3480ce8eb11aSdp (interleaved_mnodes && 3481ce8eb11aSdp PFN_2_MEM_NODE(lo) != mnode)) { 3482ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, bin, ceq_mask, 3483ce8eb11aSdp color_mask, &it); 3484ce8eb11aSdp } 34855d07b933Sdp if (hi <= lo) 34865d07b933Sdp /* mseg cannot satisfy color request */ 34875d07b933Sdp continue; 34887c478bd9Sstevel@tonic-gate } 34897c478bd9Sstevel@tonic-gate 34907c478bd9Sstevel@tonic-gate /* randomly choose a point between lo and hi to begin search */ 34917c478bd9Sstevel@tonic-gate 34927c478bd9Sstevel@tonic-gate randpfn = (pfn_t)GETTICK(); 34937c478bd9Sstevel@tonic-gate randpfn = ((randpfn % (hi - lo)) + lo) & ~(skip - 1); 3494b779d3e0Sdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, szc, &it); 3495b779d3e0Sdp if (ceq_mask || interleaved_mnodes || randpfn == (pfn_t)-1) { 3496a7c3ca36Sdp if (randpfn != (pfn_t)-1) { 3497ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(randpfn, szc, bin, 3498ce8eb11aSdp ceq_mask, color_mask, &it); 3499a7c3ca36Sdp } 3500ce8eb11aSdp if (randpfn >= hi) { 3501ce8eb11aSdp randpfn = lo; 3502b779d3e0Sdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, szc, 3503b779d3e0Sdp &it); 3504ce8eb11aSdp } 35055d07b933Sdp } 35067c478bd9Sstevel@tonic-gate randpp = mseg->pages + (randpfn - mseg->pages_base); 35077c478bd9Sstevel@tonic-gate 35087c478bd9Sstevel@tonic-gate ASSERT(randpp->p_pagenum == randpfn); 35097c478bd9Sstevel@tonic-gate 35107c478bd9Sstevel@tonic-gate pp = randpp; 351102bc52beSkchow endpp = mseg->pages + (hi - mseg->pages_base) + 1; 35127c478bd9Sstevel@tonic-gate 35137c478bd9Sstevel@tonic-gate ASSERT(randpp + szcpgcnt <= endpp); 35147c478bd9Sstevel@tonic-gate 35157c478bd9Sstevel@tonic-gate do { 35167c478bd9Sstevel@tonic-gate ASSERT(!(pp->p_pagenum & szcpgmask)); 35175d07b933Sdp ASSERT(((PP_2_BIN(pp) ^ bin) & ceq_mask) == 0); 35185d07b933Sdp 35197c478bd9Sstevel@tonic-gate if (page_trylock_contig_pages(mnode, pp, szc, flags)) { 35207c478bd9Sstevel@tonic-gate /* pages unlocked by page_claim on failure */ 35217c478bd9Sstevel@tonic-gate if (page_claim_contig_pages(pp, szc, flags)) { 35227c478bd9Sstevel@tonic-gate memsegs_unlock(0); 35237c478bd9Sstevel@tonic-gate return (pp); 35247c478bd9Sstevel@tonic-gate } 35257c478bd9Sstevel@tonic-gate } 35267c478bd9Sstevel@tonic-gate 3527ce8eb11aSdp if (ceq_mask == 0 && !interleaved_mnodes) { 35285d07b933Sdp pp += skip; 35295d07b933Sdp } else { 35305d07b933Sdp pfn_t pfn = pp->p_pagenum; 35315d07b933Sdp 35325d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfn, szc, bin, 3533ce8eb11aSdp ceq_mask, color_mask, &it); 3534ce8eb11aSdp if (pfn == (pfn_t)-1) { 3535ce8eb11aSdp pp = endpp; 3536ce8eb11aSdp } else { 3537ce8eb11aSdp pp = mseg->pages + 3538ce8eb11aSdp (pfn - mseg->pages_base); 3539ce8eb11aSdp } 35405d07b933Sdp } 35417c478bd9Sstevel@tonic-gate if (pp >= endpp) { 35427c478bd9Sstevel@tonic-gate /* start from the beginning */ 3543b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 35447c478bd9Sstevel@tonic-gate pp = mseg->pages + (lo - mseg->pages_base); 35457c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == lo); 35467c478bd9Sstevel@tonic-gate ASSERT(pp + szcpgcnt <= endpp); 35477c478bd9Sstevel@tonic-gate } 35487c478bd9Sstevel@tonic-gate } while (pp != randpp); 35497c478bd9Sstevel@tonic-gate } 35507c478bd9Sstevel@tonic-gate memsegs_unlock(0); 35517c478bd9Sstevel@tonic-gate return (NULL); 35527c478bd9Sstevel@tonic-gate } 35537c478bd9Sstevel@tonic-gate 35547c478bd9Sstevel@tonic-gate 35557c478bd9Sstevel@tonic-gate /* 35567c478bd9Sstevel@tonic-gate * controlling routine that searches through physical memory in an attempt to 35577c478bd9Sstevel@tonic-gate * claim a large page based on the input parameters. 35587c478bd9Sstevel@tonic-gate * on the page free lists. 35597c478bd9Sstevel@tonic-gate * 35607c478bd9Sstevel@tonic-gate * calls page_geti_contig_pages with an initial pfn range from the mnode 35617c478bd9Sstevel@tonic-gate * and mtype. page_geti_contig_pages will trim off the parts of the pfn range 35627c478bd9Sstevel@tonic-gate * that overlaps with the kernel cage or does not match the requested page 35637c478bd9Sstevel@tonic-gate * color if PG_MATCH_COLOR is set. Since this search is very expensive, 35647c478bd9Sstevel@tonic-gate * page_geti_contig_pages may further limit the search range based on 35657c478bd9Sstevel@tonic-gate * previous failure counts (pgcpfailcnt[]). 35667c478bd9Sstevel@tonic-gate * 35677c478bd9Sstevel@tonic-gate * for PGI_PGCPSZC0 requests, page_get_contig_pages will relocate a base 35687c478bd9Sstevel@tonic-gate * pagesize page that satisfies mtype. 35697c478bd9Sstevel@tonic-gate */ 35707c478bd9Sstevel@tonic-gate page_t * 35717c478bd9Sstevel@tonic-gate page_get_contig_pages(int mnode, uint_t bin, int mtype, uchar_t szc, 35727c478bd9Sstevel@tonic-gate uint_t flags) 35737c478bd9Sstevel@tonic-gate { 35747c478bd9Sstevel@tonic-gate pfn_t pfnlo, pfnhi; /* contig pages pfn range */ 35757c478bd9Sstevel@tonic-gate page_t *pp; 357683f9b804Skchow pgcnt_t pfnflag = 0; /* no limit on search if 0 */ 35777c478bd9Sstevel@tonic-gate 35787c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_alloc[szc]); 35797c478bd9Sstevel@tonic-gate 35800b5aa17bSmec /* no allocations from cage */ 35810b5aa17bSmec flags |= PGI_NOCAGE; 35820b5aa17bSmec 35837c478bd9Sstevel@tonic-gate /* LINTED */ 35847c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 35857c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 35867c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocempty[szc]); 35877c478bd9Sstevel@tonic-gate return (NULL); 35887c478bd9Sstevel@tonic-gate } 35897c478bd9Sstevel@tonic-gate 35907c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 35917c478bd9Sstevel@tonic-gate 35927c478bd9Sstevel@tonic-gate /* do not limit search and ignore color if hi pri */ 35937c478bd9Sstevel@tonic-gate 35947c478bd9Sstevel@tonic-gate if (pgcplimitsearch && ((flags & PGI_PGCPHIPRI) == 0)) 35957c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 35967c478bd9Sstevel@tonic-gate 35977c478bd9Sstevel@tonic-gate /* remove color match to improve chances */ 35987c478bd9Sstevel@tonic-gate 35997c478bd9Sstevel@tonic-gate if (flags & PGI_PGCPHIPRI || pfnflag) 36007c478bd9Sstevel@tonic-gate flags &= ~PG_MATCH_COLOR; 36017c478bd9Sstevel@tonic-gate 36027c478bd9Sstevel@tonic-gate do { 36037c478bd9Sstevel@tonic-gate /* get pfn range based on mnode and mtype */ 36047c478bd9Sstevel@tonic-gate MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi); 36057c478bd9Sstevel@tonic-gate 36067c478bd9Sstevel@tonic-gate ASSERT(pfnhi >= pfnlo); 36077c478bd9Sstevel@tonic-gate 36087c478bd9Sstevel@tonic-gate pp = page_geti_contig_pages(mnode, bin, szc, flags, 36097c478bd9Sstevel@tonic-gate pfnlo, pfnhi, pfnflag); 36107c478bd9Sstevel@tonic-gate 36117c478bd9Sstevel@tonic-gate if (pp != NULL) { 36127c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 36137c478bd9Sstevel@tonic-gate if (pfnflag) { 36147c478bd9Sstevel@tonic-gate /* double the search size */ 36157c478bd9Sstevel@tonic-gate pgcpfailcnt[szc] = pfnflag >> 1; 36167c478bd9Sstevel@tonic-gate } 36177c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocok[szc]); 36187c478bd9Sstevel@tonic-gate return (pp); 36197c478bd9Sstevel@tonic-gate } 3620affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3621affbd3ccSkchow } while (mtype >= 0); 36227c478bd9Sstevel@tonic-gate 36237c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocfailed[szc]); 36247c478bd9Sstevel@tonic-gate return (NULL); 36257c478bd9Sstevel@tonic-gate } 36267c478bd9Sstevel@tonic-gate 362778b03d3aSkchow #if defined(__i386) || defined(__amd64) 362878b03d3aSkchow /* 362978b03d3aSkchow * Determine the likelihood of finding/coalescing a szc page. 363078b03d3aSkchow * Return 0 if the likelihood is small otherwise return 1. 363178b03d3aSkchow * 363278b03d3aSkchow * For now, be conservative and check only 1g pages and return 0 363378b03d3aSkchow * if there had been previous coalescing failures and the szc pages 363478b03d3aSkchow * needed to satisfy request would exhaust most of freemem. 363578b03d3aSkchow */ 363678b03d3aSkchow int 363778b03d3aSkchow page_chk_freelist(uint_t szc) 363878b03d3aSkchow { 363978b03d3aSkchow pgcnt_t pgcnt; 364078b03d3aSkchow 364178b03d3aSkchow if (szc <= 1) 364278b03d3aSkchow return (1); 364378b03d3aSkchow 364478b03d3aSkchow pgcnt = page_get_pagecnt(szc); 364578b03d3aSkchow if (pgcpfailcnt[szc] && pgcnt + throttlefree >= freemem) { 364678b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_deny[szc]); 364778b03d3aSkchow return (0); 364878b03d3aSkchow } 364978b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_allow[szc]); 365078b03d3aSkchow return (1); 365178b03d3aSkchow } 365278b03d3aSkchow #endif 36537c478bd9Sstevel@tonic-gate 36547c478bd9Sstevel@tonic-gate /* 36557c478bd9Sstevel@tonic-gate * Find the `best' page on the freelist for this (vp,off) (as,vaddr) pair. 36567c478bd9Sstevel@tonic-gate * 36577c478bd9Sstevel@tonic-gate * Does its own locking and accounting. 36587c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 36597c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 36607c478bd9Sstevel@tonic-gate * 36617c478bd9Sstevel@tonic-gate * Finds a page, removes it, THEN locks it. 36627c478bd9Sstevel@tonic-gate */ 36637c478bd9Sstevel@tonic-gate 36647c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 36657c478bd9Sstevel@tonic-gate page_t * 36667c478bd9Sstevel@tonic-gate page_get_freelist(struct vnode *vp, u_offset_t off, struct seg *seg, 36677c478bd9Sstevel@tonic-gate caddr_t vaddr, size_t size, uint_t flags, struct lgrp *lgrp) 36687c478bd9Sstevel@tonic-gate { 36697c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 36707c478bd9Sstevel@tonic-gate page_t *pp = NULL; 36717c478bd9Sstevel@tonic-gate ulong_t bin; 36727c478bd9Sstevel@tonic-gate uchar_t szc; 36737c478bd9Sstevel@tonic-gate int mnode; 36747c478bd9Sstevel@tonic-gate int mtype; 36757c478bd9Sstevel@tonic-gate page_t *(*page_get_func)(int, uint_t, int, uchar_t, uint_t); 36767c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 36777c478bd9Sstevel@tonic-gate 36787c478bd9Sstevel@tonic-gate page_get_func = page_get_mnode_freelist; 36797c478bd9Sstevel@tonic-gate 36807c478bd9Sstevel@tonic-gate /* 36817c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or passed a freed lgrp 36827c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 36837c478bd9Sstevel@tonic-gate */ 36847c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 36857c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 36867c478bd9Sstevel@tonic-gate 36877c478bd9Sstevel@tonic-gate if (kcage_on) { 36887c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC)) == PG_NORELOC && 36897c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + btop(size) && 36907c478bd9Sstevel@tonic-gate curthread != kcage_cageout_thread) { 36917c478bd9Sstevel@tonic-gate /* 36927c478bd9Sstevel@tonic-gate * Set a "reserve" of kcage_throttlefree pages for 36937c478bd9Sstevel@tonic-gate * PG_PANIC and cageout thread allocations. 36947c478bd9Sstevel@tonic-gate * 36957c478bd9Sstevel@tonic-gate * Everybody else has to serialize in 36967c478bd9Sstevel@tonic-gate * page_create_get_something() to get a cage page, so 36977c478bd9Sstevel@tonic-gate * that we don't deadlock cageout! 36987c478bd9Sstevel@tonic-gate */ 36997c478bd9Sstevel@tonic-gate return (NULL); 37007c478bd9Sstevel@tonic-gate } 37017c478bd9Sstevel@tonic-gate } else { 37027c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 37037c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 37047c478bd9Sstevel@tonic-gate } 37057c478bd9Sstevel@tonic-gate 37067c478bd9Sstevel@tonic-gate /* LINTED */ 370707ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, size); 37087c478bd9Sstevel@tonic-gate 37097c478bd9Sstevel@tonic-gate /* 37107c478bd9Sstevel@tonic-gate * Convert size to page size code. 37117c478bd9Sstevel@tonic-gate */ 37127c478bd9Sstevel@tonic-gate if ((szc = page_szc(size)) == (uchar_t)-1) 37137c478bd9Sstevel@tonic-gate panic("page_get_freelist: illegal page size request"); 37147c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 37157c478bd9Sstevel@tonic-gate 37167c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_alloc[szc]); 37177c478bd9Sstevel@tonic-gate 37187c478bd9Sstevel@tonic-gate /* LINTED */ 37195d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, szc); 37207c478bd9Sstevel@tonic-gate 37215d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(szc)); 37227c478bd9Sstevel@tonic-gate 37237c478bd9Sstevel@tonic-gate /* 37247c478bd9Sstevel@tonic-gate * Try to get a local page first, but try remote if we can't 37257c478bd9Sstevel@tonic-gate * get a page of the right color. 37267c478bd9Sstevel@tonic-gate */ 37277c478bd9Sstevel@tonic-gate pgretry: 37287c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 37297c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 37307c478bd9Sstevel@tonic-gate pp = page_get_func(mnode, bin, mtype, szc, flags); 37317c478bd9Sstevel@tonic-gate if (pp != NULL) { 37327c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocok[szc]); 37337c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 37347c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 37357c478bd9Sstevel@tonic-gate int, mnode, 37367c478bd9Sstevel@tonic-gate ulong_t, bin, 37377c478bd9Sstevel@tonic-gate uint_t, flags); 37387c478bd9Sstevel@tonic-gate return (pp); 37397c478bd9Sstevel@tonic-gate } 37407c478bd9Sstevel@tonic-gate } 37417c478bd9Sstevel@tonic-gate ASSERT(pp == NULL); 37427c478bd9Sstevel@tonic-gate 37437c478bd9Sstevel@tonic-gate /* 37447c478bd9Sstevel@tonic-gate * for non-SZC0 PAGESIZE requests, check cachelist before checking 37457c478bd9Sstevel@tonic-gate * remote free lists. Caller expected to call page_get_cachelist which 37467c478bd9Sstevel@tonic-gate * will check local cache lists and remote free lists. 37477c478bd9Sstevel@tonic-gate */ 37487c478bd9Sstevel@tonic-gate if (szc == 0 && ((flags & PGI_PGCPSZC0) == 0)) { 37497c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocdeferred); 37507c478bd9Sstevel@tonic-gate return (NULL); 37517c478bd9Sstevel@tonic-gate } 37527c478bd9Sstevel@tonic-gate 37537c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 37547c478bd9Sstevel@tonic-gate 37557c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 37567c478bd9Sstevel@tonic-gate 37572cb27123Saguzovsk if (!(flags & PG_LOCAL)) { 37582cb27123Saguzovsk /* 37592cb27123Saguzovsk * Try to get a non-local freelist page. 37602cb27123Saguzovsk */ 37612cb27123Saguzovsk LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 37622cb27123Saguzovsk while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 37632cb27123Saguzovsk pp = page_get_func(mnode, bin, mtype, szc, flags); 37642cb27123Saguzovsk if (pp != NULL) { 37652cb27123Saguzovsk DTRACE_PROBE4(page__get, 37662cb27123Saguzovsk lgrp_t *, lgrp, 37672cb27123Saguzovsk int, mnode, 37682cb27123Saguzovsk ulong_t, bin, 37692cb27123Saguzovsk uint_t, flags); 37702cb27123Saguzovsk VM_STAT_ADD(vmm_vmstats.pgf_allocokrem[szc]); 37712cb27123Saguzovsk return (pp); 37722cb27123Saguzovsk } 37737c478bd9Sstevel@tonic-gate } 37742cb27123Saguzovsk ASSERT(pp == NULL); 37757c478bd9Sstevel@tonic-gate } 37767c478bd9Sstevel@tonic-gate 37777c478bd9Sstevel@tonic-gate /* 37787c478bd9Sstevel@tonic-gate * when the cage is off chances are page_get_contig_pages() will fail 37797c478bd9Sstevel@tonic-gate * to lock a large page chunk therefore when the cage is off it's not 37807c478bd9Sstevel@tonic-gate * called by default. this can be changed via /etc/system. 37817c478bd9Sstevel@tonic-gate * 37827c478bd9Sstevel@tonic-gate * page_get_contig_pages() also called to acquire a base pagesize page 37837c478bd9Sstevel@tonic-gate * for page_create_get_something(). 37847c478bd9Sstevel@tonic-gate */ 37857c478bd9Sstevel@tonic-gate if (!(flags & PG_NORELOC) && (pg_contig_disable == 0) && 37867c478bd9Sstevel@tonic-gate (kcage_on || pg_lpgcreate_nocage || szc == 0) && 37877c478bd9Sstevel@tonic-gate (page_get_func != page_get_contig_pages)) { 37887c478bd9Sstevel@tonic-gate 37897c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocretry[szc]); 37907c478bd9Sstevel@tonic-gate page_get_func = page_get_contig_pages; 37917c478bd9Sstevel@tonic-gate goto pgretry; 37927c478bd9Sstevel@tonic-gate } 37937c478bd9Sstevel@tonic-gate 37942cb27123Saguzovsk if (!(flags & PG_LOCAL) && pgcplimitsearch && 37952cb27123Saguzovsk page_get_func == page_get_contig_pages) 379683f9b804Skchow SETPGCPFAILCNT(szc); 37977c478bd9Sstevel@tonic-gate 37987c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocfailed[szc]); 37997c478bd9Sstevel@tonic-gate return (NULL); 38007c478bd9Sstevel@tonic-gate } 38017c478bd9Sstevel@tonic-gate 38027c478bd9Sstevel@tonic-gate /* 38037c478bd9Sstevel@tonic-gate * Find the `best' page on the cachelist for this (vp,off) (as,vaddr) pair. 38047c478bd9Sstevel@tonic-gate * 38057c478bd9Sstevel@tonic-gate * Does its own locking. 38067c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 38077c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 38087c478bd9Sstevel@tonic-gate * Otherwise, scan the bins for ones with pages. For each bin with pages, 38097c478bd9Sstevel@tonic-gate * try to lock one of them. If no page can be locked, try the 38107c478bd9Sstevel@tonic-gate * next bin. Return NULL if a page can not be found and locked. 38117c478bd9Sstevel@tonic-gate * 38127c478bd9Sstevel@tonic-gate * Finds a pages, trys to lock it, then removes it. 38137c478bd9Sstevel@tonic-gate */ 38147c478bd9Sstevel@tonic-gate 38157c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 38167c478bd9Sstevel@tonic-gate page_t * 38177c478bd9Sstevel@tonic-gate page_get_cachelist(struct vnode *vp, u_offset_t off, struct seg *seg, 38187c478bd9Sstevel@tonic-gate caddr_t vaddr, uint_t flags, struct lgrp *lgrp) 38197c478bd9Sstevel@tonic-gate { 38207c478bd9Sstevel@tonic-gate page_t *pp; 38217c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 38227c478bd9Sstevel@tonic-gate ulong_t bin; 38237c478bd9Sstevel@tonic-gate /*LINTED*/ 38247c478bd9Sstevel@tonic-gate int mnode; 38257c478bd9Sstevel@tonic-gate int mtype; 38267c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 38277c478bd9Sstevel@tonic-gate 38287c478bd9Sstevel@tonic-gate /* 38297c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or pasased a freed lgrp 38307c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 38317c478bd9Sstevel@tonic-gate */ 38327c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 38337c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 38347c478bd9Sstevel@tonic-gate 38357c478bd9Sstevel@tonic-gate if (!kcage_on) { 38367c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 38377c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 38387c478bd9Sstevel@tonic-gate } 38397c478bd9Sstevel@tonic-gate 38407c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == PG_NORELOC && 38417c478bd9Sstevel@tonic-gate kcage_freemem <= kcage_throttlefree) { 38427c478bd9Sstevel@tonic-gate /* 38437c478bd9Sstevel@tonic-gate * Reserve kcage_throttlefree pages for critical kernel 38447c478bd9Sstevel@tonic-gate * threads. 38457c478bd9Sstevel@tonic-gate * 38467c478bd9Sstevel@tonic-gate * Everybody else has to go to page_create_get_something() 38477c478bd9Sstevel@tonic-gate * to get a cage page, so we don't deadlock cageout. 38487c478bd9Sstevel@tonic-gate */ 38497c478bd9Sstevel@tonic-gate return (NULL); 38507c478bd9Sstevel@tonic-gate } 38517c478bd9Sstevel@tonic-gate 38527c478bd9Sstevel@tonic-gate /* LINTED */ 38535d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, 0); 38547c478bd9Sstevel@tonic-gate 38555d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(0)); 38567c478bd9Sstevel@tonic-gate 38577c478bd9Sstevel@tonic-gate /* LINTED */ 385807ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, MMU_PAGESIZE); 38597c478bd9Sstevel@tonic-gate 38607c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_alloc); 38617c478bd9Sstevel@tonic-gate 38627c478bd9Sstevel@tonic-gate /* 38637c478bd9Sstevel@tonic-gate * Try local cachelists first 38647c478bd9Sstevel@tonic-gate */ 38657c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 38667c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38677c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 38687c478bd9Sstevel@tonic-gate if (pp != NULL) { 38697c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocok); 38707c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38717c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38727c478bd9Sstevel@tonic-gate int, mnode, 38737c478bd9Sstevel@tonic-gate ulong_t, bin, 38747c478bd9Sstevel@tonic-gate uint_t, flags); 38757c478bd9Sstevel@tonic-gate return (pp); 38767c478bd9Sstevel@tonic-gate } 38777c478bd9Sstevel@tonic-gate } 38787c478bd9Sstevel@tonic-gate 38797c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 38807c478bd9Sstevel@tonic-gate 38817c478bd9Sstevel@tonic-gate /* 38827c478bd9Sstevel@tonic-gate * Try freelists/cachelists that are farther away 38837c478bd9Sstevel@tonic-gate * This is our only chance to allocate remote pages for PAGESIZE 38847c478bd9Sstevel@tonic-gate * requests. 38857c478bd9Sstevel@tonic-gate */ 38867c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 38877c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38887c478bd9Sstevel@tonic-gate pp = page_get_mnode_freelist(mnode, bin, mtype, 38897c478bd9Sstevel@tonic-gate 0, flags); 38907c478bd9Sstevel@tonic-gate if (pp != NULL) { 38917c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokdeferred); 38927c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38937c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38947c478bd9Sstevel@tonic-gate int, mnode, 38957c478bd9Sstevel@tonic-gate ulong_t, bin, 38967c478bd9Sstevel@tonic-gate uint_t, flags); 38977c478bd9Sstevel@tonic-gate return (pp); 38987c478bd9Sstevel@tonic-gate } 38997c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 39007c478bd9Sstevel@tonic-gate if (pp != NULL) { 39017c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokrem); 39027c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 39037c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 39047c478bd9Sstevel@tonic-gate int, mnode, 39057c478bd9Sstevel@tonic-gate ulong_t, bin, 39067c478bd9Sstevel@tonic-gate uint_t, flags); 39077c478bd9Sstevel@tonic-gate return (pp); 39087c478bd9Sstevel@tonic-gate } 39097c478bd9Sstevel@tonic-gate } 39107c478bd9Sstevel@tonic-gate 39117c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocfailed); 39127c478bd9Sstevel@tonic-gate return (NULL); 39137c478bd9Sstevel@tonic-gate } 39147c478bd9Sstevel@tonic-gate 39157c478bd9Sstevel@tonic-gate page_t * 39167c478bd9Sstevel@tonic-gate page_get_mnode_cachelist(uint_t bin, uint_t flags, int mnode, int mtype) 39177c478bd9Sstevel@tonic-gate { 39185d07b933Sdp kmutex_t *pcm; 39195d07b933Sdp page_t *pp, *first_pp; 39205d07b933Sdp uint_t sbin; 39215d07b933Sdp int plw_initialized; 39225d07b933Sdp page_list_walker_t plw; 39237c478bd9Sstevel@tonic-gate 39247c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_alloc); 39257c478bd9Sstevel@tonic-gate 39267c478bd9Sstevel@tonic-gate /* LINTED */ 39277c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 39287c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 39297c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocempty); 39307c478bd9Sstevel@tonic-gate return (NULL); 39317c478bd9Sstevel@tonic-gate } 39327c478bd9Sstevel@tonic-gate 39335d07b933Sdp try_again: 39347c478bd9Sstevel@tonic-gate 39355d07b933Sdp plw_initialized = 0; 39365d07b933Sdp plw.plw_ceq_dif = 1; 39377c478bd9Sstevel@tonic-gate 39387c478bd9Sstevel@tonic-gate /* 39397c478bd9Sstevel@tonic-gate * Only hold one cachelist lock at a time, that way we 39407c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 39417c478bd9Sstevel@tonic-gate * ordering. 39427c478bd9Sstevel@tonic-gate */ 39437c478bd9Sstevel@tonic-gate 39445d07b933Sdp for (plw.plw_count = 0; 39455d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 39465d07b933Sdp sbin = bin; 39475d07b933Sdp do { 39485d07b933Sdp 39495d07b933Sdp if (!PAGE_CACHELISTS(mnode, bin, mtype)) 39505d07b933Sdp goto bin_empty_1; 39517c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 39527c478bd9Sstevel@tonic-gate mutex_enter(pcm); 39537c478bd9Sstevel@tonic-gate pp = PAGE_CACHELISTS(mnode, bin, mtype); 39545d07b933Sdp if (pp == NULL) 39555d07b933Sdp goto bin_empty_0; 39567c478bd9Sstevel@tonic-gate 39575d07b933Sdp first_pp = pp; 39585d07b933Sdp ASSERT(pp->p_vnode); 39595d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39605d07b933Sdp ASSERT(pp->p_szc == 0); 39615d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 39625d07b933Sdp while (!page_trylock(pp, SE_EXCL)) { 39635d07b933Sdp pp = pp->p_next; 39645d07b933Sdp ASSERT(pp->p_szc == 0); 39655d07b933Sdp if (pp == first_pp) { 39667c478bd9Sstevel@tonic-gate /* 39675d07b933Sdp * We have searched the complete list! 39685d07b933Sdp * And all of them (might only be one) 39695d07b933Sdp * are locked. This can happen since 39705d07b933Sdp * these pages can also be found via 39715d07b933Sdp * the hash list. When found via the 39725d07b933Sdp * hash list, they are locked first, 39735d07b933Sdp * then removed. We give up to let the 39745d07b933Sdp * other thread run. 39757c478bd9Sstevel@tonic-gate */ 39765d07b933Sdp pp = NULL; 39775d07b933Sdp break; 39787c478bd9Sstevel@tonic-gate } 39795d07b933Sdp ASSERT(pp->p_vnode); 39805d07b933Sdp ASSERT(PP_ISFREE(pp)); 39815d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39825d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == 39835d07b933Sdp mnode); 39847c478bd9Sstevel@tonic-gate } 39857c478bd9Sstevel@tonic-gate 39865d07b933Sdp if (pp) { 39875d07b933Sdp page_t **ppp; 39885d07b933Sdp /* 39895d07b933Sdp * Found and locked a page. 39905d07b933Sdp * Pull it off the list. 39915d07b933Sdp */ 39925d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 39935d07b933Sdp ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 39945d07b933Sdp page_sub(ppp, pp); 39955d07b933Sdp /* 39965d07b933Sdp * Subtract counters before releasing pcm mutex 39975d07b933Sdp * to avoid a race with page_freelist_coalesce 39985d07b933Sdp * and page_freelist_split. 39995d07b933Sdp */ 40005d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 40015d07b933Sdp mutex_exit(pcm); 40025d07b933Sdp ASSERT(pp->p_vnode); 40035d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 40045d07b933Sdp #if defined(__sparc) 40055d07b933Sdp ASSERT(!kcage_on || 40065d07b933Sdp (flags & PG_NORELOC) == 0 || 40075d07b933Sdp PP_ISNORELOC(pp)); 40085d07b933Sdp if (PP_ISNORELOC(pp)) { 40095d07b933Sdp kcage_freemem_sub(1); 40107c478bd9Sstevel@tonic-gate } 40115d07b933Sdp #endif 40125d07b933Sdp VM_STAT_ADD(vmm_vmstats. pgmc_allocok); 40135d07b933Sdp return (pp); 40147c478bd9Sstevel@tonic-gate } 40155d07b933Sdp bin_empty_0: 40165d07b933Sdp mutex_exit(pcm); 40175d07b933Sdp bin_empty_1: 40185d07b933Sdp if (plw_initialized == 0) { 40195d07b933Sdp page_list_walk_init(0, flags, bin, 0, 1, &plw); 40205d07b933Sdp plw_initialized = 1; 40217c478bd9Sstevel@tonic-gate } 40225d07b933Sdp /* calculate the next bin with equivalent color */ 40235d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 40245d07b933Sdp plw.plw_ceq_mask[0], plw.plw_color_mask); 40255d07b933Sdp } while (sbin != bin); 40267c478bd9Sstevel@tonic-gate 40275d07b933Sdp if (plw.plw_ceq_dif > 1) 40285d07b933Sdp bin = page_list_walk_next_bin(0, bin, &plw); 40297c478bd9Sstevel@tonic-gate } 40307c478bd9Sstevel@tonic-gate 4031affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 4032affbd3ccSkchow if (mtype >= 0) 40335d07b933Sdp goto try_again; 4034affbd3ccSkchow 40357c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocfailed); 40367c478bd9Sstevel@tonic-gate return (NULL); 40377c478bd9Sstevel@tonic-gate } 40387c478bd9Sstevel@tonic-gate 40397c478bd9Sstevel@tonic-gate #ifdef DEBUG 40407c478bd9Sstevel@tonic-gate #define REPL_PAGE_STATS 40417c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 40427c478bd9Sstevel@tonic-gate 40437c478bd9Sstevel@tonic-gate #ifdef REPL_PAGE_STATS 40447c478bd9Sstevel@tonic-gate struct repl_page_stats { 40457c478bd9Sstevel@tonic-gate uint_t ngets; 40467c478bd9Sstevel@tonic-gate uint_t ngets_noreloc; 40477c478bd9Sstevel@tonic-gate uint_t npgr_noreloc; 40487c478bd9Sstevel@tonic-gate uint_t nnopage_first; 40497c478bd9Sstevel@tonic-gate uint_t nnopage; 40507c478bd9Sstevel@tonic-gate uint_t nhashout; 40517c478bd9Sstevel@tonic-gate uint_t nnofree; 40527c478bd9Sstevel@tonic-gate uint_t nnext_pp; 40537c478bd9Sstevel@tonic-gate } repl_page_stats; 40547c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) atomic_add_32(&repl_page_stats.v, 1) 40557c478bd9Sstevel@tonic-gate #else /* REPL_PAGE_STATS */ 40567c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) 40577c478bd9Sstevel@tonic-gate #endif /* REPL_PAGE_STATS */ 40587c478bd9Sstevel@tonic-gate 40597c478bd9Sstevel@tonic-gate int pgrppgcp; 40607c478bd9Sstevel@tonic-gate 40617c478bd9Sstevel@tonic-gate /* 40627c478bd9Sstevel@tonic-gate * The freemem accounting must be done by the caller. 40637c478bd9Sstevel@tonic-gate * First we try to get a replacement page of the same size as like_pp, 40647c478bd9Sstevel@tonic-gate * if that is not possible, then we just get a set of discontiguous 40657c478bd9Sstevel@tonic-gate * PAGESIZE pages. 40667c478bd9Sstevel@tonic-gate */ 40677c478bd9Sstevel@tonic-gate page_t * 40682dae3fb5Sjjc page_get_replacement_page(page_t *orig_like_pp, struct lgrp *lgrp_target, 40697c478bd9Sstevel@tonic-gate uint_t pgrflags) 40707c478bd9Sstevel@tonic-gate { 40717c478bd9Sstevel@tonic-gate page_t *like_pp; 40727c478bd9Sstevel@tonic-gate page_t *pp, *pplist; 40737c478bd9Sstevel@tonic-gate page_t *pl = NULL; 40747c478bd9Sstevel@tonic-gate ulong_t bin; 40757c478bd9Sstevel@tonic-gate int mnode, page_mnode; 40767c478bd9Sstevel@tonic-gate int szc; 40777c478bd9Sstevel@tonic-gate spgcnt_t npgs, pg_cnt; 40787c478bd9Sstevel@tonic-gate pfn_t pfnum; 40797c478bd9Sstevel@tonic-gate int mtype; 40807c478bd9Sstevel@tonic-gate int flags = 0; 40817c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 40822dae3fb5Sjjc lgrp_t *lgrp; 40837c478bd9Sstevel@tonic-gate 40847c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets); 40857c478bd9Sstevel@tonic-gate like_pp = orig_like_pp; 40867c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(like_pp)); 40877c478bd9Sstevel@tonic-gate 40887c478bd9Sstevel@tonic-gate szc = like_pp->p_szc; 40897c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 40907c478bd9Sstevel@tonic-gate /* 40917c478bd9Sstevel@tonic-gate * Now we reset like_pp to the base page_t. 40927c478bd9Sstevel@tonic-gate * That way, we won't walk past the end of this 'szc' page. 40937c478bd9Sstevel@tonic-gate */ 40947c478bd9Sstevel@tonic-gate pfnum = PFN_BASE(like_pp->p_pagenum, szc); 40957c478bd9Sstevel@tonic-gate like_pp = page_numtopp_nolock(pfnum); 40967c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == szc); 40977c478bd9Sstevel@tonic-gate 40987c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(like_pp)) { 40997c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 41007c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets_noreloc); 41017c478bd9Sstevel@tonic-gate flags = PGI_RELOCONLY; 41027c478bd9Sstevel@tonic-gate } else if (pgrflags & PGR_NORELOC) { 41037c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 41047c478bd9Sstevel@tonic-gate REPL_STAT_INCR(npgr_noreloc); 41057c478bd9Sstevel@tonic-gate flags = PG_NORELOC; 41067c478bd9Sstevel@tonic-gate } 41077c478bd9Sstevel@tonic-gate 41087c478bd9Sstevel@tonic-gate /* 41097c478bd9Sstevel@tonic-gate * Kernel pages must always be replaced with the same size 41107c478bd9Sstevel@tonic-gate * pages, since we cannot properly handle demotion of kernel 41117c478bd9Sstevel@tonic-gate * pages. 41127c478bd9Sstevel@tonic-gate */ 4113ad23a2dbSjohansen if (PP_ISKAS(like_pp)) 41147c478bd9Sstevel@tonic-gate pgrflags |= PGR_SAMESZC; 41157c478bd9Sstevel@tonic-gate 41167c478bd9Sstevel@tonic-gate /* LINTED */ 411707ad560dSkchow MTYPE_PGR_INIT(mtype, flags, like_pp, page_mnode, npgs); 41187c478bd9Sstevel@tonic-gate 41197c478bd9Sstevel@tonic-gate while (npgs) { 41207c478bd9Sstevel@tonic-gate pplist = NULL; 41217c478bd9Sstevel@tonic-gate for (;;) { 41227c478bd9Sstevel@tonic-gate pg_cnt = page_get_pagecnt(szc); 41237c478bd9Sstevel@tonic-gate bin = PP_2_BIN(like_pp); 41247c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == orig_like_pp->p_szc); 41257c478bd9Sstevel@tonic-gate ASSERT(pg_cnt <= npgs); 41267c478bd9Sstevel@tonic-gate 41277c478bd9Sstevel@tonic-gate /* 41287c478bd9Sstevel@tonic-gate * If an lgroup was specified, try to get the 41297c478bd9Sstevel@tonic-gate * page from that lgroup. 41302dae3fb5Sjjc * NOTE: Must be careful with code below because 41312dae3fb5Sjjc * lgroup may disappear and reappear since there 41322dae3fb5Sjjc * is no locking for lgroup here. 41337c478bd9Sstevel@tonic-gate */ 41342dae3fb5Sjjc if (LGRP_EXISTS(lgrp_target)) { 41352dae3fb5Sjjc /* 41362dae3fb5Sjjc * Keep local variable for lgroup separate 41372dae3fb5Sjjc * from lgroup argument since this code should 41382dae3fb5Sjjc * only be exercised when lgroup argument 41392dae3fb5Sjjc * exists.... 41402dae3fb5Sjjc */ 41412dae3fb5Sjjc lgrp = lgrp_target; 41422dae3fb5Sjjc 41437c478bd9Sstevel@tonic-gate /* Try the lgroup's freelists first */ 41447c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41457c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41467c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41477c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41487c478bd9Sstevel@tonic-gate != -1) { 4149ce8eb11aSdp pplist = 4150ce8eb11aSdp page_get_mnode_freelist(mnode, bin, 4151ce8eb11aSdp mtype, szc, flags); 41527c478bd9Sstevel@tonic-gate } 41537c478bd9Sstevel@tonic-gate 41547c478bd9Sstevel@tonic-gate /* 41557c478bd9Sstevel@tonic-gate * Now try it's cachelists if this is a 41567c478bd9Sstevel@tonic-gate * small page. Don't need to do it for 41577c478bd9Sstevel@tonic-gate * larger ones since page_freelist_coalesce() 41587c478bd9Sstevel@tonic-gate * already failed. 41597c478bd9Sstevel@tonic-gate */ 41607c478bd9Sstevel@tonic-gate if (pplist != NULL || szc != 0) 41617c478bd9Sstevel@tonic-gate break; 41627c478bd9Sstevel@tonic-gate 41637c478bd9Sstevel@tonic-gate /* Now try it's cachelists */ 41647c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41657c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41667c478bd9Sstevel@tonic-gate 41677c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41687c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41697c478bd9Sstevel@tonic-gate != -1) { 4170ce8eb11aSdp pplist = 4171ce8eb11aSdp page_get_mnode_cachelist(bin, flags, 4172ce8eb11aSdp mnode, mtype); 41737c478bd9Sstevel@tonic-gate } 41747c478bd9Sstevel@tonic-gate if (pplist != NULL) { 41757c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 41767c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 41777c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 41787c478bd9Sstevel@tonic-gate break; 41797c478bd9Sstevel@tonic-gate } 41807c478bd9Sstevel@tonic-gate /* Done looking in this lgroup. Bail out. */ 41817c478bd9Sstevel@tonic-gate break; 41827c478bd9Sstevel@tonic-gate } 41837c478bd9Sstevel@tonic-gate 41847c478bd9Sstevel@tonic-gate /* 41852dae3fb5Sjjc * No lgroup was specified (or lgroup was removed by 41862dae3fb5Sjjc * DR, so just try to get the page as close to 41872dae3fb5Sjjc * like_pp's mnode as possible. 41887c478bd9Sstevel@tonic-gate * First try the local freelist... 41897c478bd9Sstevel@tonic-gate */ 41907c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(like_pp); 41917c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, bin, 41927c478bd9Sstevel@tonic-gate mtype, szc, flags); 41937c478bd9Sstevel@tonic-gate if (pplist != NULL) 41947c478bd9Sstevel@tonic-gate break; 41957c478bd9Sstevel@tonic-gate 41967c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnofree); 41977c478bd9Sstevel@tonic-gate 41987c478bd9Sstevel@tonic-gate /* 41997c478bd9Sstevel@tonic-gate * ...then the local cachelist. Don't need to do it for 42007c478bd9Sstevel@tonic-gate * larger pages cause page_freelist_coalesce() already 42017c478bd9Sstevel@tonic-gate * failed there anyway. 42027c478bd9Sstevel@tonic-gate */ 42037c478bd9Sstevel@tonic-gate if (szc == 0) { 42047c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, flags, 42057c478bd9Sstevel@tonic-gate mnode, mtype); 42067c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42077c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 42087c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 42097c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 42107c478bd9Sstevel@tonic-gate break; 42117c478bd9Sstevel@tonic-gate } 42127c478bd9Sstevel@tonic-gate } 42137c478bd9Sstevel@tonic-gate 42147c478bd9Sstevel@tonic-gate /* Now try remote freelists */ 42157c478bd9Sstevel@tonic-gate page_mnode = mnode; 42167c478bd9Sstevel@tonic-gate lgrp = 42177c478bd9Sstevel@tonic-gate lgrp_hand_to_lgrp(MEM_NODE_2_LGRPHAND(page_mnode)); 42187c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42197c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42207c478bd9Sstevel@tonic-gate while (pplist == NULL && 42217c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 42227c478bd9Sstevel@tonic-gate != -1) { 42237c478bd9Sstevel@tonic-gate /* 42247c478bd9Sstevel@tonic-gate * Skip local mnode. 42257c478bd9Sstevel@tonic-gate */ 42267c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 42277c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 42287c478bd9Sstevel@tonic-gate continue; 42297c478bd9Sstevel@tonic-gate 42307c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, 42317c478bd9Sstevel@tonic-gate bin, mtype, szc, flags); 42327c478bd9Sstevel@tonic-gate } 42337c478bd9Sstevel@tonic-gate 42347c478bd9Sstevel@tonic-gate if (pplist != NULL) 42357c478bd9Sstevel@tonic-gate break; 42367c478bd9Sstevel@tonic-gate 42377c478bd9Sstevel@tonic-gate 42387c478bd9Sstevel@tonic-gate /* Now try remote cachelists */ 42397c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42407c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42417c478bd9Sstevel@tonic-gate while (pplist == NULL && szc == 0) { 42427c478bd9Sstevel@tonic-gate mnode = lgrp_memnode_choose(&lgrp_cookie); 42437c478bd9Sstevel@tonic-gate if (mnode == -1) 42447c478bd9Sstevel@tonic-gate break; 42457c478bd9Sstevel@tonic-gate /* 42467c478bd9Sstevel@tonic-gate * Skip local mnode. 42477c478bd9Sstevel@tonic-gate */ 42487c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 42497c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 42507c478bd9Sstevel@tonic-gate continue; 42517c478bd9Sstevel@tonic-gate 42527c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, 42537c478bd9Sstevel@tonic-gate flags, mnode, mtype); 42547c478bd9Sstevel@tonic-gate 42557c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42567c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 42577c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 42587c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 42597c478bd9Sstevel@tonic-gate break; 42607c478bd9Sstevel@tonic-gate } 42617c478bd9Sstevel@tonic-gate } 42627c478bd9Sstevel@tonic-gate 42637c478bd9Sstevel@tonic-gate /* 42647c478bd9Sstevel@tonic-gate * Break out of while loop under the following cases: 42657c478bd9Sstevel@tonic-gate * - If we successfully got a page. 42667c478bd9Sstevel@tonic-gate * - If pgrflags specified only returning a specific 42677c478bd9Sstevel@tonic-gate * page size and we could not find that page size. 42687c478bd9Sstevel@tonic-gate * - If we could not satisfy the request with PAGESIZE 42697c478bd9Sstevel@tonic-gate * or larger pages. 42707c478bd9Sstevel@tonic-gate */ 42717c478bd9Sstevel@tonic-gate if (pplist != NULL || szc == 0) 42727c478bd9Sstevel@tonic-gate break; 42737c478bd9Sstevel@tonic-gate 42747c478bd9Sstevel@tonic-gate if ((pgrflags & PGR_SAMESZC) || pgrppgcp) { 42757c478bd9Sstevel@tonic-gate /* try to find contig page */ 42767c478bd9Sstevel@tonic-gate 42777c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42787c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42797c478bd9Sstevel@tonic-gate 42807c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 42817c478bd9Sstevel@tonic-gate (mnode = 4282ce8eb11aSdp lgrp_memnode_choose(&lgrp_cookie)) 42837c478bd9Sstevel@tonic-gate != -1) { 42847c478bd9Sstevel@tonic-gate pplist = page_get_contig_pages( 4285ce8eb11aSdp mnode, bin, mtype, szc, 4286ce8eb11aSdp flags | PGI_PGCPHIPRI); 42877c478bd9Sstevel@tonic-gate } 42887c478bd9Sstevel@tonic-gate break; 42897c478bd9Sstevel@tonic-gate } 42907c478bd9Sstevel@tonic-gate 42917c478bd9Sstevel@tonic-gate /* 42927c478bd9Sstevel@tonic-gate * The correct thing to do here is try the next 42937c478bd9Sstevel@tonic-gate * page size down using szc--. Due to a bug 42947c478bd9Sstevel@tonic-gate * with the processing of HAT_RELOAD_SHARE 42957c478bd9Sstevel@tonic-gate * where the sfmmu_ttecnt arrays of all 42967c478bd9Sstevel@tonic-gate * hats sharing an ISM segment don't get updated, 42977c478bd9Sstevel@tonic-gate * using intermediate size pages for relocation 42987c478bd9Sstevel@tonic-gate * can lead to continuous page faults. 42997c478bd9Sstevel@tonic-gate */ 43007c478bd9Sstevel@tonic-gate szc = 0; 43017c478bd9Sstevel@tonic-gate } 43027c478bd9Sstevel@tonic-gate 43037c478bd9Sstevel@tonic-gate if (pplist != NULL) { 43047c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 43057c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 43067c478bd9Sstevel@tonic-gate int, mnode, 43077c478bd9Sstevel@tonic-gate ulong_t, bin, 43087c478bd9Sstevel@tonic-gate uint_t, flags); 43097c478bd9Sstevel@tonic-gate 43107c478bd9Sstevel@tonic-gate while (pplist != NULL && pg_cnt--) { 43117c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 43127c478bd9Sstevel@tonic-gate pp = pplist; 43137c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 43147c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 43157c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 43167c478bd9Sstevel@tonic-gate page_list_concat(&pl, &pp); 43177c478bd9Sstevel@tonic-gate npgs--; 43187c478bd9Sstevel@tonic-gate like_pp = like_pp + 1; 43197c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnext_pp); 43207c478bd9Sstevel@tonic-gate } 43217c478bd9Sstevel@tonic-gate ASSERT(pg_cnt == 0); 43227c478bd9Sstevel@tonic-gate } else { 43237c478bd9Sstevel@tonic-gate break; 43247c478bd9Sstevel@tonic-gate } 43257c478bd9Sstevel@tonic-gate } 43267c478bd9Sstevel@tonic-gate 43277c478bd9Sstevel@tonic-gate if (npgs) { 43287c478bd9Sstevel@tonic-gate /* 43297c478bd9Sstevel@tonic-gate * We were unable to allocate the necessary number 43307c478bd9Sstevel@tonic-gate * of pages. 43317c478bd9Sstevel@tonic-gate * We need to free up any pl. 43327c478bd9Sstevel@tonic-gate */ 43337c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnopage); 43347c478bd9Sstevel@tonic-gate page_free_replacement_page(pl); 43357c478bd9Sstevel@tonic-gate return (NULL); 43367c478bd9Sstevel@tonic-gate } else { 43377c478bd9Sstevel@tonic-gate return (pl); 43387c478bd9Sstevel@tonic-gate } 43397c478bd9Sstevel@tonic-gate } 43407c478bd9Sstevel@tonic-gate 43417c478bd9Sstevel@tonic-gate /* 43427c478bd9Sstevel@tonic-gate * demote a free large page to it's constituent pages 43437c478bd9Sstevel@tonic-gate */ 43447c478bd9Sstevel@tonic-gate void 43457c478bd9Sstevel@tonic-gate page_demote_free_pages(page_t *pp) 43467c478bd9Sstevel@tonic-gate { 43477c478bd9Sstevel@tonic-gate 43487c478bd9Sstevel@tonic-gate int mnode; 43497c478bd9Sstevel@tonic-gate 43507c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 43517c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 43527c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 43537c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 43547c478bd9Sstevel@tonic-gate 43557c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 43567c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 43577c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 43587c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, 4359*19397407SSherry Moore pp->p_szc), 0, pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 43607c478bd9Sstevel@tonic-gate } 43617c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 43627c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 43637c478bd9Sstevel@tonic-gate } 4364932dc8e5Sdp 4365932dc8e5Sdp /* 4366932dc8e5Sdp * Factor in colorequiv to check additional 'equivalent' bins. 4367932dc8e5Sdp * colorequiv may be set in /etc/system 4368932dc8e5Sdp */ 4369932dc8e5Sdp void 4370932dc8e5Sdp page_set_colorequiv_arr(void) 4371932dc8e5Sdp { 4372932dc8e5Sdp if (colorequiv > 1) { 4373932dc8e5Sdp int i; 4374fe70c9cfSdp uint_t sv_a = lowbit(colorequiv) - 1; 4375932dc8e5Sdp 4376fe70c9cfSdp if (sv_a > 15) 4377fe70c9cfSdp sv_a = 15; 4378932dc8e5Sdp 4379932dc8e5Sdp for (i = 0; i < MMU_PAGE_SIZES; i++) { 4380fe70c9cfSdp uint_t colors; 4381fe70c9cfSdp uint_t a = sv_a; 4382932dc8e5Sdp 4383932dc8e5Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) { 4384932dc8e5Sdp continue; 4385932dc8e5Sdp } 4386932dc8e5Sdp while ((colors >> a) == 0) 4387932dc8e5Sdp a--; 4388932dc8e5Sdp if ((a << 4) > colorequivszc[i]) { 4389932dc8e5Sdp colorequivszc[i] = (a << 4); 4390932dc8e5Sdp } 4391932dc8e5Sdp } 4392932dc8e5Sdp } 4393932dc8e5Sdp } 4394