17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50b5aa17bSmec * Common Development and Distribution License (the "License"). 60b5aa17bSmec * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22932dc8e5Sdp * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 317c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 357c478bd9Sstevel@tonic-gate 367c478bd9Sstevel@tonic-gate /* 377c478bd9Sstevel@tonic-gate * This file contains common functions to access and manage the page lists. 387c478bd9Sstevel@tonic-gate * Many of these routines originated from platform dependent modules 397c478bd9Sstevel@tonic-gate * (sun4/vm/vm_dep.c, i86pc/vm/vm_machdep.c) and modified to function in 407c478bd9Sstevel@tonic-gate * a platform independent manner. 417c478bd9Sstevel@tonic-gate * 427c478bd9Sstevel@tonic-gate * vm/vm_dep.h provides for platform specific support. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate 457c478bd9Sstevel@tonic-gate #include <sys/types.h> 467c478bd9Sstevel@tonic-gate #include <sys/debug.h> 477c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 487c478bd9Sstevel@tonic-gate #include <sys/systm.h> 497c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 507c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 517c478bd9Sstevel@tonic-gate #include <vm/as.h> 527c478bd9Sstevel@tonic-gate #include <vm/page.h> 537c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 547c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 55*78b03d3aSkchow #include <sys/vmsystm.h> 567c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 577c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 587c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 597c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 607c478bd9Sstevel@tonic-gate #include <sys/callb.h> 617c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 627c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 637c478bd9Sstevel@tonic-gate 647c478bd9Sstevel@tonic-gate extern uint_t vac_colors; 657c478bd9Sstevel@tonic-gate 666061ce8aSkchow #define MAX_PRAGMA_ALIGN 128 676061ce8aSkchow 686061ce8aSkchow /* vm_cpu_data0 for the boot cpu before kmem is initialized */ 696061ce8aSkchow 706061ce8aSkchow #if L2CACHE_ALIGN_MAX <= MAX_PRAGMA_ALIGN 71affbd3ccSkchow #pragma align L2CACHE_ALIGN_MAX(vm_cpu_data0) 726061ce8aSkchow #else 736061ce8aSkchow #pragma align MAX_PRAGMA_ALIGN(vm_cpu_data0) 746061ce8aSkchow #endif 75affbd3ccSkchow char vm_cpu_data0[VM_CPU_DATA_PADSIZE]; 76affbd3ccSkchow 777c478bd9Sstevel@tonic-gate /* 787c478bd9Sstevel@tonic-gate * number of page colors equivalent to reqested color in page_get routines. 797c478bd9Sstevel@tonic-gate * If set, keeps large pages intact longer and keeps MPO allocation 807c478bd9Sstevel@tonic-gate * from the local mnode in favor of acquiring the 'correct' page color from 817c478bd9Sstevel@tonic-gate * a demoted large page or from a remote mnode. 827c478bd9Sstevel@tonic-gate */ 835d07b933Sdp uint_t colorequiv; 845d07b933Sdp 855d07b933Sdp /* 865d07b933Sdp * color equivalency mask for each page size. 875d07b933Sdp * Mask is computed based on cpu L2$ way sizes and colorequiv global. 885d07b933Sdp * High 4 bits determine the number of high order bits of the color to ignore. 895d07b933Sdp * Low 4 bits determines number of low order bits of color to ignore (it's only 905d07b933Sdp * relevant for hashed index based page coloring). 915d07b933Sdp */ 925d07b933Sdp uchar_t colorequivszc[MMU_PAGE_SIZES]; 937c478bd9Sstevel@tonic-gate 947c478bd9Sstevel@tonic-gate /* 957c478bd9Sstevel@tonic-gate * if set, specifies the percentage of large pages that are free from within 967c478bd9Sstevel@tonic-gate * a large page region before attempting to lock those pages for 977c478bd9Sstevel@tonic-gate * page_get_contig_pages processing. 987c478bd9Sstevel@tonic-gate * 997c478bd9Sstevel@tonic-gate * Should be turned on when kpr is available when page_trylock_contig_pages 1007c478bd9Sstevel@tonic-gate * can be more selective. 1017c478bd9Sstevel@tonic-gate */ 1027c478bd9Sstevel@tonic-gate 1037c478bd9Sstevel@tonic-gate int ptcpthreshold; 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate /* 1067c478bd9Sstevel@tonic-gate * Limit page get contig page search based on failure cnts in pgcpfailcnt[]. 10783f9b804Skchow * Enabled by default via pgcplimitsearch. 10883f9b804Skchow * 10983f9b804Skchow * pgcpfailcnt[] is bounded by PGCPFAILMAX (>= 1/2 of installed 11083f9b804Skchow * memory). When reached, pgcpfailcnt[] is reset to 1/2 of this upper 11183f9b804Skchow * bound. This upper bound range guarantees: 11283f9b804Skchow * - all large page 'slots' will be searched over time 11383f9b804Skchow * - the minimum (1) large page candidates considered on each pgcp call 11483f9b804Skchow * - count doesn't wrap around to 0 1157c478bd9Sstevel@tonic-gate */ 11683f9b804Skchow pgcnt_t pgcpfailcnt[MMU_PAGE_SIZES]; 1177c478bd9Sstevel@tonic-gate int pgcplimitsearch = 1; 1187c478bd9Sstevel@tonic-gate 11983f9b804Skchow #define PGCPFAILMAX (1 << (highbit(physinstalled) - 1)) 12083f9b804Skchow #define SETPGCPFAILCNT(szc) \ 12183f9b804Skchow if (++pgcpfailcnt[szc] >= PGCPFAILMAX) \ 12283f9b804Skchow pgcpfailcnt[szc] = PGCPFAILMAX / 2; 12383f9b804Skchow 1247c478bd9Sstevel@tonic-gate #ifdef VM_STATS 1257c478bd9Sstevel@tonic-gate struct vmm_vmstats_str vmm_vmstats; 1267c478bd9Sstevel@tonic-gate 1277c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate #if defined(__sparc) 1307c478bd9Sstevel@tonic-gate #define LPGCREATE 0 1317c478bd9Sstevel@tonic-gate #else 1327c478bd9Sstevel@tonic-gate /* enable page_get_contig_pages */ 1337c478bd9Sstevel@tonic-gate #define LPGCREATE 1 1347c478bd9Sstevel@tonic-gate #endif 1357c478bd9Sstevel@tonic-gate 1367c478bd9Sstevel@tonic-gate int pg_contig_disable; 1377c478bd9Sstevel@tonic-gate int pg_lpgcreate_nocage = LPGCREATE; 1387c478bd9Sstevel@tonic-gate 1397c478bd9Sstevel@tonic-gate /* 1405d07b933Sdp * page_freelist_split pfn flag to signify no hi pfn requirement. 1417c478bd9Sstevel@tonic-gate */ 1427c478bd9Sstevel@tonic-gate #define PFNNULL 0 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate /* Flags involved in promotion and demotion routines */ 1457c478bd9Sstevel@tonic-gate #define PC_FREE 0x1 /* put page on freelist */ 1467c478bd9Sstevel@tonic-gate #define PC_ALLOC 0x2 /* return page for allocation */ 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate /* 1497c478bd9Sstevel@tonic-gate * Flag for page_demote to be used with PC_FREE to denote that we don't care 1507c478bd9Sstevel@tonic-gate * what the color is as the color parameter to the function is ignored. 1517c478bd9Sstevel@tonic-gate */ 1527c478bd9Sstevel@tonic-gate #define PC_NO_COLOR (-1) 1537c478bd9Sstevel@tonic-gate 1545d07b933Sdp /* mtype value for page_promote to use when mtype does not matter */ 1555d07b933Sdp #define PC_MTYPE_ANY (-1) 1565d07b933Sdp 1577c478bd9Sstevel@tonic-gate /* 1587c478bd9Sstevel@tonic-gate * page counters candidates info 1597c478bd9Sstevel@tonic-gate * See page_ctrs_cands comment below for more details. 1607c478bd9Sstevel@tonic-gate * fields are as follows: 1617c478bd9Sstevel@tonic-gate * pcc_pages_free: # pages which freelist coalesce can create 1627c478bd9Sstevel@tonic-gate * pcc_color_free: pointer to page free counts per color 1637c478bd9Sstevel@tonic-gate */ 1647c478bd9Sstevel@tonic-gate typedef struct pcc_info { 1657c478bd9Sstevel@tonic-gate pgcnt_t pcc_pages_free; 1667c478bd9Sstevel@tonic-gate pgcnt_t *pcc_color_free; 1677c478bd9Sstevel@tonic-gate } pcc_info_t; 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate /* 1707c478bd9Sstevel@tonic-gate * On big machines it can take a long time to check page_counters 1717c478bd9Sstevel@tonic-gate * arrays. page_ctrs_cands is a summary array whose elements are a dynamically 1727c478bd9Sstevel@tonic-gate * updated sum of all elements of the corresponding page_counters arrays. 1737c478bd9Sstevel@tonic-gate * page_freelist_coalesce() searches page_counters only if an appropriate 1747c478bd9Sstevel@tonic-gate * element of page_ctrs_cands array is greater than 0. 1757c478bd9Sstevel@tonic-gate * 1765d07b933Sdp * page_ctrs_cands is indexed by mutex (i), region (r), mnode (m), mrange (g) 1777c478bd9Sstevel@tonic-gate */ 1785d07b933Sdp pcc_info_t **page_ctrs_cands[NPC_MUTEX][MMU_PAGE_SIZES]; 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate /* 1817c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1825d07b933Sdp * for the given mnode (m), mrange (g), and region size (r) 1837c478bd9Sstevel@tonic-gate */ 1845d07b933Sdp #define PGCTRS_CANDS_GETVALUE(m, g, r, val) { \ 1857c478bd9Sstevel@tonic-gate int i; \ 1867c478bd9Sstevel@tonic-gate val = 0; \ 1877c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 1885d07b933Sdp val += page_ctrs_cands[i][(r)][(m)][(g)].pcc_pages_free; \ 1897c478bd9Sstevel@tonic-gate } \ 1907c478bd9Sstevel@tonic-gate } 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate /* 1937c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1945d07b933Sdp * for the given mnode (m), mrange (g), region size (r), and color (c) 1957c478bd9Sstevel@tonic-gate */ 1965d07b933Sdp #define PGCTRS_CANDS_GETVALUECOLOR(m, g, r, c, val) { \ 1977c478bd9Sstevel@tonic-gate int i; \ 1987c478bd9Sstevel@tonic-gate val = 0; \ 1995d07b933Sdp ASSERT((c) < PAGE_GET_PAGECOLORS(r)); \ 2007c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 2015d07b933Sdp val += \ 2025d07b933Sdp page_ctrs_cands[i][(r)][(m)][(g)].pcc_color_free[(c)]; \ 2037c478bd9Sstevel@tonic-gate } \ 2047c478bd9Sstevel@tonic-gate } 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate /* 2077c478bd9Sstevel@tonic-gate * We can only allow a single thread to update a counter within the physical 2087c478bd9Sstevel@tonic-gate * range of the largest supported page size. That is the finest granularity 2097c478bd9Sstevel@tonic-gate * possible since the counter values are dependent on each other 2107c478bd9Sstevel@tonic-gate * as you move accross region sizes. PP_CTR_LOCK_INDX is used to determine the 2117c478bd9Sstevel@tonic-gate * ctr_mutex lock index for a particular physical range. 2127c478bd9Sstevel@tonic-gate */ 2137c478bd9Sstevel@tonic-gate static kmutex_t *ctr_mutex[NPC_MUTEX]; 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate #define PP_CTR_LOCK_INDX(pp) \ 2165d07b933Sdp (((pp)->p_pagenum >> \ 2177c478bd9Sstevel@tonic-gate (PAGE_BSZS_SHIFT(mmu_page_sizes - 1))) & (NPC_MUTEX - 1)) 2187c478bd9Sstevel@tonic-gate 2195d07b933Sdp #define INVALID_COLOR 0xffffffff 2205d07b933Sdp #define INVALID_MASK 0xffffffff 2215d07b933Sdp 2227c478bd9Sstevel@tonic-gate /* 2237c478bd9Sstevel@tonic-gate * Local functions prototypes. 2247c478bd9Sstevel@tonic-gate */ 2257c478bd9Sstevel@tonic-gate 226affbd3ccSkchow void page_ctr_add(int, int, page_t *, int); 227affbd3ccSkchow void page_ctr_add_internal(int, int, page_t *, int); 228affbd3ccSkchow void page_ctr_sub(int, int, page_t *, int); 2295d07b933Sdp void page_ctr_sub_internal(int, int, page_t *, int); 2307c478bd9Sstevel@tonic-gate void page_freelist_lock(int); 2317c478bd9Sstevel@tonic-gate void page_freelist_unlock(int); 2325d07b933Sdp page_t *page_promote(int, pfn_t, uchar_t, int, int); 2337c478bd9Sstevel@tonic-gate page_t *page_demote(int, pfn_t, uchar_t, uchar_t, int, int); 2345d07b933Sdp page_t *page_freelist_split(uchar_t, 2355d07b933Sdp uint_t, int, int, pfn_t, page_list_walker_t *); 2367c478bd9Sstevel@tonic-gate page_t *page_get_mnode_cachelist(uint_t, uint_t, int, int); 2377c478bd9Sstevel@tonic-gate static int page_trylock_cons(page_t *pp, se_t se); 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * The page_counters array below is used to keep track of free contiguous 2417c478bd9Sstevel@tonic-gate * physical memory. A hw_page_map_t will be allocated per mnode per szc. 2427c478bd9Sstevel@tonic-gate * This contains an array of counters, the size of the array, a shift value 2437c478bd9Sstevel@tonic-gate * used to convert a pagenum into a counter array index or vice versa, as 2447c478bd9Sstevel@tonic-gate * well as a cache of the last successful index to be promoted to a larger 2457c478bd9Sstevel@tonic-gate * page size. As an optimization, we keep track of the last successful index 2467c478bd9Sstevel@tonic-gate * to be promoted per page color for the given size region, and this is 2477c478bd9Sstevel@tonic-gate * allocated dynamically based upon the number of colors for a given 2487c478bd9Sstevel@tonic-gate * region size. 2497c478bd9Sstevel@tonic-gate * 2507c478bd9Sstevel@tonic-gate * Conceptually, the page counters are represented as: 2517c478bd9Sstevel@tonic-gate * 2527c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode] 2537c478bd9Sstevel@tonic-gate * 2547c478bd9Sstevel@tonic-gate * region_size: size code of a candidate larger page made up 2557c478bd9Sstevel@tonic-gate * of contiguous free smaller pages. 2567c478bd9Sstevel@tonic-gate * 2577c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode].hpm_counters[index]: 2587c478bd9Sstevel@tonic-gate * represents how many (region_size - 1) pages either 2597c478bd9Sstevel@tonic-gate * exist or can be created within the given index range. 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * Let's look at a sparc example: 2627c478bd9Sstevel@tonic-gate * If we want to create a free 512k page, we look at region_size 2 2637c478bd9Sstevel@tonic-gate * for the mnode we want. We calculate the index and look at a specific 2647c478bd9Sstevel@tonic-gate * hpm_counters location. If we see 8 (FULL_REGION_CNT on sparc) at 2657c478bd9Sstevel@tonic-gate * this location, it means that 8 64k pages either exist or can be created 2667c478bd9Sstevel@tonic-gate * from 8K pages in order to make a single free 512k page at the given 2677c478bd9Sstevel@tonic-gate * index. Note that when a region is full, it will contribute to the 2687c478bd9Sstevel@tonic-gate * counts in the region above it. Thus we will not know what page 2697c478bd9Sstevel@tonic-gate * size the free pages will be which can be promoted to this new free 2707c478bd9Sstevel@tonic-gate * page unless we look at all regions below the current region. 2717c478bd9Sstevel@tonic-gate */ 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate /* 2747c478bd9Sstevel@tonic-gate * Note: hpmctr_t is defined in platform vm_dep.h 2757c478bd9Sstevel@tonic-gate * hw_page_map_t contains all the information needed for the page_counters 2767c478bd9Sstevel@tonic-gate * logic. The fields are as follows: 2777c478bd9Sstevel@tonic-gate * 2787c478bd9Sstevel@tonic-gate * hpm_counters: dynamically allocated array to hold counter data 2797c478bd9Sstevel@tonic-gate * hpm_entries: entries in hpm_counters 2807c478bd9Sstevel@tonic-gate * hpm_shift: shift for pnum/array index conv 2817c478bd9Sstevel@tonic-gate * hpm_base: PFN mapped to counter index 0 2827c478bd9Sstevel@tonic-gate * hpm_color_current: last index in counter array for this color at 2837c478bd9Sstevel@tonic-gate * which we successfully created a large page 2847c478bd9Sstevel@tonic-gate */ 2857c478bd9Sstevel@tonic-gate typedef struct hw_page_map { 2867c478bd9Sstevel@tonic-gate hpmctr_t *hpm_counters; 2877c478bd9Sstevel@tonic-gate size_t hpm_entries; 2887c478bd9Sstevel@tonic-gate int hpm_shift; 2897c478bd9Sstevel@tonic-gate pfn_t hpm_base; 2905d07b933Sdp size_t *hpm_color_current[MAX_MNODE_MRANGES]; 2917c478bd9Sstevel@tonic-gate } hw_page_map_t; 2927c478bd9Sstevel@tonic-gate 2937c478bd9Sstevel@tonic-gate /* 2947c478bd9Sstevel@tonic-gate * Element zero is not used, but is allocated for convenience. 2957c478bd9Sstevel@tonic-gate */ 2967c478bd9Sstevel@tonic-gate static hw_page_map_t *page_counters[MMU_PAGE_SIZES]; 2977c478bd9Sstevel@tonic-gate 2985d07b933Sdp /* 2995d07b933Sdp * Cached value of MNODE_RANGE_CNT(mnode). 3005d07b933Sdp * This is a function call in x86. 3015d07b933Sdp */ 3025d07b933Sdp static int mnode_nranges[MAX_MEM_NODES]; 3035d07b933Sdp static int mnode_maxmrange[MAX_MEM_NODES]; 3045d07b933Sdp 3057c478bd9Sstevel@tonic-gate /* 3067c478bd9Sstevel@tonic-gate * The following macros are convenient ways to get access to the individual 3077c478bd9Sstevel@tonic-gate * elements of the page_counters arrays. They can be used on both 3087c478bd9Sstevel@tonic-gate * the left side and right side of equations. 3097c478bd9Sstevel@tonic-gate */ 3107c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS(mnode, rg_szc, idx) \ 3117c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters[(idx)]) 3127c478bd9Sstevel@tonic-gate 3137c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_COUNTERS(mnode, rg_szc) \ 3147c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters) 3157c478bd9Sstevel@tonic-gate 3167c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_SHIFT(mnode, rg_szc) \ 3177c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_shift) 3187c478bd9Sstevel@tonic-gate 3197c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_ENTRIES(mnode, rg_szc) \ 3207c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_entries) 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_BASE(mnode, rg_szc) \ 3237c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_base) 3247c478bd9Sstevel@tonic-gate 3255d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, rg_szc, g) \ 3265d07b933Sdp (page_counters[(rg_szc)][(mnode)].hpm_color_current[(g)]) 3277c478bd9Sstevel@tonic-gate 3285d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR(mnode, rg_szc, color, mrange) \ 3295d07b933Sdp (page_counters[(rg_szc)][(mnode)]. \ 3305d07b933Sdp hpm_color_current[(mrange)][(color)]) 3317c478bd9Sstevel@tonic-gate 3327c478bd9Sstevel@tonic-gate #define PNUM_TO_IDX(mnode, rg_szc, pnum) \ 3337c478bd9Sstevel@tonic-gate (((pnum) - PAGE_COUNTERS_BASE((mnode), (rg_szc))) >> \ 3347c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT((mnode), (rg_szc))) 3357c478bd9Sstevel@tonic-gate 3367c478bd9Sstevel@tonic-gate #define IDX_TO_PNUM(mnode, rg_szc, index) \ 3377c478bd9Sstevel@tonic-gate (PAGE_COUNTERS_BASE((mnode), (rg_szc)) + \ 3387c478bd9Sstevel@tonic-gate ((index) << PAGE_COUNTERS_SHIFT((mnode), (rg_szc)))) 3397c478bd9Sstevel@tonic-gate 3407c478bd9Sstevel@tonic-gate /* 3417c478bd9Sstevel@tonic-gate * Protects the hpm_counters and hpm_color_current memory from changing while 3427c478bd9Sstevel@tonic-gate * looking at page counters information. 3437c478bd9Sstevel@tonic-gate * Grab the write lock to modify what these fields point at. 3447c478bd9Sstevel@tonic-gate * Grab the read lock to prevent any pointers from changing. 3457c478bd9Sstevel@tonic-gate * The write lock can not be held during memory allocation due to a possible 3467c478bd9Sstevel@tonic-gate * recursion deadlock with trying to grab the read lock while the 3477c478bd9Sstevel@tonic-gate * write lock is already held. 3487c478bd9Sstevel@tonic-gate */ 3497c478bd9Sstevel@tonic-gate krwlock_t page_ctrs_rwlock[MAX_MEM_NODES]; 3507c478bd9Sstevel@tonic-gate 351affbd3ccSkchow 352affbd3ccSkchow /* 353affbd3ccSkchow * initialize cpu_vm_data to point at cache aligned vm_cpu_data_t. 354affbd3ccSkchow */ 355affbd3ccSkchow void 356affbd3ccSkchow cpu_vm_data_init(struct cpu *cp) 357affbd3ccSkchow { 358affbd3ccSkchow if (cp == CPU0) { 359affbd3ccSkchow cp->cpu_vm_data = (void *)&vm_cpu_data0; 360affbd3ccSkchow } else { 361affbd3ccSkchow void *kmptr; 3626061ce8aSkchow int align; 3636061ce8aSkchow size_t sz; 364affbd3ccSkchow 3656061ce8aSkchow align = (L2CACHE_ALIGN) ? L2CACHE_ALIGN : L2CACHE_ALIGN_MAX; 3666061ce8aSkchow sz = P2ROUNDUP(sizeof (vm_cpu_data_t), align) + align; 3676061ce8aSkchow kmptr = kmem_zalloc(sz, KM_SLEEP); 368affbd3ccSkchow cp->cpu_vm_data = (void *) P2ROUNDUP((uintptr_t)kmptr, align); 369affbd3ccSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr = kmptr; 3706061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize = sz; 371affbd3ccSkchow } 372affbd3ccSkchow } 373affbd3ccSkchow 374affbd3ccSkchow /* 375affbd3ccSkchow * free cpu_vm_data 376affbd3ccSkchow */ 377affbd3ccSkchow void 378affbd3ccSkchow cpu_vm_data_destroy(struct cpu *cp) 379affbd3ccSkchow { 380affbd3ccSkchow if (cp->cpu_seqid && cp->cpu_vm_data) { 381affbd3ccSkchow ASSERT(cp != CPU0); 382affbd3ccSkchow kmem_free(((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr, 3836061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize); 384affbd3ccSkchow } 385affbd3ccSkchow cp->cpu_vm_data = NULL; 386affbd3ccSkchow } 387affbd3ccSkchow 388affbd3ccSkchow 3897c478bd9Sstevel@tonic-gate /* 3907c478bd9Sstevel@tonic-gate * page size to page size code 3917c478bd9Sstevel@tonic-gate */ 3927c478bd9Sstevel@tonic-gate int 3937c478bd9Sstevel@tonic-gate page_szc(size_t pagesize) 3947c478bd9Sstevel@tonic-gate { 3957c478bd9Sstevel@tonic-gate int i = 0; 3967c478bd9Sstevel@tonic-gate 3977c478bd9Sstevel@tonic-gate while (hw_page_array[i].hp_size) { 3987c478bd9Sstevel@tonic-gate if (pagesize == hw_page_array[i].hp_size) 3997c478bd9Sstevel@tonic-gate return (i); 4007c478bd9Sstevel@tonic-gate i++; 4017c478bd9Sstevel@tonic-gate } 4027c478bd9Sstevel@tonic-gate return (-1); 4037c478bd9Sstevel@tonic-gate } 4047c478bd9Sstevel@tonic-gate 4057c478bd9Sstevel@tonic-gate /* 4064abce959Smec * page size to page size code with the restriction that it be a supported 4074abce959Smec * user page size. If it's not a supported user page size, -1 will be returned. 4087c478bd9Sstevel@tonic-gate */ 4097c478bd9Sstevel@tonic-gate int 4104abce959Smec page_szc_user_filtered(size_t pagesize) 4117c478bd9Sstevel@tonic-gate { 4127c478bd9Sstevel@tonic-gate int szc = page_szc(pagesize); 4134abce959Smec if ((szc != -1) && (SZC_2_USERSZC(szc) != -1)) { 4144abce959Smec return (szc); 4154abce959Smec } 4167c478bd9Sstevel@tonic-gate return (-1); 4177c478bd9Sstevel@tonic-gate } 4187c478bd9Sstevel@tonic-gate 4197c478bd9Sstevel@tonic-gate /* 4207c478bd9Sstevel@tonic-gate * Return how many page sizes are available for the user to use. This is 4217c478bd9Sstevel@tonic-gate * what the hardware supports and not based upon how the OS implements the 4227c478bd9Sstevel@tonic-gate * support of different page sizes. 42302bc52beSkchow * 42402bc52beSkchow * If legacy is non-zero, return the number of pagesizes available to legacy 42502bc52beSkchow * applications. The number of legacy page sizes might be less than the 42602bc52beSkchow * exported user page sizes. This is to prevent legacy applications that 42702bc52beSkchow * use the largest page size returned from getpagesizes(3c) from inadvertantly 42802bc52beSkchow * using the 'new' large pagesizes. 4297c478bd9Sstevel@tonic-gate */ 4307c478bd9Sstevel@tonic-gate uint_t 43102bc52beSkchow page_num_user_pagesizes(int legacy) 4327c478bd9Sstevel@tonic-gate { 43302bc52beSkchow if (legacy) 43402bc52beSkchow return (mmu_legacy_page_sizes); 4357c478bd9Sstevel@tonic-gate return (mmu_exported_page_sizes); 4367c478bd9Sstevel@tonic-gate } 4377c478bd9Sstevel@tonic-gate 4387c478bd9Sstevel@tonic-gate uint_t 4397c478bd9Sstevel@tonic-gate page_num_pagesizes(void) 4407c478bd9Sstevel@tonic-gate { 4417c478bd9Sstevel@tonic-gate return (mmu_page_sizes); 4427c478bd9Sstevel@tonic-gate } 4437c478bd9Sstevel@tonic-gate 4447c478bd9Sstevel@tonic-gate /* 4457c478bd9Sstevel@tonic-gate * returns the count of the number of base pagesize pages associated with szc 4467c478bd9Sstevel@tonic-gate */ 4477c478bd9Sstevel@tonic-gate pgcnt_t 4487c478bd9Sstevel@tonic-gate page_get_pagecnt(uint_t szc) 4497c478bd9Sstevel@tonic-gate { 4507c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4517c478bd9Sstevel@tonic-gate panic("page_get_pagecnt: out of range %d", szc); 4527c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_pgcnt); 4537c478bd9Sstevel@tonic-gate } 4547c478bd9Sstevel@tonic-gate 4557c478bd9Sstevel@tonic-gate size_t 4567c478bd9Sstevel@tonic-gate page_get_pagesize(uint_t szc) 4577c478bd9Sstevel@tonic-gate { 4587c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4597c478bd9Sstevel@tonic-gate panic("page_get_pagesize: out of range %d", szc); 4607c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4617c478bd9Sstevel@tonic-gate } 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate /* 4647c478bd9Sstevel@tonic-gate * Return the size of a page based upon the index passed in. An index of 4657c478bd9Sstevel@tonic-gate * zero refers to the smallest page size in the system, and as index increases 4667c478bd9Sstevel@tonic-gate * it refers to the next larger supported page size in the system. 4677c478bd9Sstevel@tonic-gate * Note that szc and userszc may not be the same due to unsupported szc's on 4687c478bd9Sstevel@tonic-gate * some systems. 4697c478bd9Sstevel@tonic-gate */ 4707c478bd9Sstevel@tonic-gate size_t 4717c478bd9Sstevel@tonic-gate page_get_user_pagesize(uint_t userszc) 4727c478bd9Sstevel@tonic-gate { 4737c478bd9Sstevel@tonic-gate uint_t szc = USERSZC_2_SZC(userszc); 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4767c478bd9Sstevel@tonic-gate panic("page_get_user_pagesize: out of range %d", szc); 4777c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4787c478bd9Sstevel@tonic-gate } 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate uint_t 4817c478bd9Sstevel@tonic-gate page_get_shift(uint_t szc) 4827c478bd9Sstevel@tonic-gate { 4837c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4847c478bd9Sstevel@tonic-gate panic("page_get_shift: out of range %d", szc); 4855d07b933Sdp return (PAGE_GET_SHIFT(szc)); 4867c478bd9Sstevel@tonic-gate } 4877c478bd9Sstevel@tonic-gate 4887c478bd9Sstevel@tonic-gate uint_t 4897c478bd9Sstevel@tonic-gate page_get_pagecolors(uint_t szc) 4907c478bd9Sstevel@tonic-gate { 4915d07b933Sdp if (szc >= mmu_page_sizes) 4925d07b933Sdp panic("page_get_pagecolors: out of range %d", szc); 4935d07b933Sdp return (PAGE_GET_PAGECOLORS(szc)); 4945d07b933Sdp } 4955d07b933Sdp 4965d07b933Sdp /* 4975d07b933Sdp * this assigns the desired equivalent color after a split 4985d07b933Sdp */ 4995d07b933Sdp uint_t 5005d07b933Sdp page_correct_color(uchar_t szc, uchar_t nszc, uint_t color, 5015d07b933Sdp uint_t ncolor, uint_t ceq_mask) 5025d07b933Sdp { 5035d07b933Sdp ASSERT(nszc > szc); 5045d07b933Sdp ASSERT(szc < mmu_page_sizes); 5055d07b933Sdp ASSERT(color < PAGE_GET_PAGECOLORS(szc)); 5065d07b933Sdp ASSERT(ncolor < PAGE_GET_PAGECOLORS(nszc)); 5075d07b933Sdp 5085d07b933Sdp color &= ceq_mask; 509ce8eb11aSdp ncolor = PAGE_CONVERT_COLOR(ncolor, szc, nszc); 5105d07b933Sdp return (color | (ncolor & ~ceq_mask)); 5117c478bd9Sstevel@tonic-gate } 5127c478bd9Sstevel@tonic-gate 513ce8eb11aSdp /* 514ce8eb11aSdp * The interleaved_mnodes flag is set when mnodes overlap in 515ce8eb11aSdp * the physbase..physmax range, but have disjoint slices. 516ce8eb11aSdp * In this case hpm_counters is shared by all mnodes. 517ce8eb11aSdp * This flag is set dynamically by the platform. 518ce8eb11aSdp */ 519ce8eb11aSdp int interleaved_mnodes = 0; 520ce8eb11aSdp 5217c478bd9Sstevel@tonic-gate /* 5227c478bd9Sstevel@tonic-gate * Called by startup(). 5237c478bd9Sstevel@tonic-gate * Size up the per page size free list counters based on physmax 5247c478bd9Sstevel@tonic-gate * of each node and max_mem_nodes. 525ce8eb11aSdp * 526ce8eb11aSdp * If interleaved_mnodes is set we need to find the first mnode that 527ce8eb11aSdp * exists. hpm_counters for the first mnode will then be shared by 528ce8eb11aSdp * all other mnodes. If interleaved_mnodes is not set, just set 529ce8eb11aSdp * first=mnode each time. That means there will be no sharing. 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate size_t 5327c478bd9Sstevel@tonic-gate page_ctrs_sz(void) 5337c478bd9Sstevel@tonic-gate { 5347c478bd9Sstevel@tonic-gate int r; /* region size */ 5357c478bd9Sstevel@tonic-gate int mnode; 536ce8eb11aSdp int firstmn; /* first mnode that exists */ 5375d07b933Sdp int nranges; 538ce8eb11aSdp pfn_t physbase; 539ce8eb11aSdp pfn_t physmax; 5407c478bd9Sstevel@tonic-gate uint_t ctrs_sz = 0; 5417c478bd9Sstevel@tonic-gate int i; 5427c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 5437c478bd9Sstevel@tonic-gate 5447c478bd9Sstevel@tonic-gate /* 5457c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 5467c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 5477c478bd9Sstevel@tonic-gate * arrays. 5487c478bd9Sstevel@tonic-gate */ 5495d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 5505d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 5517c478bd9Sstevel@tonic-gate } 5527c478bd9Sstevel@tonic-gate 553ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 5567c478bd9Sstevel@tonic-gate pfn_t r_base; 5577c478bd9Sstevel@tonic-gate pgcnt_t r_align; 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 5607c478bd9Sstevel@tonic-gate continue; 5617c478bd9Sstevel@tonic-gate 562ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 5635d07b933Sdp nranges = MNODE_RANGE_CNT(mnode); 5645d07b933Sdp mnode_nranges[mnode] = nranges; 5655d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 5665d07b933Sdp 5677c478bd9Sstevel@tonic-gate /* 5687c478bd9Sstevel@tonic-gate * determine size needed for page counter arrays with 5697c478bd9Sstevel@tonic-gate * base aligned to large page size. 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 572ce8eb11aSdp /* add in space for hpm_color_current */ 573ce8eb11aSdp ctrs_sz += sizeof (size_t) * 574ce8eb11aSdp colors_per_szc[r] * nranges; 575ce8eb11aSdp 576ce8eb11aSdp if (firstmn != mnode) 577ce8eb11aSdp continue; 578ce8eb11aSdp 5797c478bd9Sstevel@tonic-gate /* add in space for hpm_counters */ 5807c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 581ce8eb11aSdp r_base = physbase; 5827c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 583ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 584ce8eb11aSdp 5857c478bd9Sstevel@tonic-gate /* 5867c478bd9Sstevel@tonic-gate * Round up to always allocate on pointer sized 5877c478bd9Sstevel@tonic-gate * boundaries. 5887c478bd9Sstevel@tonic-gate */ 5897c478bd9Sstevel@tonic-gate ctrs_sz += P2ROUNDUP((r_pgcnt * sizeof (hpmctr_t)), 5907c478bd9Sstevel@tonic-gate sizeof (hpmctr_t *)); 5917c478bd9Sstevel@tonic-gate } 5927c478bd9Sstevel@tonic-gate } 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 5957c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * sizeof (hw_page_map_t)); 5965d07b933Sdp } 5975d07b933Sdp 5985d07b933Sdp /* add in space for page_ctrs_cands and pcc_color_free */ 5995d07b933Sdp ctrs_sz += sizeof (pcc_info_t *) * max_mem_nodes * 6005d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6015d07b933Sdp 6025d07b933Sdp for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6035d07b933Sdp 6045d07b933Sdp if (mem_node_config[mnode].exists == 0) 6055d07b933Sdp continue; 6067c478bd9Sstevel@tonic-gate 6075d07b933Sdp nranges = mnode_nranges[mnode]; 6085d07b933Sdp ctrs_sz += sizeof (pcc_info_t) * nranges * 6095d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6105d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6115d07b933Sdp ctrs_sz += sizeof (pgcnt_t) * nranges * 6125d07b933Sdp colors_per_szc[r] * NPC_MUTEX; 6135d07b933Sdp } 6147c478bd9Sstevel@tonic-gate } 6157c478bd9Sstevel@tonic-gate 6167c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6177c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * NPC_MUTEX * sizeof (kmutex_t)); 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate /* size for page list counts */ 6207c478bd9Sstevel@tonic-gate PLCNT_SZ(ctrs_sz); 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate /* 6237c478bd9Sstevel@tonic-gate * add some slop for roundups. page_ctrs_alloc will roundup the start 6247c478bd9Sstevel@tonic-gate * address of the counters to ecache_alignsize boundary for every 6257c478bd9Sstevel@tonic-gate * memory node. 6267c478bd9Sstevel@tonic-gate */ 6277c478bd9Sstevel@tonic-gate return (ctrs_sz + max_mem_nodes * L2CACHE_ALIGN); 6287c478bd9Sstevel@tonic-gate } 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate caddr_t 6317c478bd9Sstevel@tonic-gate page_ctrs_alloc(caddr_t alloc_base) 6327c478bd9Sstevel@tonic-gate { 6337c478bd9Sstevel@tonic-gate int mnode; 6345d07b933Sdp int mrange, nranges; 6357c478bd9Sstevel@tonic-gate int r; /* region size */ 6367c478bd9Sstevel@tonic-gate int i; 637ce8eb11aSdp int firstmn; /* first mnode that exists */ 638ce8eb11aSdp pfn_t physbase; 639ce8eb11aSdp pfn_t physmax; 6407c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate /* 6437c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 6447c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 6457c478bd9Sstevel@tonic-gate * arrays. 6467c478bd9Sstevel@tonic-gate */ 6475d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 6485d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 6497c478bd9Sstevel@tonic-gate } 6507c478bd9Sstevel@tonic-gate 6517c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 6527c478bd9Sstevel@tonic-gate page_counters[r] = (hw_page_map_t *)alloc_base; 6537c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (hw_page_map_t)); 6547c478bd9Sstevel@tonic-gate } 6557c478bd9Sstevel@tonic-gate 6565d07b933Sdp /* page_ctrs_cands and pcc_color_free array */ 6575d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 6585d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6597c478bd9Sstevel@tonic-gate 6605d07b933Sdp page_ctrs_cands[i][r] = (pcc_info_t **)alloc_base; 6615d07b933Sdp alloc_base += sizeof (pcc_info_t *) * max_mem_nodes; 6627c478bd9Sstevel@tonic-gate 6637c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6645d07b933Sdp pcc_info_t *pi; 6655d07b933Sdp 6665d07b933Sdp if (mem_node_config[mnode].exists == 0) 6675d07b933Sdp continue; 6685d07b933Sdp 6695d07b933Sdp nranges = mnode_nranges[mnode]; 6705d07b933Sdp 6715d07b933Sdp pi = (pcc_info_t *)alloc_base; 6725d07b933Sdp alloc_base += sizeof (pcc_info_t) * nranges; 6735d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 6745d07b933Sdp 6755d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 6765d07b933Sdp pi->pcc_color_free = 6775d07b933Sdp (pgcnt_t *)alloc_base; 6785d07b933Sdp alloc_base += sizeof (pgcnt_t) * 6795d07b933Sdp colors_per_szc[r]; 6805d07b933Sdp pi++; 6815d07b933Sdp } 6827c478bd9Sstevel@tonic-gate } 6837c478bd9Sstevel@tonic-gate } 6847c478bd9Sstevel@tonic-gate } 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6877c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 6887c478bd9Sstevel@tonic-gate ctr_mutex[i] = (kmutex_t *)alloc_base; 6897c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (kmutex_t)); 6907c478bd9Sstevel@tonic-gate } 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate /* initialize page list counts */ 6937c478bd9Sstevel@tonic-gate PLCNT_INIT(alloc_base); 6947c478bd9Sstevel@tonic-gate 695ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 6967c478bd9Sstevel@tonic-gate 6977c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 6987c478bd9Sstevel@tonic-gate pfn_t r_base; 6997c478bd9Sstevel@tonic-gate pgcnt_t r_align; 7007c478bd9Sstevel@tonic-gate int r_shift; 7015d07b933Sdp int nranges = mnode_nranges[mnode]; 7027c478bd9Sstevel@tonic-gate 7037c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 7047c478bd9Sstevel@tonic-gate continue; 7057c478bd9Sstevel@tonic-gate 706ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 707ce8eb11aSdp 7087c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 7097c478bd9Sstevel@tonic-gate /* 7107c478bd9Sstevel@tonic-gate * the page_counters base has to be aligned to the 7117c478bd9Sstevel@tonic-gate * page count of page size code r otherwise the counts 7127c478bd9Sstevel@tonic-gate * will cross large page boundaries. 7137c478bd9Sstevel@tonic-gate */ 7147c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 715ce8eb11aSdp r_base = physbase; 7167c478bd9Sstevel@tonic-gate /* base needs to be aligned - lower to aligned value */ 7177c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 718ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 7197c478bd9Sstevel@tonic-gate r_shift = PAGE_BSZS_SHIFT(r); 7207c478bd9Sstevel@tonic-gate 7217c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = r_shift; 7227c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = r_pgcnt; 7237c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = r_base; 7245d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7255d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 7265d07b933Sdp r, mrange) = (size_t *)alloc_base; 7275d07b933Sdp alloc_base += sizeof (size_t) * 7285d07b933Sdp colors_per_szc[r]; 7295d07b933Sdp } 7307c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 7315d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 7325d07b933Sdp pfn_t pfnum = r_base; 7335d07b933Sdp size_t idx; 7345d07b933Sdp int mrange; 735ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 7365d07b933Sdp 737ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, &it); 738ce8eb11aSdp ASSERT(pfnum != (pfn_t)-1); 7395d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 740ce8eb11aSdp color_mask, color_mask, &it); 7415d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 7425d07b933Sdp idx = (idx >= r_pgcnt) ? 0 : idx; 7435d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7445d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, 7455d07b933Sdp r, i, mrange) = idx; 7465d07b933Sdp } 7477c478bd9Sstevel@tonic-gate } 748ce8eb11aSdp 749ce8eb11aSdp /* hpm_counters may be shared by all mnodes */ 750ce8eb11aSdp if (firstmn == mnode) { 751ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 752ce8eb11aSdp (hpmctr_t *)alloc_base; 753ce8eb11aSdp alloc_base += 754ce8eb11aSdp P2ROUNDUP((sizeof (hpmctr_t) * r_pgcnt), 755ce8eb11aSdp sizeof (hpmctr_t *)); 756ce8eb11aSdp } else { 757ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 758ce8eb11aSdp PAGE_COUNTERS_COUNTERS(firstmn, r); 759ce8eb11aSdp } 7607c478bd9Sstevel@tonic-gate 7617c478bd9Sstevel@tonic-gate /* 7627c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 7637c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 7647c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 7657c478bd9Sstevel@tonic-gate * and get consistent values. 7667c478bd9Sstevel@tonic-gate */ 7677c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 7687c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 7697c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 7707c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, r_base))) == r_base); 7717c478bd9Sstevel@tonic-gate } 7727c478bd9Sstevel@tonic-gate /* 7737c478bd9Sstevel@tonic-gate * Roundup the start address of the page_counters to 7747c478bd9Sstevel@tonic-gate * cache aligned boundary for every memory node. 7757c478bd9Sstevel@tonic-gate * page_ctrs_sz() has added some slop for these roundups. 7767c478bd9Sstevel@tonic-gate */ 7777c478bd9Sstevel@tonic-gate alloc_base = (caddr_t)P2ROUNDUP((uintptr_t)alloc_base, 778ce8eb11aSdp L2CACHE_ALIGN); 7797c478bd9Sstevel@tonic-gate } 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate /* Initialize other page counter specific data structures. */ 7827c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < MAX_MEM_NODES; mnode++) { 7837c478bd9Sstevel@tonic-gate rw_init(&page_ctrs_rwlock[mnode], NULL, RW_DEFAULT, NULL); 7847c478bd9Sstevel@tonic-gate } 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate return (alloc_base); 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate 7897c478bd9Sstevel@tonic-gate /* 7907c478bd9Sstevel@tonic-gate * Functions to adjust region counters for each size free list. 7917c478bd9Sstevel@tonic-gate * Caller is responsible to acquire the ctr_mutex lock if necessary and 7927c478bd9Sstevel@tonic-gate * thus can be called during startup without locks. 7937c478bd9Sstevel@tonic-gate */ 7947c478bd9Sstevel@tonic-gate /* ARGSUSED */ 7957c478bd9Sstevel@tonic-gate void 796affbd3ccSkchow page_ctr_add_internal(int mnode, int mtype, page_t *pp, int flags) 7977c478bd9Sstevel@tonic-gate { 7987c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 7997c478bd9Sstevel@tonic-gate ssize_t idx; 8007c478bd9Sstevel@tonic-gate pfn_t pfnum; 8017c478bd9Sstevel@tonic-gate int lckidx; 8027c478bd9Sstevel@tonic-gate 803affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 804affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 805affbd3ccSkchow 8067c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8077c478bd9Sstevel@tonic-gate 808affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 8097c478bd9Sstevel@tonic-gate 8107c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8117c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8127c478bd9Sstevel@tonic-gate return; 8137c478bd9Sstevel@tonic-gate } 8147c478bd9Sstevel@tonic-gate 8157c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8167c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8177c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8187c478bd9Sstevel@tonic-gate 8197c478bd9Sstevel@tonic-gate /* 8207c478bd9Sstevel@tonic-gate * Increment the count of free pages for the current 8217c478bd9Sstevel@tonic-gate * region. Continue looping up in region size incrementing 8227c478bd9Sstevel@tonic-gate * count if the preceeding region is full. 8237c478bd9Sstevel@tonic-gate */ 8247c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8257c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8267c478bd9Sstevel@tonic-gate 8277c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8287c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) < FULL_REGION_CNT(r)); 8297c478bd9Sstevel@tonic-gate 8305d07b933Sdp if (++PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r)) { 8317c478bd9Sstevel@tonic-gate break; 8325d07b933Sdp } else { 8335d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 8345d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 8355d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 8367c478bd9Sstevel@tonic-gate 8375d07b933Sdp cand->pcc_pages_free++; 8385d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]++; 8395d07b933Sdp } 8407c478bd9Sstevel@tonic-gate r++; 8417c478bd9Sstevel@tonic-gate } 8427c478bd9Sstevel@tonic-gate } 8437c478bd9Sstevel@tonic-gate 8447c478bd9Sstevel@tonic-gate void 845affbd3ccSkchow page_ctr_add(int mnode, int mtype, page_t *pp, int flags) 8467c478bd9Sstevel@tonic-gate { 8477c478bd9Sstevel@tonic-gate int lckidx = PP_CTR_LOCK_INDX(pp); 8487c478bd9Sstevel@tonic-gate kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate mutex_enter(lock); 851affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 8527c478bd9Sstevel@tonic-gate mutex_exit(lock); 8537c478bd9Sstevel@tonic-gate } 8547c478bd9Sstevel@tonic-gate 8557c478bd9Sstevel@tonic-gate void 8565d07b933Sdp page_ctr_sub_internal(int mnode, int mtype, page_t *pp, int flags) 8577c478bd9Sstevel@tonic-gate { 8587c478bd9Sstevel@tonic-gate int lckidx; 8597c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8607c478bd9Sstevel@tonic-gate ssize_t idx; 8617c478bd9Sstevel@tonic-gate pfn_t pfnum; 8627c478bd9Sstevel@tonic-gate 863affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 864affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 865affbd3ccSkchow 8667c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8677c478bd9Sstevel@tonic-gate 868affbd3ccSkchow PLCNT_DECR(pp, mnode, mtype, pp->p_szc, flags); 8697c478bd9Sstevel@tonic-gate 8707c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8717c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8727c478bd9Sstevel@tonic-gate return; 8737c478bd9Sstevel@tonic-gate } 8747c478bd9Sstevel@tonic-gate 8757c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8767c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8777c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate /* 8807c478bd9Sstevel@tonic-gate * Decrement the count of free pages for the current 8817c478bd9Sstevel@tonic-gate * region. Continue looping up in region size decrementing 8827c478bd9Sstevel@tonic-gate * count if the preceeding region was full. 8837c478bd9Sstevel@tonic-gate */ 8847c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8857c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8887c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) > 0); 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate if (--PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r) - 1) { 8917c478bd9Sstevel@tonic-gate break; 8925d07b933Sdp } else { 8935d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 8945d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 8955d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 8967c478bd9Sstevel@tonic-gate 8975d07b933Sdp ASSERT(cand->pcc_pages_free != 0); 8985d07b933Sdp ASSERT(cand->pcc_color_free[PP_2_BIN_SZC(pp, r)] != 0); 8995d07b933Sdp 9005d07b933Sdp cand->pcc_pages_free--; 9015d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]--; 9025d07b933Sdp } 9037c478bd9Sstevel@tonic-gate r++; 9047c478bd9Sstevel@tonic-gate } 9055d07b933Sdp } 9065d07b933Sdp 9075d07b933Sdp void 9085d07b933Sdp page_ctr_sub(int mnode, int mtype, page_t *pp, int flags) 9095d07b933Sdp { 9105d07b933Sdp int lckidx = PP_CTR_LOCK_INDX(pp); 9115d07b933Sdp kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 9125d07b933Sdp 9135d07b933Sdp mutex_enter(lock); 9145d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 9157c478bd9Sstevel@tonic-gate mutex_exit(lock); 9167c478bd9Sstevel@tonic-gate } 9177c478bd9Sstevel@tonic-gate 9187c478bd9Sstevel@tonic-gate /* 9197c478bd9Sstevel@tonic-gate * Adjust page counters following a memory attach, since typically the 9207c478bd9Sstevel@tonic-gate * size of the array needs to change, and the PFN to counter index 9217c478bd9Sstevel@tonic-gate * mapping needs to change. 9225d07b933Sdp * 9235d07b933Sdp * It is possible this mnode did not exist at startup. In that case 9245d07b933Sdp * allocate pcc_info_t and pcc_color_free arrays. Also, allow for nranges 9255d07b933Sdp * to change (a theoretical possibility on x86), which means pcc_color_free 9265d07b933Sdp * arrays must be extended. 9277c478bd9Sstevel@tonic-gate */ 9287c478bd9Sstevel@tonic-gate uint_t 9297c478bd9Sstevel@tonic-gate page_ctrs_adjust(int mnode) 9307c478bd9Sstevel@tonic-gate { 9317c478bd9Sstevel@tonic-gate pgcnt_t npgs; 9327c478bd9Sstevel@tonic-gate int r; /* region size */ 9337c478bd9Sstevel@tonic-gate int i; 9347c478bd9Sstevel@tonic-gate size_t pcsz, old_csz; 9357c478bd9Sstevel@tonic-gate hpmctr_t *new_ctr, *old_ctr; 9367c478bd9Sstevel@tonic-gate pfn_t oldbase, newbase; 937ce8eb11aSdp pfn_t physbase, physmax; 9387c478bd9Sstevel@tonic-gate size_t old_npgs; 9397c478bd9Sstevel@tonic-gate hpmctr_t *ctr_cache[MMU_PAGE_SIZES]; 9407c478bd9Sstevel@tonic-gate size_t size_cache[MMU_PAGE_SIZES]; 9415d07b933Sdp size_t *color_cache[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 9425d07b933Sdp size_t *old_color_array[MAX_MNODE_MRANGES]; 9437c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 9445d07b933Sdp pcc_info_t **cands_cache; 9455d07b933Sdp pcc_info_t *old_pi, *pi; 9465d07b933Sdp pgcnt_t *pgcntp; 9475d07b933Sdp int nr, old_nranges, mrange, nranges = MNODE_RANGE_CNT(mnode); 9485d07b933Sdp int cands_cache_nranges; 9495d07b933Sdp int old_maxmrange, new_maxmrange; 9505d07b933Sdp int rc = 0; 9517c478bd9Sstevel@tonic-gate 9525d07b933Sdp cands_cache = kmem_zalloc(sizeof (pcc_info_t *) * NPC_MUTEX * 9535d07b933Sdp MMU_PAGE_SIZES, KM_NOSLEEP); 9545d07b933Sdp if (cands_cache == NULL) 9555d07b933Sdp return (ENOMEM); 9565d07b933Sdp 957ce8eb11aSdp i = -1; 958ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, i); 959ce8eb11aSdp 960ce8eb11aSdp newbase = physbase & ~PC_BASE_ALIGN_MASK; 961ce8eb11aSdp npgs = roundup(physmax, PC_BASE_ALIGN) - newbase; 962ce8eb11aSdp 9635d07b933Sdp /* prepare to free non-null pointers on the way out */ 9645d07b933Sdp cands_cache_nranges = nranges; 9655d07b933Sdp bzero(ctr_cache, sizeof (ctr_cache)); 9665d07b933Sdp bzero(color_cache, sizeof (color_cache)); 9675d07b933Sdp 9687c478bd9Sstevel@tonic-gate /* 9697c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 9707c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 9717c478bd9Sstevel@tonic-gate * arrays. 9727c478bd9Sstevel@tonic-gate */ 9735d07b933Sdp for (r = 0; r < mmu_page_sizes; r++) { 9745d07b933Sdp colors_per_szc[r] = PAGE_GET_PAGECOLORS(r); 9757c478bd9Sstevel@tonic-gate } 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate /* 9787c478bd9Sstevel@tonic-gate * Preallocate all of the new hpm_counters arrays as we can't 9797c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 9807c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 9817c478bd9Sstevel@tonic-gate * and return failure. 9827c478bd9Sstevel@tonic-gate */ 9837c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 9847c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_BSZS_SHIFT(r); 9855d07b933Sdp size_cache[r] = pcsz; 9867c478bd9Sstevel@tonic-gate ctr_cache[r] = kmem_zalloc(pcsz * 9877c478bd9Sstevel@tonic-gate sizeof (hpmctr_t), KM_NOSLEEP); 9887c478bd9Sstevel@tonic-gate if (ctr_cache[r] == NULL) { 9895d07b933Sdp rc = ENOMEM; 9905d07b933Sdp goto cleanup; 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate } 9935d07b933Sdp 9947c478bd9Sstevel@tonic-gate /* 9957c478bd9Sstevel@tonic-gate * Preallocate all of the new color current arrays as we can't 9967c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 9977c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 9987c478bd9Sstevel@tonic-gate * and return failure. 9997c478bd9Sstevel@tonic-gate */ 10007c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10015d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 10025d07b933Sdp color_cache[r][mrange] = kmem_zalloc(sizeof (size_t) * 10035d07b933Sdp colors_per_szc[r], KM_NOSLEEP); 10045d07b933Sdp if (color_cache[r][mrange] == NULL) { 10055d07b933Sdp rc = ENOMEM; 10065d07b933Sdp goto cleanup; 10075d07b933Sdp } 10085d07b933Sdp } 10095d07b933Sdp } 10105d07b933Sdp 10115d07b933Sdp /* 10125d07b933Sdp * Preallocate all of the new pcc_info_t arrays as we can't 10135d07b933Sdp * hold the page_ctrs_rwlock as a writer and allocate memory. 10145d07b933Sdp * If we can't allocate all of the arrays, undo our work so far 10155d07b933Sdp * and return failure. 10165d07b933Sdp */ 10175d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 10185d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 10195d07b933Sdp pi = kmem_zalloc(nranges * sizeof (pcc_info_t), 10205d07b933Sdp KM_NOSLEEP); 10215d07b933Sdp if (pi == NULL) { 10225d07b933Sdp rc = ENOMEM; 10235d07b933Sdp goto cleanup; 10247c478bd9Sstevel@tonic-gate } 10255d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = pi; 10265d07b933Sdp 10275d07b933Sdp for (mrange = 0; mrange < nranges; mrange++, pi++) { 10285d07b933Sdp pgcntp = kmem_zalloc(colors_per_szc[r] * 10295d07b933Sdp sizeof (pgcnt_t), KM_NOSLEEP); 10305d07b933Sdp if (pgcntp == NULL) { 10315d07b933Sdp rc = ENOMEM; 10325d07b933Sdp goto cleanup; 10335d07b933Sdp } 10345d07b933Sdp pi->pcc_color_free = pgcntp; 10357c478bd9Sstevel@tonic-gate } 10367c478bd9Sstevel@tonic-gate } 10377c478bd9Sstevel@tonic-gate } 10387c478bd9Sstevel@tonic-gate 10397c478bd9Sstevel@tonic-gate /* 10407c478bd9Sstevel@tonic-gate * Grab the write lock to prevent others from walking these arrays 10417c478bd9Sstevel@tonic-gate * while we are modifying them. 10427c478bd9Sstevel@tonic-gate */ 1043ce8eb11aSdp PAGE_CTRS_WRITE_LOCK(mnode); 10445d07b933Sdp 10455d07b933Sdp old_nranges = mnode_nranges[mnode]; 10465d07b933Sdp cands_cache_nranges = old_nranges; 10475d07b933Sdp mnode_nranges[mnode] = nranges; 10485d07b933Sdp old_maxmrange = mnode_maxmrange[mnode]; 10495d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 10505d07b933Sdp new_maxmrange = mnode_maxmrange[mnode]; 10515d07b933Sdp 10527c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10537c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = PAGE_BSZS_SHIFT(r); 10547c478bd9Sstevel@tonic-gate old_ctr = PAGE_COUNTERS_COUNTERS(mnode, r); 10557c478bd9Sstevel@tonic-gate old_csz = PAGE_COUNTERS_ENTRIES(mnode, r); 10567c478bd9Sstevel@tonic-gate oldbase = PAGE_COUNTERS_BASE(mnode, r); 10577c478bd9Sstevel@tonic-gate old_npgs = old_csz << PAGE_COUNTERS_SHIFT(mnode, r); 10585d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 10595d07b933Sdp old_color_array[mrange] = 10605d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 1061ce8eb11aSdp r, mrange); 10625d07b933Sdp } 10637c478bd9Sstevel@tonic-gate 10647c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_COUNTERS_SHIFT(mnode, r); 10657c478bd9Sstevel@tonic-gate new_ctr = ctr_cache[r]; 10667c478bd9Sstevel@tonic-gate ctr_cache[r] = NULL; 10677c478bd9Sstevel@tonic-gate if (old_ctr != NULL && 10687c478bd9Sstevel@tonic-gate (oldbase + old_npgs > newbase) && 10697c478bd9Sstevel@tonic-gate (newbase + npgs > oldbase)) { 10707c478bd9Sstevel@tonic-gate /* 10717c478bd9Sstevel@tonic-gate * Map the intersection of the old and new 10727c478bd9Sstevel@tonic-gate * counters into the new array. 10737c478bd9Sstevel@tonic-gate */ 10747c478bd9Sstevel@tonic-gate size_t offset; 10757c478bd9Sstevel@tonic-gate if (newbase > oldbase) { 10767c478bd9Sstevel@tonic-gate offset = (newbase - oldbase) >> 10777c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 10787c478bd9Sstevel@tonic-gate bcopy(old_ctr + offset, new_ctr, 10797c478bd9Sstevel@tonic-gate MIN(pcsz, (old_csz - offset)) * 10807c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 10817c478bd9Sstevel@tonic-gate } else { 10827c478bd9Sstevel@tonic-gate offset = (oldbase - newbase) >> 10837c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 10847c478bd9Sstevel@tonic-gate bcopy(old_ctr, new_ctr + offset, 10857c478bd9Sstevel@tonic-gate MIN(pcsz - offset, old_csz) * 10867c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 10877c478bd9Sstevel@tonic-gate } 10887c478bd9Sstevel@tonic-gate } 10897c478bd9Sstevel@tonic-gate 10907c478bd9Sstevel@tonic-gate PAGE_COUNTERS_COUNTERS(mnode, r) = new_ctr; 10917c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = pcsz; 10927c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = newbase; 1093ce8eb11aSdp 1094ce8eb11aSdp /* update shared hpm_counters in other mnodes */ 1095ce8eb11aSdp if (interleaved_mnodes) { 1096ce8eb11aSdp for (i = 0; i < max_mem_nodes; i++) { 1097ce8eb11aSdp if (i == mnode) 1098ce8eb11aSdp continue; 1099ce8eb11aSdp if (mem_node_config[i].exists == 0) 1100ce8eb11aSdp continue; 1101ce8eb11aSdp ASSERT(PAGE_COUNTERS_COUNTERS(i, r) == old_ctr); 1102ce8eb11aSdp PAGE_COUNTERS_COUNTERS(i, r) = new_ctr; 1103ce8eb11aSdp PAGE_COUNTERS_ENTRIES(i, r) = pcsz; 1104ce8eb11aSdp PAGE_COUNTERS_BASE(i, r) = newbase; 1105ce8eb11aSdp } 1106ce8eb11aSdp } 1107ce8eb11aSdp 11085d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11095d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, r, mrange) = 11105d07b933Sdp color_cache[r][mrange]; 11115d07b933Sdp color_cache[r][mrange] = NULL; 11125d07b933Sdp } 11137c478bd9Sstevel@tonic-gate /* 11147c478bd9Sstevel@tonic-gate * for now, just reset on these events as it's probably 11157c478bd9Sstevel@tonic-gate * not worthwhile to try and optimize this. 11167c478bd9Sstevel@tonic-gate */ 11177c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 11185d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 1119ce8eb11aSdp int mlo = interleaved_mnodes ? 0 : mnode; 1120ce8eb11aSdp int mhi = interleaved_mnodes ? max_mem_nodes : 1121ce8eb11aSdp (mnode + 1); 1122ce8eb11aSdp int m; 11235d07b933Sdp pfn_t pfnum = newbase; 11245d07b933Sdp size_t idx; 1125ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 11265d07b933Sdp 1127ce8eb11aSdp for (m = mlo; m < mhi; m++) { 1128ce8eb11aSdp if (mem_node_config[m].exists == 0) 1129ce8eb11aSdp continue; 1130ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, m, &it); 1131ce8eb11aSdp ASSERT(pfnum != (pfn_t)-1); 1132ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, color_mask, 1133ce8eb11aSdp color_mask, &it); 1134ce8eb11aSdp idx = PNUM_TO_IDX(m, r, pfnum); 1135ce8eb11aSdp idx = (idx < pcsz) ? idx : 0; 1136ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 1137ce8eb11aSdp PAGE_COUNTERS_CURRENT_COLOR(m, 1138ce8eb11aSdp r, i, mrange) = idx; 1139ce8eb11aSdp } 11405d07b933Sdp } 11417c478bd9Sstevel@tonic-gate } 11427c478bd9Sstevel@tonic-gate 11437c478bd9Sstevel@tonic-gate /* cache info for freeing out of the critical path */ 11447c478bd9Sstevel@tonic-gate if ((caddr_t)old_ctr >= kernelheap && 11457c478bd9Sstevel@tonic-gate (caddr_t)old_ctr < ekernelheap) { 11467c478bd9Sstevel@tonic-gate ctr_cache[r] = old_ctr; 11477c478bd9Sstevel@tonic-gate size_cache[r] = old_csz; 11487c478bd9Sstevel@tonic-gate } 11495d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11505d07b933Sdp size_t *tmp = old_color_array[mrange]; 11515d07b933Sdp if ((caddr_t)tmp >= kernelheap && 11525d07b933Sdp (caddr_t)tmp < ekernelheap) { 11535d07b933Sdp color_cache[r][mrange] = tmp; 11545d07b933Sdp } 11557c478bd9Sstevel@tonic-gate } 11567c478bd9Sstevel@tonic-gate /* 11577c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 11587c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 11597c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 11607c478bd9Sstevel@tonic-gate * and get consistent values. 11617c478bd9Sstevel@tonic-gate */ 11627c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 11637c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 11647c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 11657c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, newbase))) == newbase); 11665d07b933Sdp 11675d07b933Sdp /* pcc_info_t and pcc_color_free */ 11685d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 11695d07b933Sdp pcc_info_t *epi; 11705d07b933Sdp pcc_info_t *eold_pi; 11715d07b933Sdp 11725d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 11735d07b933Sdp old_pi = page_ctrs_cands[i][r][mnode]; 11745d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 11755d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = old_pi; 11765d07b933Sdp 11775d07b933Sdp /* preserve old pcc_color_free values, if any */ 11785d07b933Sdp if (old_pi == NULL) 11795d07b933Sdp continue; 11805d07b933Sdp 11815d07b933Sdp /* 11825d07b933Sdp * when/if x86 does DR, must account for 11835d07b933Sdp * possible change in range index when 11845d07b933Sdp * preserving pcc_info 11855d07b933Sdp */ 11865d07b933Sdp epi = &pi[nranges]; 11875d07b933Sdp eold_pi = &old_pi[old_nranges]; 11885d07b933Sdp if (new_maxmrange > old_maxmrange) { 11895d07b933Sdp pi += new_maxmrange - old_maxmrange; 11905d07b933Sdp } else if (new_maxmrange < old_maxmrange) { 11915d07b933Sdp old_pi += old_maxmrange - new_maxmrange; 11925d07b933Sdp } 11935d07b933Sdp for (; pi < epi && old_pi < eold_pi; pi++, old_pi++) { 11945d07b933Sdp pcc_info_t tmp = *pi; 11955d07b933Sdp *pi = *old_pi; 11965d07b933Sdp *old_pi = tmp; 11975d07b933Sdp } 11985d07b933Sdp } 11997c478bd9Sstevel@tonic-gate } 1200ce8eb11aSdp PAGE_CTRS_WRITE_UNLOCK(mnode); 12017c478bd9Sstevel@tonic-gate 12027c478bd9Sstevel@tonic-gate /* 12037c478bd9Sstevel@tonic-gate * Now that we have dropped the write lock, it is safe to free all 12047c478bd9Sstevel@tonic-gate * of the memory we have cached above. 12055d07b933Sdp * We come thru here to free memory when pre-alloc fails, and also to 12065d07b933Sdp * free old pointers which were recorded while locked. 12077c478bd9Sstevel@tonic-gate */ 12085d07b933Sdp cleanup: 12097c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 12107c478bd9Sstevel@tonic-gate if (ctr_cache[r] != NULL) { 12117c478bd9Sstevel@tonic-gate kmem_free(ctr_cache[r], 12127c478bd9Sstevel@tonic-gate size_cache[r] * sizeof (hpmctr_t)); 12137c478bd9Sstevel@tonic-gate } 12145d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 12155d07b933Sdp if (color_cache[r][mrange] != NULL) { 12165d07b933Sdp kmem_free(color_cache[r][mrange], 12175d07b933Sdp colors_per_szc[r] * sizeof (size_t)); 12185d07b933Sdp } 12195d07b933Sdp } 12205d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 12215d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12225d07b933Sdp if (pi == NULL) 12235d07b933Sdp continue; 12245d07b933Sdp nr = cands_cache_nranges; 12255d07b933Sdp for (mrange = 0; mrange < nr; mrange++, pi++) { 12265d07b933Sdp pgcntp = pi->pcc_color_free; 12275d07b933Sdp if (pgcntp == NULL) 12285d07b933Sdp continue; 12295d07b933Sdp if ((caddr_t)pgcntp >= kernelheap && 12305d07b933Sdp (caddr_t)pgcntp < ekernelheap) { 12315d07b933Sdp kmem_free(pgcntp, 12325d07b933Sdp colors_per_szc[r] * 12335d07b933Sdp sizeof (pgcnt_t)); 12345d07b933Sdp } 12355d07b933Sdp } 12365d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12375d07b933Sdp if ((caddr_t)pi >= kernelheap && 12385d07b933Sdp (caddr_t)pi < ekernelheap) { 12395d07b933Sdp kmem_free(pi, nr * sizeof (pcc_info_t)); 12405d07b933Sdp } 12417c478bd9Sstevel@tonic-gate } 12427c478bd9Sstevel@tonic-gate } 12437c478bd9Sstevel@tonic-gate 12445d07b933Sdp kmem_free(cands_cache, 12455d07b933Sdp sizeof (pcc_info_t *) * NPC_MUTEX * MMU_PAGE_SIZES); 12465d07b933Sdp return (rc); 12477c478bd9Sstevel@tonic-gate } 12487c478bd9Sstevel@tonic-gate 12495d07b933Sdp 12507c478bd9Sstevel@tonic-gate #ifdef DEBUG 12517c478bd9Sstevel@tonic-gate 12527c478bd9Sstevel@tonic-gate /* 12537c478bd9Sstevel@tonic-gate * confirm pp is a large page corresponding to szc 12547c478bd9Sstevel@tonic-gate */ 12557c478bd9Sstevel@tonic-gate void 12567c478bd9Sstevel@tonic-gate chk_lpg(page_t *pp, uchar_t szc) 12577c478bd9Sstevel@tonic-gate { 12587c478bd9Sstevel@tonic-gate spgcnt_t npgs = page_get_pagecnt(pp->p_szc); 12597c478bd9Sstevel@tonic-gate uint_t noreloc; 12607c478bd9Sstevel@tonic-gate 12617c478bd9Sstevel@tonic-gate if (npgs == 1) { 12627c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 12637c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == pp); 12647c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == pp); 12657c478bd9Sstevel@tonic-gate return; 12667c478bd9Sstevel@tonic-gate } 12677c478bd9Sstevel@tonic-gate 12687c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12697c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pp->p_pagenum, npgs)); 12727c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == (pp->p_next->p_pagenum - 1)); 12737c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev->p_pagenum == (pp->p_pagenum + (npgs - 1))); 12747c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == (pp + (npgs - 1))); 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate /* 12777c478bd9Sstevel@tonic-gate * Check list of pages. 12787c478bd9Sstevel@tonic-gate */ 12797c478bd9Sstevel@tonic-gate noreloc = PP_ISNORELOC(pp); 12807c478bd9Sstevel@tonic-gate while (npgs--) { 12817c478bd9Sstevel@tonic-gate if (npgs != 0) { 12827c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == pp->p_next->p_pagenum - 1); 12837c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == (pp + 1)); 12847c478bd9Sstevel@tonic-gate } 12857c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 12867c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 12877c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 12887c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12897c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 12907c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 12917c478bd9Sstevel@tonic-gate ASSERT(PP_ISNORELOC(pp) == noreloc); 12927c478bd9Sstevel@tonic-gate 12937c478bd9Sstevel@tonic-gate pp = pp->p_next; 12947c478bd9Sstevel@tonic-gate } 12957c478bd9Sstevel@tonic-gate } 12967c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate void 12997c478bd9Sstevel@tonic-gate page_freelist_lock(int mnode) 13007c478bd9Sstevel@tonic-gate { 13017c478bd9Sstevel@tonic-gate int i; 13027c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13037c478bd9Sstevel@tonic-gate mutex_enter(FPC_MUTEX(mnode, i)); 13047c478bd9Sstevel@tonic-gate mutex_enter(CPC_MUTEX(mnode, i)); 13057c478bd9Sstevel@tonic-gate } 13067c478bd9Sstevel@tonic-gate } 13077c478bd9Sstevel@tonic-gate 13087c478bd9Sstevel@tonic-gate void 13097c478bd9Sstevel@tonic-gate page_freelist_unlock(int mnode) 13107c478bd9Sstevel@tonic-gate { 13117c478bd9Sstevel@tonic-gate int i; 13127c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13137c478bd9Sstevel@tonic-gate mutex_exit(FPC_MUTEX(mnode, i)); 13147c478bd9Sstevel@tonic-gate mutex_exit(CPC_MUTEX(mnode, i)); 13157c478bd9Sstevel@tonic-gate } 13167c478bd9Sstevel@tonic-gate } 13177c478bd9Sstevel@tonic-gate 13187c478bd9Sstevel@tonic-gate /* 13197c478bd9Sstevel@tonic-gate * add pp to the specified page list. Defaults to head of the page list 13207c478bd9Sstevel@tonic-gate * unless PG_LIST_TAIL is specified. 13217c478bd9Sstevel@tonic-gate */ 13227c478bd9Sstevel@tonic-gate void 13237c478bd9Sstevel@tonic-gate page_list_add(page_t *pp, int flags) 13247c478bd9Sstevel@tonic-gate { 13257c478bd9Sstevel@tonic-gate page_t **ppp; 13267c478bd9Sstevel@tonic-gate kmutex_t *pcm; 13277c478bd9Sstevel@tonic-gate uint_t bin, mtype; 13287c478bd9Sstevel@tonic-gate int mnode; 13297c478bd9Sstevel@tonic-gate 13307c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 13317c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 13327c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 13337c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(pp) == 0); 13347c478bd9Sstevel@tonic-gate 13357c478bd9Sstevel@tonic-gate /* 13367c478bd9Sstevel@tonic-gate * Large pages should be freed via page_list_add_pages(). 13377c478bd9Sstevel@tonic-gate */ 13387c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 13397c478bd9Sstevel@tonic-gate 13407c478bd9Sstevel@tonic-gate /* 13417c478bd9Sstevel@tonic-gate * Don't need to lock the freelist first here 13427c478bd9Sstevel@tonic-gate * because the page isn't on the freelist yet. 13437c478bd9Sstevel@tonic-gate * This means p_szc can't change on us. 13447c478bd9Sstevel@tonic-gate */ 13457c478bd9Sstevel@tonic-gate 13467c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 13477c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 13487c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 13517c478bd9Sstevel@tonic-gate /* 13527c478bd9Sstevel@tonic-gate * PG_LIST_ISINIT is set during system startup (ie. single 13537c478bd9Sstevel@tonic-gate * threaded), add a page to the free list and add to the 13547c478bd9Sstevel@tonic-gate * the free region counters w/o any locking 13557c478bd9Sstevel@tonic-gate */ 13567c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13577c478bd9Sstevel@tonic-gate 13587c478bd9Sstevel@tonic-gate /* inline version of page_add() */ 13597c478bd9Sstevel@tonic-gate if (*ppp != NULL) { 13607c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 13617c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 13627c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 13637c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 13647c478bd9Sstevel@tonic-gate } else 13657c478bd9Sstevel@tonic-gate *ppp = pp; 13667c478bd9Sstevel@tonic-gate 1367affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 1368affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13697c478bd9Sstevel@tonic-gate } else { 13707c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 13717c478bd9Sstevel@tonic-gate 13727c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1373affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13747c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 13757c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13767c478bd9Sstevel@tonic-gate 13777c478bd9Sstevel@tonic-gate } else { 1378affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_cache); 13797c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 13807c478bd9Sstevel@tonic-gate ASSERT((pp->p_offset & PAGEOFFSET) == 0); 13817c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 13827c478bd9Sstevel@tonic-gate } 13837c478bd9Sstevel@tonic-gate mutex_enter(pcm); 13847c478bd9Sstevel@tonic-gate page_add(ppp, pp); 13857c478bd9Sstevel@tonic-gate 13867c478bd9Sstevel@tonic-gate if (flags & PG_LIST_TAIL) 13877c478bd9Sstevel@tonic-gate *ppp = (*ppp)->p_next; 13887c478bd9Sstevel@tonic-gate /* 13897c478bd9Sstevel@tonic-gate * Add counters before releasing pcm mutex to avoid a race with 13905d07b933Sdp * page_freelist_coalesce and page_freelist_split. 13917c478bd9Sstevel@tonic-gate */ 1392affbd3ccSkchow page_ctr_add(mnode, mtype, pp, flags); 13937c478bd9Sstevel@tonic-gate mutex_exit(pcm); 13947c478bd9Sstevel@tonic-gate } 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate #if defined(__sparc) 13987c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 13997c478bd9Sstevel@tonic-gate kcage_freemem_add(1); 14007c478bd9Sstevel@tonic-gate } 14017c478bd9Sstevel@tonic-gate #endif 14027c478bd9Sstevel@tonic-gate /* 14037c478bd9Sstevel@tonic-gate * It is up to the caller to unlock the page! 14047c478bd9Sstevel@tonic-gate */ 14057c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 14067c478bd9Sstevel@tonic-gate } 14077c478bd9Sstevel@tonic-gate 14087c478bd9Sstevel@tonic-gate 14097c478bd9Sstevel@tonic-gate #ifdef __sparc 14107c478bd9Sstevel@tonic-gate /* 14117c478bd9Sstevel@tonic-gate * This routine is only used by kcage_init during system startup. 14127c478bd9Sstevel@tonic-gate * It performs the function of page_list_sub/PP_SETNORELOC/page_list_add 14137c478bd9Sstevel@tonic-gate * without the overhead of taking locks and updating counters. 14147c478bd9Sstevel@tonic-gate */ 14157c478bd9Sstevel@tonic-gate void 14167c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 14177c478bd9Sstevel@tonic-gate { 14187c478bd9Sstevel@tonic-gate page_t **ppp; 14197c478bd9Sstevel@tonic-gate uint_t bin; 14207c478bd9Sstevel@tonic-gate int mnode; 14217c478bd9Sstevel@tonic-gate int mtype; 1422e21bae1bSkchow int flags = 0; 14237c478bd9Sstevel@tonic-gate 14247c478bd9Sstevel@tonic-gate /* 14257c478bd9Sstevel@tonic-gate * If this is a large page on the freelist then 14267c478bd9Sstevel@tonic-gate * break it up into smaller pages. 14277c478bd9Sstevel@tonic-gate */ 14287c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 14297c478bd9Sstevel@tonic-gate page_boot_demote(pp); 14307c478bd9Sstevel@tonic-gate 14317c478bd9Sstevel@tonic-gate /* 14327c478bd9Sstevel@tonic-gate * Get list page is currently on. 14337c478bd9Sstevel@tonic-gate */ 14347c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 14357c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 14367c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14377c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_RELOC); 14387c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 14397c478bd9Sstevel@tonic-gate 14407c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14417c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14427c478bd9Sstevel@tonic-gate flags |= PG_FREE_LIST; 14437c478bd9Sstevel@tonic-gate } else { 14447c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14457c478bd9Sstevel@tonic-gate flags |= PG_CACHE_LIST; 14467c478bd9Sstevel@tonic-gate } 14477c478bd9Sstevel@tonic-gate 14487c478bd9Sstevel@tonic-gate ASSERT(*ppp != NULL); 14497c478bd9Sstevel@tonic-gate 14507c478bd9Sstevel@tonic-gate /* 14517c478bd9Sstevel@tonic-gate * Delete page from current list. 14527c478bd9Sstevel@tonic-gate */ 14537c478bd9Sstevel@tonic-gate if (*ppp == pp) 14547c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 14557c478bd9Sstevel@tonic-gate if (*ppp == pp) { 14567c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 14577c478bd9Sstevel@tonic-gate } else { 14587c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 14597c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 14607c478bd9Sstevel@tonic-gate } 14617c478bd9Sstevel@tonic-gate 14625d07b933Sdp /* 14635d07b933Sdp * Decrement page counters 14645d07b933Sdp */ 14655d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 14667c478bd9Sstevel@tonic-gate 14677c478bd9Sstevel@tonic-gate /* 14687c478bd9Sstevel@tonic-gate * Set no reloc for cage initted pages. 14697c478bd9Sstevel@tonic-gate */ 14707c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14737c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_NORELOC); 14747c478bd9Sstevel@tonic-gate 14757c478bd9Sstevel@tonic-gate /* 14767c478bd9Sstevel@tonic-gate * Get new list for page. 14777c478bd9Sstevel@tonic-gate */ 14787c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14797c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14807c478bd9Sstevel@tonic-gate } else { 14817c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14827c478bd9Sstevel@tonic-gate } 14837c478bd9Sstevel@tonic-gate 14847c478bd9Sstevel@tonic-gate /* 14857c478bd9Sstevel@tonic-gate * Insert page on new list. 14867c478bd9Sstevel@tonic-gate */ 14877c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 14887c478bd9Sstevel@tonic-gate *ppp = pp; 14897c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 14907c478bd9Sstevel@tonic-gate } else { 14917c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 14927c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 14937c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 14947c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 14957c478bd9Sstevel@tonic-gate } 14967c478bd9Sstevel@tonic-gate 14975d07b933Sdp /* 14985d07b933Sdp * Increment page counters 14995d07b933Sdp */ 15005d07b933Sdp page_ctr_add_internal(mnode, mtype, pp, flags); 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate /* 15037c478bd9Sstevel@tonic-gate * Update cage freemem counter 15047c478bd9Sstevel@tonic-gate */ 15057c478bd9Sstevel@tonic-gate atomic_add_long(&kcage_freemem, 1); 15067c478bd9Sstevel@tonic-gate } 15077c478bd9Sstevel@tonic-gate #else /* __sparc */ 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate /* ARGSUSED */ 15107c478bd9Sstevel@tonic-gate void 15117c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 15127c478bd9Sstevel@tonic-gate { 15137c478bd9Sstevel@tonic-gate panic("page_list_noreloc_startup: should be here only for sparc"); 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate #endif 15167c478bd9Sstevel@tonic-gate 15177c478bd9Sstevel@tonic-gate void 15187c478bd9Sstevel@tonic-gate page_list_add_pages(page_t *pp, int flags) 15197c478bd9Sstevel@tonic-gate { 15207c478bd9Sstevel@tonic-gate kmutex_t *pcm; 15217c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 15227c478bd9Sstevel@tonic-gate uint_t bin, mtype, i; 15237c478bd9Sstevel@tonic-gate int mnode; 15247c478bd9Sstevel@tonic-gate 15257c478bd9Sstevel@tonic-gate /* default to freelist/head */ 15267c478bd9Sstevel@tonic-gate ASSERT((flags & (PG_CACHE_LIST | PG_LIST_TAIL)) == 0); 15277c478bd9Sstevel@tonic-gate 15287c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 1529affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[pp->p_szc]); 15307c478bd9Sstevel@tonic-gate 15317c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 15327c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 15337c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 15347c478bd9Sstevel@tonic-gate 15357c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 15367c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == mmu_page_sizes - 1); 15377c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 15387c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 1539affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 15407c478bd9Sstevel@tonic-gate } else { 15417c478bd9Sstevel@tonic-gate 15427c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 15437c478bd9Sstevel@tonic-gate 15447c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 15457c478bd9Sstevel@tonic-gate 15467c478bd9Sstevel@tonic-gate mutex_enter(pcm); 15477c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 1548affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 15497c478bd9Sstevel@tonic-gate mutex_exit(pcm); 15507c478bd9Sstevel@tonic-gate 15517c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 15527c478bd9Sstevel@tonic-gate #if defined(__sparc) 15537c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) 15547c478bd9Sstevel@tonic-gate kcage_freemem_add(pgcnt); 15557c478bd9Sstevel@tonic-gate #endif 15567c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++, pp++) 15578b464eb8Smec page_unlock_nocapture(pp); 15587c478bd9Sstevel@tonic-gate } 15597c478bd9Sstevel@tonic-gate } 15607c478bd9Sstevel@tonic-gate 15617c478bd9Sstevel@tonic-gate /* 15627c478bd9Sstevel@tonic-gate * During boot, need to demote a large page to base 15637c478bd9Sstevel@tonic-gate * pagesize pages for seg_kmem for use in boot_alloc() 15647c478bd9Sstevel@tonic-gate */ 15657c478bd9Sstevel@tonic-gate void 15667c478bd9Sstevel@tonic-gate page_boot_demote(page_t *pp) 15677c478bd9Sstevel@tonic-gate { 15687c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0); 15697c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15707c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 15717c478bd9Sstevel@tonic-gate 15727c478bd9Sstevel@tonic-gate (void) page_demote(PP_2_MEM_NODE(pp), 15737c478bd9Sstevel@tonic-gate PFN_BASE(pp->p_pagenum, pp->p_szc), pp->p_szc, 0, PC_NO_COLOR, 15747c478bd9Sstevel@tonic-gate PC_FREE); 15757c478bd9Sstevel@tonic-gate 15767c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15777c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 15787c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 15797c478bd9Sstevel@tonic-gate } 15807c478bd9Sstevel@tonic-gate 15817c478bd9Sstevel@tonic-gate /* 15827c478bd9Sstevel@tonic-gate * Take a particular page off of whatever freelist the page 15837c478bd9Sstevel@tonic-gate * is claimed to be on. 15847c478bd9Sstevel@tonic-gate * 15857c478bd9Sstevel@tonic-gate * NOTE: Only used for PAGESIZE pages. 15867c478bd9Sstevel@tonic-gate */ 15877c478bd9Sstevel@tonic-gate void 15887c478bd9Sstevel@tonic-gate page_list_sub(page_t *pp, int flags) 15897c478bd9Sstevel@tonic-gate { 15907c478bd9Sstevel@tonic-gate int bin; 15917c478bd9Sstevel@tonic-gate uint_t mtype; 15927c478bd9Sstevel@tonic-gate int mnode; 15937c478bd9Sstevel@tonic-gate kmutex_t *pcm; 15947c478bd9Sstevel@tonic-gate page_t **ppp; 15957c478bd9Sstevel@tonic-gate 15967c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 15977c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15987c478bd9Sstevel@tonic-gate 15997c478bd9Sstevel@tonic-gate /* 16007c478bd9Sstevel@tonic-gate * The p_szc field can only be changed by page_promote() 16017c478bd9Sstevel@tonic-gate * and page_demote(). Only free pages can be promoted and 16027c478bd9Sstevel@tonic-gate * demoted and the free list MUST be locked during these 16037c478bd9Sstevel@tonic-gate * operations. So to prevent a race in page_list_sub() 16047c478bd9Sstevel@tonic-gate * between computing which bin of the freelist lock to 16057c478bd9Sstevel@tonic-gate * grab and actually grabing the lock we check again that 16067c478bd9Sstevel@tonic-gate * the bin we locked is still the correct one. Notice that 16077c478bd9Sstevel@tonic-gate * the p_szc field could have actually changed on us but 16087c478bd9Sstevel@tonic-gate * if the bin happens to still be the same we are safe. 16097c478bd9Sstevel@tonic-gate */ 16107c478bd9Sstevel@tonic-gate try_again: 16117c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 16127c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 16137c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 16147c478bd9Sstevel@tonic-gate mutex_enter(pcm); 16157c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 16167c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16177c478bd9Sstevel@tonic-gate goto try_again; 16187c478bd9Sstevel@tonic-gate } 16197c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 16207c478bd9Sstevel@tonic-gate 16217c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1622affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[0]); 16237c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16247c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 16257c478bd9Sstevel@tonic-gate } else { 1626affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_cache); 16277c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 16287c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 16297c478bd9Sstevel@tonic-gate } 16307c478bd9Sstevel@tonic-gate 16317c478bd9Sstevel@tonic-gate /* 16327c478bd9Sstevel@tonic-gate * Common PAGESIZE case. 16337c478bd9Sstevel@tonic-gate * 16347c478bd9Sstevel@tonic-gate * Note that we locked the freelist. This prevents 16357c478bd9Sstevel@tonic-gate * any page promotion/demotion operations. Therefore 16367c478bd9Sstevel@tonic-gate * the p_szc will not change until we drop pcm mutex. 16377c478bd9Sstevel@tonic-gate */ 16387c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 16397c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 16407c478bd9Sstevel@tonic-gate /* 16417c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16427c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16437c478bd9Sstevel@tonic-gate */ 1644affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 16457c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16467c478bd9Sstevel@tonic-gate 16477c478bd9Sstevel@tonic-gate #if defined(__sparc) 16487c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 16497c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 16507c478bd9Sstevel@tonic-gate } 16517c478bd9Sstevel@tonic-gate #endif 16527c478bd9Sstevel@tonic-gate return; 16537c478bd9Sstevel@tonic-gate } 16547c478bd9Sstevel@tonic-gate 16557c478bd9Sstevel@tonic-gate /* 16567c478bd9Sstevel@tonic-gate * Large pages on the cache list are not supported. 16577c478bd9Sstevel@tonic-gate */ 16587c478bd9Sstevel@tonic-gate if (flags & PG_CACHE_LIST) 16597c478bd9Sstevel@tonic-gate panic("page_list_sub: large page on cachelist"); 16607c478bd9Sstevel@tonic-gate 16617c478bd9Sstevel@tonic-gate /* 16627c478bd9Sstevel@tonic-gate * Slow but rare. 16637c478bd9Sstevel@tonic-gate * 16647c478bd9Sstevel@tonic-gate * Somebody wants this particular page which is part 16657c478bd9Sstevel@tonic-gate * of a large page. In this case we just demote the page 16667c478bd9Sstevel@tonic-gate * if it's on the freelist. 16677c478bd9Sstevel@tonic-gate * 16687c478bd9Sstevel@tonic-gate * We have to drop pcm before locking the entire freelist. 16697c478bd9Sstevel@tonic-gate * Once we have re-locked the freelist check to make sure 16707c478bd9Sstevel@tonic-gate * the page hasn't already been demoted or completely 16717c478bd9Sstevel@tonic-gate * freed. 16727c478bd9Sstevel@tonic-gate */ 16737c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16747c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 16757c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 16767c478bd9Sstevel@tonic-gate /* 16777c478bd9Sstevel@tonic-gate * Large page is on freelist. 16787c478bd9Sstevel@tonic-gate */ 16797c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, pp->p_szc), 16807c478bd9Sstevel@tonic-gate pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 16817c478bd9Sstevel@tonic-gate } 16827c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16837c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16847c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 16857c478bd9Sstevel@tonic-gate 16867c478bd9Sstevel@tonic-gate /* 16877c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16887c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16897c478bd9Sstevel@tonic-gate */ 16907c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 16917c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 16927c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 16937c478bd9Sstevel@tonic-gate 16947c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 1695affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 16967c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 16977c478bd9Sstevel@tonic-gate 16987c478bd9Sstevel@tonic-gate #if defined(__sparc) 16997c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17007c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 17017c478bd9Sstevel@tonic-gate } 17027c478bd9Sstevel@tonic-gate #endif 17037c478bd9Sstevel@tonic-gate } 17047c478bd9Sstevel@tonic-gate 17057c478bd9Sstevel@tonic-gate void 17067c478bd9Sstevel@tonic-gate page_list_sub_pages(page_t *pp, uint_t szc) 17077c478bd9Sstevel@tonic-gate { 17087c478bd9Sstevel@tonic-gate kmutex_t *pcm; 17097c478bd9Sstevel@tonic-gate uint_t bin, mtype; 17107c478bd9Sstevel@tonic-gate int mnode; 17117c478bd9Sstevel@tonic-gate 17127c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 17137c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17147c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17157c478bd9Sstevel@tonic-gate 17167c478bd9Sstevel@tonic-gate /* 17177c478bd9Sstevel@tonic-gate * See comment in page_list_sub(). 17187c478bd9Sstevel@tonic-gate */ 17197c478bd9Sstevel@tonic-gate try_again: 17207c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17217c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 17227c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 17237c478bd9Sstevel@tonic-gate mutex_enter(pcm); 17247c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 17257c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17267c478bd9Sstevel@tonic-gate goto try_again; 17277c478bd9Sstevel@tonic-gate } 17287c478bd9Sstevel@tonic-gate 17297c478bd9Sstevel@tonic-gate /* 17307c478bd9Sstevel@tonic-gate * If we're called with a page larger than szc or it got 17317c478bd9Sstevel@tonic-gate * promoted above szc before we locked the freelist then 17327c478bd9Sstevel@tonic-gate * drop pcm and re-lock entire freelist. If page still larger 17337c478bd9Sstevel@tonic-gate * than szc then demote it. 17347c478bd9Sstevel@tonic-gate */ 17357c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 17367c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17377c478bd9Sstevel@tonic-gate pcm = NULL; 17387c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 17397c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 1740affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szcbig); 17417c478bd9Sstevel@tonic-gate (void) page_demote(mnode, 17427c478bd9Sstevel@tonic-gate PFN_BASE(pp->p_pagenum, pp->p_szc), 17437c478bd9Sstevel@tonic-gate pp->p_szc, szc, PC_NO_COLOR, PC_FREE); 17447c478bd9Sstevel@tonic-gate } 17457c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17467c478bd9Sstevel@tonic-gate } 17477c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17487c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17497c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc <= szc); 17507c478bd9Sstevel@tonic-gate ASSERT(pp == PP_PAGEROOT(pp)); 17517c478bd9Sstevel@tonic-gate 1752affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[pp->p_szc]); 1753affbd3ccSkchow 17547c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 17557c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 17567c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17577c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 17587c478bd9Sstevel@tonic-gate } else { 1759affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szc0); 17607c478bd9Sstevel@tonic-gate page_sub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17617c478bd9Sstevel@tonic-gate } 1762affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 17637c478bd9Sstevel@tonic-gate 17647c478bd9Sstevel@tonic-gate if (pcm != NULL) { 17657c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17667c478bd9Sstevel@tonic-gate } else { 17677c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17687c478bd9Sstevel@tonic-gate } 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate #if defined(__sparc) 17717c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17727c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 17737c478bd9Sstevel@tonic-gate 17747c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 17757c478bd9Sstevel@tonic-gate kcage_freemem_sub(pgcnt); 17767c478bd9Sstevel@tonic-gate } 17777c478bd9Sstevel@tonic-gate #endif 17787c478bd9Sstevel@tonic-gate } 17797c478bd9Sstevel@tonic-gate 17807c478bd9Sstevel@tonic-gate /* 17817c478bd9Sstevel@tonic-gate * Add the page to the front of a linked list of pages 17827c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 17837c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 17847c478bd9Sstevel@tonic-gate */ 17857c478bd9Sstevel@tonic-gate void 17867c478bd9Sstevel@tonic-gate mach_page_add(page_t **ppp, page_t *pp) 17877c478bd9Sstevel@tonic-gate { 17887c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 17897c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 17907c478bd9Sstevel@tonic-gate } else { 17917c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 17927c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 17937c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 17947c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 17957c478bd9Sstevel@tonic-gate } 17967c478bd9Sstevel@tonic-gate *ppp = pp; 17977c478bd9Sstevel@tonic-gate } 17987c478bd9Sstevel@tonic-gate 17997c478bd9Sstevel@tonic-gate /* 18007c478bd9Sstevel@tonic-gate * Remove this page from a linked list of pages 18017c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 18027c478bd9Sstevel@tonic-gate * 18037c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 18047c478bd9Sstevel@tonic-gate */ 18057c478bd9Sstevel@tonic-gate void 18067c478bd9Sstevel@tonic-gate mach_page_sub(page_t **ppp, page_t *pp) 18077c478bd9Sstevel@tonic-gate { 18087c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 18097c478bd9Sstevel@tonic-gate 18107c478bd9Sstevel@tonic-gate if (*ppp == NULL || pp == NULL) 18117c478bd9Sstevel@tonic-gate panic("mach_page_sub"); 18127c478bd9Sstevel@tonic-gate 18137c478bd9Sstevel@tonic-gate if (*ppp == pp) 18147c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 18157c478bd9Sstevel@tonic-gate 18167c478bd9Sstevel@tonic-gate if (*ppp == pp) 18177c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 18187c478bd9Sstevel@tonic-gate else { 18197c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 18207c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 18217c478bd9Sstevel@tonic-gate } 18227c478bd9Sstevel@tonic-gate pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 18237c478bd9Sstevel@tonic-gate } 18247c478bd9Sstevel@tonic-gate 18257c478bd9Sstevel@tonic-gate /* 18267c478bd9Sstevel@tonic-gate * Routine fsflush uses to gradually coalesce the free list into larger pages. 18277c478bd9Sstevel@tonic-gate */ 18287c478bd9Sstevel@tonic-gate void 18297c478bd9Sstevel@tonic-gate page_promote_size(page_t *pp, uint_t cur_szc) 18307c478bd9Sstevel@tonic-gate { 18317c478bd9Sstevel@tonic-gate pfn_t pfn; 18327c478bd9Sstevel@tonic-gate int mnode; 18337c478bd9Sstevel@tonic-gate int idx; 18347c478bd9Sstevel@tonic-gate int new_szc = cur_szc + 1; 18357c478bd9Sstevel@tonic-gate int full = FULL_REGION_CNT(new_szc); 18367c478bd9Sstevel@tonic-gate 18377c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 18387c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(pfn); 18397c478bd9Sstevel@tonic-gate 18407c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 18417c478bd9Sstevel@tonic-gate 18427c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, new_szc, pfn); 18437c478bd9Sstevel@tonic-gate if (PAGE_COUNTERS(mnode, new_szc, idx) == full) 18445d07b933Sdp (void) page_promote(mnode, pfn, new_szc, PC_FREE, PC_MTYPE_ANY); 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 18477c478bd9Sstevel@tonic-gate } 18487c478bd9Sstevel@tonic-gate 18497c478bd9Sstevel@tonic-gate static uint_t page_promote_err; 18507c478bd9Sstevel@tonic-gate static uint_t page_promote_noreloc_err; 18517c478bd9Sstevel@tonic-gate 18527c478bd9Sstevel@tonic-gate /* 18537c478bd9Sstevel@tonic-gate * Create a single larger page (of szc new_szc) from smaller contiguous pages 18547c478bd9Sstevel@tonic-gate * for the given mnode starting at pfnum. Pages involved are on the freelist 18557c478bd9Sstevel@tonic-gate * before the call and may be returned to the caller if requested, otherwise 18567c478bd9Sstevel@tonic-gate * they will be placed back on the freelist. 18577c478bd9Sstevel@tonic-gate * If flags is PC_ALLOC, then the large page will be returned to the user in 18587c478bd9Sstevel@tonic-gate * a state which is consistent with a page being taken off the freelist. If 18597c478bd9Sstevel@tonic-gate * we failed to lock the new large page, then we will return NULL to the 18607c478bd9Sstevel@tonic-gate * caller and put the large page on the freelist instead. 18617c478bd9Sstevel@tonic-gate * If flags is PC_FREE, then the large page will be placed on the freelist, 18627c478bd9Sstevel@tonic-gate * and NULL will be returned. 18637c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 18647c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 18657c478bd9Sstevel@tonic-gate * 18667c478bd9Sstevel@tonic-gate * RFE: For performance pass in pp instead of pfnum so 18677c478bd9Sstevel@tonic-gate * we can avoid excessive calls to page_numtopp_nolock(). 18687c478bd9Sstevel@tonic-gate * This would depend on an assumption that all contiguous 18697c478bd9Sstevel@tonic-gate * pages are in the same memseg so we can just add/dec 18707c478bd9Sstevel@tonic-gate * our pp. 18717c478bd9Sstevel@tonic-gate * 18727c478bd9Sstevel@tonic-gate * Lock ordering: 18737c478bd9Sstevel@tonic-gate * 18747c478bd9Sstevel@tonic-gate * There is a potential but rare deadlock situation 18757c478bd9Sstevel@tonic-gate * for page promotion and demotion operations. The problem 18767c478bd9Sstevel@tonic-gate * is there are two paths into the freelist manager and 18777c478bd9Sstevel@tonic-gate * they have different lock orders: 18787c478bd9Sstevel@tonic-gate * 18797c478bd9Sstevel@tonic-gate * page_create() 18807c478bd9Sstevel@tonic-gate * lock freelist 18817c478bd9Sstevel@tonic-gate * page_lock(EXCL) 18827c478bd9Sstevel@tonic-gate * unlock freelist 18837c478bd9Sstevel@tonic-gate * return 18847c478bd9Sstevel@tonic-gate * caller drops page_lock 18857c478bd9Sstevel@tonic-gate * 18867c478bd9Sstevel@tonic-gate * page_free() and page_reclaim() 18877c478bd9Sstevel@tonic-gate * caller grabs page_lock(EXCL) 18887c478bd9Sstevel@tonic-gate * 18897c478bd9Sstevel@tonic-gate * lock freelist 18907c478bd9Sstevel@tonic-gate * unlock freelist 18917c478bd9Sstevel@tonic-gate * drop page_lock 18927c478bd9Sstevel@tonic-gate * 18937c478bd9Sstevel@tonic-gate * What prevents a thread in page_create() from deadlocking 18947c478bd9Sstevel@tonic-gate * with a thread freeing or reclaiming the same page is the 18957c478bd9Sstevel@tonic-gate * page_trylock() in page_get_freelist(). If the trylock fails 18967c478bd9Sstevel@tonic-gate * it skips the page. 18977c478bd9Sstevel@tonic-gate * 18987c478bd9Sstevel@tonic-gate * The lock ordering for promotion and demotion is the same as 18997c478bd9Sstevel@tonic-gate * for page_create(). Since the same deadlock could occur during 19007c478bd9Sstevel@tonic-gate * page promotion and freeing or reclaiming of a page on the 19017c478bd9Sstevel@tonic-gate * cache list we might have to fail the operation and undo what 19027c478bd9Sstevel@tonic-gate * have done so far. Again this is rare. 19037c478bd9Sstevel@tonic-gate */ 19047c478bd9Sstevel@tonic-gate page_t * 19055d07b933Sdp page_promote(int mnode, pfn_t pfnum, uchar_t new_szc, int flags, int mtype) 19067c478bd9Sstevel@tonic-gate { 19077c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *tpp, *start_pp; 19087c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, npgs; 19097c478bd9Sstevel@tonic-gate uint_t bin; 19107c478bd9Sstevel@tonic-gate pgcnt_t tmpnpgs, pages_left; 19117c478bd9Sstevel@tonic-gate uint_t noreloc; 19127c478bd9Sstevel@tonic-gate int which_list; 19137c478bd9Sstevel@tonic-gate ulong_t index; 19147c478bd9Sstevel@tonic-gate kmutex_t *phm; 19157c478bd9Sstevel@tonic-gate 19167c478bd9Sstevel@tonic-gate /* 19177c478bd9Sstevel@tonic-gate * General algorithm: 19187c478bd9Sstevel@tonic-gate * Find the starting page 19197c478bd9Sstevel@tonic-gate * Walk each page struct removing it from the freelist, 19207c478bd9Sstevel@tonic-gate * and linking it to all the other pages removed. 19217c478bd9Sstevel@tonic-gate * Once all pages are off the freelist, 19227c478bd9Sstevel@tonic-gate * walk the list, modifying p_szc to new_szc and what 19237c478bd9Sstevel@tonic-gate * ever other info needs to be done to create a large free page. 19247c478bd9Sstevel@tonic-gate * According to the flags, either return the page or put it 19257c478bd9Sstevel@tonic-gate * on the freelist. 19267c478bd9Sstevel@tonic-gate */ 19277c478bd9Sstevel@tonic-gate 19287c478bd9Sstevel@tonic-gate start_pp = page_numtopp_nolock(pfnum); 19297c478bd9Sstevel@tonic-gate ASSERT(start_pp && (start_pp->p_pagenum == pfnum)); 19307c478bd9Sstevel@tonic-gate new_npgs = page_get_pagecnt(new_szc); 19317c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pfnum, new_npgs)); 19327c478bd9Sstevel@tonic-gate 19335d07b933Sdp /* don't return page of the wrong mtype */ 19345d07b933Sdp if (mtype != PC_MTYPE_ANY && mtype != PP_2_MTYPE(start_pp)) 19355d07b933Sdp return (NULL); 19365d07b933Sdp 19377c478bd9Sstevel@tonic-gate /* 19387c478bd9Sstevel@tonic-gate * Loop through smaller pages to confirm that all pages 19397c478bd9Sstevel@tonic-gate * give the same result for PP_ISNORELOC(). 19407c478bd9Sstevel@tonic-gate * We can check this reliably here as the protocol for setting 19417c478bd9Sstevel@tonic-gate * P_NORELOC requires pages to be taken off the free list first. 19427c478bd9Sstevel@tonic-gate */ 19435d07b933Sdp noreloc = PP_ISNORELOC(start_pp); 19445d07b933Sdp for (pp = start_pp + new_npgs; --pp > start_pp; ) { 19455d07b933Sdp if (noreloc != PP_ISNORELOC(pp)) { 19467c478bd9Sstevel@tonic-gate page_promote_noreloc_err++; 19477c478bd9Sstevel@tonic-gate page_promote_err++; 19487c478bd9Sstevel@tonic-gate return (NULL); 19497c478bd9Sstevel@tonic-gate } 19507c478bd9Sstevel@tonic-gate } 19517c478bd9Sstevel@tonic-gate 19527c478bd9Sstevel@tonic-gate pages_left = new_npgs; 19537c478bd9Sstevel@tonic-gate pplist = NULL; 19547c478bd9Sstevel@tonic-gate pp = start_pp; 19557c478bd9Sstevel@tonic-gate 19567c478bd9Sstevel@tonic-gate /* Loop around coalescing the smaller pages into a big page. */ 19577c478bd9Sstevel@tonic-gate while (pages_left) { 19587c478bd9Sstevel@tonic-gate /* 19597c478bd9Sstevel@tonic-gate * Remove from the freelist. 19607c478bd9Sstevel@tonic-gate */ 19617c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 19627c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 19637c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 19647c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 19657c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 19667c478bd9Sstevel@tonic-gate 19677c478bd9Sstevel@tonic-gate /* 19687c478bd9Sstevel@tonic-gate * PG_FREE_LIST 19697c478bd9Sstevel@tonic-gate */ 19707c478bd9Sstevel@tonic-gate if (pp->p_szc) { 19717c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, 19727c478bd9Sstevel@tonic-gate pp->p_szc, bin, mtype), pp); 19737c478bd9Sstevel@tonic-gate } else { 19747c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_FREELISTS(mnode, 0, 19757c478bd9Sstevel@tonic-gate bin, mtype), pp); 19767c478bd9Sstevel@tonic-gate } 19777c478bd9Sstevel@tonic-gate which_list = PG_FREE_LIST; 19787c478bd9Sstevel@tonic-gate } else { 19797c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 19807c478bd9Sstevel@tonic-gate 19817c478bd9Sstevel@tonic-gate /* 19827c478bd9Sstevel@tonic-gate * PG_CACHE_LIST 19837c478bd9Sstevel@tonic-gate * 19847c478bd9Sstevel@tonic-gate * Since this page comes from the 19857c478bd9Sstevel@tonic-gate * cachelist, we must destroy the 19867c478bd9Sstevel@tonic-gate * vnode association. 19877c478bd9Sstevel@tonic-gate */ 19887c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 19897c478bd9Sstevel@tonic-gate goto fail_promote; 19907c478bd9Sstevel@tonic-gate } 19917c478bd9Sstevel@tonic-gate 19927c478bd9Sstevel@tonic-gate /* 19937c478bd9Sstevel@tonic-gate * We need to be careful not to deadlock 19947c478bd9Sstevel@tonic-gate * with another thread in page_lookup(). 19957c478bd9Sstevel@tonic-gate * The page_lookup() thread could be holding 19967c478bd9Sstevel@tonic-gate * the same phm that we need if the two 19977c478bd9Sstevel@tonic-gate * pages happen to hash to the same phm lock. 19987c478bd9Sstevel@tonic-gate * At this point we have locked the entire 19997c478bd9Sstevel@tonic-gate * freelist and page_lookup() could be trying 20007c478bd9Sstevel@tonic-gate * to grab a freelist lock. 20017c478bd9Sstevel@tonic-gate */ 20027c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(pp->p_vnode, pp->p_offset); 20037c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 20047c478bd9Sstevel@tonic-gate if (!mutex_tryenter(phm)) { 20058b464eb8Smec page_unlock_nocapture(pp); 20067c478bd9Sstevel@tonic-gate goto fail_promote; 20077c478bd9Sstevel@tonic-gate } 20087c478bd9Sstevel@tonic-gate 20097c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_CACHELISTS(mnode, bin, mtype), pp); 20107c478bd9Sstevel@tonic-gate page_hashout(pp, phm); 20117c478bd9Sstevel@tonic-gate mutex_exit(phm); 20127c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 20138b464eb8Smec page_unlock_nocapture(pp); 20147c478bd9Sstevel@tonic-gate which_list = PG_CACHE_LIST; 20157c478bd9Sstevel@tonic-gate } 2016affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, which_list); 20177c478bd9Sstevel@tonic-gate 20187c478bd9Sstevel@tonic-gate /* 20197c478bd9Sstevel@tonic-gate * Concatenate the smaller page(s) onto 20207c478bd9Sstevel@tonic-gate * the large page list. 20217c478bd9Sstevel@tonic-gate */ 20227c478bd9Sstevel@tonic-gate tmpnpgs = npgs = page_get_pagecnt(pp->p_szc); 20237c478bd9Sstevel@tonic-gate pages_left -= npgs; 20247c478bd9Sstevel@tonic-gate tpp = pp; 20257c478bd9Sstevel@tonic-gate while (npgs--) { 20267c478bd9Sstevel@tonic-gate tpp->p_szc = new_szc; 20277c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 20287c478bd9Sstevel@tonic-gate } 20297c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 20307c478bd9Sstevel@tonic-gate pp += tmpnpgs; 20317c478bd9Sstevel@tonic-gate } 20327c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 20337c478bd9Sstevel@tonic-gate 20347c478bd9Sstevel@tonic-gate /* 20357c478bd9Sstevel@tonic-gate * return the page to the user if requested 20367c478bd9Sstevel@tonic-gate * in the properly locked state. 20377c478bd9Sstevel@tonic-gate */ 20387c478bd9Sstevel@tonic-gate if (flags == PC_ALLOC && (page_trylock_cons(pplist, SE_EXCL))) { 20397c478bd9Sstevel@tonic-gate return (pplist); 20407c478bd9Sstevel@tonic-gate } 20417c478bd9Sstevel@tonic-gate 20427c478bd9Sstevel@tonic-gate /* 20437c478bd9Sstevel@tonic-gate * Otherwise place the new large page on the freelist 20447c478bd9Sstevel@tonic-gate */ 20457c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 20467c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pplist); 20477c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 20487c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, bin, mtype), pplist); 20497c478bd9Sstevel@tonic-gate 2050affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, PG_FREE_LIST); 20517c478bd9Sstevel@tonic-gate return (NULL); 20527c478bd9Sstevel@tonic-gate 20537c478bd9Sstevel@tonic-gate fail_promote: 20547c478bd9Sstevel@tonic-gate /* 20557c478bd9Sstevel@tonic-gate * A thread must have still been freeing or 20567c478bd9Sstevel@tonic-gate * reclaiming the page on the cachelist. 20577c478bd9Sstevel@tonic-gate * To prevent a deadlock undo what we have 20587c478bd9Sstevel@tonic-gate * done sofar and return failure. This 20597c478bd9Sstevel@tonic-gate * situation can only happen while promoting 20607c478bd9Sstevel@tonic-gate * PAGESIZE pages. 20617c478bd9Sstevel@tonic-gate */ 20627c478bd9Sstevel@tonic-gate page_promote_err++; 20637c478bd9Sstevel@tonic-gate while (pplist) { 20647c478bd9Sstevel@tonic-gate pp = pplist; 20657c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 20667c478bd9Sstevel@tonic-gate pp->p_szc = 0; 20677c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 20687c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 20697c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, mtype), pp); 2070affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 20717c478bd9Sstevel@tonic-gate } 20727c478bd9Sstevel@tonic-gate return (NULL); 20737c478bd9Sstevel@tonic-gate 20747c478bd9Sstevel@tonic-gate } 20757c478bd9Sstevel@tonic-gate 20767c478bd9Sstevel@tonic-gate /* 20777c478bd9Sstevel@tonic-gate * Break up a large page into smaller size pages. 20787c478bd9Sstevel@tonic-gate * Pages involved are on the freelist before the call and may 20797c478bd9Sstevel@tonic-gate * be returned to the caller if requested, otherwise they will 20807c478bd9Sstevel@tonic-gate * be placed back on the freelist. 20817c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 20827c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 20837c478bd9Sstevel@tonic-gate * If flags is not PC_ALLOC, the color argument is ignored, and thus 20847c478bd9Sstevel@tonic-gate * technically, any value may be passed in but PC_NO_COLOR is the standard 20857c478bd9Sstevel@tonic-gate * which should be followed for clarity's sake. 20867c478bd9Sstevel@tonic-gate */ 20877c478bd9Sstevel@tonic-gate page_t * 20887c478bd9Sstevel@tonic-gate page_demote(int mnode, pfn_t pfnum, uchar_t cur_szc, uchar_t new_szc, 20897c478bd9Sstevel@tonic-gate int color, int flags) 20907c478bd9Sstevel@tonic-gate { 20917c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *npplist; 20927c478bd9Sstevel@tonic-gate pgcnt_t npgs, n; 20937c478bd9Sstevel@tonic-gate uint_t bin; 20947c478bd9Sstevel@tonic-gate uint_t mtype; 20957c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 20967c478bd9Sstevel@tonic-gate 20977c478bd9Sstevel@tonic-gate ASSERT(cur_szc != 0); 20987c478bd9Sstevel@tonic-gate ASSERT(new_szc < cur_szc); 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate pplist = page_numtopp_nolock(pfnum); 21017c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate ASSERT(pplist->p_szc == cur_szc); 21047c478bd9Sstevel@tonic-gate 21057c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 21067c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pplist)); 21077c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 21087c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, cur_szc, bin, mtype), pplist); 21097c478bd9Sstevel@tonic-gate 21107c478bd9Sstevel@tonic-gate CHK_LPG(pplist, cur_szc); 2111affbd3ccSkchow page_ctr_sub(mnode, mtype, pplist, PG_FREE_LIST); 21127c478bd9Sstevel@tonic-gate 21137c478bd9Sstevel@tonic-gate /* 21147c478bd9Sstevel@tonic-gate * Number of PAGESIZE pages for smaller new_szc 21157c478bd9Sstevel@tonic-gate * page. 21167c478bd9Sstevel@tonic-gate */ 21177c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(new_szc); 21187c478bd9Sstevel@tonic-gate 21197c478bd9Sstevel@tonic-gate while (pplist) { 21207c478bd9Sstevel@tonic-gate pp = pplist; 21217c478bd9Sstevel@tonic-gate 21227c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21237c478bd9Sstevel@tonic-gate 21247c478bd9Sstevel@tonic-gate /* 21257c478bd9Sstevel@tonic-gate * We either break it up into PAGESIZE pages or larger. 21267c478bd9Sstevel@tonic-gate */ 21277c478bd9Sstevel@tonic-gate if (npgs == 1) { /* PAGESIZE case */ 21287c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 21297c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21307c478bd9Sstevel@tonic-gate ASSERT(new_szc == 0); 21317c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 21327c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21337c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 21347c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 21357c478bd9Sstevel@tonic-gate (ret_pp == NULL) && 21367c478bd9Sstevel@tonic-gate page_trylock_cons(pp, SE_EXCL)) { 21377c478bd9Sstevel@tonic-gate ret_pp = pp; 21387c478bd9Sstevel@tonic-gate } else { 21397c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21407c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, 21417c478bd9Sstevel@tonic-gate mtype), pp); 2142affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 21437c478bd9Sstevel@tonic-gate } 21447c478bd9Sstevel@tonic-gate } else { 21457c478bd9Sstevel@tonic-gate 21467c478bd9Sstevel@tonic-gate /* 21477c478bd9Sstevel@tonic-gate * Break down into smaller lists of pages. 21487c478bd9Sstevel@tonic-gate */ 21497c478bd9Sstevel@tonic-gate page_list_break(&pplist, &npplist, npgs); 21507c478bd9Sstevel@tonic-gate 21517c478bd9Sstevel@tonic-gate pp = pplist; 21527c478bd9Sstevel@tonic-gate n = npgs; 21537c478bd9Sstevel@tonic-gate while (n--) { 21547c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21557c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21567c478bd9Sstevel@tonic-gate pp = pp->p_next; 21577c478bd9Sstevel@tonic-gate } 21587c478bd9Sstevel@tonic-gate 21597c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 21607c478bd9Sstevel@tonic-gate 21617c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 21627c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 21637c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 21647c478bd9Sstevel@tonic-gate (ret_pp == NULL) && 21657c478bd9Sstevel@tonic-gate page_trylock_cons(pp, SE_EXCL)) { 21667c478bd9Sstevel@tonic-gate ret_pp = pp; 21677c478bd9Sstevel@tonic-gate } else { 21687c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21697c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, 21707c478bd9Sstevel@tonic-gate bin, mtype), pplist); 21717c478bd9Sstevel@tonic-gate 2172affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, 2173affbd3ccSkchow PG_FREE_LIST); 21747c478bd9Sstevel@tonic-gate } 21757c478bd9Sstevel@tonic-gate pplist = npplist; 21767c478bd9Sstevel@tonic-gate } 21777c478bd9Sstevel@tonic-gate } 21787c478bd9Sstevel@tonic-gate return (ret_pp); 21797c478bd9Sstevel@tonic-gate } 21807c478bd9Sstevel@tonic-gate 21817c478bd9Sstevel@tonic-gate int mpss_coalesce_disable = 0; 21827c478bd9Sstevel@tonic-gate 21837c478bd9Sstevel@tonic-gate /* 21847c478bd9Sstevel@tonic-gate * Coalesce free pages into a page of the given szc and color if possible. 21857c478bd9Sstevel@tonic-gate * Return the pointer to the page created, otherwise, return NULL. 21865d07b933Sdp * 21875d07b933Sdp * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 21887c478bd9Sstevel@tonic-gate */ 21895d07b933Sdp page_t * 21905d07b933Sdp page_freelist_coalesce(int mnode, uchar_t szc, uint_t color, uint_t ceq_mask, 21915d07b933Sdp int mtype, pfn_t pfnhi) 21927c478bd9Sstevel@tonic-gate { 21935d07b933Sdp int r = szc; /* region size */ 21945d07b933Sdp int mrange; 21955d07b933Sdp uint_t full, bin, color_mask, wrap = 0; 21965d07b933Sdp pfn_t pfnum, lo, hi; 21975d07b933Sdp size_t len, idx, idx0; 21985d07b933Sdp pgcnt_t cands = 0, szcpgcnt = page_get_pagecnt(szc); 21997c478bd9Sstevel@tonic-gate page_t *ret_pp; 2200ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 22015d07b933Sdp #if defined(__sparc) 22025d07b933Sdp pfn_t pfnum0, nlo, nhi; 22035d07b933Sdp #endif 22047c478bd9Sstevel@tonic-gate 22057c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 22065d07b933Sdp ASSERT(szc < MMU_PAGE_SIZES); 22075d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[szc][0]); 22087c478bd9Sstevel@tonic-gate return (NULL); 22097c478bd9Sstevel@tonic-gate } 22107c478bd9Sstevel@tonic-gate 22115d07b933Sdp ASSERT(szc < mmu_page_sizes); 22125d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(szc) - 1; 22135d07b933Sdp ASSERT(ceq_mask <= color_mask); 22145d07b933Sdp ASSERT(color <= color_mask); 22155d07b933Sdp color &= ceq_mask; 22165d07b933Sdp 22175d07b933Sdp /* Prevent page_counters dynamic memory from being freed */ 22185d07b933Sdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 22195d07b933Sdp 22205d07b933Sdp mrange = MTYPE_2_MRANGE(mnode, mtype); 22215d07b933Sdp ASSERT(mrange < mnode_nranges[mnode]); 22225d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[r][mrange]); 22235d07b933Sdp 22245d07b933Sdp /* get pfn range for mtype */ 22255d07b933Sdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 22265d07b933Sdp #if defined(__sparc) 22275d07b933Sdp lo = PAGE_COUNTERS_BASE(mnode, r); 22285d07b933Sdp hi = IDX_TO_PNUM(mnode, r, len); 22295d07b933Sdp #else 22305d07b933Sdp MNODETYPE_2_PFN(mnode, mtype, lo, hi); 22315d07b933Sdp hi++; 22325d07b933Sdp #endif 22335d07b933Sdp 22345d07b933Sdp /* use lower limit if given */ 22355d07b933Sdp if (pfnhi != PFNNULL && pfnhi < hi) 22365d07b933Sdp hi = pfnhi; 22375d07b933Sdp 22385d07b933Sdp /* round to szcpgcnt boundaries */ 22395d07b933Sdp lo = P2ROUNDUP(lo, szcpgcnt); 2240ce8eb11aSdp MEM_NODE_ITERATOR_INIT(lo, mnode, &it); 2241ce8eb11aSdp ASSERT(lo != (pfn_t)-1); 22425d07b933Sdp hi = hi & ~(szcpgcnt - 1); 22435d07b933Sdp 22445d07b933Sdp /* set lo to the closest pfn of the right color */ 2245ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ color) & ceq_mask) || 2246ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(lo) != mnode)) { 2247ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, color, ceq_mask, color_mask, 2248ce8eb11aSdp &it); 22495d07b933Sdp } 22505d07b933Sdp 22515d07b933Sdp if (hi <= lo) { 22525d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 22537c478bd9Sstevel@tonic-gate return (NULL); 22547c478bd9Sstevel@tonic-gate } 22555d07b933Sdp 22567c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 22577c478bd9Sstevel@tonic-gate 22585d07b933Sdp /* calculate the number of page candidates and initial search index */ 22595d07b933Sdp bin = color; 22605d07b933Sdp idx0 = (size_t)(-1); 22615d07b933Sdp do { 22625d07b933Sdp pgcnt_t acand; 22635d07b933Sdp 22645d07b933Sdp PGCTRS_CANDS_GETVALUECOLOR(mnode, mrange, r, bin, acand); 22655d07b933Sdp if (acand) { 22665d07b933Sdp idx = PAGE_COUNTERS_CURRENT_COLOR(mnode, 22675d07b933Sdp r, bin, mrange); 22685d07b933Sdp idx0 = MIN(idx0, idx); 22695d07b933Sdp cands += acand; 22705d07b933Sdp } 22715d07b933Sdp bin = ADD_MASKED(bin, 1, ceq_mask, color_mask); 22725d07b933Sdp } while (bin != color); 22735d07b933Sdp 22745d07b933Sdp if (cands == 0) { 22755d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_cands_skip[r][mrange]); 22765d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 22775d07b933Sdp return (NULL); 22785d07b933Sdp } 22795d07b933Sdp 22805d07b933Sdp pfnum = IDX_TO_PNUM(mnode, r, idx0); 22815d07b933Sdp if (pfnum < lo || pfnum >= hi) { 22825d07b933Sdp pfnum = lo; 2283ce8eb11aSdp } else { 2284ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, &it); 2285ce8eb11aSdp if (pfnum == (pfn_t)-1) { 2286ce8eb11aSdp pfnum = lo; 2287ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, &it); 2288ce8eb11aSdp ASSERT(pfnum != (pfn_t)-1); 2289ce8eb11aSdp } else if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & ceq_mask || 2290ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(pfnum) != mnode)) { 2291ce8eb11aSdp /* invalid color, get the closest correct pfn */ 2292ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2293ce8eb11aSdp color_mask, &it); 2294ce8eb11aSdp if (pfnum >= hi) { 2295ce8eb11aSdp pfnum = lo; 2296ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, &it); 2297ce8eb11aSdp } 2298ce8eb11aSdp } 22995d07b933Sdp } 23005d07b933Sdp 23015d07b933Sdp /* set starting index */ 23025d07b933Sdp idx0 = PNUM_TO_IDX(mnode, r, pfnum); 23035d07b933Sdp ASSERT(idx0 < len); 23045d07b933Sdp 23055d07b933Sdp #if defined(__sparc) 23065d07b933Sdp pfnum0 = pfnum; /* page corresponding to idx0 */ 23075d07b933Sdp nhi = 0; /* search kcage ranges */ 23085d07b933Sdp #endif 23095d07b933Sdp 23105d07b933Sdp for (idx = idx0; wrap == 0 || (idx < idx0 && wrap < 2); ) { 23115d07b933Sdp 23125d07b933Sdp #if defined(__sparc) 23135d07b933Sdp /* 23145d07b933Sdp * Find lowest intersection of kcage ranges and mnode. 23155d07b933Sdp * MTYPE_NORELOC means look in the cage, otherwise outside. 23165d07b933Sdp */ 23175d07b933Sdp if (nhi <= pfnum) { 23185d07b933Sdp if (kcage_next_range(mtype == MTYPE_NORELOC, pfnum, 23195d07b933Sdp (wrap == 0 ? hi : pfnum0), &nlo, &nhi)) 23205d07b933Sdp goto wrapit; 23215d07b933Sdp 23225d07b933Sdp /* jump to the next page in the range */ 23235d07b933Sdp if (pfnum < nlo) { 23245d07b933Sdp pfnum = P2ROUNDUP(nlo, szcpgcnt); 2325ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, &it); 23265d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 23275d07b933Sdp if (idx >= len || pfnum >= hi) 23285d07b933Sdp goto wrapit; 2329ce8eb11aSdp if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & 23305d07b933Sdp ceq_mask) 23315d07b933Sdp goto next; 2332ce8eb11aSdp if (interleaved_mnodes && 2333ce8eb11aSdp PFN_2_MEM_NODE(pfnum) != mnode) 2334ce8eb11aSdp goto next; 23357c478bd9Sstevel@tonic-gate } 23365d07b933Sdp } 23375d07b933Sdp #endif 23385d07b933Sdp 23395d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) != full) 23405d07b933Sdp goto next; 23415d07b933Sdp 23425d07b933Sdp /* 23435d07b933Sdp * RFE: For performance maybe we can do something less 23445d07b933Sdp * brutal than locking the entire freelist. So far 23455d07b933Sdp * this doesn't seem to be a performance problem? 23465d07b933Sdp */ 23475d07b933Sdp page_freelist_lock(mnode); 23485d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 23495d07b933Sdp ret_pp = 23505d07b933Sdp page_promote(mnode, pfnum, r, PC_ALLOC, mtype); 23517c478bd9Sstevel@tonic-gate if (ret_pp != NULL) { 23525d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfc_coalok[r][mrange]); 23535d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, 2354ce8eb11aSdp PFN_2_COLOR(pfnum, szc, &it), mrange) = idx; 23557c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 23567c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 23577c478bd9Sstevel@tonic-gate #if defined(__sparc) 23587c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 23597c478bd9Sstevel@tonic-gate pgcnt_t npgs; 23607c478bd9Sstevel@tonic-gate 23617c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(ret_pp->p_szc); 23627c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 23637c478bd9Sstevel@tonic-gate } 23647c478bd9Sstevel@tonic-gate #endif 23657c478bd9Sstevel@tonic-gate return (ret_pp); 23667c478bd9Sstevel@tonic-gate } 23675d07b933Sdp } else { 23685d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_changed[r][mrange]); 23695d07b933Sdp } 23705d07b933Sdp 23715d07b933Sdp page_freelist_unlock(mnode); 23725d07b933Sdp /* 23735d07b933Sdp * No point looking for another page if we've 23745d07b933Sdp * already tried all of the ones that 23755d07b933Sdp * page_ctr_cands indicated. Stash off where we left 23765d07b933Sdp * off. 23775d07b933Sdp * Note: this is not exact since we don't hold the 23785d07b933Sdp * page_freelist_locks before we initially get the 23795d07b933Sdp * value of cands for performance reasons, but should 23805d07b933Sdp * be a decent approximation. 23815d07b933Sdp */ 23825d07b933Sdp if (--cands == 0) { 23835d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, color, mrange) = 23845d07b933Sdp idx; 23855d07b933Sdp break; 23865d07b933Sdp } 23875d07b933Sdp next: 23885d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2389ce8eb11aSdp color_mask, &it); 23905d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 23915d07b933Sdp if (idx >= len || pfnum >= hi) { 23925d07b933Sdp wrapit: 23935d07b933Sdp pfnum = lo; 2394ce8eb11aSdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, &it); 23955d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 23965d07b933Sdp wrap++; 23975d07b933Sdp #if defined(__sparc) 23985d07b933Sdp nhi = 0; /* search kcage ranges */ 23995d07b933Sdp #endif 24007c478bd9Sstevel@tonic-gate } 24017c478bd9Sstevel@tonic-gate } 24025d07b933Sdp 24037c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 24045d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_failed[r][mrange]); 24057c478bd9Sstevel@tonic-gate return (NULL); 24067c478bd9Sstevel@tonic-gate } 24077c478bd9Sstevel@tonic-gate 24087c478bd9Sstevel@tonic-gate /* 24097c478bd9Sstevel@tonic-gate * For the given mnode, promote as many small pages to large pages as possible. 2410ce8eb11aSdp * mnode can be -1, which means do them all 24117c478bd9Sstevel@tonic-gate */ 24127c478bd9Sstevel@tonic-gate void 24137c478bd9Sstevel@tonic-gate page_freelist_coalesce_all(int mnode) 24147c478bd9Sstevel@tonic-gate { 24157c478bd9Sstevel@tonic-gate int r; /* region size */ 24167c478bd9Sstevel@tonic-gate int idx, full; 24177c478bd9Sstevel@tonic-gate size_t len; 2418ce8eb11aSdp int doall = interleaved_mnodes || mnode < 0; 2419ce8eb11aSdp int mlo = doall ? 0 : mnode; 2420ce8eb11aSdp int mhi = doall ? max_mem_nodes : (mnode + 1); 24217c478bd9Sstevel@tonic-gate 24227c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce_all); 24237c478bd9Sstevel@tonic-gate 24247c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 24257c478bd9Sstevel@tonic-gate return; 24267c478bd9Sstevel@tonic-gate } 24277c478bd9Sstevel@tonic-gate 24287c478bd9Sstevel@tonic-gate /* 24297c478bd9Sstevel@tonic-gate * Lock the entire freelist and coalesce what we can. 24307c478bd9Sstevel@tonic-gate * 24317c478bd9Sstevel@tonic-gate * Always promote to the largest page possible 24327c478bd9Sstevel@tonic-gate * first to reduce the number of page promotions. 24337c478bd9Sstevel@tonic-gate */ 2434ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2435ce8eb11aSdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 2436ce8eb11aSdp page_freelist_lock(mnode); 2437ce8eb11aSdp } 24387c478bd9Sstevel@tonic-gate for (r = mmu_page_sizes - 1; r > 0; r--) { 2439ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2440ce8eb11aSdp pgcnt_t cands = 0; 2441ce8eb11aSdp int mrange, nranges = mnode_nranges[mnode]; 24427c478bd9Sstevel@tonic-gate 2443ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 2444ce8eb11aSdp PGCTRS_CANDS_GETVALUE(mnode, mrange, r, cands); 2445ce8eb11aSdp if (cands != 0) 2446ce8eb11aSdp break; 2447ce8eb11aSdp } 2448ce8eb11aSdp if (cands == 0) { 2449ce8eb11aSdp VM_STAT_ADD(vmm_vmstats. 2450ce8eb11aSdp page_ctrs_cands_skip_all); 2451ce8eb11aSdp continue; 2452ce8eb11aSdp } 24537c478bd9Sstevel@tonic-gate 2454ce8eb11aSdp full = FULL_REGION_CNT(r); 2455ce8eb11aSdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 2456ce8eb11aSdp 2457ce8eb11aSdp for (idx = 0; idx < len; idx++) { 2458ce8eb11aSdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 2459ce8eb11aSdp pfn_t pfnum = 2460ce8eb11aSdp IDX_TO_PNUM(mnode, r, idx); 2461ce8eb11aSdp int tmnode = interleaved_mnodes ? 2462ce8eb11aSdp PFN_2_MEM_NODE(pfnum) : mnode; 2463ce8eb11aSdp 2464ce8eb11aSdp ASSERT(pfnum >= 2465ce8eb11aSdp mem_node_config[tmnode].physbase && 2466ce8eb11aSdp pfnum < 2467ce8eb11aSdp mem_node_config[tmnode].physmax); 2468ce8eb11aSdp 2469ce8eb11aSdp (void) page_promote(tmnode, 2470ce8eb11aSdp pfnum, r, PC_FREE, PC_MTYPE_ANY); 2471ce8eb11aSdp } 24727c478bd9Sstevel@tonic-gate } 2473ce8eb11aSdp /* shared hpm_counters covers all mnodes, so we quit */ 2474ce8eb11aSdp if (interleaved_mnodes) 2475ce8eb11aSdp break; 24767c478bd9Sstevel@tonic-gate } 24777c478bd9Sstevel@tonic-gate } 2478ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2479ce8eb11aSdp page_freelist_unlock(mnode); 2480ce8eb11aSdp rw_exit(&page_ctrs_rwlock[mnode]); 2481ce8eb11aSdp } 24827c478bd9Sstevel@tonic-gate } 24837c478bd9Sstevel@tonic-gate 24847c478bd9Sstevel@tonic-gate /* 24857c478bd9Sstevel@tonic-gate * This is where all polices for moving pages around 24867c478bd9Sstevel@tonic-gate * to different page size free lists is implemented. 24877c478bd9Sstevel@tonic-gate * Returns 1 on success, 0 on failure. 24887c478bd9Sstevel@tonic-gate * 24897c478bd9Sstevel@tonic-gate * So far these are the priorities for this algorithm in descending 24907c478bd9Sstevel@tonic-gate * order: 24917c478bd9Sstevel@tonic-gate * 24927c478bd9Sstevel@tonic-gate * 1) When servicing a request try to do so with a free page 24937c478bd9Sstevel@tonic-gate * from next size up. Helps defer fragmentation as long 24947c478bd9Sstevel@tonic-gate * as possible. 24957c478bd9Sstevel@tonic-gate * 24967c478bd9Sstevel@tonic-gate * 2) Page coalesce on demand. Only when a freelist 24977c478bd9Sstevel@tonic-gate * larger than PAGESIZE is empty and step 1 24987c478bd9Sstevel@tonic-gate * will not work since all larger size lists are 24997c478bd9Sstevel@tonic-gate * also empty. 25007c478bd9Sstevel@tonic-gate * 25017c478bd9Sstevel@tonic-gate * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 25027c478bd9Sstevel@tonic-gate */ 25035d07b933Sdp 25047c478bd9Sstevel@tonic-gate page_t * 25055d07b933Sdp page_freelist_split(uchar_t szc, uint_t color, int mnode, int mtype, 25065d07b933Sdp pfn_t pfnhi, page_list_walker_t *plw) 25077c478bd9Sstevel@tonic-gate { 25087c478bd9Sstevel@tonic-gate uchar_t nszc = szc + 1; 25095d07b933Sdp uint_t bin, sbin, bin_prev; 25107c478bd9Sstevel@tonic-gate page_t *pp, *firstpp; 25117c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 25125d07b933Sdp uint_t color_mask; 25137c478bd9Sstevel@tonic-gate 25145d07b933Sdp if (nszc == mmu_page_sizes) 25155d07b933Sdp return (NULL); 25167c478bd9Sstevel@tonic-gate 25175d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25185d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nszc) - 1; 25195d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(szc, color); 25205d07b933Sdp bin_prev = (plw->plw_bin_split_prev == color) ? INVALID_COLOR : 25215d07b933Sdp PAGE_GET_NSZ_COLOR(szc, plw->plw_bin_split_prev); 25225d07b933Sdp 25235d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_req[szc]); 25247c478bd9Sstevel@tonic-gate /* 25255d07b933Sdp * First try to break up a larger page to fill current size freelist. 25267c478bd9Sstevel@tonic-gate */ 25275d07b933Sdp while (plw->plw_bins[nszc] != 0) { 25285d07b933Sdp 25295d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25305d07b933Sdp 25317c478bd9Sstevel@tonic-gate /* 25327c478bd9Sstevel@tonic-gate * If page found then demote it. 25337c478bd9Sstevel@tonic-gate */ 25347c478bd9Sstevel@tonic-gate if (PAGE_FREELISTS(mnode, nszc, bin, mtype)) { 25357c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 25367c478bd9Sstevel@tonic-gate firstpp = pp = PAGE_FREELISTS(mnode, nszc, bin, mtype); 25377c478bd9Sstevel@tonic-gate 25387c478bd9Sstevel@tonic-gate /* 25397c478bd9Sstevel@tonic-gate * If pfnhi is not PFNNULL, look for large page below 25407c478bd9Sstevel@tonic-gate * pfnhi. PFNNULL signifies no pfn requirement. 25417c478bd9Sstevel@tonic-gate */ 25427c478bd9Sstevel@tonic-gate if (pfnhi != PFNNULL && pp->p_pagenum >= pfnhi) { 25437c478bd9Sstevel@tonic-gate do { 25447c478bd9Sstevel@tonic-gate pp = pp->p_vpnext; 25457c478bd9Sstevel@tonic-gate if (pp == firstpp) { 25467c478bd9Sstevel@tonic-gate pp = NULL; 25477c478bd9Sstevel@tonic-gate break; 25487c478bd9Sstevel@tonic-gate } 25497c478bd9Sstevel@tonic-gate } while (pp->p_pagenum >= pfnhi); 25507c478bd9Sstevel@tonic-gate } 25517c478bd9Sstevel@tonic-gate if (pp) { 25525d07b933Sdp uint_t ccolor = page_correct_color(szc, nszc, 25535d07b933Sdp color, bin, plw->plw_ceq_mask[szc]); 25545d07b933Sdp 25557c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == nszc); 25565d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_demote[nszc]); 25577c478bd9Sstevel@tonic-gate ret_pp = page_demote(mnode, pp->p_pagenum, 25585d07b933Sdp pp->p_szc, szc, ccolor, PC_ALLOC); 25597c478bd9Sstevel@tonic-gate if (ret_pp) { 25607c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 25617c478bd9Sstevel@tonic-gate #if defined(__sparc) 25627c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 25637c478bd9Sstevel@tonic-gate pgcnt_t npgs; 25647c478bd9Sstevel@tonic-gate 25657c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt( 25667c478bd9Sstevel@tonic-gate ret_pp->p_szc); 25677c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 25687c478bd9Sstevel@tonic-gate } 25697c478bd9Sstevel@tonic-gate #endif 25707c478bd9Sstevel@tonic-gate return (ret_pp); 25717c478bd9Sstevel@tonic-gate } 25727c478bd9Sstevel@tonic-gate } 25737c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 25747c478bd9Sstevel@tonic-gate } 25757c478bd9Sstevel@tonic-gate 25765d07b933Sdp /* loop through next size bins */ 25775d07b933Sdp bin = ADD_MASKED(bin, 1, plw->plw_ceq_mask[nszc], color_mask); 25785d07b933Sdp plw->plw_bins[nszc]--; 25795d07b933Sdp 25805d07b933Sdp if (bin == sbin) { 25815d07b933Sdp uchar_t nnszc = nszc + 1; 25825d07b933Sdp 25835d07b933Sdp /* we are done with this page size - check next */ 25845d07b933Sdp if (plw->plw_bins[nnszc] == 0) 25855d07b933Sdp /* we have already checked next size bins */ 25865d07b933Sdp break; 25875d07b933Sdp 25885d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(nszc, bin); 25895d07b933Sdp if (bin_prev != INVALID_COLOR) { 25905d07b933Sdp bin_prev = PAGE_GET_NSZ_COLOR(nszc, bin_prev); 25915d07b933Sdp if (!((bin ^ bin_prev) & 25925d07b933Sdp plw->plw_ceq_mask[nnszc])) 25935d07b933Sdp break; 25945d07b933Sdp } 25955d07b933Sdp ASSERT(nnszc < mmu_page_sizes); 25965d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nnszc) - 1; 25975d07b933Sdp nszc = nnszc; 25985d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25995d07b933Sdp } 26007c478bd9Sstevel@tonic-gate } 26017c478bd9Sstevel@tonic-gate 26027c478bd9Sstevel@tonic-gate return (ret_pp); 26037c478bd9Sstevel@tonic-gate } 26047c478bd9Sstevel@tonic-gate 26057c478bd9Sstevel@tonic-gate /* 26067c478bd9Sstevel@tonic-gate * Helper routine used only by the freelist code to lock 26077c478bd9Sstevel@tonic-gate * a page. If the page is a large page then it succeeds in 26087c478bd9Sstevel@tonic-gate * locking all the constituent pages or none at all. 26097c478bd9Sstevel@tonic-gate * Returns 1 on sucess, 0 on failure. 26107c478bd9Sstevel@tonic-gate */ 26117c478bd9Sstevel@tonic-gate static int 26127c478bd9Sstevel@tonic-gate page_trylock_cons(page_t *pp, se_t se) 26137c478bd9Sstevel@tonic-gate { 26147c478bd9Sstevel@tonic-gate page_t *tpp, *first_pp = pp; 26157c478bd9Sstevel@tonic-gate 26167c478bd9Sstevel@tonic-gate /* 26177c478bd9Sstevel@tonic-gate * Fail if can't lock first or only page. 26187c478bd9Sstevel@tonic-gate */ 26197c478bd9Sstevel@tonic-gate if (!page_trylock(pp, se)) { 26207c478bd9Sstevel@tonic-gate return (0); 26217c478bd9Sstevel@tonic-gate } 26227c478bd9Sstevel@tonic-gate 26237c478bd9Sstevel@tonic-gate /* 26247c478bd9Sstevel@tonic-gate * PAGESIZE: common case. 26257c478bd9Sstevel@tonic-gate */ 26267c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 26277c478bd9Sstevel@tonic-gate return (1); 26287c478bd9Sstevel@tonic-gate } 26297c478bd9Sstevel@tonic-gate 26307c478bd9Sstevel@tonic-gate /* 26317c478bd9Sstevel@tonic-gate * Large page case. 26327c478bd9Sstevel@tonic-gate */ 26337c478bd9Sstevel@tonic-gate tpp = pp->p_next; 26347c478bd9Sstevel@tonic-gate while (tpp != pp) { 26357c478bd9Sstevel@tonic-gate if (!page_trylock(tpp, se)) { 26367c478bd9Sstevel@tonic-gate /* 26378b464eb8Smec * On failure unlock what we have locked so far. 26388b464eb8Smec * We want to avoid attempting to capture these 26398b464eb8Smec * pages as the pcm mutex may be held which could 26408b464eb8Smec * lead to a recursive mutex panic. 26417c478bd9Sstevel@tonic-gate */ 26427c478bd9Sstevel@tonic-gate while (first_pp != tpp) { 26438b464eb8Smec page_unlock_nocapture(first_pp); 26447c478bd9Sstevel@tonic-gate first_pp = first_pp->p_next; 26457c478bd9Sstevel@tonic-gate } 26467c478bd9Sstevel@tonic-gate return (0); 26477c478bd9Sstevel@tonic-gate } 26487c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 26497c478bd9Sstevel@tonic-gate } 26507c478bd9Sstevel@tonic-gate return (1); 26517c478bd9Sstevel@tonic-gate } 26527c478bd9Sstevel@tonic-gate 26535d07b933Sdp /* 26545d07b933Sdp * init context for walking page lists 26555d07b933Sdp * Called when a page of the given szc in unavailable. Sets markers 26565d07b933Sdp * for the beginning of the search to detect when search has 26575d07b933Sdp * completed a full cycle. Sets flags for splitting larger pages 26585d07b933Sdp * and coalescing smaller pages. Page walking procedes until a page 26595d07b933Sdp * of the desired equivalent color is found. 26605d07b933Sdp */ 26615d07b933Sdp void 26625d07b933Sdp page_list_walk_init(uchar_t szc, uint_t flags, uint_t bin, int can_split, 26635d07b933Sdp int use_ceq, page_list_walker_t *plw) 26647c478bd9Sstevel@tonic-gate { 26655d07b933Sdp uint_t nszc, ceq_mask, colors; 26665d07b933Sdp uchar_t ceq = use_ceq ? colorequivszc[szc] : 0; 26677c478bd9Sstevel@tonic-gate 26687c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 26695d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 26707c478bd9Sstevel@tonic-gate 26715d07b933Sdp plw->plw_colors = colors; 26725d07b933Sdp plw->plw_color_mask = colors - 1; 26735d07b933Sdp plw->plw_bin_marker = plw->plw_bin0 = bin; 26745d07b933Sdp plw->plw_bin_split_prev = bin; 26755d07b933Sdp plw->plw_bin_step = (szc == 0) ? vac_colors : 1; 26767c478bd9Sstevel@tonic-gate 26775d07b933Sdp /* 26785d07b933Sdp * if vac aliasing is possible make sure lower order color 26795d07b933Sdp * bits are never ignored 26805d07b933Sdp */ 26815d07b933Sdp if (vac_colors > 1) 26825d07b933Sdp ceq &= 0xf0; 26837c478bd9Sstevel@tonic-gate 26847c478bd9Sstevel@tonic-gate /* 26855d07b933Sdp * calculate the number of non-equivalent colors and 26865d07b933Sdp * color equivalency mask 26877c478bd9Sstevel@tonic-gate */ 26885d07b933Sdp plw->plw_ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 26895d07b933Sdp ASSERT(szc > 0 || plw->plw_ceq_dif >= vac_colors); 26905d07b933Sdp ASSERT(plw->plw_ceq_dif > 0); 26915d07b933Sdp plw->plw_ceq_mask[szc] = (plw->plw_ceq_dif - 1) << (ceq & 0xf); 26927c478bd9Sstevel@tonic-gate 26935d07b933Sdp if (flags & PG_MATCH_COLOR) { 26945d07b933Sdp if (cpu_page_colors < 0) { 26955d07b933Sdp /* 26965d07b933Sdp * this is a heterogeneous machine with different CPUs 26975d07b933Sdp * having different size e$ (not supported for ni2/rock 26985d07b933Sdp */ 26995d07b933Sdp uint_t cpucolors = CPUSETSIZE() >> PAGE_GET_SHIFT(szc); 27005d07b933Sdp cpucolors = MAX(cpucolors, 1); 27015d07b933Sdp ceq_mask = plw->plw_color_mask & (cpucolors - 1); 27025d07b933Sdp plw->plw_ceq_mask[szc] = 27035d07b933Sdp MIN(ceq_mask, plw->plw_ceq_mask[szc]); 27045d07b933Sdp } 27055d07b933Sdp plw->plw_ceq_dif = 1; 27065d07b933Sdp } 27077c478bd9Sstevel@tonic-gate 27085d07b933Sdp /* we can split pages in the freelist, but not the cachelist */ 27095d07b933Sdp if (can_split) { 2710ce8eb11aSdp plw->plw_do_split = (szc + 1 < mmu_page_sizes) ? 1 : 0; 27117c478bd9Sstevel@tonic-gate 2712ce8eb11aSdp /* set next szc color masks and number of free list bins */ 2713ce8eb11aSdp for (nszc = szc + 1; nszc < mmu_page_sizes; nszc++, szc++) { 2714ce8eb11aSdp plw->plw_ceq_mask[nszc] = PAGE_GET_NSZ_MASK(szc, 2715ce8eb11aSdp plw->plw_ceq_mask[szc]); 2716ce8eb11aSdp plw->plw_bins[nszc] = PAGE_GET_PAGECOLORS(nszc); 2717ce8eb11aSdp } 2718ce8eb11aSdp plw->plw_ceq_mask[nszc] = INVALID_MASK; 2719ce8eb11aSdp plw->plw_bins[nszc] = 0; 27207c478bd9Sstevel@tonic-gate 27215d07b933Sdp } else { 2722ce8eb11aSdp ASSERT(szc == 0); 2723ce8eb11aSdp plw->plw_do_split = 0; 2724ce8eb11aSdp plw->plw_bins[1] = 0; 2725ce8eb11aSdp plw->plw_ceq_mask[1] = INVALID_MASK; 27267c478bd9Sstevel@tonic-gate } 27275d07b933Sdp } 27287c478bd9Sstevel@tonic-gate 27295d07b933Sdp /* 27305d07b933Sdp * set mark to flag where next split should occur 27315d07b933Sdp */ 27325d07b933Sdp #define PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw) { \ 27335d07b933Sdp uint_t bin_nsz = PAGE_GET_NSZ_COLOR(szc, bin); \ 27345d07b933Sdp uint_t bin0_nsz = PAGE_GET_NSZ_COLOR(szc, plw->plw_bin0); \ 27355d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[nszc] & plw->plw_color_mask; \ 27365d07b933Sdp plw->plw_split_next = \ 27375d07b933Sdp INC_MASKED(bin_nsz, neq_mask, plw->plw_color_mask); \ 27385d07b933Sdp if (!((plw->plw_split_next ^ bin0_nsz) & plw->plw_ceq_mask[nszc])) { \ 27395d07b933Sdp plw->plw_split_next = \ 27405d07b933Sdp INC_MASKED(plw->plw_split_next, \ 27415d07b933Sdp neq_mask, plw->plw_color_mask); \ 27425d07b933Sdp } \ 27435d07b933Sdp } 27447c478bd9Sstevel@tonic-gate 27455d07b933Sdp uint_t 27465d07b933Sdp page_list_walk_next_bin(uchar_t szc, uint_t bin, page_list_walker_t *plw) 27475d07b933Sdp { 27485d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[szc] & plw->plw_color_mask; 27495d07b933Sdp uint_t bin0_nsz, nbin_nsz, nbin0, nbin; 27505d07b933Sdp uchar_t nszc = szc + 1; 27515d07b933Sdp 27525d07b933Sdp nbin = ADD_MASKED(bin, 27535d07b933Sdp plw->plw_bin_step, neq_mask, plw->plw_color_mask); 27545d07b933Sdp 27555d07b933Sdp if (plw->plw_do_split) { 27565d07b933Sdp plw->plw_bin_split_prev = bin; 27575d07b933Sdp PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw); 27585d07b933Sdp plw->plw_do_split = 0; 27595d07b933Sdp } 27605d07b933Sdp 27615d07b933Sdp if (szc == 0) { 27625d07b933Sdp if (plw->plw_count != 0 || plw->plw_ceq_dif == vac_colors) { 27635d07b933Sdp if (nbin == plw->plw_bin0 && 27645d07b933Sdp (vac_colors == 1 || nbin != plw->plw_bin_marker)) { 27655d07b933Sdp nbin = ADD_MASKED(nbin, plw->plw_bin_step, 27665d07b933Sdp neq_mask, plw->plw_color_mask); 27675d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 27685d07b933Sdp } 27695d07b933Sdp 27705d07b933Sdp if (vac_colors > 1 && nbin == plw->plw_bin_marker) { 27715d07b933Sdp plw->plw_bin_marker = 27725d07b933Sdp nbin = INC_MASKED(nbin, neq_mask, 2773ce8eb11aSdp plw->plw_color_mask); 27745d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 27755d07b933Sdp /* 27765d07b933Sdp * large pages all have the same vac color 27775d07b933Sdp * so by now we should be done with next 27785d07b933Sdp * size page splitting process 27795d07b933Sdp */ 27805d07b933Sdp ASSERT(plw->plw_bins[1] == 0); 27815d07b933Sdp plw->plw_do_split = 0; 27825d07b933Sdp return (nbin); 27835d07b933Sdp } 27845d07b933Sdp 27855d07b933Sdp } else { 27865d07b933Sdp uint_t bin_jump = (vac_colors == 1) ? 27875d07b933Sdp (BIN_STEP & ~3) - (plw->plw_bin0 & 3) : BIN_STEP; 27885d07b933Sdp 27895d07b933Sdp bin_jump &= ~(vac_colors - 1); 27905d07b933Sdp 27915d07b933Sdp nbin0 = ADD_MASKED(plw->plw_bin0, bin_jump, neq_mask, 27925d07b933Sdp plw->plw_color_mask); 27935d07b933Sdp 27945d07b933Sdp if ((nbin0 ^ plw->plw_bin0) & plw->plw_ceq_mask[szc]) { 27955d07b933Sdp 27965d07b933Sdp plw->plw_bin_marker = nbin = nbin0; 27975d07b933Sdp 27985d07b933Sdp if (plw->plw_bins[nszc] != 0) { 27995d07b933Sdp /* 28005d07b933Sdp * check if next page size bin is the 28015d07b933Sdp * same as the next page size bin for 28025d07b933Sdp * bin0 28035d07b933Sdp */ 28045d07b933Sdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, 28055d07b933Sdp nbin); 28065d07b933Sdp bin0_nsz = PAGE_GET_NSZ_COLOR(szc, 28075d07b933Sdp plw->plw_bin0); 28085d07b933Sdp 28095d07b933Sdp if ((bin0_nsz ^ nbin_nsz) & 28105d07b933Sdp plw->plw_ceq_mask[nszc]) 28115d07b933Sdp plw->plw_do_split = 1; 28125d07b933Sdp } 28135d07b933Sdp return (nbin); 28145d07b933Sdp } 28155d07b933Sdp } 28165d07b933Sdp } 28175d07b933Sdp 28185d07b933Sdp if (plw->plw_bins[nszc] != 0) { 2819ce8eb11aSdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, nbin); 2820ce8eb11aSdp if (!((plw->plw_split_next ^ nbin_nsz) & 2821ce8eb11aSdp plw->plw_ceq_mask[nszc])) 2822ce8eb11aSdp plw->plw_do_split = 1; 28235d07b933Sdp } 28245d07b933Sdp 28255d07b933Sdp return (nbin); 28265d07b933Sdp } 28275d07b933Sdp 28285d07b933Sdp page_t * 28295d07b933Sdp page_get_mnode_freelist(int mnode, uint_t bin, int mtype, uchar_t szc, 28305d07b933Sdp uint_t flags) 28315d07b933Sdp { 28325d07b933Sdp kmutex_t *pcm; 28335d07b933Sdp page_t *pp, *first_pp; 28345d07b933Sdp uint_t sbin; 28355d07b933Sdp int plw_initialized; 28365d07b933Sdp page_list_walker_t plw; 28375d07b933Sdp 28385d07b933Sdp ASSERT(szc < mmu_page_sizes); 28395d07b933Sdp 28405d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_alloc[szc]); 28415d07b933Sdp 28425d07b933Sdp MTYPE_START(mnode, mtype, flags); 28435d07b933Sdp if (mtype < 0) { /* mnode does not have memory in mtype range */ 28445d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocempty[szc]); 28455d07b933Sdp return (NULL); 28465d07b933Sdp } 28475d07b933Sdp try_again: 28485d07b933Sdp 28495d07b933Sdp plw_initialized = 0; 28505d07b933Sdp plw.plw_ceq_dif = 1; 28517c478bd9Sstevel@tonic-gate 28527c478bd9Sstevel@tonic-gate /* 28537c478bd9Sstevel@tonic-gate * Only hold one freelist lock at a time, that way we 28547c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 28557c478bd9Sstevel@tonic-gate * ordering. 28567c478bd9Sstevel@tonic-gate */ 28575d07b933Sdp for (plw.plw_count = 0; 28585d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 28595d07b933Sdp sbin = bin; 28605d07b933Sdp do { 28615d07b933Sdp if (!PAGE_FREELISTS(mnode, szc, bin, mtype)) 28625d07b933Sdp goto bin_empty_1; 28635d07b933Sdp 28647c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 28657c478bd9Sstevel@tonic-gate mutex_enter(pcm); 28667c478bd9Sstevel@tonic-gate pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 28675d07b933Sdp if (pp == NULL) 28685d07b933Sdp goto bin_empty_0; 28695d07b933Sdp 28705d07b933Sdp /* 28715d07b933Sdp * These were set before the page 28725d07b933Sdp * was put on the free list, 28735d07b933Sdp * they must still be set. 28745d07b933Sdp */ 28755d07b933Sdp ASSERT(PP_ISFREE(pp)); 28765d07b933Sdp ASSERT(PP_ISAGED(pp)); 28775d07b933Sdp ASSERT(pp->p_vnode == NULL); 28785d07b933Sdp ASSERT(pp->p_hash == NULL); 28795d07b933Sdp ASSERT(pp->p_offset == (u_offset_t)-1); 28805d07b933Sdp ASSERT(pp->p_szc == szc); 28815d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 28825d07b933Sdp 28835d07b933Sdp /* 28845d07b933Sdp * Walk down the hash chain. 28855d07b933Sdp * 8k pages are linked on p_next 28865d07b933Sdp * and p_prev fields. Large pages 28875d07b933Sdp * are a contiguous group of 28885d07b933Sdp * constituent pages linked together 28895d07b933Sdp * on their p_next and p_prev fields. 28905d07b933Sdp * The large pages are linked together 28915d07b933Sdp * on the hash chain using p_vpnext 28925d07b933Sdp * p_vpprev of the base constituent 28935d07b933Sdp * page of each large page. 28945d07b933Sdp */ 28955d07b933Sdp first_pp = pp; 28965d07b933Sdp while (!page_trylock_cons(pp, SE_EXCL)) { 28975d07b933Sdp if (szc == 0) { 28985d07b933Sdp pp = pp->p_next; 28995d07b933Sdp } else { 29005d07b933Sdp pp = pp->p_vpnext; 29015d07b933Sdp } 29025d07b933Sdp 29037c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 29047c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 29057c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 29067c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL); 29077c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1); 29087c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 29097c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 29107c478bd9Sstevel@tonic-gate 29115d07b933Sdp if (pp == first_pp) 29125d07b933Sdp goto bin_empty_0; 29135d07b933Sdp } 29147c478bd9Sstevel@tonic-gate 29155d07b933Sdp ASSERT(pp != NULL); 29165d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 29175d07b933Sdp ASSERT(pp->p_szc == szc); 29185d07b933Sdp if (szc == 0) { 29195d07b933Sdp page_sub(&PAGE_FREELISTS(mnode, 29205d07b933Sdp szc, bin, mtype), pp); 29215d07b933Sdp } else { 29225d07b933Sdp page_vpsub(&PAGE_FREELISTS(mnode, 29235d07b933Sdp szc, bin, mtype), pp); 29245d07b933Sdp CHK_LPG(pp, szc); 29255d07b933Sdp } 29265d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 29277c478bd9Sstevel@tonic-gate 29285d07b933Sdp if ((PP_ISFREE(pp) == 0) || (PP_ISAGED(pp) == 0)) 29295d07b933Sdp panic("free page is not. pp %p", (void *)pp); 29305d07b933Sdp mutex_exit(pcm); 29317c478bd9Sstevel@tonic-gate 29327c478bd9Sstevel@tonic-gate #if defined(__sparc) 29335d07b933Sdp ASSERT(!kcage_on || PP_ISNORELOC(pp) || 29345d07b933Sdp (flags & PG_NORELOC) == 0); 29357c478bd9Sstevel@tonic-gate 29365d07b933Sdp if (PP_ISNORELOC(pp)) 29375d07b933Sdp kcage_freemem_sub(page_get_pagecnt(szc)); 29387c478bd9Sstevel@tonic-gate #endif 29395d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocok[szc]); 29405d07b933Sdp return (pp); 29417c478bd9Sstevel@tonic-gate 29425d07b933Sdp bin_empty_0: 29435d07b933Sdp mutex_exit(pcm); 29445d07b933Sdp bin_empty_1: 29455d07b933Sdp if (plw_initialized == 0) { 29465d07b933Sdp page_list_walk_init(szc, flags, bin, 1, 1, 29475d07b933Sdp &plw); 29485d07b933Sdp plw_initialized = 1; 29495d07b933Sdp ASSERT(plw.plw_colors <= 29505d07b933Sdp PAGE_GET_PAGECOLORS(szc)); 29515d07b933Sdp ASSERT(plw.plw_colors > 0); 29525d07b933Sdp ASSERT((plw.plw_colors & 29535d07b933Sdp (plw.plw_colors - 1)) == 0); 29545d07b933Sdp ASSERT(bin < plw.plw_colors); 29555d07b933Sdp ASSERT(plw.plw_ceq_mask[szc] < plw.plw_colors); 29567c478bd9Sstevel@tonic-gate } 29575d07b933Sdp /* calculate the next bin with equivalent color */ 29585d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 29595d07b933Sdp plw.plw_ceq_mask[szc], plw.plw_color_mask); 29605d07b933Sdp } while (sbin != bin); 29617c478bd9Sstevel@tonic-gate 29627c478bd9Sstevel@tonic-gate /* 29635d07b933Sdp * color bins are all empty if color match. Try and 29645d07b933Sdp * satisfy the request by breaking up or coalescing 29655d07b933Sdp * pages from a different size freelist of the correct 29665d07b933Sdp * color that satisfies the ORIGINAL color requested. 29675d07b933Sdp * If that fails then try pages of the same size but 29685d07b933Sdp * different colors assuming we are not called with 29697c478bd9Sstevel@tonic-gate * PG_MATCH_COLOR. 29707c478bd9Sstevel@tonic-gate */ 29715d07b933Sdp if (plw.plw_do_split && 29725d07b933Sdp (pp = page_freelist_split(szc, bin, mnode, 2973ce8eb11aSdp mtype, PFNNULL, &plw)) != NULL) 2974ce8eb11aSdp return (pp); 29757c478bd9Sstevel@tonic-gate 29765d07b933Sdp if (szc > 0 && (pp = page_freelist_coalesce(mnode, szc, 29775d07b933Sdp bin, plw.plw_ceq_mask[szc], mtype, PFNNULL)) != NULL) 29785d07b933Sdp return (pp); 29797c478bd9Sstevel@tonic-gate 29805d07b933Sdp if (plw.plw_ceq_dif > 1) 29815d07b933Sdp bin = page_list_walk_next_bin(szc, bin, &plw); 29827c478bd9Sstevel@tonic-gate } 29837c478bd9Sstevel@tonic-gate 2984affbd3ccSkchow /* if allowed, cycle through additional mtypes */ 2985affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 2986affbd3ccSkchow if (mtype >= 0) 29875d07b933Sdp goto try_again; 2988affbd3ccSkchow 29897c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmf_allocfailed[szc]); 29907c478bd9Sstevel@tonic-gate 29917c478bd9Sstevel@tonic-gate return (NULL); 29927c478bd9Sstevel@tonic-gate } 29937c478bd9Sstevel@tonic-gate 29947c478bd9Sstevel@tonic-gate /* 29957c478bd9Sstevel@tonic-gate * Returns the count of free pages for 'pp' with size code 'szc'. 29967c478bd9Sstevel@tonic-gate * Note: This function does not return an exact value as the page freelist 29977c478bd9Sstevel@tonic-gate * locks are not held and thus the values in the page_counters may be 29987c478bd9Sstevel@tonic-gate * changing as we walk through the data. 29997c478bd9Sstevel@tonic-gate */ 30007c478bd9Sstevel@tonic-gate static int 30017c478bd9Sstevel@tonic-gate page_freecnt(int mnode, page_t *pp, uchar_t szc) 30027c478bd9Sstevel@tonic-gate { 30037c478bd9Sstevel@tonic-gate pgcnt_t pgfree; 30047c478bd9Sstevel@tonic-gate pgcnt_t cnt; 30057c478bd9Sstevel@tonic-gate ssize_t r = szc; /* region size */ 30067c478bd9Sstevel@tonic-gate ssize_t idx; 30077c478bd9Sstevel@tonic-gate int i; 30087c478bd9Sstevel@tonic-gate int full, range; 30097c478bd9Sstevel@tonic-gate 30107c478bd9Sstevel@tonic-gate /* Make sure pagenum passed in is aligned properly */ 30117c478bd9Sstevel@tonic-gate ASSERT((pp->p_pagenum & (PNUM_SIZE(szc) - 1)) == 0); 30127c478bd9Sstevel@tonic-gate ASSERT(szc > 0); 30137c478bd9Sstevel@tonic-gate 30147c478bd9Sstevel@tonic-gate /* Prevent page_counters dynamic memory from being freed */ 30157c478bd9Sstevel@tonic-gate rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 30167c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30177c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30187c478bd9Sstevel@tonic-gate pgfree = cnt << PNUM_SHIFT(r - 1); 30197c478bd9Sstevel@tonic-gate range = FULL_REGION_CNT(szc); 30207c478bd9Sstevel@tonic-gate 30217c478bd9Sstevel@tonic-gate /* Check for completely full region */ 30227c478bd9Sstevel@tonic-gate if (cnt == range) { 30237c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30247c478bd9Sstevel@tonic-gate return (pgfree); 30257c478bd9Sstevel@tonic-gate } 30267c478bd9Sstevel@tonic-gate 30277c478bd9Sstevel@tonic-gate while (--r > 0) { 30287c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30297c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 30307c478bd9Sstevel@tonic-gate for (i = 0; i < range; i++, idx++) { 30317c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30327c478bd9Sstevel@tonic-gate /* 30337c478bd9Sstevel@tonic-gate * If cnt here is full, that means we have already 30347c478bd9Sstevel@tonic-gate * accounted for these pages earlier. 30357c478bd9Sstevel@tonic-gate */ 30367c478bd9Sstevel@tonic-gate if (cnt != full) { 30377c478bd9Sstevel@tonic-gate pgfree += (cnt << PNUM_SHIFT(r - 1)); 30387c478bd9Sstevel@tonic-gate } 30397c478bd9Sstevel@tonic-gate } 30407c478bd9Sstevel@tonic-gate range *= full; 30417c478bd9Sstevel@tonic-gate } 30427c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30437c478bd9Sstevel@tonic-gate return (pgfree); 30447c478bd9Sstevel@tonic-gate } 30457c478bd9Sstevel@tonic-gate 30467c478bd9Sstevel@tonic-gate /* 30477c478bd9Sstevel@tonic-gate * Called from page_geti_contig_pages to exclusively lock constituent pages 30487c478bd9Sstevel@tonic-gate * starting from 'spp' for page size code 'szc'. 30497c478bd9Sstevel@tonic-gate * 30507c478bd9Sstevel@tonic-gate * If 'ptcpthreshold' is set, the number of free pages needed in the 'szc' 30517c478bd9Sstevel@tonic-gate * region needs to be greater than or equal to the threshold. 30527c478bd9Sstevel@tonic-gate */ 30537c478bd9Sstevel@tonic-gate static int 30547c478bd9Sstevel@tonic-gate page_trylock_contig_pages(int mnode, page_t *spp, uchar_t szc, int flags) 30557c478bd9Sstevel@tonic-gate { 30567c478bd9Sstevel@tonic-gate pgcnt_t pgcnt = PNUM_SIZE(szc); 30577c478bd9Sstevel@tonic-gate pgcnt_t pgfree, i; 30587c478bd9Sstevel@tonic-gate page_t *pp; 30597c478bd9Sstevel@tonic-gate 30607c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcp[szc]); 30617c478bd9Sstevel@tonic-gate 30627c478bd9Sstevel@tonic-gate 30637c478bd9Sstevel@tonic-gate if ((ptcpthreshold == 0) || (flags & PGI_PGCPHIPRI)) 30647c478bd9Sstevel@tonic-gate goto skipptcpcheck; 30657c478bd9Sstevel@tonic-gate /* 30667c478bd9Sstevel@tonic-gate * check if there are sufficient free pages available before attempting 30677c478bd9Sstevel@tonic-gate * to trylock. Count is approximate as page counters can change. 30687c478bd9Sstevel@tonic-gate */ 30697c478bd9Sstevel@tonic-gate pgfree = page_freecnt(mnode, spp, szc); 30707c478bd9Sstevel@tonic-gate 30717c478bd9Sstevel@tonic-gate /* attempt to trylock if there are sufficient already free pages */ 30727c478bd9Sstevel@tonic-gate if (pgfree < pgcnt/ptcpthreshold) { 30737c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfreethresh[szc]); 30747c478bd9Sstevel@tonic-gate return (0); 30757c478bd9Sstevel@tonic-gate } 30767c478bd9Sstevel@tonic-gate 30777c478bd9Sstevel@tonic-gate skipptcpcheck: 30787c478bd9Sstevel@tonic-gate 30797c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++) { 30807c478bd9Sstevel@tonic-gate pp = &spp[i]; 30817c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 30827c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailexcl[szc]); 30837c478bd9Sstevel@tonic-gate while (--i != (pgcnt_t)-1) { 30847c478bd9Sstevel@tonic-gate pp = &spp[i]; 30857c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 30868b464eb8Smec page_unlock_nocapture(pp); 30877c478bd9Sstevel@tonic-gate } 30887c478bd9Sstevel@tonic-gate return (0); 30897c478bd9Sstevel@tonic-gate } 30907c478bd9Sstevel@tonic-gate ASSERT(spp[i].p_pagenum == spp->p_pagenum + i); 30917c478bd9Sstevel@tonic-gate if ((pp->p_szc > szc || (szc && pp->p_szc == szc)) && 30927c478bd9Sstevel@tonic-gate !PP_ISFREE(pp)) { 30937c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailszc[szc]); 30947c478bd9Sstevel@tonic-gate ASSERT(i == 0); 30958b464eb8Smec page_unlock_nocapture(pp); 30967c478bd9Sstevel@tonic-gate return (0); 30977c478bd9Sstevel@tonic-gate } 30987c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 30997c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailcage[szc]); 31007c478bd9Sstevel@tonic-gate while (i != (pgcnt_t)-1) { 31017c478bd9Sstevel@tonic-gate pp = &spp[i]; 31027c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31038b464eb8Smec page_unlock_nocapture(pp); 31047c478bd9Sstevel@tonic-gate i--; 31057c478bd9Sstevel@tonic-gate } 31067c478bd9Sstevel@tonic-gate return (0); 31077c478bd9Sstevel@tonic-gate } 31087c478bd9Sstevel@tonic-gate } 31097c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpok[szc]); 31107c478bd9Sstevel@tonic-gate return (1); 31117c478bd9Sstevel@tonic-gate } 31127c478bd9Sstevel@tonic-gate 31137c478bd9Sstevel@tonic-gate /* 31147c478bd9Sstevel@tonic-gate * Claim large page pointed to by 'pp'. 'pp' is the starting set 31157c478bd9Sstevel@tonic-gate * of 'szc' constituent pages that had been locked exclusively previously. 31167c478bd9Sstevel@tonic-gate * Will attempt to relocate constituent pages in use. 31177c478bd9Sstevel@tonic-gate */ 31187c478bd9Sstevel@tonic-gate static page_t * 31197c478bd9Sstevel@tonic-gate page_claim_contig_pages(page_t *pp, uchar_t szc, int flags) 31207c478bd9Sstevel@tonic-gate { 31217c478bd9Sstevel@tonic-gate spgcnt_t pgcnt, npgs, i; 31227c478bd9Sstevel@tonic-gate page_t *targpp, *rpp, *hpp; 31237c478bd9Sstevel@tonic-gate page_t *replpp = NULL; 31247c478bd9Sstevel@tonic-gate page_t *pplist = NULL; 31257c478bd9Sstevel@tonic-gate 31267c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 31277c478bd9Sstevel@tonic-gate 31287c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(szc); 31297c478bd9Sstevel@tonic-gate while (pgcnt) { 31307c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31317c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 31327c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 31337c478bd9Sstevel@tonic-gate /* 31347c478bd9Sstevel@tonic-gate * If this is a PG_FREE_LIST page then its 31357c478bd9Sstevel@tonic-gate * size code can change underneath us due to 31367c478bd9Sstevel@tonic-gate * page promotion or demotion. As an optimzation 31377c478bd9Sstevel@tonic-gate * use page_list_sub_pages() instead of 31387c478bd9Sstevel@tonic-gate * page_list_sub(). 31397c478bd9Sstevel@tonic-gate */ 31407c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 31417c478bd9Sstevel@tonic-gate page_list_sub_pages(pp, szc); 31427c478bd9Sstevel@tonic-gate if (pp->p_szc == szc) { 31437c478bd9Sstevel@tonic-gate return (pp); 31447c478bd9Sstevel@tonic-gate } 31457c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < szc); 31467c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31477c478bd9Sstevel@tonic-gate hpp = pp; 31487c478bd9Sstevel@tonic-gate for (i = 0; i < npgs; i++, pp++) { 31497c478bd9Sstevel@tonic-gate pp->p_szc = szc; 31507c478bd9Sstevel@tonic-gate } 31517c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &hpp); 31527c478bd9Sstevel@tonic-gate pgcnt -= npgs; 31537c478bd9Sstevel@tonic-gate continue; 31547c478bd9Sstevel@tonic-gate } 31557c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 31567c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 31577c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 31587c478bd9Sstevel@tonic-gate page_hashout(pp, NULL); 31597c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 31607c478bd9Sstevel@tonic-gate pp->p_szc = szc; 31617c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 31627c478bd9Sstevel@tonic-gate pp++; 31637c478bd9Sstevel@tonic-gate pgcnt--; 31647c478bd9Sstevel@tonic-gate continue; 31657c478bd9Sstevel@tonic-gate } 31667c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31677c478bd9Sstevel@tonic-gate 31687c478bd9Sstevel@tonic-gate /* 31697c478bd9Sstevel@tonic-gate * page_create_wait freemem accounting done by caller of 31707c478bd9Sstevel@tonic-gate * page_get_freelist and not necessary to call it prior to 31717c478bd9Sstevel@tonic-gate * calling page_get_replacement_page. 31727c478bd9Sstevel@tonic-gate * 31737c478bd9Sstevel@tonic-gate * page_get_replacement_page can call page_get_contig_pages 31747c478bd9Sstevel@tonic-gate * to acquire a large page (szc > 0); the replacement must be 31757c478bd9Sstevel@tonic-gate * smaller than the contig page size to avoid looping or 31767c478bd9Sstevel@tonic-gate * szc == 0 and PGI_PGCPSZC0 is set. 31777c478bd9Sstevel@tonic-gate */ 31787c478bd9Sstevel@tonic-gate if (pp->p_szc < szc || (szc == 0 && (flags & PGI_PGCPSZC0))) { 31797c478bd9Sstevel@tonic-gate replpp = page_get_replacement_page(pp, NULL, 0); 31807c478bd9Sstevel@tonic-gate if (replpp) { 31817c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31827c478bd9Sstevel@tonic-gate ASSERT(npgs <= pgcnt); 31837c478bd9Sstevel@tonic-gate targpp = pp; 31847c478bd9Sstevel@tonic-gate } 31857c478bd9Sstevel@tonic-gate } 31867c478bd9Sstevel@tonic-gate 31877c478bd9Sstevel@tonic-gate /* 31887c478bd9Sstevel@tonic-gate * If replacement is NULL or do_page_relocate fails, fail 31897c478bd9Sstevel@tonic-gate * coalescing of pages. 31907c478bd9Sstevel@tonic-gate */ 31917c478bd9Sstevel@tonic-gate if (replpp == NULL || (do_page_relocate(&targpp, &replpp, 0, 31927c478bd9Sstevel@tonic-gate &npgs, NULL) != 0)) { 31937c478bd9Sstevel@tonic-gate /* 31947c478bd9Sstevel@tonic-gate * Unlock un-processed target list 31957c478bd9Sstevel@tonic-gate */ 31967c478bd9Sstevel@tonic-gate while (pgcnt--) { 31977c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31988b464eb8Smec page_unlock_nocapture(pp); 31997c478bd9Sstevel@tonic-gate pp++; 32007c478bd9Sstevel@tonic-gate } 32017c478bd9Sstevel@tonic-gate /* 32027c478bd9Sstevel@tonic-gate * Free the processed target list. 32037c478bd9Sstevel@tonic-gate */ 32047c478bd9Sstevel@tonic-gate while (pplist) { 32057c478bd9Sstevel@tonic-gate pp = pplist; 32067c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 32077c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32087c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 32097c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 32107c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 32117c478bd9Sstevel@tonic-gate pp->p_szc = 0; 32127c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 32138b464eb8Smec page_unlock_nocapture(pp); 32147c478bd9Sstevel@tonic-gate } 32157c478bd9Sstevel@tonic-gate 32167c478bd9Sstevel@tonic-gate if (replpp != NULL) 32177c478bd9Sstevel@tonic-gate page_free_replacement_page(replpp); 32187c478bd9Sstevel@tonic-gate 32197c478bd9Sstevel@tonic-gate return (NULL); 32207c478bd9Sstevel@tonic-gate } 32217c478bd9Sstevel@tonic-gate ASSERT(pp == targpp); 32227c478bd9Sstevel@tonic-gate 32237c478bd9Sstevel@tonic-gate /* LINTED */ 32247c478bd9Sstevel@tonic-gate ASSERT(hpp = pp); /* That's right, it's an assignment */ 32257c478bd9Sstevel@tonic-gate 32267c478bd9Sstevel@tonic-gate pp += npgs; 32277c478bd9Sstevel@tonic-gate pgcnt -= npgs; 32287c478bd9Sstevel@tonic-gate 32297c478bd9Sstevel@tonic-gate while (npgs--) { 32307c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(targpp)); 32317c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(targpp)); 32327c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(targpp)); 32337c478bd9Sstevel@tonic-gate PP_SETFREE(targpp); 32347c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(targpp)); 32357c478bd9Sstevel@tonic-gate ASSERT(targpp->p_szc < szc || (szc == 0 && 32367c478bd9Sstevel@tonic-gate (flags & PGI_PGCPSZC0))); 32377c478bd9Sstevel@tonic-gate targpp->p_szc = szc; 32387c478bd9Sstevel@tonic-gate targpp = targpp->p_next; 32397c478bd9Sstevel@tonic-gate 32407c478bd9Sstevel@tonic-gate rpp = replpp; 32417c478bd9Sstevel@tonic-gate ASSERT(rpp != NULL); 32427c478bd9Sstevel@tonic-gate page_sub(&replpp, rpp); 32437c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(rpp)); 32447c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(rpp)); 32458b464eb8Smec page_unlock_nocapture(rpp); 32467c478bd9Sstevel@tonic-gate } 32477c478bd9Sstevel@tonic-gate ASSERT(targpp == hpp); 32487c478bd9Sstevel@tonic-gate ASSERT(replpp == NULL); 32497c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &targpp); 32507c478bd9Sstevel@tonic-gate } 32517c478bd9Sstevel@tonic-gate CHK_LPG(pplist, szc); 32527c478bd9Sstevel@tonic-gate return (pplist); 32537c478bd9Sstevel@tonic-gate } 32547c478bd9Sstevel@tonic-gate 32557c478bd9Sstevel@tonic-gate /* 32567c478bd9Sstevel@tonic-gate * Trim kernel cage from pfnlo-pfnhi and store result in lo-hi. Return code 32577c478bd9Sstevel@tonic-gate * of 0 means nothing left after trim. 32587c478bd9Sstevel@tonic-gate */ 32597c478bd9Sstevel@tonic-gate int 32607c478bd9Sstevel@tonic-gate trimkcage(struct memseg *mseg, pfn_t *lo, pfn_t *hi, pfn_t pfnlo, pfn_t pfnhi) 32617c478bd9Sstevel@tonic-gate { 32627c478bd9Sstevel@tonic-gate pfn_t kcagepfn; 32637c478bd9Sstevel@tonic-gate int decr; 32647c478bd9Sstevel@tonic-gate int rc = 0; 32657c478bd9Sstevel@tonic-gate 32667c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->pages)) { 32677c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1) == 0) { 32687c478bd9Sstevel@tonic-gate 32697c478bd9Sstevel@tonic-gate /* lower part of this mseg inside kernel cage */ 32707c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 32717c478bd9Sstevel@tonic-gate 32727c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 32737c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 32747c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 32757c478bd9Sstevel@tonic-gate ASSERT(decr == 0); 3276*78b03d3aSkchow *lo = MAX(kcagepfn, pfnlo); 3277*78b03d3aSkchow *hi = MIN(pfnhi, (mseg->pages_end - 1)); 32787c478bd9Sstevel@tonic-gate rc = 1; 32797c478bd9Sstevel@tonic-gate } 32807c478bd9Sstevel@tonic-gate } 32817c478bd9Sstevel@tonic-gate /* else entire mseg in the cage */ 32827c478bd9Sstevel@tonic-gate } else { 32837c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1)) { 32847c478bd9Sstevel@tonic-gate 32857c478bd9Sstevel@tonic-gate /* upper part of this mseg inside kernel cage */ 32867c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 32877c478bd9Sstevel@tonic-gate 32887c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 32897c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 32907c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 32917c478bd9Sstevel@tonic-gate ASSERT(decr); 3292*78b03d3aSkchow *hi = MIN(kcagepfn, pfnhi); 32937c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 32947c478bd9Sstevel@tonic-gate rc = 1; 32957c478bd9Sstevel@tonic-gate } 32967c478bd9Sstevel@tonic-gate } else { 32977c478bd9Sstevel@tonic-gate /* entire mseg outside of kernel cage */ 32987c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 32997c478bd9Sstevel@tonic-gate *hi = MIN(pfnhi, (mseg->pages_end - 1)); 33007c478bd9Sstevel@tonic-gate rc = 1; 33017c478bd9Sstevel@tonic-gate } 33027c478bd9Sstevel@tonic-gate } 33037c478bd9Sstevel@tonic-gate return (rc); 33047c478bd9Sstevel@tonic-gate } 33057c478bd9Sstevel@tonic-gate 33067c478bd9Sstevel@tonic-gate /* 33075d07b933Sdp * called from page_get_contig_pages to search 'pfnlo' thru 'pfnhi' to claim a 33087c478bd9Sstevel@tonic-gate * page with size code 'szc'. Claiming such a page requires acquiring 33097c478bd9Sstevel@tonic-gate * exclusive locks on all constituent pages (page_trylock_contig_pages), 33107c478bd9Sstevel@tonic-gate * relocating pages in use and concatenating these constituent pages into a 33117c478bd9Sstevel@tonic-gate * large page. 33127c478bd9Sstevel@tonic-gate * 33135d07b933Sdp * The page lists do not have such a large page and page_freelist_split has 33147c478bd9Sstevel@tonic-gate * already failed to demote larger pages and/or coalesce smaller free pages. 33157c478bd9Sstevel@tonic-gate * 33167c478bd9Sstevel@tonic-gate * 'flags' may specify PG_COLOR_MATCH which would limit the search of large 33177c478bd9Sstevel@tonic-gate * pages with the same color as 'bin'. 33187c478bd9Sstevel@tonic-gate * 33197c478bd9Sstevel@tonic-gate * 'pfnflag' specifies the subset of the pfn range to search. 33207c478bd9Sstevel@tonic-gate */ 33217c478bd9Sstevel@tonic-gate 33227c478bd9Sstevel@tonic-gate static page_t * 33237c478bd9Sstevel@tonic-gate page_geti_contig_pages(int mnode, uint_t bin, uchar_t szc, int flags, 332483f9b804Skchow pfn_t pfnlo, pfn_t pfnhi, pgcnt_t pfnflag) 33257c478bd9Sstevel@tonic-gate { 33267c478bd9Sstevel@tonic-gate struct memseg *mseg; 33277c478bd9Sstevel@tonic-gate pgcnt_t szcpgcnt = page_get_pagecnt(szc); 33287c478bd9Sstevel@tonic-gate pgcnt_t szcpgmask = szcpgcnt - 1; 33297c478bd9Sstevel@tonic-gate pfn_t randpfn; 33307c478bd9Sstevel@tonic-gate page_t *pp, *randpp, *endpp; 33315d07b933Sdp uint_t colors, ceq_mask; 33325d07b933Sdp /* LINTED : set but not used in function */ 33335d07b933Sdp uint_t color_mask; 33347c478bd9Sstevel@tonic-gate pfn_t hi, lo; 33357c478bd9Sstevel@tonic-gate uint_t skip; 3336ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 33377c478bd9Sstevel@tonic-gate 33387c478bd9Sstevel@tonic-gate ASSERT(szc != 0 || (flags & PGI_PGCPSZC0)); 33397c478bd9Sstevel@tonic-gate 334002bc52beSkchow pfnlo = P2ROUNDUP(pfnlo, szcpgcnt); 334102bc52beSkchow 334202bc52beSkchow if ((pfnhi - pfnlo) + 1 < szcpgcnt || pfnlo >= pfnhi) 33437c478bd9Sstevel@tonic-gate return (NULL); 33447c478bd9Sstevel@tonic-gate 33457c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 33467c478bd9Sstevel@tonic-gate 33475d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 33485d07b933Sdp color_mask = colors - 1; 33495d07b933Sdp if ((colors > 1) && (flags & PG_MATCH_COLOR)) { 33505d07b933Sdp uchar_t ceq = colorequivszc[szc]; 33515d07b933Sdp uint_t ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 33525d07b933Sdp 33535d07b933Sdp ASSERT(ceq_dif > 0); 33545d07b933Sdp ceq_mask = (ceq_dif - 1) << (ceq & 0xf); 33555d07b933Sdp } else { 33565d07b933Sdp ceq_mask = 0; 33575d07b933Sdp } 33587c478bd9Sstevel@tonic-gate 33597c478bd9Sstevel@tonic-gate ASSERT(bin < colors); 33607c478bd9Sstevel@tonic-gate 33615d07b933Sdp /* clear "non-significant" color bits */ 33625d07b933Sdp bin &= ceq_mask; 33635d07b933Sdp 33647c478bd9Sstevel@tonic-gate /* 33657c478bd9Sstevel@tonic-gate * trim the pfn range to search based on pfnflag. pfnflag is set 33667c478bd9Sstevel@tonic-gate * when there have been previous page_get_contig_page failures to 33677c478bd9Sstevel@tonic-gate * limit the search. 33687c478bd9Sstevel@tonic-gate * 33697c478bd9Sstevel@tonic-gate * The high bit in pfnflag specifies the number of 'slots' in the 33707c478bd9Sstevel@tonic-gate * pfn range and the remainder of pfnflag specifies which slot. 33717c478bd9Sstevel@tonic-gate * For example, a value of 1010b would mean the second slot of 33727c478bd9Sstevel@tonic-gate * the pfn range that has been divided into 8 slots. 33737c478bd9Sstevel@tonic-gate */ 33747c478bd9Sstevel@tonic-gate if (pfnflag > 1) { 33757c478bd9Sstevel@tonic-gate int slots = 1 << (highbit(pfnflag) - 1); 33767c478bd9Sstevel@tonic-gate int slotid = pfnflag & (slots - 1); 33777c478bd9Sstevel@tonic-gate pgcnt_t szcpages; 33787c478bd9Sstevel@tonic-gate int slotlen; 33797c478bd9Sstevel@tonic-gate 338002bc52beSkchow pfnhi = P2ALIGN((pfnhi + 1), szcpgcnt) - 1; 33817c478bd9Sstevel@tonic-gate szcpages = ((pfnhi - pfnlo) + 1) / szcpgcnt; 33827c478bd9Sstevel@tonic-gate slotlen = howmany(szcpages, slots); 338302bc52beSkchow /* skip if 'slotid' slot is empty */ 338402bc52beSkchow if (slotid * slotlen >= szcpages) 338502bc52beSkchow return (NULL); 33867c478bd9Sstevel@tonic-gate pfnlo = pfnlo + (((slotid * slotlen) % szcpages) * szcpgcnt); 33877c478bd9Sstevel@tonic-gate ASSERT(pfnlo < pfnhi); 33887c478bd9Sstevel@tonic-gate if (pfnhi > pfnlo + (slotlen * szcpgcnt)) 338902bc52beSkchow pfnhi = pfnlo + (slotlen * szcpgcnt) - 1; 33907c478bd9Sstevel@tonic-gate } 33917c478bd9Sstevel@tonic-gate 33927c478bd9Sstevel@tonic-gate memsegs_lock(0); 33937c478bd9Sstevel@tonic-gate 33947c478bd9Sstevel@tonic-gate /* 33957c478bd9Sstevel@tonic-gate * loop through memsegs to look for contig page candidates 33967c478bd9Sstevel@tonic-gate */ 33977c478bd9Sstevel@tonic-gate 33987c478bd9Sstevel@tonic-gate for (mseg = memsegs; mseg != NULL; mseg = mseg->next) { 33997c478bd9Sstevel@tonic-gate if (pfnhi < mseg->pages_base || pfnlo >= mseg->pages_end) { 34007c478bd9Sstevel@tonic-gate /* no overlap */ 34017c478bd9Sstevel@tonic-gate continue; 34027c478bd9Sstevel@tonic-gate } 34037c478bd9Sstevel@tonic-gate 34047c478bd9Sstevel@tonic-gate if (mseg->pages_end - mseg->pages_base < szcpgcnt) 34057c478bd9Sstevel@tonic-gate /* mseg too small */ 34067c478bd9Sstevel@tonic-gate continue; 34077c478bd9Sstevel@tonic-gate 3408*78b03d3aSkchow /* 3409*78b03d3aSkchow * trim off kernel cage pages from pfn range and check for 3410*78b03d3aSkchow * a trimmed pfn range returned that does not span the 3411*78b03d3aSkchow * desired large page size. 3412*78b03d3aSkchow */ 34137c478bd9Sstevel@tonic-gate if (kcage_on) { 3414*78b03d3aSkchow if (trimkcage(mseg, &lo, &hi, pfnlo, pfnhi) == 0 || 3415*78b03d3aSkchow ((hi - lo) + 1) < szcpgcnt) 34167c478bd9Sstevel@tonic-gate continue; 34177c478bd9Sstevel@tonic-gate } else { 34187c478bd9Sstevel@tonic-gate lo = MAX(pfnlo, mseg->pages_base); 34197c478bd9Sstevel@tonic-gate hi = MIN(pfnhi, (mseg->pages_end - 1)); 34207c478bd9Sstevel@tonic-gate } 34217c478bd9Sstevel@tonic-gate 34227c478bd9Sstevel@tonic-gate /* round to szcpgcnt boundaries */ 34237c478bd9Sstevel@tonic-gate lo = P2ROUNDUP(lo, szcpgcnt); 342402bc52beSkchow 3425ce8eb11aSdp MEM_NODE_ITERATOR_INIT(lo, mnode, &it); 342602bc52beSkchow hi = P2ALIGN((hi + 1), szcpgcnt) - 1; 34277c478bd9Sstevel@tonic-gate 34287c478bd9Sstevel@tonic-gate if (hi <= lo) 34297c478bd9Sstevel@tonic-gate continue; 34307c478bd9Sstevel@tonic-gate 34317c478bd9Sstevel@tonic-gate /* 34327c478bd9Sstevel@tonic-gate * set lo to point to the pfn for the desired bin. Large 34337c478bd9Sstevel@tonic-gate * page sizes may only have a single page color 34347c478bd9Sstevel@tonic-gate */ 34355d07b933Sdp skip = szcpgcnt; 3436ce8eb11aSdp if (ceq_mask > 0 || interleaved_mnodes) { 34375d07b933Sdp /* set lo to point at appropriate color */ 3438ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ bin) & ceq_mask) || 3439ce8eb11aSdp (interleaved_mnodes && 3440ce8eb11aSdp PFN_2_MEM_NODE(lo) != mnode)) { 3441ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, bin, ceq_mask, 3442ce8eb11aSdp color_mask, &it); 3443ce8eb11aSdp } 34445d07b933Sdp if (hi <= lo) 34455d07b933Sdp /* mseg cannot satisfy color request */ 34465d07b933Sdp continue; 34477c478bd9Sstevel@tonic-gate } 34487c478bd9Sstevel@tonic-gate 34497c478bd9Sstevel@tonic-gate /* randomly choose a point between lo and hi to begin search */ 34507c478bd9Sstevel@tonic-gate 34517c478bd9Sstevel@tonic-gate randpfn = (pfn_t)GETTICK(); 34527c478bd9Sstevel@tonic-gate randpfn = ((randpfn % (hi - lo)) + lo) & ~(skip - 1); 3453ce8eb11aSdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, &it); 3454ce8eb11aSdp if (ceq_mask || interleaved_mnodes) { 3455ce8eb11aSdp if (randpfn != (pfn_t)-1) 3456ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(randpfn, szc, bin, 3457ce8eb11aSdp ceq_mask, color_mask, &it); 3458ce8eb11aSdp if (randpfn >= hi) { 3459ce8eb11aSdp randpfn = lo; 3460ce8eb11aSdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, &it); 3461ce8eb11aSdp } 34625d07b933Sdp } 34637c478bd9Sstevel@tonic-gate randpp = mseg->pages + (randpfn - mseg->pages_base); 34647c478bd9Sstevel@tonic-gate 34657c478bd9Sstevel@tonic-gate ASSERT(randpp->p_pagenum == randpfn); 34667c478bd9Sstevel@tonic-gate 34677c478bd9Sstevel@tonic-gate pp = randpp; 346802bc52beSkchow endpp = mseg->pages + (hi - mseg->pages_base) + 1; 34697c478bd9Sstevel@tonic-gate 34707c478bd9Sstevel@tonic-gate ASSERT(randpp + szcpgcnt <= endpp); 34717c478bd9Sstevel@tonic-gate 34727c478bd9Sstevel@tonic-gate do { 34737c478bd9Sstevel@tonic-gate ASSERT(!(pp->p_pagenum & szcpgmask)); 34745d07b933Sdp ASSERT(((PP_2_BIN(pp) ^ bin) & ceq_mask) == 0); 34755d07b933Sdp 34767c478bd9Sstevel@tonic-gate if (page_trylock_contig_pages(mnode, pp, szc, flags)) { 34777c478bd9Sstevel@tonic-gate /* pages unlocked by page_claim on failure */ 34787c478bd9Sstevel@tonic-gate if (page_claim_contig_pages(pp, szc, flags)) { 34797c478bd9Sstevel@tonic-gate memsegs_unlock(0); 34807c478bd9Sstevel@tonic-gate return (pp); 34817c478bd9Sstevel@tonic-gate } 34827c478bd9Sstevel@tonic-gate } 34837c478bd9Sstevel@tonic-gate 3484ce8eb11aSdp if (ceq_mask == 0 && !interleaved_mnodes) { 34855d07b933Sdp pp += skip; 34865d07b933Sdp } else { 34875d07b933Sdp pfn_t pfn = pp->p_pagenum; 34885d07b933Sdp 34895d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfn, szc, bin, 3490ce8eb11aSdp ceq_mask, color_mask, &it); 3491ce8eb11aSdp if (pfn == (pfn_t)-1) { 3492ce8eb11aSdp pp = endpp; 3493ce8eb11aSdp } else { 3494ce8eb11aSdp pp = mseg->pages + 3495ce8eb11aSdp (pfn - mseg->pages_base); 3496ce8eb11aSdp } 34975d07b933Sdp } 34987c478bd9Sstevel@tonic-gate if (pp >= endpp) { 34997c478bd9Sstevel@tonic-gate /* start from the beginning */ 3500ce8eb11aSdp MEM_NODE_ITERATOR_INIT(lo, mnode, &it); 35017c478bd9Sstevel@tonic-gate pp = mseg->pages + (lo - mseg->pages_base); 35027c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == lo); 35037c478bd9Sstevel@tonic-gate ASSERT(pp + szcpgcnt <= endpp); 35047c478bd9Sstevel@tonic-gate } 35057c478bd9Sstevel@tonic-gate } while (pp != randpp); 35067c478bd9Sstevel@tonic-gate } 35077c478bd9Sstevel@tonic-gate memsegs_unlock(0); 35087c478bd9Sstevel@tonic-gate return (NULL); 35097c478bd9Sstevel@tonic-gate } 35107c478bd9Sstevel@tonic-gate 35117c478bd9Sstevel@tonic-gate 35127c478bd9Sstevel@tonic-gate /* 35137c478bd9Sstevel@tonic-gate * controlling routine that searches through physical memory in an attempt to 35147c478bd9Sstevel@tonic-gate * claim a large page based on the input parameters. 35157c478bd9Sstevel@tonic-gate * on the page free lists. 35167c478bd9Sstevel@tonic-gate * 35177c478bd9Sstevel@tonic-gate * calls page_geti_contig_pages with an initial pfn range from the mnode 35187c478bd9Sstevel@tonic-gate * and mtype. page_geti_contig_pages will trim off the parts of the pfn range 35197c478bd9Sstevel@tonic-gate * that overlaps with the kernel cage or does not match the requested page 35207c478bd9Sstevel@tonic-gate * color if PG_MATCH_COLOR is set. Since this search is very expensive, 35217c478bd9Sstevel@tonic-gate * page_geti_contig_pages may further limit the search range based on 35227c478bd9Sstevel@tonic-gate * previous failure counts (pgcpfailcnt[]). 35237c478bd9Sstevel@tonic-gate * 35247c478bd9Sstevel@tonic-gate * for PGI_PGCPSZC0 requests, page_get_contig_pages will relocate a base 35257c478bd9Sstevel@tonic-gate * pagesize page that satisfies mtype. 35267c478bd9Sstevel@tonic-gate */ 35277c478bd9Sstevel@tonic-gate page_t * 35287c478bd9Sstevel@tonic-gate page_get_contig_pages(int mnode, uint_t bin, int mtype, uchar_t szc, 35297c478bd9Sstevel@tonic-gate uint_t flags) 35307c478bd9Sstevel@tonic-gate { 35317c478bd9Sstevel@tonic-gate pfn_t pfnlo, pfnhi; /* contig pages pfn range */ 35327c478bd9Sstevel@tonic-gate page_t *pp; 353383f9b804Skchow pgcnt_t pfnflag = 0; /* no limit on search if 0 */ 35347c478bd9Sstevel@tonic-gate 35357c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_alloc[szc]); 35367c478bd9Sstevel@tonic-gate 35370b5aa17bSmec /* no allocations from cage */ 35380b5aa17bSmec flags |= PGI_NOCAGE; 35390b5aa17bSmec 35407c478bd9Sstevel@tonic-gate /* LINTED */ 35417c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 35427c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 35437c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocempty[szc]); 35447c478bd9Sstevel@tonic-gate return (NULL); 35457c478bd9Sstevel@tonic-gate } 35467c478bd9Sstevel@tonic-gate 35477c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 35487c478bd9Sstevel@tonic-gate 35497c478bd9Sstevel@tonic-gate /* do not limit search and ignore color if hi pri */ 35507c478bd9Sstevel@tonic-gate 35517c478bd9Sstevel@tonic-gate if (pgcplimitsearch && ((flags & PGI_PGCPHIPRI) == 0)) 35527c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 35537c478bd9Sstevel@tonic-gate 35547c478bd9Sstevel@tonic-gate /* remove color match to improve chances */ 35557c478bd9Sstevel@tonic-gate 35567c478bd9Sstevel@tonic-gate if (flags & PGI_PGCPHIPRI || pfnflag) 35577c478bd9Sstevel@tonic-gate flags &= ~PG_MATCH_COLOR; 35587c478bd9Sstevel@tonic-gate 35597c478bd9Sstevel@tonic-gate do { 35607c478bd9Sstevel@tonic-gate /* get pfn range based on mnode and mtype */ 35617c478bd9Sstevel@tonic-gate MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi); 35627c478bd9Sstevel@tonic-gate 35637c478bd9Sstevel@tonic-gate ASSERT(pfnhi >= pfnlo); 35647c478bd9Sstevel@tonic-gate 35657c478bd9Sstevel@tonic-gate pp = page_geti_contig_pages(mnode, bin, szc, flags, 35667c478bd9Sstevel@tonic-gate pfnlo, pfnhi, pfnflag); 35677c478bd9Sstevel@tonic-gate 35687c478bd9Sstevel@tonic-gate if (pp != NULL) { 35697c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 35707c478bd9Sstevel@tonic-gate if (pfnflag) { 35717c478bd9Sstevel@tonic-gate /* double the search size */ 35727c478bd9Sstevel@tonic-gate pgcpfailcnt[szc] = pfnflag >> 1; 35737c478bd9Sstevel@tonic-gate } 35747c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocok[szc]); 35757c478bd9Sstevel@tonic-gate return (pp); 35767c478bd9Sstevel@tonic-gate } 3577affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3578affbd3ccSkchow } while (mtype >= 0); 35797c478bd9Sstevel@tonic-gate 35807c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocfailed[szc]); 35817c478bd9Sstevel@tonic-gate return (NULL); 35827c478bd9Sstevel@tonic-gate } 35837c478bd9Sstevel@tonic-gate 3584*78b03d3aSkchow #if defined(__i386) || defined(__amd64) 3585*78b03d3aSkchow /* 3586*78b03d3aSkchow * Determine the likelihood of finding/coalescing a szc page. 3587*78b03d3aSkchow * Return 0 if the likelihood is small otherwise return 1. 3588*78b03d3aSkchow * 3589*78b03d3aSkchow * For now, be conservative and check only 1g pages and return 0 3590*78b03d3aSkchow * if there had been previous coalescing failures and the szc pages 3591*78b03d3aSkchow * needed to satisfy request would exhaust most of freemem. 3592*78b03d3aSkchow */ 3593*78b03d3aSkchow int 3594*78b03d3aSkchow page_chk_freelist(uint_t szc) 3595*78b03d3aSkchow { 3596*78b03d3aSkchow pgcnt_t pgcnt; 3597*78b03d3aSkchow 3598*78b03d3aSkchow if (szc <= 1) 3599*78b03d3aSkchow return (1); 3600*78b03d3aSkchow 3601*78b03d3aSkchow pgcnt = page_get_pagecnt(szc); 3602*78b03d3aSkchow if (pgcpfailcnt[szc] && pgcnt + throttlefree >= freemem) { 3603*78b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_deny[szc]); 3604*78b03d3aSkchow return (0); 3605*78b03d3aSkchow } 3606*78b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_allow[szc]); 3607*78b03d3aSkchow return (1); 3608*78b03d3aSkchow } 3609*78b03d3aSkchow #endif 36107c478bd9Sstevel@tonic-gate 36117c478bd9Sstevel@tonic-gate /* 36127c478bd9Sstevel@tonic-gate * Find the `best' page on the freelist for this (vp,off) (as,vaddr) pair. 36137c478bd9Sstevel@tonic-gate * 36147c478bd9Sstevel@tonic-gate * Does its own locking and accounting. 36157c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 36167c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 36177c478bd9Sstevel@tonic-gate * 36187c478bd9Sstevel@tonic-gate * Finds a page, removes it, THEN locks it. 36197c478bd9Sstevel@tonic-gate */ 36207c478bd9Sstevel@tonic-gate 36217c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 36227c478bd9Sstevel@tonic-gate page_t * 36237c478bd9Sstevel@tonic-gate page_get_freelist(struct vnode *vp, u_offset_t off, struct seg *seg, 36247c478bd9Sstevel@tonic-gate caddr_t vaddr, size_t size, uint_t flags, struct lgrp *lgrp) 36257c478bd9Sstevel@tonic-gate { 36267c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 36277c478bd9Sstevel@tonic-gate page_t *pp = NULL; 36287c478bd9Sstevel@tonic-gate ulong_t bin; 36297c478bd9Sstevel@tonic-gate uchar_t szc; 36307c478bd9Sstevel@tonic-gate int mnode; 36317c478bd9Sstevel@tonic-gate int mtype; 36327c478bd9Sstevel@tonic-gate page_t *(*page_get_func)(int, uint_t, int, uchar_t, uint_t); 36337c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 36347c478bd9Sstevel@tonic-gate 36357c478bd9Sstevel@tonic-gate page_get_func = page_get_mnode_freelist; 36367c478bd9Sstevel@tonic-gate 36377c478bd9Sstevel@tonic-gate /* 36387c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or passed a freed lgrp 36397c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 36407c478bd9Sstevel@tonic-gate */ 36417c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 36427c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 36437c478bd9Sstevel@tonic-gate 36447c478bd9Sstevel@tonic-gate if (kcage_on) { 36457c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC)) == PG_NORELOC && 36467c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + btop(size) && 36477c478bd9Sstevel@tonic-gate curthread != kcage_cageout_thread) { 36487c478bd9Sstevel@tonic-gate /* 36497c478bd9Sstevel@tonic-gate * Set a "reserve" of kcage_throttlefree pages for 36507c478bd9Sstevel@tonic-gate * PG_PANIC and cageout thread allocations. 36517c478bd9Sstevel@tonic-gate * 36527c478bd9Sstevel@tonic-gate * Everybody else has to serialize in 36537c478bd9Sstevel@tonic-gate * page_create_get_something() to get a cage page, so 36547c478bd9Sstevel@tonic-gate * that we don't deadlock cageout! 36557c478bd9Sstevel@tonic-gate */ 36567c478bd9Sstevel@tonic-gate return (NULL); 36577c478bd9Sstevel@tonic-gate } 36587c478bd9Sstevel@tonic-gate } else { 36597c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 36607c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 36617c478bd9Sstevel@tonic-gate } 36627c478bd9Sstevel@tonic-gate 36637c478bd9Sstevel@tonic-gate /* LINTED */ 366407ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, size); 36657c478bd9Sstevel@tonic-gate 36667c478bd9Sstevel@tonic-gate /* 36677c478bd9Sstevel@tonic-gate * Convert size to page size code. 36687c478bd9Sstevel@tonic-gate */ 36697c478bd9Sstevel@tonic-gate if ((szc = page_szc(size)) == (uchar_t)-1) 36707c478bd9Sstevel@tonic-gate panic("page_get_freelist: illegal page size request"); 36717c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 36727c478bd9Sstevel@tonic-gate 36737c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_alloc[szc]); 36747c478bd9Sstevel@tonic-gate 36757c478bd9Sstevel@tonic-gate /* LINTED */ 36765d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, szc); 36777c478bd9Sstevel@tonic-gate 36785d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(szc)); 36797c478bd9Sstevel@tonic-gate 36807c478bd9Sstevel@tonic-gate /* 36817c478bd9Sstevel@tonic-gate * Try to get a local page first, but try remote if we can't 36827c478bd9Sstevel@tonic-gate * get a page of the right color. 36837c478bd9Sstevel@tonic-gate */ 36847c478bd9Sstevel@tonic-gate pgretry: 36857c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 36867c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 36877c478bd9Sstevel@tonic-gate pp = page_get_func(mnode, bin, mtype, szc, flags); 36887c478bd9Sstevel@tonic-gate if (pp != NULL) { 36897c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocok[szc]); 36907c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 36917c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 36927c478bd9Sstevel@tonic-gate int, mnode, 36937c478bd9Sstevel@tonic-gate ulong_t, bin, 36947c478bd9Sstevel@tonic-gate uint_t, flags); 36957c478bd9Sstevel@tonic-gate return (pp); 36967c478bd9Sstevel@tonic-gate } 36977c478bd9Sstevel@tonic-gate } 36987c478bd9Sstevel@tonic-gate ASSERT(pp == NULL); 36997c478bd9Sstevel@tonic-gate 37007c478bd9Sstevel@tonic-gate /* 37017c478bd9Sstevel@tonic-gate * for non-SZC0 PAGESIZE requests, check cachelist before checking 37027c478bd9Sstevel@tonic-gate * remote free lists. Caller expected to call page_get_cachelist which 37037c478bd9Sstevel@tonic-gate * will check local cache lists and remote free lists. 37047c478bd9Sstevel@tonic-gate */ 37057c478bd9Sstevel@tonic-gate if (szc == 0 && ((flags & PGI_PGCPSZC0) == 0)) { 37067c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocdeferred); 37077c478bd9Sstevel@tonic-gate return (NULL); 37087c478bd9Sstevel@tonic-gate } 37097c478bd9Sstevel@tonic-gate 37107c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 37117c478bd9Sstevel@tonic-gate 37127c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 37137c478bd9Sstevel@tonic-gate 37142cb27123Saguzovsk if (!(flags & PG_LOCAL)) { 37152cb27123Saguzovsk /* 37162cb27123Saguzovsk * Try to get a non-local freelist page. 37172cb27123Saguzovsk */ 37182cb27123Saguzovsk LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 37192cb27123Saguzovsk while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 37202cb27123Saguzovsk pp = page_get_func(mnode, bin, mtype, szc, flags); 37212cb27123Saguzovsk if (pp != NULL) { 37222cb27123Saguzovsk DTRACE_PROBE4(page__get, 37232cb27123Saguzovsk lgrp_t *, lgrp, 37242cb27123Saguzovsk int, mnode, 37252cb27123Saguzovsk ulong_t, bin, 37262cb27123Saguzovsk uint_t, flags); 37272cb27123Saguzovsk VM_STAT_ADD(vmm_vmstats.pgf_allocokrem[szc]); 37282cb27123Saguzovsk return (pp); 37292cb27123Saguzovsk } 37307c478bd9Sstevel@tonic-gate } 37312cb27123Saguzovsk ASSERT(pp == NULL); 37327c478bd9Sstevel@tonic-gate } 37337c478bd9Sstevel@tonic-gate 37347c478bd9Sstevel@tonic-gate /* 37357c478bd9Sstevel@tonic-gate * when the cage is off chances are page_get_contig_pages() will fail 37367c478bd9Sstevel@tonic-gate * to lock a large page chunk therefore when the cage is off it's not 37377c478bd9Sstevel@tonic-gate * called by default. this can be changed via /etc/system. 37387c478bd9Sstevel@tonic-gate * 37397c478bd9Sstevel@tonic-gate * page_get_contig_pages() also called to acquire a base pagesize page 37407c478bd9Sstevel@tonic-gate * for page_create_get_something(). 37417c478bd9Sstevel@tonic-gate */ 37427c478bd9Sstevel@tonic-gate if (!(flags & PG_NORELOC) && (pg_contig_disable == 0) && 37437c478bd9Sstevel@tonic-gate (kcage_on || pg_lpgcreate_nocage || szc == 0) && 37447c478bd9Sstevel@tonic-gate (page_get_func != page_get_contig_pages)) { 37457c478bd9Sstevel@tonic-gate 37467c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocretry[szc]); 37477c478bd9Sstevel@tonic-gate page_get_func = page_get_contig_pages; 37487c478bd9Sstevel@tonic-gate goto pgretry; 37497c478bd9Sstevel@tonic-gate } 37507c478bd9Sstevel@tonic-gate 37512cb27123Saguzovsk if (!(flags & PG_LOCAL) && pgcplimitsearch && 37522cb27123Saguzovsk page_get_func == page_get_contig_pages) 375383f9b804Skchow SETPGCPFAILCNT(szc); 37547c478bd9Sstevel@tonic-gate 37557c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocfailed[szc]); 37567c478bd9Sstevel@tonic-gate return (NULL); 37577c478bd9Sstevel@tonic-gate } 37587c478bd9Sstevel@tonic-gate 37597c478bd9Sstevel@tonic-gate /* 37607c478bd9Sstevel@tonic-gate * Find the `best' page on the cachelist for this (vp,off) (as,vaddr) pair. 37617c478bd9Sstevel@tonic-gate * 37627c478bd9Sstevel@tonic-gate * Does its own locking. 37637c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 37647c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 37657c478bd9Sstevel@tonic-gate * Otherwise, scan the bins for ones with pages. For each bin with pages, 37667c478bd9Sstevel@tonic-gate * try to lock one of them. If no page can be locked, try the 37677c478bd9Sstevel@tonic-gate * next bin. Return NULL if a page can not be found and locked. 37687c478bd9Sstevel@tonic-gate * 37697c478bd9Sstevel@tonic-gate * Finds a pages, trys to lock it, then removes it. 37707c478bd9Sstevel@tonic-gate */ 37717c478bd9Sstevel@tonic-gate 37727c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 37737c478bd9Sstevel@tonic-gate page_t * 37747c478bd9Sstevel@tonic-gate page_get_cachelist(struct vnode *vp, u_offset_t off, struct seg *seg, 37757c478bd9Sstevel@tonic-gate caddr_t vaddr, uint_t flags, struct lgrp *lgrp) 37767c478bd9Sstevel@tonic-gate { 37777c478bd9Sstevel@tonic-gate page_t *pp; 37787c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 37797c478bd9Sstevel@tonic-gate ulong_t bin; 37807c478bd9Sstevel@tonic-gate /*LINTED*/ 37817c478bd9Sstevel@tonic-gate int mnode; 37827c478bd9Sstevel@tonic-gate int mtype; 37837c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 37847c478bd9Sstevel@tonic-gate 37857c478bd9Sstevel@tonic-gate /* 37867c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or pasased a freed lgrp 37877c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 37887c478bd9Sstevel@tonic-gate */ 37897c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 37907c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 37917c478bd9Sstevel@tonic-gate 37927c478bd9Sstevel@tonic-gate if (!kcage_on) { 37937c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 37947c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 37957c478bd9Sstevel@tonic-gate } 37967c478bd9Sstevel@tonic-gate 37977c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == PG_NORELOC && 37987c478bd9Sstevel@tonic-gate kcage_freemem <= kcage_throttlefree) { 37997c478bd9Sstevel@tonic-gate /* 38007c478bd9Sstevel@tonic-gate * Reserve kcage_throttlefree pages for critical kernel 38017c478bd9Sstevel@tonic-gate * threads. 38027c478bd9Sstevel@tonic-gate * 38037c478bd9Sstevel@tonic-gate * Everybody else has to go to page_create_get_something() 38047c478bd9Sstevel@tonic-gate * to get a cage page, so we don't deadlock cageout. 38057c478bd9Sstevel@tonic-gate */ 38067c478bd9Sstevel@tonic-gate return (NULL); 38077c478bd9Sstevel@tonic-gate } 38087c478bd9Sstevel@tonic-gate 38097c478bd9Sstevel@tonic-gate /* LINTED */ 38105d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, 0); 38117c478bd9Sstevel@tonic-gate 38125d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(0)); 38137c478bd9Sstevel@tonic-gate 38147c478bd9Sstevel@tonic-gate /* LINTED */ 381507ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, MMU_PAGESIZE); 38167c478bd9Sstevel@tonic-gate 38177c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_alloc); 38187c478bd9Sstevel@tonic-gate 38197c478bd9Sstevel@tonic-gate /* 38207c478bd9Sstevel@tonic-gate * Try local cachelists first 38217c478bd9Sstevel@tonic-gate */ 38227c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 38237c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38247c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 38257c478bd9Sstevel@tonic-gate if (pp != NULL) { 38267c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocok); 38277c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38287c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38297c478bd9Sstevel@tonic-gate int, mnode, 38307c478bd9Sstevel@tonic-gate ulong_t, bin, 38317c478bd9Sstevel@tonic-gate uint_t, flags); 38327c478bd9Sstevel@tonic-gate return (pp); 38337c478bd9Sstevel@tonic-gate } 38347c478bd9Sstevel@tonic-gate } 38357c478bd9Sstevel@tonic-gate 38367c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 38377c478bd9Sstevel@tonic-gate 38387c478bd9Sstevel@tonic-gate /* 38397c478bd9Sstevel@tonic-gate * Try freelists/cachelists that are farther away 38407c478bd9Sstevel@tonic-gate * This is our only chance to allocate remote pages for PAGESIZE 38417c478bd9Sstevel@tonic-gate * requests. 38427c478bd9Sstevel@tonic-gate */ 38437c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 38447c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38457c478bd9Sstevel@tonic-gate pp = page_get_mnode_freelist(mnode, bin, mtype, 38467c478bd9Sstevel@tonic-gate 0, flags); 38477c478bd9Sstevel@tonic-gate if (pp != NULL) { 38487c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokdeferred); 38497c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38507c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38517c478bd9Sstevel@tonic-gate int, mnode, 38527c478bd9Sstevel@tonic-gate ulong_t, bin, 38537c478bd9Sstevel@tonic-gate uint_t, flags); 38547c478bd9Sstevel@tonic-gate return (pp); 38557c478bd9Sstevel@tonic-gate } 38567c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 38577c478bd9Sstevel@tonic-gate if (pp != NULL) { 38587c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokrem); 38597c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38607c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38617c478bd9Sstevel@tonic-gate int, mnode, 38627c478bd9Sstevel@tonic-gate ulong_t, bin, 38637c478bd9Sstevel@tonic-gate uint_t, flags); 38647c478bd9Sstevel@tonic-gate return (pp); 38657c478bd9Sstevel@tonic-gate } 38667c478bd9Sstevel@tonic-gate } 38677c478bd9Sstevel@tonic-gate 38687c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocfailed); 38697c478bd9Sstevel@tonic-gate return (NULL); 38707c478bd9Sstevel@tonic-gate } 38717c478bd9Sstevel@tonic-gate 38727c478bd9Sstevel@tonic-gate page_t * 38737c478bd9Sstevel@tonic-gate page_get_mnode_cachelist(uint_t bin, uint_t flags, int mnode, int mtype) 38747c478bd9Sstevel@tonic-gate { 38755d07b933Sdp kmutex_t *pcm; 38765d07b933Sdp page_t *pp, *first_pp; 38775d07b933Sdp uint_t sbin; 38785d07b933Sdp int plw_initialized; 38795d07b933Sdp page_list_walker_t plw; 38807c478bd9Sstevel@tonic-gate 38817c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_alloc); 38827c478bd9Sstevel@tonic-gate 38837c478bd9Sstevel@tonic-gate /* LINTED */ 38847c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 38857c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 38867c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocempty); 38877c478bd9Sstevel@tonic-gate return (NULL); 38887c478bd9Sstevel@tonic-gate } 38897c478bd9Sstevel@tonic-gate 38905d07b933Sdp try_again: 38917c478bd9Sstevel@tonic-gate 38925d07b933Sdp plw_initialized = 0; 38935d07b933Sdp plw.plw_ceq_dif = 1; 38947c478bd9Sstevel@tonic-gate 38957c478bd9Sstevel@tonic-gate /* 38967c478bd9Sstevel@tonic-gate * Only hold one cachelist lock at a time, that way we 38977c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 38987c478bd9Sstevel@tonic-gate * ordering. 38997c478bd9Sstevel@tonic-gate */ 39007c478bd9Sstevel@tonic-gate 39015d07b933Sdp for (plw.plw_count = 0; 39025d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 39035d07b933Sdp sbin = bin; 39045d07b933Sdp do { 39055d07b933Sdp 39065d07b933Sdp if (!PAGE_CACHELISTS(mnode, bin, mtype)) 39075d07b933Sdp goto bin_empty_1; 39087c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 39097c478bd9Sstevel@tonic-gate mutex_enter(pcm); 39107c478bd9Sstevel@tonic-gate pp = PAGE_CACHELISTS(mnode, bin, mtype); 39115d07b933Sdp if (pp == NULL) 39125d07b933Sdp goto bin_empty_0; 39137c478bd9Sstevel@tonic-gate 39145d07b933Sdp first_pp = pp; 39155d07b933Sdp ASSERT(pp->p_vnode); 39165d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39175d07b933Sdp ASSERT(pp->p_szc == 0); 39185d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 39195d07b933Sdp while (!page_trylock(pp, SE_EXCL)) { 39205d07b933Sdp pp = pp->p_next; 39215d07b933Sdp ASSERT(pp->p_szc == 0); 39225d07b933Sdp if (pp == first_pp) { 39237c478bd9Sstevel@tonic-gate /* 39245d07b933Sdp * We have searched the complete list! 39255d07b933Sdp * And all of them (might only be one) 39265d07b933Sdp * are locked. This can happen since 39275d07b933Sdp * these pages can also be found via 39285d07b933Sdp * the hash list. When found via the 39295d07b933Sdp * hash list, they are locked first, 39305d07b933Sdp * then removed. We give up to let the 39315d07b933Sdp * other thread run. 39327c478bd9Sstevel@tonic-gate */ 39335d07b933Sdp pp = NULL; 39345d07b933Sdp break; 39357c478bd9Sstevel@tonic-gate } 39365d07b933Sdp ASSERT(pp->p_vnode); 39375d07b933Sdp ASSERT(PP_ISFREE(pp)); 39385d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39395d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == 39405d07b933Sdp mnode); 39417c478bd9Sstevel@tonic-gate } 39427c478bd9Sstevel@tonic-gate 39435d07b933Sdp if (pp) { 39445d07b933Sdp page_t **ppp; 39455d07b933Sdp /* 39465d07b933Sdp * Found and locked a page. 39475d07b933Sdp * Pull it off the list. 39485d07b933Sdp */ 39495d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 39505d07b933Sdp ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 39515d07b933Sdp page_sub(ppp, pp); 39525d07b933Sdp /* 39535d07b933Sdp * Subtract counters before releasing pcm mutex 39545d07b933Sdp * to avoid a race with page_freelist_coalesce 39555d07b933Sdp * and page_freelist_split. 39565d07b933Sdp */ 39575d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 39585d07b933Sdp mutex_exit(pcm); 39595d07b933Sdp ASSERT(pp->p_vnode); 39605d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39615d07b933Sdp #if defined(__sparc) 39625d07b933Sdp ASSERT(!kcage_on || 39635d07b933Sdp (flags & PG_NORELOC) == 0 || 39645d07b933Sdp PP_ISNORELOC(pp)); 39655d07b933Sdp if (PP_ISNORELOC(pp)) { 39665d07b933Sdp kcage_freemem_sub(1); 39677c478bd9Sstevel@tonic-gate } 39685d07b933Sdp #endif 39695d07b933Sdp VM_STAT_ADD(vmm_vmstats. pgmc_allocok); 39705d07b933Sdp return (pp); 39717c478bd9Sstevel@tonic-gate } 39725d07b933Sdp bin_empty_0: 39735d07b933Sdp mutex_exit(pcm); 39745d07b933Sdp bin_empty_1: 39755d07b933Sdp if (plw_initialized == 0) { 39765d07b933Sdp page_list_walk_init(0, flags, bin, 0, 1, &plw); 39775d07b933Sdp plw_initialized = 1; 39787c478bd9Sstevel@tonic-gate } 39795d07b933Sdp /* calculate the next bin with equivalent color */ 39805d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 39815d07b933Sdp plw.plw_ceq_mask[0], plw.plw_color_mask); 39825d07b933Sdp } while (sbin != bin); 39837c478bd9Sstevel@tonic-gate 39845d07b933Sdp if (plw.plw_ceq_dif > 1) 39855d07b933Sdp bin = page_list_walk_next_bin(0, bin, &plw); 39867c478bd9Sstevel@tonic-gate } 39877c478bd9Sstevel@tonic-gate 3988affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3989affbd3ccSkchow if (mtype >= 0) 39905d07b933Sdp goto try_again; 3991affbd3ccSkchow 39927c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocfailed); 39937c478bd9Sstevel@tonic-gate return (NULL); 39947c478bd9Sstevel@tonic-gate } 39957c478bd9Sstevel@tonic-gate 39967c478bd9Sstevel@tonic-gate #ifdef DEBUG 39977c478bd9Sstevel@tonic-gate #define REPL_PAGE_STATS 39987c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 39997c478bd9Sstevel@tonic-gate 40007c478bd9Sstevel@tonic-gate #ifdef REPL_PAGE_STATS 40017c478bd9Sstevel@tonic-gate struct repl_page_stats { 40027c478bd9Sstevel@tonic-gate uint_t ngets; 40037c478bd9Sstevel@tonic-gate uint_t ngets_noreloc; 40047c478bd9Sstevel@tonic-gate uint_t npgr_noreloc; 40057c478bd9Sstevel@tonic-gate uint_t nnopage_first; 40067c478bd9Sstevel@tonic-gate uint_t nnopage; 40077c478bd9Sstevel@tonic-gate uint_t nhashout; 40087c478bd9Sstevel@tonic-gate uint_t nnofree; 40097c478bd9Sstevel@tonic-gate uint_t nnext_pp; 40107c478bd9Sstevel@tonic-gate } repl_page_stats; 40117c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) atomic_add_32(&repl_page_stats.v, 1) 40127c478bd9Sstevel@tonic-gate #else /* REPL_PAGE_STATS */ 40137c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) 40147c478bd9Sstevel@tonic-gate #endif /* REPL_PAGE_STATS */ 40157c478bd9Sstevel@tonic-gate 40167c478bd9Sstevel@tonic-gate int pgrppgcp; 40177c478bd9Sstevel@tonic-gate 40187c478bd9Sstevel@tonic-gate /* 40197c478bd9Sstevel@tonic-gate * The freemem accounting must be done by the caller. 40207c478bd9Sstevel@tonic-gate * First we try to get a replacement page of the same size as like_pp, 40217c478bd9Sstevel@tonic-gate * if that is not possible, then we just get a set of discontiguous 40227c478bd9Sstevel@tonic-gate * PAGESIZE pages. 40237c478bd9Sstevel@tonic-gate */ 40247c478bd9Sstevel@tonic-gate page_t * 40252dae3fb5Sjjc page_get_replacement_page(page_t *orig_like_pp, struct lgrp *lgrp_target, 40267c478bd9Sstevel@tonic-gate uint_t pgrflags) 40277c478bd9Sstevel@tonic-gate { 40287c478bd9Sstevel@tonic-gate page_t *like_pp; 40297c478bd9Sstevel@tonic-gate page_t *pp, *pplist; 40307c478bd9Sstevel@tonic-gate page_t *pl = NULL; 40317c478bd9Sstevel@tonic-gate ulong_t bin; 40327c478bd9Sstevel@tonic-gate int mnode, page_mnode; 40337c478bd9Sstevel@tonic-gate int szc; 40347c478bd9Sstevel@tonic-gate spgcnt_t npgs, pg_cnt; 40357c478bd9Sstevel@tonic-gate pfn_t pfnum; 40367c478bd9Sstevel@tonic-gate int mtype; 40377c478bd9Sstevel@tonic-gate int flags = 0; 40387c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 40392dae3fb5Sjjc lgrp_t *lgrp; 40407c478bd9Sstevel@tonic-gate 40417c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets); 40427c478bd9Sstevel@tonic-gate like_pp = orig_like_pp; 40437c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(like_pp)); 40447c478bd9Sstevel@tonic-gate 40457c478bd9Sstevel@tonic-gate szc = like_pp->p_szc; 40467c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 40477c478bd9Sstevel@tonic-gate /* 40487c478bd9Sstevel@tonic-gate * Now we reset like_pp to the base page_t. 40497c478bd9Sstevel@tonic-gate * That way, we won't walk past the end of this 'szc' page. 40507c478bd9Sstevel@tonic-gate */ 40517c478bd9Sstevel@tonic-gate pfnum = PFN_BASE(like_pp->p_pagenum, szc); 40527c478bd9Sstevel@tonic-gate like_pp = page_numtopp_nolock(pfnum); 40537c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == szc); 40547c478bd9Sstevel@tonic-gate 40557c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(like_pp)) { 40567c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 40577c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets_noreloc); 40587c478bd9Sstevel@tonic-gate flags = PGI_RELOCONLY; 40597c478bd9Sstevel@tonic-gate } else if (pgrflags & PGR_NORELOC) { 40607c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 40617c478bd9Sstevel@tonic-gate REPL_STAT_INCR(npgr_noreloc); 40627c478bd9Sstevel@tonic-gate flags = PG_NORELOC; 40637c478bd9Sstevel@tonic-gate } 40647c478bd9Sstevel@tonic-gate 40657c478bd9Sstevel@tonic-gate /* 40667c478bd9Sstevel@tonic-gate * Kernel pages must always be replaced with the same size 40677c478bd9Sstevel@tonic-gate * pages, since we cannot properly handle demotion of kernel 40687c478bd9Sstevel@tonic-gate * pages. 40697c478bd9Sstevel@tonic-gate */ 4070ad23a2dbSjohansen if (PP_ISKAS(like_pp)) 40717c478bd9Sstevel@tonic-gate pgrflags |= PGR_SAMESZC; 40727c478bd9Sstevel@tonic-gate 40737c478bd9Sstevel@tonic-gate /* LINTED */ 407407ad560dSkchow MTYPE_PGR_INIT(mtype, flags, like_pp, page_mnode, npgs); 40757c478bd9Sstevel@tonic-gate 40767c478bd9Sstevel@tonic-gate while (npgs) { 40777c478bd9Sstevel@tonic-gate pplist = NULL; 40787c478bd9Sstevel@tonic-gate for (;;) { 40797c478bd9Sstevel@tonic-gate pg_cnt = page_get_pagecnt(szc); 40807c478bd9Sstevel@tonic-gate bin = PP_2_BIN(like_pp); 40817c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == orig_like_pp->p_szc); 40827c478bd9Sstevel@tonic-gate ASSERT(pg_cnt <= npgs); 40837c478bd9Sstevel@tonic-gate 40847c478bd9Sstevel@tonic-gate /* 40857c478bd9Sstevel@tonic-gate * If an lgroup was specified, try to get the 40867c478bd9Sstevel@tonic-gate * page from that lgroup. 40872dae3fb5Sjjc * NOTE: Must be careful with code below because 40882dae3fb5Sjjc * lgroup may disappear and reappear since there 40892dae3fb5Sjjc * is no locking for lgroup here. 40907c478bd9Sstevel@tonic-gate */ 40912dae3fb5Sjjc if (LGRP_EXISTS(lgrp_target)) { 40922dae3fb5Sjjc /* 40932dae3fb5Sjjc * Keep local variable for lgroup separate 40942dae3fb5Sjjc * from lgroup argument since this code should 40952dae3fb5Sjjc * only be exercised when lgroup argument 40962dae3fb5Sjjc * exists.... 40972dae3fb5Sjjc */ 40982dae3fb5Sjjc lgrp = lgrp_target; 40992dae3fb5Sjjc 41007c478bd9Sstevel@tonic-gate /* Try the lgroup's freelists first */ 41017c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41027c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41037c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41047c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41057c478bd9Sstevel@tonic-gate != -1) { 4106ce8eb11aSdp pplist = 4107ce8eb11aSdp page_get_mnode_freelist(mnode, bin, 4108ce8eb11aSdp mtype, szc, flags); 41097c478bd9Sstevel@tonic-gate } 41107c478bd9Sstevel@tonic-gate 41117c478bd9Sstevel@tonic-gate /* 41127c478bd9Sstevel@tonic-gate * Now try it's cachelists if this is a 41137c478bd9Sstevel@tonic-gate * small page. Don't need to do it for 41147c478bd9Sstevel@tonic-gate * larger ones since page_freelist_coalesce() 41157c478bd9Sstevel@tonic-gate * already failed. 41167c478bd9Sstevel@tonic-gate */ 41177c478bd9Sstevel@tonic-gate if (pplist != NULL || szc != 0) 41187c478bd9Sstevel@tonic-gate break; 41197c478bd9Sstevel@tonic-gate 41207c478bd9Sstevel@tonic-gate /* Now try it's cachelists */ 41217c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41227c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41237c478bd9Sstevel@tonic-gate 41247c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41257c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41267c478bd9Sstevel@tonic-gate != -1) { 4127ce8eb11aSdp pplist = 4128ce8eb11aSdp page_get_mnode_cachelist(bin, flags, 4129ce8eb11aSdp mnode, mtype); 41307c478bd9Sstevel@tonic-gate } 41317c478bd9Sstevel@tonic-gate if (pplist != NULL) { 41327c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 41337c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 41347c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 41357c478bd9Sstevel@tonic-gate break; 41367c478bd9Sstevel@tonic-gate } 41377c478bd9Sstevel@tonic-gate /* Done looking in this lgroup. Bail out. */ 41387c478bd9Sstevel@tonic-gate break; 41397c478bd9Sstevel@tonic-gate } 41407c478bd9Sstevel@tonic-gate 41417c478bd9Sstevel@tonic-gate /* 41422dae3fb5Sjjc * No lgroup was specified (or lgroup was removed by 41432dae3fb5Sjjc * DR, so just try to get the page as close to 41442dae3fb5Sjjc * like_pp's mnode as possible. 41457c478bd9Sstevel@tonic-gate * First try the local freelist... 41467c478bd9Sstevel@tonic-gate */ 41477c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(like_pp); 41487c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, bin, 41497c478bd9Sstevel@tonic-gate mtype, szc, flags); 41507c478bd9Sstevel@tonic-gate if (pplist != NULL) 41517c478bd9Sstevel@tonic-gate break; 41527c478bd9Sstevel@tonic-gate 41537c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnofree); 41547c478bd9Sstevel@tonic-gate 41557c478bd9Sstevel@tonic-gate /* 41567c478bd9Sstevel@tonic-gate * ...then the local cachelist. Don't need to do it for 41577c478bd9Sstevel@tonic-gate * larger pages cause page_freelist_coalesce() already 41587c478bd9Sstevel@tonic-gate * failed there anyway. 41597c478bd9Sstevel@tonic-gate */ 41607c478bd9Sstevel@tonic-gate if (szc == 0) { 41617c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, flags, 41627c478bd9Sstevel@tonic-gate mnode, mtype); 41637c478bd9Sstevel@tonic-gate if (pplist != NULL) { 41647c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 41657c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 41667c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 41677c478bd9Sstevel@tonic-gate break; 41687c478bd9Sstevel@tonic-gate } 41697c478bd9Sstevel@tonic-gate } 41707c478bd9Sstevel@tonic-gate 41717c478bd9Sstevel@tonic-gate /* Now try remote freelists */ 41727c478bd9Sstevel@tonic-gate page_mnode = mnode; 41737c478bd9Sstevel@tonic-gate lgrp = 41747c478bd9Sstevel@tonic-gate lgrp_hand_to_lgrp(MEM_NODE_2_LGRPHAND(page_mnode)); 41757c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41767c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 41777c478bd9Sstevel@tonic-gate while (pplist == NULL && 41787c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41797c478bd9Sstevel@tonic-gate != -1) { 41807c478bd9Sstevel@tonic-gate /* 41817c478bd9Sstevel@tonic-gate * Skip local mnode. 41827c478bd9Sstevel@tonic-gate */ 41837c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 41847c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 41857c478bd9Sstevel@tonic-gate continue; 41867c478bd9Sstevel@tonic-gate 41877c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, 41887c478bd9Sstevel@tonic-gate bin, mtype, szc, flags); 41897c478bd9Sstevel@tonic-gate } 41907c478bd9Sstevel@tonic-gate 41917c478bd9Sstevel@tonic-gate if (pplist != NULL) 41927c478bd9Sstevel@tonic-gate break; 41937c478bd9Sstevel@tonic-gate 41947c478bd9Sstevel@tonic-gate 41957c478bd9Sstevel@tonic-gate /* Now try remote cachelists */ 41967c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41977c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 41987c478bd9Sstevel@tonic-gate while (pplist == NULL && szc == 0) { 41997c478bd9Sstevel@tonic-gate mnode = lgrp_memnode_choose(&lgrp_cookie); 42007c478bd9Sstevel@tonic-gate if (mnode == -1) 42017c478bd9Sstevel@tonic-gate break; 42027c478bd9Sstevel@tonic-gate /* 42037c478bd9Sstevel@tonic-gate * Skip local mnode. 42047c478bd9Sstevel@tonic-gate */ 42057c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 42067c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 42077c478bd9Sstevel@tonic-gate continue; 42087c478bd9Sstevel@tonic-gate 42097c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, 42107c478bd9Sstevel@tonic-gate flags, mnode, mtype); 42117c478bd9Sstevel@tonic-gate 42127c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42137c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 42147c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 42157c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 42167c478bd9Sstevel@tonic-gate break; 42177c478bd9Sstevel@tonic-gate } 42187c478bd9Sstevel@tonic-gate } 42197c478bd9Sstevel@tonic-gate 42207c478bd9Sstevel@tonic-gate /* 42217c478bd9Sstevel@tonic-gate * Break out of while loop under the following cases: 42227c478bd9Sstevel@tonic-gate * - If we successfully got a page. 42237c478bd9Sstevel@tonic-gate * - If pgrflags specified only returning a specific 42247c478bd9Sstevel@tonic-gate * page size and we could not find that page size. 42257c478bd9Sstevel@tonic-gate * - If we could not satisfy the request with PAGESIZE 42267c478bd9Sstevel@tonic-gate * or larger pages. 42277c478bd9Sstevel@tonic-gate */ 42287c478bd9Sstevel@tonic-gate if (pplist != NULL || szc == 0) 42297c478bd9Sstevel@tonic-gate break; 42307c478bd9Sstevel@tonic-gate 42317c478bd9Sstevel@tonic-gate if ((pgrflags & PGR_SAMESZC) || pgrppgcp) { 42327c478bd9Sstevel@tonic-gate /* try to find contig page */ 42337c478bd9Sstevel@tonic-gate 42347c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42357c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42367c478bd9Sstevel@tonic-gate 42377c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 42387c478bd9Sstevel@tonic-gate (mnode = 4239ce8eb11aSdp lgrp_memnode_choose(&lgrp_cookie)) 42407c478bd9Sstevel@tonic-gate != -1) { 42417c478bd9Sstevel@tonic-gate pplist = page_get_contig_pages( 4242ce8eb11aSdp mnode, bin, mtype, szc, 4243ce8eb11aSdp flags | PGI_PGCPHIPRI); 42447c478bd9Sstevel@tonic-gate } 42457c478bd9Sstevel@tonic-gate break; 42467c478bd9Sstevel@tonic-gate } 42477c478bd9Sstevel@tonic-gate 42487c478bd9Sstevel@tonic-gate /* 42497c478bd9Sstevel@tonic-gate * The correct thing to do here is try the next 42507c478bd9Sstevel@tonic-gate * page size down using szc--. Due to a bug 42517c478bd9Sstevel@tonic-gate * with the processing of HAT_RELOAD_SHARE 42527c478bd9Sstevel@tonic-gate * where the sfmmu_ttecnt arrays of all 42537c478bd9Sstevel@tonic-gate * hats sharing an ISM segment don't get updated, 42547c478bd9Sstevel@tonic-gate * using intermediate size pages for relocation 42557c478bd9Sstevel@tonic-gate * can lead to continuous page faults. 42567c478bd9Sstevel@tonic-gate */ 42577c478bd9Sstevel@tonic-gate szc = 0; 42587c478bd9Sstevel@tonic-gate } 42597c478bd9Sstevel@tonic-gate 42607c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42617c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 42627c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 42637c478bd9Sstevel@tonic-gate int, mnode, 42647c478bd9Sstevel@tonic-gate ulong_t, bin, 42657c478bd9Sstevel@tonic-gate uint_t, flags); 42667c478bd9Sstevel@tonic-gate 42677c478bd9Sstevel@tonic-gate while (pplist != NULL && pg_cnt--) { 42687c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 42697c478bd9Sstevel@tonic-gate pp = pplist; 42707c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 42717c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 42727c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 42737c478bd9Sstevel@tonic-gate page_list_concat(&pl, &pp); 42747c478bd9Sstevel@tonic-gate npgs--; 42757c478bd9Sstevel@tonic-gate like_pp = like_pp + 1; 42767c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnext_pp); 42777c478bd9Sstevel@tonic-gate } 42787c478bd9Sstevel@tonic-gate ASSERT(pg_cnt == 0); 42797c478bd9Sstevel@tonic-gate } else { 42807c478bd9Sstevel@tonic-gate break; 42817c478bd9Sstevel@tonic-gate } 42827c478bd9Sstevel@tonic-gate } 42837c478bd9Sstevel@tonic-gate 42847c478bd9Sstevel@tonic-gate if (npgs) { 42857c478bd9Sstevel@tonic-gate /* 42867c478bd9Sstevel@tonic-gate * We were unable to allocate the necessary number 42877c478bd9Sstevel@tonic-gate * of pages. 42887c478bd9Sstevel@tonic-gate * We need to free up any pl. 42897c478bd9Sstevel@tonic-gate */ 42907c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnopage); 42917c478bd9Sstevel@tonic-gate page_free_replacement_page(pl); 42927c478bd9Sstevel@tonic-gate return (NULL); 42937c478bd9Sstevel@tonic-gate } else { 42947c478bd9Sstevel@tonic-gate return (pl); 42957c478bd9Sstevel@tonic-gate } 42967c478bd9Sstevel@tonic-gate } 42977c478bd9Sstevel@tonic-gate 42987c478bd9Sstevel@tonic-gate /* 42997c478bd9Sstevel@tonic-gate * demote a free large page to it's constituent pages 43007c478bd9Sstevel@tonic-gate */ 43017c478bd9Sstevel@tonic-gate void 43027c478bd9Sstevel@tonic-gate page_demote_free_pages(page_t *pp) 43037c478bd9Sstevel@tonic-gate { 43047c478bd9Sstevel@tonic-gate 43057c478bd9Sstevel@tonic-gate int mnode; 43067c478bd9Sstevel@tonic-gate 43077c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 43087c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 43097c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 43107c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 43117c478bd9Sstevel@tonic-gate 43127c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 43137c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 43147c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 43157c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, 43167c478bd9Sstevel@tonic-gate pp->p_szc), pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 43177c478bd9Sstevel@tonic-gate } 43187c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 43197c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 43207c478bd9Sstevel@tonic-gate } 4321932dc8e5Sdp 4322932dc8e5Sdp /* 4323932dc8e5Sdp * Factor in colorequiv to check additional 'equivalent' bins. 4324932dc8e5Sdp * colorequiv may be set in /etc/system 4325932dc8e5Sdp */ 4326932dc8e5Sdp void 4327932dc8e5Sdp page_set_colorequiv_arr(void) 4328932dc8e5Sdp { 4329932dc8e5Sdp if (colorequiv > 1) { 4330932dc8e5Sdp int i; 4331fe70c9cfSdp uint_t sv_a = lowbit(colorequiv) - 1; 4332932dc8e5Sdp 4333fe70c9cfSdp if (sv_a > 15) 4334fe70c9cfSdp sv_a = 15; 4335932dc8e5Sdp 4336932dc8e5Sdp for (i = 0; i < MMU_PAGE_SIZES; i++) { 4337fe70c9cfSdp uint_t colors; 4338fe70c9cfSdp uint_t a = sv_a; 4339932dc8e5Sdp 4340932dc8e5Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) { 4341932dc8e5Sdp continue; 4342932dc8e5Sdp } 4343932dc8e5Sdp while ((colors >> a) == 0) 4344932dc8e5Sdp a--; 4345932dc8e5Sdp if ((a << 4) > colorequivszc[i]) { 4346932dc8e5Sdp colorequivszc[i] = (a << 4); 4347932dc8e5Sdp } 4348932dc8e5Sdp } 4349932dc8e5Sdp } 4350932dc8e5Sdp } 4351