17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50b5aa17bSmec * Common Development and Distribution License (the "License"). 60b5aa17bSmec * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22a7c3ca36Sdp * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 277c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 4.3 BSD 317c478bd9Sstevel@tonic-gate * under license from the Regents of the University of California. 327c478bd9Sstevel@tonic-gate */ 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 357c478bd9Sstevel@tonic-gate 367c478bd9Sstevel@tonic-gate /* 377c478bd9Sstevel@tonic-gate * This file contains common functions to access and manage the page lists. 387c478bd9Sstevel@tonic-gate * Many of these routines originated from platform dependent modules 397c478bd9Sstevel@tonic-gate * (sun4/vm/vm_dep.c, i86pc/vm/vm_machdep.c) and modified to function in 407c478bd9Sstevel@tonic-gate * a platform independent manner. 417c478bd9Sstevel@tonic-gate * 427c478bd9Sstevel@tonic-gate * vm/vm_dep.h provides for platform specific support. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate 457c478bd9Sstevel@tonic-gate #include <sys/types.h> 467c478bd9Sstevel@tonic-gate #include <sys/debug.h> 477c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 487c478bd9Sstevel@tonic-gate #include <sys/systm.h> 497c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 507c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 517c478bd9Sstevel@tonic-gate #include <vm/as.h> 527c478bd9Sstevel@tonic-gate #include <vm/page.h> 537c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 547c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 5578b03d3aSkchow #include <sys/vmsystm.h> 567c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 577c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 587c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 597c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> 607c478bd9Sstevel@tonic-gate #include <sys/callb.h> 617c478bd9Sstevel@tonic-gate #include <sys/mem_cage.h> 627c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 637c478bd9Sstevel@tonic-gate 647c478bd9Sstevel@tonic-gate extern uint_t vac_colors; 657c478bd9Sstevel@tonic-gate 666061ce8aSkchow #define MAX_PRAGMA_ALIGN 128 676061ce8aSkchow 686061ce8aSkchow /* vm_cpu_data0 for the boot cpu before kmem is initialized */ 696061ce8aSkchow 706061ce8aSkchow #if L2CACHE_ALIGN_MAX <= MAX_PRAGMA_ALIGN 71affbd3ccSkchow #pragma align L2CACHE_ALIGN_MAX(vm_cpu_data0) 726061ce8aSkchow #else 736061ce8aSkchow #pragma align MAX_PRAGMA_ALIGN(vm_cpu_data0) 746061ce8aSkchow #endif 75affbd3ccSkchow char vm_cpu_data0[VM_CPU_DATA_PADSIZE]; 76affbd3ccSkchow 777c478bd9Sstevel@tonic-gate /* 787c478bd9Sstevel@tonic-gate * number of page colors equivalent to reqested color in page_get routines. 797c478bd9Sstevel@tonic-gate * If set, keeps large pages intact longer and keeps MPO allocation 807c478bd9Sstevel@tonic-gate * from the local mnode in favor of acquiring the 'correct' page color from 817c478bd9Sstevel@tonic-gate * a demoted large page or from a remote mnode. 827c478bd9Sstevel@tonic-gate */ 835d07b933Sdp uint_t colorequiv; 845d07b933Sdp 855d07b933Sdp /* 865d07b933Sdp * color equivalency mask for each page size. 875d07b933Sdp * Mask is computed based on cpu L2$ way sizes and colorequiv global. 885d07b933Sdp * High 4 bits determine the number of high order bits of the color to ignore. 895d07b933Sdp * Low 4 bits determines number of low order bits of color to ignore (it's only 905d07b933Sdp * relevant for hashed index based page coloring). 915d07b933Sdp */ 925d07b933Sdp uchar_t colorequivszc[MMU_PAGE_SIZES]; 937c478bd9Sstevel@tonic-gate 947c478bd9Sstevel@tonic-gate /* 957c478bd9Sstevel@tonic-gate * if set, specifies the percentage of large pages that are free from within 967c478bd9Sstevel@tonic-gate * a large page region before attempting to lock those pages for 977c478bd9Sstevel@tonic-gate * page_get_contig_pages processing. 987c478bd9Sstevel@tonic-gate * 997c478bd9Sstevel@tonic-gate * Should be turned on when kpr is available when page_trylock_contig_pages 1007c478bd9Sstevel@tonic-gate * can be more selective. 1017c478bd9Sstevel@tonic-gate */ 1027c478bd9Sstevel@tonic-gate 1037c478bd9Sstevel@tonic-gate int ptcpthreshold; 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate /* 1067c478bd9Sstevel@tonic-gate * Limit page get contig page search based on failure cnts in pgcpfailcnt[]. 10783f9b804Skchow * Enabled by default via pgcplimitsearch. 10883f9b804Skchow * 10983f9b804Skchow * pgcpfailcnt[] is bounded by PGCPFAILMAX (>= 1/2 of installed 11083f9b804Skchow * memory). When reached, pgcpfailcnt[] is reset to 1/2 of this upper 11183f9b804Skchow * bound. This upper bound range guarantees: 11283f9b804Skchow * - all large page 'slots' will be searched over time 11383f9b804Skchow * - the minimum (1) large page candidates considered on each pgcp call 11483f9b804Skchow * - count doesn't wrap around to 0 1157c478bd9Sstevel@tonic-gate */ 11683f9b804Skchow pgcnt_t pgcpfailcnt[MMU_PAGE_SIZES]; 1177c478bd9Sstevel@tonic-gate int pgcplimitsearch = 1; 1187c478bd9Sstevel@tonic-gate 11983f9b804Skchow #define PGCPFAILMAX (1 << (highbit(physinstalled) - 1)) 12083f9b804Skchow #define SETPGCPFAILCNT(szc) \ 12183f9b804Skchow if (++pgcpfailcnt[szc] >= PGCPFAILMAX) \ 12283f9b804Skchow pgcpfailcnt[szc] = PGCPFAILMAX / 2; 12383f9b804Skchow 1247c478bd9Sstevel@tonic-gate #ifdef VM_STATS 1257c478bd9Sstevel@tonic-gate struct vmm_vmstats_str vmm_vmstats; 1267c478bd9Sstevel@tonic-gate 1277c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate #if defined(__sparc) 1307c478bd9Sstevel@tonic-gate #define LPGCREATE 0 1317c478bd9Sstevel@tonic-gate #else 1327c478bd9Sstevel@tonic-gate /* enable page_get_contig_pages */ 1337c478bd9Sstevel@tonic-gate #define LPGCREATE 1 1347c478bd9Sstevel@tonic-gate #endif 1357c478bd9Sstevel@tonic-gate 1367c478bd9Sstevel@tonic-gate int pg_contig_disable; 1377c478bd9Sstevel@tonic-gate int pg_lpgcreate_nocage = LPGCREATE; 1387c478bd9Sstevel@tonic-gate 1397c478bd9Sstevel@tonic-gate /* 1405d07b933Sdp * page_freelist_split pfn flag to signify no hi pfn requirement. 1417c478bd9Sstevel@tonic-gate */ 1427c478bd9Sstevel@tonic-gate #define PFNNULL 0 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate /* Flags involved in promotion and demotion routines */ 1457c478bd9Sstevel@tonic-gate #define PC_FREE 0x1 /* put page on freelist */ 1467c478bd9Sstevel@tonic-gate #define PC_ALLOC 0x2 /* return page for allocation */ 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate /* 1497c478bd9Sstevel@tonic-gate * Flag for page_demote to be used with PC_FREE to denote that we don't care 1507c478bd9Sstevel@tonic-gate * what the color is as the color parameter to the function is ignored. 1517c478bd9Sstevel@tonic-gate */ 1527c478bd9Sstevel@tonic-gate #define PC_NO_COLOR (-1) 1537c478bd9Sstevel@tonic-gate 1545d07b933Sdp /* mtype value for page_promote to use when mtype does not matter */ 1555d07b933Sdp #define PC_MTYPE_ANY (-1) 1565d07b933Sdp 1577c478bd9Sstevel@tonic-gate /* 1587c478bd9Sstevel@tonic-gate * page counters candidates info 1597c478bd9Sstevel@tonic-gate * See page_ctrs_cands comment below for more details. 1607c478bd9Sstevel@tonic-gate * fields are as follows: 1617c478bd9Sstevel@tonic-gate * pcc_pages_free: # pages which freelist coalesce can create 1627c478bd9Sstevel@tonic-gate * pcc_color_free: pointer to page free counts per color 1637c478bd9Sstevel@tonic-gate */ 1647c478bd9Sstevel@tonic-gate typedef struct pcc_info { 1657c478bd9Sstevel@tonic-gate pgcnt_t pcc_pages_free; 1667c478bd9Sstevel@tonic-gate pgcnt_t *pcc_color_free; 1677c478bd9Sstevel@tonic-gate } pcc_info_t; 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate /* 1707c478bd9Sstevel@tonic-gate * On big machines it can take a long time to check page_counters 1717c478bd9Sstevel@tonic-gate * arrays. page_ctrs_cands is a summary array whose elements are a dynamically 1727c478bd9Sstevel@tonic-gate * updated sum of all elements of the corresponding page_counters arrays. 1737c478bd9Sstevel@tonic-gate * page_freelist_coalesce() searches page_counters only if an appropriate 1747c478bd9Sstevel@tonic-gate * element of page_ctrs_cands array is greater than 0. 1757c478bd9Sstevel@tonic-gate * 1765d07b933Sdp * page_ctrs_cands is indexed by mutex (i), region (r), mnode (m), mrange (g) 1777c478bd9Sstevel@tonic-gate */ 1785d07b933Sdp pcc_info_t **page_ctrs_cands[NPC_MUTEX][MMU_PAGE_SIZES]; 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate /* 1817c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1825d07b933Sdp * for the given mnode (m), mrange (g), and region size (r) 1837c478bd9Sstevel@tonic-gate */ 1845d07b933Sdp #define PGCTRS_CANDS_GETVALUE(m, g, r, val) { \ 1857c478bd9Sstevel@tonic-gate int i; \ 1867c478bd9Sstevel@tonic-gate val = 0; \ 1877c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 1885d07b933Sdp val += page_ctrs_cands[i][(r)][(m)][(g)].pcc_pages_free; \ 1897c478bd9Sstevel@tonic-gate } \ 1907c478bd9Sstevel@tonic-gate } 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate /* 1937c478bd9Sstevel@tonic-gate * Return in val the total number of free pages which can be created 1945d07b933Sdp * for the given mnode (m), mrange (g), region size (r), and color (c) 1957c478bd9Sstevel@tonic-gate */ 1965d07b933Sdp #define PGCTRS_CANDS_GETVALUECOLOR(m, g, r, c, val) { \ 1977c478bd9Sstevel@tonic-gate int i; \ 1987c478bd9Sstevel@tonic-gate val = 0; \ 1995d07b933Sdp ASSERT((c) < PAGE_GET_PAGECOLORS(r)); \ 2007c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { \ 2015d07b933Sdp val += \ 2025d07b933Sdp page_ctrs_cands[i][(r)][(m)][(g)].pcc_color_free[(c)]; \ 2037c478bd9Sstevel@tonic-gate } \ 2047c478bd9Sstevel@tonic-gate } 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate /* 2077c478bd9Sstevel@tonic-gate * We can only allow a single thread to update a counter within the physical 2087c478bd9Sstevel@tonic-gate * range of the largest supported page size. That is the finest granularity 2097c478bd9Sstevel@tonic-gate * possible since the counter values are dependent on each other 2107c478bd9Sstevel@tonic-gate * as you move accross region sizes. PP_CTR_LOCK_INDX is used to determine the 2117c478bd9Sstevel@tonic-gate * ctr_mutex lock index for a particular physical range. 2127c478bd9Sstevel@tonic-gate */ 2137c478bd9Sstevel@tonic-gate static kmutex_t *ctr_mutex[NPC_MUTEX]; 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate #define PP_CTR_LOCK_INDX(pp) \ 2165d07b933Sdp (((pp)->p_pagenum >> \ 2177c478bd9Sstevel@tonic-gate (PAGE_BSZS_SHIFT(mmu_page_sizes - 1))) & (NPC_MUTEX - 1)) 2187c478bd9Sstevel@tonic-gate 2195d07b933Sdp #define INVALID_COLOR 0xffffffff 2205d07b933Sdp #define INVALID_MASK 0xffffffff 2215d07b933Sdp 2227c478bd9Sstevel@tonic-gate /* 2237c478bd9Sstevel@tonic-gate * Local functions prototypes. 2247c478bd9Sstevel@tonic-gate */ 2257c478bd9Sstevel@tonic-gate 226affbd3ccSkchow void page_ctr_add(int, int, page_t *, int); 227affbd3ccSkchow void page_ctr_add_internal(int, int, page_t *, int); 228affbd3ccSkchow void page_ctr_sub(int, int, page_t *, int); 2295d07b933Sdp void page_ctr_sub_internal(int, int, page_t *, int); 2307c478bd9Sstevel@tonic-gate void page_freelist_lock(int); 2317c478bd9Sstevel@tonic-gate void page_freelist_unlock(int); 2325d07b933Sdp page_t *page_promote(int, pfn_t, uchar_t, int, int); 2337c478bd9Sstevel@tonic-gate page_t *page_demote(int, pfn_t, uchar_t, uchar_t, int, int); 2345d07b933Sdp page_t *page_freelist_split(uchar_t, 2355d07b933Sdp uint_t, int, int, pfn_t, page_list_walker_t *); 2367c478bd9Sstevel@tonic-gate page_t *page_get_mnode_cachelist(uint_t, uint_t, int, int); 2377c478bd9Sstevel@tonic-gate static int page_trylock_cons(page_t *pp, se_t se); 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * The page_counters array below is used to keep track of free contiguous 2417c478bd9Sstevel@tonic-gate * physical memory. A hw_page_map_t will be allocated per mnode per szc. 2427c478bd9Sstevel@tonic-gate * This contains an array of counters, the size of the array, a shift value 2437c478bd9Sstevel@tonic-gate * used to convert a pagenum into a counter array index or vice versa, as 2447c478bd9Sstevel@tonic-gate * well as a cache of the last successful index to be promoted to a larger 2457c478bd9Sstevel@tonic-gate * page size. As an optimization, we keep track of the last successful index 2467c478bd9Sstevel@tonic-gate * to be promoted per page color for the given size region, and this is 2477c478bd9Sstevel@tonic-gate * allocated dynamically based upon the number of colors for a given 2487c478bd9Sstevel@tonic-gate * region size. 2497c478bd9Sstevel@tonic-gate * 2507c478bd9Sstevel@tonic-gate * Conceptually, the page counters are represented as: 2517c478bd9Sstevel@tonic-gate * 2527c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode] 2537c478bd9Sstevel@tonic-gate * 2547c478bd9Sstevel@tonic-gate * region_size: size code of a candidate larger page made up 2557c478bd9Sstevel@tonic-gate * of contiguous free smaller pages. 2567c478bd9Sstevel@tonic-gate * 2577c478bd9Sstevel@tonic-gate * page_counters[region_size][mnode].hpm_counters[index]: 2587c478bd9Sstevel@tonic-gate * represents how many (region_size - 1) pages either 2597c478bd9Sstevel@tonic-gate * exist or can be created within the given index range. 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * Let's look at a sparc example: 2627c478bd9Sstevel@tonic-gate * If we want to create a free 512k page, we look at region_size 2 2637c478bd9Sstevel@tonic-gate * for the mnode we want. We calculate the index and look at a specific 2647c478bd9Sstevel@tonic-gate * hpm_counters location. If we see 8 (FULL_REGION_CNT on sparc) at 2657c478bd9Sstevel@tonic-gate * this location, it means that 8 64k pages either exist or can be created 2667c478bd9Sstevel@tonic-gate * from 8K pages in order to make a single free 512k page at the given 2677c478bd9Sstevel@tonic-gate * index. Note that when a region is full, it will contribute to the 2687c478bd9Sstevel@tonic-gate * counts in the region above it. Thus we will not know what page 2697c478bd9Sstevel@tonic-gate * size the free pages will be which can be promoted to this new free 2707c478bd9Sstevel@tonic-gate * page unless we look at all regions below the current region. 2717c478bd9Sstevel@tonic-gate */ 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate /* 2747c478bd9Sstevel@tonic-gate * Note: hpmctr_t is defined in platform vm_dep.h 2757c478bd9Sstevel@tonic-gate * hw_page_map_t contains all the information needed for the page_counters 2767c478bd9Sstevel@tonic-gate * logic. The fields are as follows: 2777c478bd9Sstevel@tonic-gate * 2787c478bd9Sstevel@tonic-gate * hpm_counters: dynamically allocated array to hold counter data 2797c478bd9Sstevel@tonic-gate * hpm_entries: entries in hpm_counters 2807c478bd9Sstevel@tonic-gate * hpm_shift: shift for pnum/array index conv 2817c478bd9Sstevel@tonic-gate * hpm_base: PFN mapped to counter index 0 2827c478bd9Sstevel@tonic-gate * hpm_color_current: last index in counter array for this color at 2837c478bd9Sstevel@tonic-gate * which we successfully created a large page 2847c478bd9Sstevel@tonic-gate */ 2857c478bd9Sstevel@tonic-gate typedef struct hw_page_map { 2867c478bd9Sstevel@tonic-gate hpmctr_t *hpm_counters; 2877c478bd9Sstevel@tonic-gate size_t hpm_entries; 2887c478bd9Sstevel@tonic-gate int hpm_shift; 2897c478bd9Sstevel@tonic-gate pfn_t hpm_base; 2905d07b933Sdp size_t *hpm_color_current[MAX_MNODE_MRANGES]; 2917c478bd9Sstevel@tonic-gate } hw_page_map_t; 2927c478bd9Sstevel@tonic-gate 2937c478bd9Sstevel@tonic-gate /* 2947c478bd9Sstevel@tonic-gate * Element zero is not used, but is allocated for convenience. 2957c478bd9Sstevel@tonic-gate */ 2967c478bd9Sstevel@tonic-gate static hw_page_map_t *page_counters[MMU_PAGE_SIZES]; 2977c478bd9Sstevel@tonic-gate 2985d07b933Sdp /* 2995d07b933Sdp * Cached value of MNODE_RANGE_CNT(mnode). 3005d07b933Sdp * This is a function call in x86. 3015d07b933Sdp */ 3025d07b933Sdp static int mnode_nranges[MAX_MEM_NODES]; 3035d07b933Sdp static int mnode_maxmrange[MAX_MEM_NODES]; 3045d07b933Sdp 3057c478bd9Sstevel@tonic-gate /* 3067c478bd9Sstevel@tonic-gate * The following macros are convenient ways to get access to the individual 3077c478bd9Sstevel@tonic-gate * elements of the page_counters arrays. They can be used on both 3087c478bd9Sstevel@tonic-gate * the left side and right side of equations. 3097c478bd9Sstevel@tonic-gate */ 3107c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS(mnode, rg_szc, idx) \ 3117c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters[(idx)]) 3127c478bd9Sstevel@tonic-gate 3137c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_COUNTERS(mnode, rg_szc) \ 3147c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_counters) 3157c478bd9Sstevel@tonic-gate 3167c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_SHIFT(mnode, rg_szc) \ 3177c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_shift) 3187c478bd9Sstevel@tonic-gate 3197c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_ENTRIES(mnode, rg_szc) \ 3207c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_entries) 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate #define PAGE_COUNTERS_BASE(mnode, rg_szc) \ 3237c478bd9Sstevel@tonic-gate (page_counters[(rg_szc)][(mnode)].hpm_base) 3247c478bd9Sstevel@tonic-gate 3255d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, rg_szc, g) \ 3265d07b933Sdp (page_counters[(rg_szc)][(mnode)].hpm_color_current[(g)]) 3277c478bd9Sstevel@tonic-gate 3285d07b933Sdp #define PAGE_COUNTERS_CURRENT_COLOR(mnode, rg_szc, color, mrange) \ 3295d07b933Sdp (page_counters[(rg_szc)][(mnode)]. \ 3305d07b933Sdp hpm_color_current[(mrange)][(color)]) 3317c478bd9Sstevel@tonic-gate 3327c478bd9Sstevel@tonic-gate #define PNUM_TO_IDX(mnode, rg_szc, pnum) \ 3337c478bd9Sstevel@tonic-gate (((pnum) - PAGE_COUNTERS_BASE((mnode), (rg_szc))) >> \ 3347c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT((mnode), (rg_szc))) 3357c478bd9Sstevel@tonic-gate 3367c478bd9Sstevel@tonic-gate #define IDX_TO_PNUM(mnode, rg_szc, index) \ 3377c478bd9Sstevel@tonic-gate (PAGE_COUNTERS_BASE((mnode), (rg_szc)) + \ 3387c478bd9Sstevel@tonic-gate ((index) << PAGE_COUNTERS_SHIFT((mnode), (rg_szc)))) 3397c478bd9Sstevel@tonic-gate 3407c478bd9Sstevel@tonic-gate /* 3417c478bd9Sstevel@tonic-gate * Protects the hpm_counters and hpm_color_current memory from changing while 3427c478bd9Sstevel@tonic-gate * looking at page counters information. 3437c478bd9Sstevel@tonic-gate * Grab the write lock to modify what these fields point at. 3447c478bd9Sstevel@tonic-gate * Grab the read lock to prevent any pointers from changing. 3457c478bd9Sstevel@tonic-gate * The write lock can not be held during memory allocation due to a possible 3467c478bd9Sstevel@tonic-gate * recursion deadlock with trying to grab the read lock while the 3477c478bd9Sstevel@tonic-gate * write lock is already held. 3487c478bd9Sstevel@tonic-gate */ 3497c478bd9Sstevel@tonic-gate krwlock_t page_ctrs_rwlock[MAX_MEM_NODES]; 3507c478bd9Sstevel@tonic-gate 351affbd3ccSkchow 352affbd3ccSkchow /* 353affbd3ccSkchow * initialize cpu_vm_data to point at cache aligned vm_cpu_data_t. 354affbd3ccSkchow */ 355affbd3ccSkchow void 356affbd3ccSkchow cpu_vm_data_init(struct cpu *cp) 357affbd3ccSkchow { 358affbd3ccSkchow if (cp == CPU0) { 359affbd3ccSkchow cp->cpu_vm_data = (void *)&vm_cpu_data0; 360affbd3ccSkchow } else { 361affbd3ccSkchow void *kmptr; 3626061ce8aSkchow int align; 3636061ce8aSkchow size_t sz; 364affbd3ccSkchow 3656061ce8aSkchow align = (L2CACHE_ALIGN) ? L2CACHE_ALIGN : L2CACHE_ALIGN_MAX; 3666061ce8aSkchow sz = P2ROUNDUP(sizeof (vm_cpu_data_t), align) + align; 3676061ce8aSkchow kmptr = kmem_zalloc(sz, KM_SLEEP); 368affbd3ccSkchow cp->cpu_vm_data = (void *) P2ROUNDUP((uintptr_t)kmptr, align); 369affbd3ccSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr = kmptr; 3706061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize = sz; 371affbd3ccSkchow } 372affbd3ccSkchow } 373affbd3ccSkchow 374affbd3ccSkchow /* 375affbd3ccSkchow * free cpu_vm_data 376affbd3ccSkchow */ 377affbd3ccSkchow void 378affbd3ccSkchow cpu_vm_data_destroy(struct cpu *cp) 379affbd3ccSkchow { 380affbd3ccSkchow if (cp->cpu_seqid && cp->cpu_vm_data) { 381affbd3ccSkchow ASSERT(cp != CPU0); 382affbd3ccSkchow kmem_free(((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmptr, 3836061ce8aSkchow ((vm_cpu_data_t *)cp->cpu_vm_data)->vc_kmsize); 384affbd3ccSkchow } 385affbd3ccSkchow cp->cpu_vm_data = NULL; 386affbd3ccSkchow } 387affbd3ccSkchow 388affbd3ccSkchow 3897c478bd9Sstevel@tonic-gate /* 3907c478bd9Sstevel@tonic-gate * page size to page size code 3917c478bd9Sstevel@tonic-gate */ 3927c478bd9Sstevel@tonic-gate int 3937c478bd9Sstevel@tonic-gate page_szc(size_t pagesize) 3947c478bd9Sstevel@tonic-gate { 3957c478bd9Sstevel@tonic-gate int i = 0; 3967c478bd9Sstevel@tonic-gate 3977c478bd9Sstevel@tonic-gate while (hw_page_array[i].hp_size) { 3987c478bd9Sstevel@tonic-gate if (pagesize == hw_page_array[i].hp_size) 3997c478bd9Sstevel@tonic-gate return (i); 4007c478bd9Sstevel@tonic-gate i++; 4017c478bd9Sstevel@tonic-gate } 4027c478bd9Sstevel@tonic-gate return (-1); 4037c478bd9Sstevel@tonic-gate } 4047c478bd9Sstevel@tonic-gate 4057c478bd9Sstevel@tonic-gate /* 4064abce959Smec * page size to page size code with the restriction that it be a supported 4074abce959Smec * user page size. If it's not a supported user page size, -1 will be returned. 4087c478bd9Sstevel@tonic-gate */ 4097c478bd9Sstevel@tonic-gate int 4104abce959Smec page_szc_user_filtered(size_t pagesize) 4117c478bd9Sstevel@tonic-gate { 4127c478bd9Sstevel@tonic-gate int szc = page_szc(pagesize); 4134abce959Smec if ((szc != -1) && (SZC_2_USERSZC(szc) != -1)) { 4144abce959Smec return (szc); 4154abce959Smec } 4167c478bd9Sstevel@tonic-gate return (-1); 4177c478bd9Sstevel@tonic-gate } 4187c478bd9Sstevel@tonic-gate 4197c478bd9Sstevel@tonic-gate /* 4207c478bd9Sstevel@tonic-gate * Return how many page sizes are available for the user to use. This is 4217c478bd9Sstevel@tonic-gate * what the hardware supports and not based upon how the OS implements the 4227c478bd9Sstevel@tonic-gate * support of different page sizes. 42302bc52beSkchow * 42402bc52beSkchow * If legacy is non-zero, return the number of pagesizes available to legacy 42502bc52beSkchow * applications. The number of legacy page sizes might be less than the 42602bc52beSkchow * exported user page sizes. This is to prevent legacy applications that 42702bc52beSkchow * use the largest page size returned from getpagesizes(3c) from inadvertantly 42802bc52beSkchow * using the 'new' large pagesizes. 4297c478bd9Sstevel@tonic-gate */ 4307c478bd9Sstevel@tonic-gate uint_t 43102bc52beSkchow page_num_user_pagesizes(int legacy) 4327c478bd9Sstevel@tonic-gate { 43302bc52beSkchow if (legacy) 43402bc52beSkchow return (mmu_legacy_page_sizes); 4357c478bd9Sstevel@tonic-gate return (mmu_exported_page_sizes); 4367c478bd9Sstevel@tonic-gate } 4377c478bd9Sstevel@tonic-gate 4387c478bd9Sstevel@tonic-gate uint_t 4397c478bd9Sstevel@tonic-gate page_num_pagesizes(void) 4407c478bd9Sstevel@tonic-gate { 4417c478bd9Sstevel@tonic-gate return (mmu_page_sizes); 4427c478bd9Sstevel@tonic-gate } 4437c478bd9Sstevel@tonic-gate 4447c478bd9Sstevel@tonic-gate /* 4457c478bd9Sstevel@tonic-gate * returns the count of the number of base pagesize pages associated with szc 4467c478bd9Sstevel@tonic-gate */ 4477c478bd9Sstevel@tonic-gate pgcnt_t 4487c478bd9Sstevel@tonic-gate page_get_pagecnt(uint_t szc) 4497c478bd9Sstevel@tonic-gate { 4507c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4517c478bd9Sstevel@tonic-gate panic("page_get_pagecnt: out of range %d", szc); 4527c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_pgcnt); 4537c478bd9Sstevel@tonic-gate } 4547c478bd9Sstevel@tonic-gate 4557c478bd9Sstevel@tonic-gate size_t 4567c478bd9Sstevel@tonic-gate page_get_pagesize(uint_t szc) 4577c478bd9Sstevel@tonic-gate { 4587c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4597c478bd9Sstevel@tonic-gate panic("page_get_pagesize: out of range %d", szc); 4607c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4617c478bd9Sstevel@tonic-gate } 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate /* 4647c478bd9Sstevel@tonic-gate * Return the size of a page based upon the index passed in. An index of 4657c478bd9Sstevel@tonic-gate * zero refers to the smallest page size in the system, and as index increases 4667c478bd9Sstevel@tonic-gate * it refers to the next larger supported page size in the system. 4677c478bd9Sstevel@tonic-gate * Note that szc and userszc may not be the same due to unsupported szc's on 4687c478bd9Sstevel@tonic-gate * some systems. 4697c478bd9Sstevel@tonic-gate */ 4707c478bd9Sstevel@tonic-gate size_t 4717c478bd9Sstevel@tonic-gate page_get_user_pagesize(uint_t userszc) 4727c478bd9Sstevel@tonic-gate { 4737c478bd9Sstevel@tonic-gate uint_t szc = USERSZC_2_SZC(userszc); 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4767c478bd9Sstevel@tonic-gate panic("page_get_user_pagesize: out of range %d", szc); 4777c478bd9Sstevel@tonic-gate return (hw_page_array[szc].hp_size); 4787c478bd9Sstevel@tonic-gate } 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate uint_t 4817c478bd9Sstevel@tonic-gate page_get_shift(uint_t szc) 4827c478bd9Sstevel@tonic-gate { 4837c478bd9Sstevel@tonic-gate if (szc >= mmu_page_sizes) 4847c478bd9Sstevel@tonic-gate panic("page_get_shift: out of range %d", szc); 4855d07b933Sdp return (PAGE_GET_SHIFT(szc)); 4867c478bd9Sstevel@tonic-gate } 4877c478bd9Sstevel@tonic-gate 4887c478bd9Sstevel@tonic-gate uint_t 4897c478bd9Sstevel@tonic-gate page_get_pagecolors(uint_t szc) 4907c478bd9Sstevel@tonic-gate { 4915d07b933Sdp if (szc >= mmu_page_sizes) 4925d07b933Sdp panic("page_get_pagecolors: out of range %d", szc); 4935d07b933Sdp return (PAGE_GET_PAGECOLORS(szc)); 4945d07b933Sdp } 4955d07b933Sdp 4965d07b933Sdp /* 4975d07b933Sdp * this assigns the desired equivalent color after a split 4985d07b933Sdp */ 4995d07b933Sdp uint_t 5005d07b933Sdp page_correct_color(uchar_t szc, uchar_t nszc, uint_t color, 5015d07b933Sdp uint_t ncolor, uint_t ceq_mask) 5025d07b933Sdp { 5035d07b933Sdp ASSERT(nszc > szc); 5045d07b933Sdp ASSERT(szc < mmu_page_sizes); 5055d07b933Sdp ASSERT(color < PAGE_GET_PAGECOLORS(szc)); 5065d07b933Sdp ASSERT(ncolor < PAGE_GET_PAGECOLORS(nszc)); 5075d07b933Sdp 5085d07b933Sdp color &= ceq_mask; 509ce8eb11aSdp ncolor = PAGE_CONVERT_COLOR(ncolor, szc, nszc); 5105d07b933Sdp return (color | (ncolor & ~ceq_mask)); 5117c478bd9Sstevel@tonic-gate } 5127c478bd9Sstevel@tonic-gate 513ce8eb11aSdp /* 514ce8eb11aSdp * The interleaved_mnodes flag is set when mnodes overlap in 515ce8eb11aSdp * the physbase..physmax range, but have disjoint slices. 516ce8eb11aSdp * In this case hpm_counters is shared by all mnodes. 517ce8eb11aSdp * This flag is set dynamically by the platform. 518ce8eb11aSdp */ 519ce8eb11aSdp int interleaved_mnodes = 0; 520ce8eb11aSdp 5217c478bd9Sstevel@tonic-gate /* 5227c478bd9Sstevel@tonic-gate * Called by startup(). 5237c478bd9Sstevel@tonic-gate * Size up the per page size free list counters based on physmax 5247c478bd9Sstevel@tonic-gate * of each node and max_mem_nodes. 525ce8eb11aSdp * 526ce8eb11aSdp * If interleaved_mnodes is set we need to find the first mnode that 527ce8eb11aSdp * exists. hpm_counters for the first mnode will then be shared by 528ce8eb11aSdp * all other mnodes. If interleaved_mnodes is not set, just set 529ce8eb11aSdp * first=mnode each time. That means there will be no sharing. 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate size_t 5327c478bd9Sstevel@tonic-gate page_ctrs_sz(void) 5337c478bd9Sstevel@tonic-gate { 5347c478bd9Sstevel@tonic-gate int r; /* region size */ 5357c478bd9Sstevel@tonic-gate int mnode; 536ce8eb11aSdp int firstmn; /* first mnode that exists */ 5375d07b933Sdp int nranges; 538ce8eb11aSdp pfn_t physbase; 539ce8eb11aSdp pfn_t physmax; 5407c478bd9Sstevel@tonic-gate uint_t ctrs_sz = 0; 5417c478bd9Sstevel@tonic-gate int i; 5427c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 5437c478bd9Sstevel@tonic-gate 5447c478bd9Sstevel@tonic-gate /* 5457c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 5467c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 5477c478bd9Sstevel@tonic-gate * arrays. 5487c478bd9Sstevel@tonic-gate */ 5495d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 5505d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 5517c478bd9Sstevel@tonic-gate } 5527c478bd9Sstevel@tonic-gate 553ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 5567c478bd9Sstevel@tonic-gate pfn_t r_base; 5577c478bd9Sstevel@tonic-gate pgcnt_t r_align; 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 5607c478bd9Sstevel@tonic-gate continue; 5617c478bd9Sstevel@tonic-gate 562ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 5635d07b933Sdp nranges = MNODE_RANGE_CNT(mnode); 5645d07b933Sdp mnode_nranges[mnode] = nranges; 5655d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 5665d07b933Sdp 5677c478bd9Sstevel@tonic-gate /* 5687c478bd9Sstevel@tonic-gate * determine size needed for page counter arrays with 5697c478bd9Sstevel@tonic-gate * base aligned to large page size. 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 572ce8eb11aSdp /* add in space for hpm_color_current */ 573ce8eb11aSdp ctrs_sz += sizeof (size_t) * 574ce8eb11aSdp colors_per_szc[r] * nranges; 575ce8eb11aSdp 576ce8eb11aSdp if (firstmn != mnode) 577ce8eb11aSdp continue; 578ce8eb11aSdp 5797c478bd9Sstevel@tonic-gate /* add in space for hpm_counters */ 5807c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 581ce8eb11aSdp r_base = physbase; 5827c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 583ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 584ce8eb11aSdp 5857c478bd9Sstevel@tonic-gate /* 5867c478bd9Sstevel@tonic-gate * Round up to always allocate on pointer sized 5877c478bd9Sstevel@tonic-gate * boundaries. 5887c478bd9Sstevel@tonic-gate */ 5897c478bd9Sstevel@tonic-gate ctrs_sz += P2ROUNDUP((r_pgcnt * sizeof (hpmctr_t)), 5907c478bd9Sstevel@tonic-gate sizeof (hpmctr_t *)); 5917c478bd9Sstevel@tonic-gate } 5927c478bd9Sstevel@tonic-gate } 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 5957c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * sizeof (hw_page_map_t)); 5965d07b933Sdp } 5975d07b933Sdp 5985d07b933Sdp /* add in space for page_ctrs_cands and pcc_color_free */ 5995d07b933Sdp ctrs_sz += sizeof (pcc_info_t *) * max_mem_nodes * 6005d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6015d07b933Sdp 6025d07b933Sdp for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6035d07b933Sdp 6045d07b933Sdp if (mem_node_config[mnode].exists == 0) 6055d07b933Sdp continue; 6067c478bd9Sstevel@tonic-gate 6075d07b933Sdp nranges = mnode_nranges[mnode]; 6085d07b933Sdp ctrs_sz += sizeof (pcc_info_t) * nranges * 6095d07b933Sdp mmu_page_sizes * NPC_MUTEX; 6105d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6115d07b933Sdp ctrs_sz += sizeof (pgcnt_t) * nranges * 6125d07b933Sdp colors_per_szc[r] * NPC_MUTEX; 6135d07b933Sdp } 6147c478bd9Sstevel@tonic-gate } 6157c478bd9Sstevel@tonic-gate 6167c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6177c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * NPC_MUTEX * sizeof (kmutex_t)); 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate /* size for page list counts */ 6207c478bd9Sstevel@tonic-gate PLCNT_SZ(ctrs_sz); 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate /* 6237c478bd9Sstevel@tonic-gate * add some slop for roundups. page_ctrs_alloc will roundup the start 6247c478bd9Sstevel@tonic-gate * address of the counters to ecache_alignsize boundary for every 6257c478bd9Sstevel@tonic-gate * memory node. 6267c478bd9Sstevel@tonic-gate */ 6277c478bd9Sstevel@tonic-gate return (ctrs_sz + max_mem_nodes * L2CACHE_ALIGN); 6287c478bd9Sstevel@tonic-gate } 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate caddr_t 6317c478bd9Sstevel@tonic-gate page_ctrs_alloc(caddr_t alloc_base) 6327c478bd9Sstevel@tonic-gate { 6337c478bd9Sstevel@tonic-gate int mnode; 6345d07b933Sdp int mrange, nranges; 6357c478bd9Sstevel@tonic-gate int r; /* region size */ 6367c478bd9Sstevel@tonic-gate int i; 637ce8eb11aSdp int firstmn; /* first mnode that exists */ 638ce8eb11aSdp pfn_t physbase; 639ce8eb11aSdp pfn_t physmax; 6407c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate /* 6437c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 6447c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 6457c478bd9Sstevel@tonic-gate * arrays. 6467c478bd9Sstevel@tonic-gate */ 6475d07b933Sdp for (i = 0; i < mmu_page_sizes; i++) { 6485d07b933Sdp colors_per_szc[i] = PAGE_GET_PAGECOLORS(i); 6497c478bd9Sstevel@tonic-gate } 6507c478bd9Sstevel@tonic-gate 6517c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 6527c478bd9Sstevel@tonic-gate page_counters[r] = (hw_page_map_t *)alloc_base; 6537c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (hw_page_map_t)); 6547c478bd9Sstevel@tonic-gate } 6557c478bd9Sstevel@tonic-gate 6565d07b933Sdp /* page_ctrs_cands and pcc_color_free array */ 6575d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 6585d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 6597c478bd9Sstevel@tonic-gate 6605d07b933Sdp page_ctrs_cands[i][r] = (pcc_info_t **)alloc_base; 6615d07b933Sdp alloc_base += sizeof (pcc_info_t *) * max_mem_nodes; 6627c478bd9Sstevel@tonic-gate 6637c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) { 6645d07b933Sdp pcc_info_t *pi; 6655d07b933Sdp 6665d07b933Sdp if (mem_node_config[mnode].exists == 0) 6675d07b933Sdp continue; 6685d07b933Sdp 6695d07b933Sdp nranges = mnode_nranges[mnode]; 6705d07b933Sdp 6715d07b933Sdp pi = (pcc_info_t *)alloc_base; 6725d07b933Sdp alloc_base += sizeof (pcc_info_t) * nranges; 6735d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 6745d07b933Sdp 6755d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 6765d07b933Sdp pi->pcc_color_free = 6775d07b933Sdp (pgcnt_t *)alloc_base; 6785d07b933Sdp alloc_base += sizeof (pgcnt_t) * 6795d07b933Sdp colors_per_szc[r]; 6805d07b933Sdp pi++; 6815d07b933Sdp } 6827c478bd9Sstevel@tonic-gate } 6837c478bd9Sstevel@tonic-gate } 6847c478bd9Sstevel@tonic-gate } 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate /* ctr_mutex */ 6877c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 6887c478bd9Sstevel@tonic-gate ctr_mutex[i] = (kmutex_t *)alloc_base; 6897c478bd9Sstevel@tonic-gate alloc_base += (max_mem_nodes * sizeof (kmutex_t)); 6907c478bd9Sstevel@tonic-gate } 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate /* initialize page list counts */ 6937c478bd9Sstevel@tonic-gate PLCNT_INIT(alloc_base); 6947c478bd9Sstevel@tonic-gate 695ce8eb11aSdp for (firstmn = -1, mnode = 0; mnode < max_mem_nodes; mnode++) { 6967c478bd9Sstevel@tonic-gate 6977c478bd9Sstevel@tonic-gate pgcnt_t r_pgcnt; 6987c478bd9Sstevel@tonic-gate pfn_t r_base; 6997c478bd9Sstevel@tonic-gate pgcnt_t r_align; 7007c478bd9Sstevel@tonic-gate int r_shift; 7015d07b933Sdp int nranges = mnode_nranges[mnode]; 7027c478bd9Sstevel@tonic-gate 7037c478bd9Sstevel@tonic-gate if (mem_node_config[mnode].exists == 0) 7047c478bd9Sstevel@tonic-gate continue; 7057c478bd9Sstevel@tonic-gate 706ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, firstmn); 707ce8eb11aSdp 7087c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 7097c478bd9Sstevel@tonic-gate /* 7107c478bd9Sstevel@tonic-gate * the page_counters base has to be aligned to the 7117c478bd9Sstevel@tonic-gate * page count of page size code r otherwise the counts 7127c478bd9Sstevel@tonic-gate * will cross large page boundaries. 7137c478bd9Sstevel@tonic-gate */ 7147c478bd9Sstevel@tonic-gate r_align = page_get_pagecnt(r); 715ce8eb11aSdp r_base = physbase; 7167c478bd9Sstevel@tonic-gate /* base needs to be aligned - lower to aligned value */ 7177c478bd9Sstevel@tonic-gate r_base &= ~(r_align - 1); 718ce8eb11aSdp r_pgcnt = howmany(physmax - r_base + 1, r_align); 7197c478bd9Sstevel@tonic-gate r_shift = PAGE_BSZS_SHIFT(r); 7207c478bd9Sstevel@tonic-gate 7217c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = r_shift; 7227c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = r_pgcnt; 7237c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = r_base; 7245d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7255d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 7265d07b933Sdp r, mrange) = (size_t *)alloc_base; 7275d07b933Sdp alloc_base += sizeof (size_t) * 7285d07b933Sdp colors_per_szc[r]; 7295d07b933Sdp } 7307c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 7315d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 7325d07b933Sdp pfn_t pfnum = r_base; 7335d07b933Sdp size_t idx; 7345d07b933Sdp int mrange; 735ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 7365d07b933Sdp 737*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, r, &it); 738*b779d3e0Sdp if (pfnum == (pfn_t)-1) { 739*b779d3e0Sdp idx = 0; 740*b779d3e0Sdp } else { 741*b779d3e0Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 742*b779d3e0Sdp color_mask, color_mask, &it); 743*b779d3e0Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 744*b779d3e0Sdp idx = (idx >= r_pgcnt) ? 0 : idx; 745*b779d3e0Sdp } 7465d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 7475d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, 7485d07b933Sdp r, i, mrange) = idx; 7495d07b933Sdp } 7507c478bd9Sstevel@tonic-gate } 751ce8eb11aSdp 752ce8eb11aSdp /* hpm_counters may be shared by all mnodes */ 753ce8eb11aSdp if (firstmn == mnode) { 754ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 755ce8eb11aSdp (hpmctr_t *)alloc_base; 756ce8eb11aSdp alloc_base += 757ce8eb11aSdp P2ROUNDUP((sizeof (hpmctr_t) * r_pgcnt), 758ce8eb11aSdp sizeof (hpmctr_t *)); 759ce8eb11aSdp } else { 760ce8eb11aSdp PAGE_COUNTERS_COUNTERS(mnode, r) = 761ce8eb11aSdp PAGE_COUNTERS_COUNTERS(firstmn, r); 762ce8eb11aSdp } 7637c478bd9Sstevel@tonic-gate 7647c478bd9Sstevel@tonic-gate /* 7657c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 7667c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 7677c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 7687c478bd9Sstevel@tonic-gate * and get consistent values. 7697c478bd9Sstevel@tonic-gate */ 7707c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 7717c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 7727c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 7737c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, r_base))) == r_base); 7747c478bd9Sstevel@tonic-gate } 7757c478bd9Sstevel@tonic-gate /* 7767c478bd9Sstevel@tonic-gate * Roundup the start address of the page_counters to 7777c478bd9Sstevel@tonic-gate * cache aligned boundary for every memory node. 7787c478bd9Sstevel@tonic-gate * page_ctrs_sz() has added some slop for these roundups. 7797c478bd9Sstevel@tonic-gate */ 7807c478bd9Sstevel@tonic-gate alloc_base = (caddr_t)P2ROUNDUP((uintptr_t)alloc_base, 781ce8eb11aSdp L2CACHE_ALIGN); 7827c478bd9Sstevel@tonic-gate } 7837c478bd9Sstevel@tonic-gate 7847c478bd9Sstevel@tonic-gate /* Initialize other page counter specific data structures. */ 7857c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < MAX_MEM_NODES; mnode++) { 7867c478bd9Sstevel@tonic-gate rw_init(&page_ctrs_rwlock[mnode], NULL, RW_DEFAULT, NULL); 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate 7897c478bd9Sstevel@tonic-gate return (alloc_base); 7907c478bd9Sstevel@tonic-gate } 7917c478bd9Sstevel@tonic-gate 7927c478bd9Sstevel@tonic-gate /* 7937c478bd9Sstevel@tonic-gate * Functions to adjust region counters for each size free list. 7947c478bd9Sstevel@tonic-gate * Caller is responsible to acquire the ctr_mutex lock if necessary and 7957c478bd9Sstevel@tonic-gate * thus can be called during startup without locks. 7967c478bd9Sstevel@tonic-gate */ 7977c478bd9Sstevel@tonic-gate /* ARGSUSED */ 7987c478bd9Sstevel@tonic-gate void 799affbd3ccSkchow page_ctr_add_internal(int mnode, int mtype, page_t *pp, int flags) 8007c478bd9Sstevel@tonic-gate { 8017c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8027c478bd9Sstevel@tonic-gate ssize_t idx; 8037c478bd9Sstevel@tonic-gate pfn_t pfnum; 8047c478bd9Sstevel@tonic-gate int lckidx; 8057c478bd9Sstevel@tonic-gate 806affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 807affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 808affbd3ccSkchow 8097c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8107c478bd9Sstevel@tonic-gate 811affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8147c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8157c478bd9Sstevel@tonic-gate return; 8167c478bd9Sstevel@tonic-gate } 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8197c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8207c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate /* 8237c478bd9Sstevel@tonic-gate * Increment the count of free pages for the current 8247c478bd9Sstevel@tonic-gate * region. Continue looping up in region size incrementing 8257c478bd9Sstevel@tonic-gate * count if the preceeding region is full. 8267c478bd9Sstevel@tonic-gate */ 8277c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8287c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8297c478bd9Sstevel@tonic-gate 8307c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8317c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) < FULL_REGION_CNT(r)); 8327c478bd9Sstevel@tonic-gate 8335d07b933Sdp if (++PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r)) { 8347c478bd9Sstevel@tonic-gate break; 8355d07b933Sdp } else { 8365d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 8375d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 8385d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 8397c478bd9Sstevel@tonic-gate 8405d07b933Sdp cand->pcc_pages_free++; 8415d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]++; 8425d07b933Sdp } 8437c478bd9Sstevel@tonic-gate r++; 8447c478bd9Sstevel@tonic-gate } 8457c478bd9Sstevel@tonic-gate } 8467c478bd9Sstevel@tonic-gate 8477c478bd9Sstevel@tonic-gate void 848affbd3ccSkchow page_ctr_add(int mnode, int mtype, page_t *pp, int flags) 8497c478bd9Sstevel@tonic-gate { 8507c478bd9Sstevel@tonic-gate int lckidx = PP_CTR_LOCK_INDX(pp); 8517c478bd9Sstevel@tonic-gate kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 8527c478bd9Sstevel@tonic-gate 8537c478bd9Sstevel@tonic-gate mutex_enter(lock); 854affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 8557c478bd9Sstevel@tonic-gate mutex_exit(lock); 8567c478bd9Sstevel@tonic-gate } 8577c478bd9Sstevel@tonic-gate 8587c478bd9Sstevel@tonic-gate void 8595d07b933Sdp page_ctr_sub_internal(int mnode, int mtype, page_t *pp, int flags) 8607c478bd9Sstevel@tonic-gate { 8617c478bd9Sstevel@tonic-gate int lckidx; 8627c478bd9Sstevel@tonic-gate ssize_t r; /* region size */ 8637c478bd9Sstevel@tonic-gate ssize_t idx; 8647c478bd9Sstevel@tonic-gate pfn_t pfnum; 8657c478bd9Sstevel@tonic-gate 866affbd3ccSkchow ASSERT(mnode == PP_2_MEM_NODE(pp)); 867affbd3ccSkchow ASSERT(mtype == PP_2_MTYPE(pp)); 868affbd3ccSkchow 8697c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < mmu_page_sizes); 8707c478bd9Sstevel@tonic-gate 871affbd3ccSkchow PLCNT_DECR(pp, mnode, mtype, pp->p_szc, flags); 8727c478bd9Sstevel@tonic-gate 8737c478bd9Sstevel@tonic-gate /* no counter update needed for largest page size */ 8747c478bd9Sstevel@tonic-gate if (pp->p_szc >= mmu_page_sizes - 1) { 8757c478bd9Sstevel@tonic-gate return; 8767c478bd9Sstevel@tonic-gate } 8777c478bd9Sstevel@tonic-gate 8787c478bd9Sstevel@tonic-gate r = pp->p_szc + 1; 8797c478bd9Sstevel@tonic-gate pfnum = pp->p_pagenum; 8807c478bd9Sstevel@tonic-gate lckidx = PP_CTR_LOCK_INDX(pp); 8817c478bd9Sstevel@tonic-gate 8827c478bd9Sstevel@tonic-gate /* 8837c478bd9Sstevel@tonic-gate * Decrement the count of free pages for the current 8847c478bd9Sstevel@tonic-gate * region. Continue looping up in region size decrementing 8857c478bd9Sstevel@tonic-gate * count if the preceeding region was full. 8867c478bd9Sstevel@tonic-gate */ 8877c478bd9Sstevel@tonic-gate while (r < mmu_page_sizes) { 8887c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pfnum); 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate ASSERT(idx < PAGE_COUNTERS_ENTRIES(mnode, r)); 8917c478bd9Sstevel@tonic-gate ASSERT(PAGE_COUNTERS(mnode, r, idx) > 0); 8927c478bd9Sstevel@tonic-gate 8937c478bd9Sstevel@tonic-gate if (--PAGE_COUNTERS(mnode, r, idx) != FULL_REGION_CNT(r) - 1) { 8947c478bd9Sstevel@tonic-gate break; 8955d07b933Sdp } else { 8965d07b933Sdp int root_mtype = PP_2_MTYPE(PP_GROUPLEADER(pp, r)); 8975d07b933Sdp pcc_info_t *cand = &page_ctrs_cands[lckidx][r][mnode] 8985d07b933Sdp [MTYPE_2_MRANGE(mnode, root_mtype)]; 8997c478bd9Sstevel@tonic-gate 9005d07b933Sdp ASSERT(cand->pcc_pages_free != 0); 9015d07b933Sdp ASSERT(cand->pcc_color_free[PP_2_BIN_SZC(pp, r)] != 0); 9025d07b933Sdp 9035d07b933Sdp cand->pcc_pages_free--; 9045d07b933Sdp cand->pcc_color_free[PP_2_BIN_SZC(pp, r)]--; 9055d07b933Sdp } 9067c478bd9Sstevel@tonic-gate r++; 9077c478bd9Sstevel@tonic-gate } 9085d07b933Sdp } 9095d07b933Sdp 9105d07b933Sdp void 9115d07b933Sdp page_ctr_sub(int mnode, int mtype, page_t *pp, int flags) 9125d07b933Sdp { 9135d07b933Sdp int lckidx = PP_CTR_LOCK_INDX(pp); 9145d07b933Sdp kmutex_t *lock = &ctr_mutex[lckidx][mnode]; 9155d07b933Sdp 9165d07b933Sdp mutex_enter(lock); 9175d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 9187c478bd9Sstevel@tonic-gate mutex_exit(lock); 9197c478bd9Sstevel@tonic-gate } 9207c478bd9Sstevel@tonic-gate 9217c478bd9Sstevel@tonic-gate /* 9227c478bd9Sstevel@tonic-gate * Adjust page counters following a memory attach, since typically the 9237c478bd9Sstevel@tonic-gate * size of the array needs to change, and the PFN to counter index 9247c478bd9Sstevel@tonic-gate * mapping needs to change. 9255d07b933Sdp * 9265d07b933Sdp * It is possible this mnode did not exist at startup. In that case 9275d07b933Sdp * allocate pcc_info_t and pcc_color_free arrays. Also, allow for nranges 9285d07b933Sdp * to change (a theoretical possibility on x86), which means pcc_color_free 9295d07b933Sdp * arrays must be extended. 9307c478bd9Sstevel@tonic-gate */ 9317c478bd9Sstevel@tonic-gate uint_t 9327c478bd9Sstevel@tonic-gate page_ctrs_adjust(int mnode) 9337c478bd9Sstevel@tonic-gate { 9347c478bd9Sstevel@tonic-gate pgcnt_t npgs; 9357c478bd9Sstevel@tonic-gate int r; /* region size */ 9367c478bd9Sstevel@tonic-gate int i; 9377c478bd9Sstevel@tonic-gate size_t pcsz, old_csz; 9387c478bd9Sstevel@tonic-gate hpmctr_t *new_ctr, *old_ctr; 9397c478bd9Sstevel@tonic-gate pfn_t oldbase, newbase; 940ce8eb11aSdp pfn_t physbase, physmax; 9417c478bd9Sstevel@tonic-gate size_t old_npgs; 9427c478bd9Sstevel@tonic-gate hpmctr_t *ctr_cache[MMU_PAGE_SIZES]; 9437c478bd9Sstevel@tonic-gate size_t size_cache[MMU_PAGE_SIZES]; 9445d07b933Sdp size_t *color_cache[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 9455d07b933Sdp size_t *old_color_array[MAX_MNODE_MRANGES]; 9467c478bd9Sstevel@tonic-gate pgcnt_t colors_per_szc[MMU_PAGE_SIZES]; 9475d07b933Sdp pcc_info_t **cands_cache; 9485d07b933Sdp pcc_info_t *old_pi, *pi; 9495d07b933Sdp pgcnt_t *pgcntp; 9505d07b933Sdp int nr, old_nranges, mrange, nranges = MNODE_RANGE_CNT(mnode); 9515d07b933Sdp int cands_cache_nranges; 9525d07b933Sdp int old_maxmrange, new_maxmrange; 9535d07b933Sdp int rc = 0; 9547c478bd9Sstevel@tonic-gate 9555d07b933Sdp cands_cache = kmem_zalloc(sizeof (pcc_info_t *) * NPC_MUTEX * 9565d07b933Sdp MMU_PAGE_SIZES, KM_NOSLEEP); 9575d07b933Sdp if (cands_cache == NULL) 9585d07b933Sdp return (ENOMEM); 9595d07b933Sdp 960ce8eb11aSdp i = -1; 961ce8eb11aSdp HPM_COUNTERS_LIMITS(mnode, physbase, physmax, i); 962ce8eb11aSdp 963ce8eb11aSdp newbase = physbase & ~PC_BASE_ALIGN_MASK; 964ce8eb11aSdp npgs = roundup(physmax, PC_BASE_ALIGN) - newbase; 965ce8eb11aSdp 9665d07b933Sdp /* prepare to free non-null pointers on the way out */ 9675d07b933Sdp cands_cache_nranges = nranges; 9685d07b933Sdp bzero(ctr_cache, sizeof (ctr_cache)); 9695d07b933Sdp bzero(color_cache, sizeof (color_cache)); 9705d07b933Sdp 9717c478bd9Sstevel@tonic-gate /* 9727c478bd9Sstevel@tonic-gate * We need to determine how many page colors there are for each 9737c478bd9Sstevel@tonic-gate * page size in order to allocate memory for any color specific 9747c478bd9Sstevel@tonic-gate * arrays. 9757c478bd9Sstevel@tonic-gate */ 9765d07b933Sdp for (r = 0; r < mmu_page_sizes; r++) { 9775d07b933Sdp colors_per_szc[r] = PAGE_GET_PAGECOLORS(r); 9787c478bd9Sstevel@tonic-gate } 9797c478bd9Sstevel@tonic-gate 9807c478bd9Sstevel@tonic-gate /* 9817c478bd9Sstevel@tonic-gate * Preallocate all of the new hpm_counters arrays as we can't 9827c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 9837c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 9847c478bd9Sstevel@tonic-gate * and return failure. 9857c478bd9Sstevel@tonic-gate */ 9867c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 9877c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_BSZS_SHIFT(r); 9885d07b933Sdp size_cache[r] = pcsz; 9897c478bd9Sstevel@tonic-gate ctr_cache[r] = kmem_zalloc(pcsz * 9907c478bd9Sstevel@tonic-gate sizeof (hpmctr_t), KM_NOSLEEP); 9917c478bd9Sstevel@tonic-gate if (ctr_cache[r] == NULL) { 9925d07b933Sdp rc = ENOMEM; 9935d07b933Sdp goto cleanup; 9947c478bd9Sstevel@tonic-gate } 9957c478bd9Sstevel@tonic-gate } 9965d07b933Sdp 9977c478bd9Sstevel@tonic-gate /* 9987c478bd9Sstevel@tonic-gate * Preallocate all of the new color current arrays as we can't 9997c478bd9Sstevel@tonic-gate * hold the page_ctrs_rwlock as a writer and allocate memory. 10007c478bd9Sstevel@tonic-gate * If we can't allocate all of the arrays, undo our work so far 10017c478bd9Sstevel@tonic-gate * and return failure. 10027c478bd9Sstevel@tonic-gate */ 10037c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10045d07b933Sdp for (mrange = 0; mrange < nranges; mrange++) { 10055d07b933Sdp color_cache[r][mrange] = kmem_zalloc(sizeof (size_t) * 10065d07b933Sdp colors_per_szc[r], KM_NOSLEEP); 10075d07b933Sdp if (color_cache[r][mrange] == NULL) { 10085d07b933Sdp rc = ENOMEM; 10095d07b933Sdp goto cleanup; 10105d07b933Sdp } 10115d07b933Sdp } 10125d07b933Sdp } 10135d07b933Sdp 10145d07b933Sdp /* 10155d07b933Sdp * Preallocate all of the new pcc_info_t arrays as we can't 10165d07b933Sdp * hold the page_ctrs_rwlock as a writer and allocate memory. 10175d07b933Sdp * If we can't allocate all of the arrays, undo our work so far 10185d07b933Sdp * and return failure. 10195d07b933Sdp */ 10205d07b933Sdp for (r = 1; r < mmu_page_sizes; r++) { 10215d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 10225d07b933Sdp pi = kmem_zalloc(nranges * sizeof (pcc_info_t), 10235d07b933Sdp KM_NOSLEEP); 10245d07b933Sdp if (pi == NULL) { 10255d07b933Sdp rc = ENOMEM; 10265d07b933Sdp goto cleanup; 10277c478bd9Sstevel@tonic-gate } 10285d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = pi; 10295d07b933Sdp 10305d07b933Sdp for (mrange = 0; mrange < nranges; mrange++, pi++) { 10315d07b933Sdp pgcntp = kmem_zalloc(colors_per_szc[r] * 10325d07b933Sdp sizeof (pgcnt_t), KM_NOSLEEP); 10335d07b933Sdp if (pgcntp == NULL) { 10345d07b933Sdp rc = ENOMEM; 10355d07b933Sdp goto cleanup; 10365d07b933Sdp } 10375d07b933Sdp pi->pcc_color_free = pgcntp; 10387c478bd9Sstevel@tonic-gate } 10397c478bd9Sstevel@tonic-gate } 10407c478bd9Sstevel@tonic-gate } 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate /* 10437c478bd9Sstevel@tonic-gate * Grab the write lock to prevent others from walking these arrays 10447c478bd9Sstevel@tonic-gate * while we are modifying them. 10457c478bd9Sstevel@tonic-gate */ 1046ce8eb11aSdp PAGE_CTRS_WRITE_LOCK(mnode); 10475d07b933Sdp 10485d07b933Sdp old_nranges = mnode_nranges[mnode]; 10495d07b933Sdp cands_cache_nranges = old_nranges; 10505d07b933Sdp mnode_nranges[mnode] = nranges; 10515d07b933Sdp old_maxmrange = mnode_maxmrange[mnode]; 10525d07b933Sdp mnode_maxmrange[mnode] = MNODE_MAX_MRANGE(mnode); 10535d07b933Sdp new_maxmrange = mnode_maxmrange[mnode]; 10545d07b933Sdp 10557c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 10567c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r) = PAGE_BSZS_SHIFT(r); 10577c478bd9Sstevel@tonic-gate old_ctr = PAGE_COUNTERS_COUNTERS(mnode, r); 10587c478bd9Sstevel@tonic-gate old_csz = PAGE_COUNTERS_ENTRIES(mnode, r); 10597c478bd9Sstevel@tonic-gate oldbase = PAGE_COUNTERS_BASE(mnode, r); 10607c478bd9Sstevel@tonic-gate old_npgs = old_csz << PAGE_COUNTERS_SHIFT(mnode, r); 10615d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 10625d07b933Sdp old_color_array[mrange] = 10635d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, 1064ce8eb11aSdp r, mrange); 10655d07b933Sdp } 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate pcsz = npgs >> PAGE_COUNTERS_SHIFT(mnode, r); 10687c478bd9Sstevel@tonic-gate new_ctr = ctr_cache[r]; 10697c478bd9Sstevel@tonic-gate ctr_cache[r] = NULL; 10707c478bd9Sstevel@tonic-gate if (old_ctr != NULL && 10717c478bd9Sstevel@tonic-gate (oldbase + old_npgs > newbase) && 10727c478bd9Sstevel@tonic-gate (newbase + npgs > oldbase)) { 10737c478bd9Sstevel@tonic-gate /* 10747c478bd9Sstevel@tonic-gate * Map the intersection of the old and new 10757c478bd9Sstevel@tonic-gate * counters into the new array. 10767c478bd9Sstevel@tonic-gate */ 10777c478bd9Sstevel@tonic-gate size_t offset; 10787c478bd9Sstevel@tonic-gate if (newbase > oldbase) { 10797c478bd9Sstevel@tonic-gate offset = (newbase - oldbase) >> 10807c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 10817c478bd9Sstevel@tonic-gate bcopy(old_ctr + offset, new_ctr, 10827c478bd9Sstevel@tonic-gate MIN(pcsz, (old_csz - offset)) * 10837c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 10847c478bd9Sstevel@tonic-gate } else { 10857c478bd9Sstevel@tonic-gate offset = (oldbase - newbase) >> 10867c478bd9Sstevel@tonic-gate PAGE_COUNTERS_SHIFT(mnode, r); 10877c478bd9Sstevel@tonic-gate bcopy(old_ctr, new_ctr + offset, 10887c478bd9Sstevel@tonic-gate MIN(pcsz - offset, old_csz) * 10897c478bd9Sstevel@tonic-gate sizeof (hpmctr_t)); 10907c478bd9Sstevel@tonic-gate } 10917c478bd9Sstevel@tonic-gate } 10927c478bd9Sstevel@tonic-gate 10937c478bd9Sstevel@tonic-gate PAGE_COUNTERS_COUNTERS(mnode, r) = new_ctr; 10947c478bd9Sstevel@tonic-gate PAGE_COUNTERS_ENTRIES(mnode, r) = pcsz; 10957c478bd9Sstevel@tonic-gate PAGE_COUNTERS_BASE(mnode, r) = newbase; 1096ce8eb11aSdp 1097ce8eb11aSdp /* update shared hpm_counters in other mnodes */ 1098ce8eb11aSdp if (interleaved_mnodes) { 1099ce8eb11aSdp for (i = 0; i < max_mem_nodes; i++) { 1100ce8eb11aSdp if (i == mnode) 1101ce8eb11aSdp continue; 1102ce8eb11aSdp if (mem_node_config[i].exists == 0) 1103ce8eb11aSdp continue; 1104ce8eb11aSdp ASSERT(PAGE_COUNTERS_COUNTERS(i, r) == old_ctr); 1105ce8eb11aSdp PAGE_COUNTERS_COUNTERS(i, r) = new_ctr; 1106ce8eb11aSdp PAGE_COUNTERS_ENTRIES(i, r) = pcsz; 1107ce8eb11aSdp PAGE_COUNTERS_BASE(i, r) = newbase; 1108ce8eb11aSdp } 1109ce8eb11aSdp } 1110ce8eb11aSdp 11115d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11125d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR_ARRAY(mnode, r, mrange) = 11135d07b933Sdp color_cache[r][mrange]; 11145d07b933Sdp color_cache[r][mrange] = NULL; 11155d07b933Sdp } 11167c478bd9Sstevel@tonic-gate /* 11177c478bd9Sstevel@tonic-gate * for now, just reset on these events as it's probably 11187c478bd9Sstevel@tonic-gate * not worthwhile to try and optimize this. 11197c478bd9Sstevel@tonic-gate */ 11207c478bd9Sstevel@tonic-gate for (i = 0; i < colors_per_szc[r]; i++) { 11215d07b933Sdp uint_t color_mask = colors_per_szc[r] - 1; 1122ce8eb11aSdp int mlo = interleaved_mnodes ? 0 : mnode; 1123ce8eb11aSdp int mhi = interleaved_mnodes ? max_mem_nodes : 1124ce8eb11aSdp (mnode + 1); 1125ce8eb11aSdp int m; 11265d07b933Sdp pfn_t pfnum = newbase; 11275d07b933Sdp size_t idx; 1128ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 11295d07b933Sdp 1130ce8eb11aSdp for (m = mlo; m < mhi; m++) { 1131ce8eb11aSdp if (mem_node_config[m].exists == 0) 1132ce8eb11aSdp continue; 1133*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, m, r, &it); 1134*b779d3e0Sdp if (pfnum == (pfn_t)-1) { 1135*b779d3e0Sdp idx = 0; 1136*b779d3e0Sdp } else { 1137*b779d3e0Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, r, i, 1138*b779d3e0Sdp color_mask, color_mask, &it); 1139*b779d3e0Sdp idx = PNUM_TO_IDX(m, r, pfnum); 1140*b779d3e0Sdp idx = (idx < pcsz) ? idx : 0; 1141*b779d3e0Sdp } 1142ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 1143ce8eb11aSdp PAGE_COUNTERS_CURRENT_COLOR(m, 1144ce8eb11aSdp r, i, mrange) = idx; 1145ce8eb11aSdp } 11465d07b933Sdp } 11477c478bd9Sstevel@tonic-gate } 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate /* cache info for freeing out of the critical path */ 11507c478bd9Sstevel@tonic-gate if ((caddr_t)old_ctr >= kernelheap && 11517c478bd9Sstevel@tonic-gate (caddr_t)old_ctr < ekernelheap) { 11527c478bd9Sstevel@tonic-gate ctr_cache[r] = old_ctr; 11537c478bd9Sstevel@tonic-gate size_cache[r] = old_csz; 11547c478bd9Sstevel@tonic-gate } 11555d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 11565d07b933Sdp size_t *tmp = old_color_array[mrange]; 11575d07b933Sdp if ((caddr_t)tmp >= kernelheap && 11585d07b933Sdp (caddr_t)tmp < ekernelheap) { 11595d07b933Sdp color_cache[r][mrange] = tmp; 11605d07b933Sdp } 11617c478bd9Sstevel@tonic-gate } 11627c478bd9Sstevel@tonic-gate /* 11637c478bd9Sstevel@tonic-gate * Verify that PNUM_TO_IDX and IDX_TO_PNUM 11647c478bd9Sstevel@tonic-gate * satisfy the identity requirement. 11657c478bd9Sstevel@tonic-gate * We should be able to go from one to the other 11667c478bd9Sstevel@tonic-gate * and get consistent values. 11677c478bd9Sstevel@tonic-gate */ 11687c478bd9Sstevel@tonic-gate ASSERT(PNUM_TO_IDX(mnode, r, 11697c478bd9Sstevel@tonic-gate (IDX_TO_PNUM(mnode, r, 0))) == 0); 11707c478bd9Sstevel@tonic-gate ASSERT(IDX_TO_PNUM(mnode, r, 11717c478bd9Sstevel@tonic-gate (PNUM_TO_IDX(mnode, r, newbase))) == newbase); 11725d07b933Sdp 11735d07b933Sdp /* pcc_info_t and pcc_color_free */ 11745d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 11755d07b933Sdp pcc_info_t *epi; 11765d07b933Sdp pcc_info_t *eold_pi; 11775d07b933Sdp 11785d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 11795d07b933Sdp old_pi = page_ctrs_cands[i][r][mnode]; 11805d07b933Sdp page_ctrs_cands[i][r][mnode] = pi; 11815d07b933Sdp cands_cache[i * MMU_PAGE_SIZES + r] = old_pi; 11825d07b933Sdp 11835d07b933Sdp /* preserve old pcc_color_free values, if any */ 11845d07b933Sdp if (old_pi == NULL) 11855d07b933Sdp continue; 11865d07b933Sdp 11875d07b933Sdp /* 11885d07b933Sdp * when/if x86 does DR, must account for 11895d07b933Sdp * possible change in range index when 11905d07b933Sdp * preserving pcc_info 11915d07b933Sdp */ 11925d07b933Sdp epi = &pi[nranges]; 11935d07b933Sdp eold_pi = &old_pi[old_nranges]; 11945d07b933Sdp if (new_maxmrange > old_maxmrange) { 11955d07b933Sdp pi += new_maxmrange - old_maxmrange; 11965d07b933Sdp } else if (new_maxmrange < old_maxmrange) { 11975d07b933Sdp old_pi += old_maxmrange - new_maxmrange; 11985d07b933Sdp } 11995d07b933Sdp for (; pi < epi && old_pi < eold_pi; pi++, old_pi++) { 12005d07b933Sdp pcc_info_t tmp = *pi; 12015d07b933Sdp *pi = *old_pi; 12025d07b933Sdp *old_pi = tmp; 12035d07b933Sdp } 12045d07b933Sdp } 12057c478bd9Sstevel@tonic-gate } 1206ce8eb11aSdp PAGE_CTRS_WRITE_UNLOCK(mnode); 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate /* 12097c478bd9Sstevel@tonic-gate * Now that we have dropped the write lock, it is safe to free all 12107c478bd9Sstevel@tonic-gate * of the memory we have cached above. 12115d07b933Sdp * We come thru here to free memory when pre-alloc fails, and also to 12125d07b933Sdp * free old pointers which were recorded while locked. 12137c478bd9Sstevel@tonic-gate */ 12145d07b933Sdp cleanup: 12157c478bd9Sstevel@tonic-gate for (r = 1; r < mmu_page_sizes; r++) { 12167c478bd9Sstevel@tonic-gate if (ctr_cache[r] != NULL) { 12177c478bd9Sstevel@tonic-gate kmem_free(ctr_cache[r], 12187c478bd9Sstevel@tonic-gate size_cache[r] * sizeof (hpmctr_t)); 12197c478bd9Sstevel@tonic-gate } 12205d07b933Sdp for (mrange = 0; mrange < MAX_MNODE_MRANGES; mrange++) { 12215d07b933Sdp if (color_cache[r][mrange] != NULL) { 12225d07b933Sdp kmem_free(color_cache[r][mrange], 12235d07b933Sdp colors_per_szc[r] * sizeof (size_t)); 12245d07b933Sdp } 12255d07b933Sdp } 12265d07b933Sdp for (i = 0; i < NPC_MUTEX; i++) { 12275d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12285d07b933Sdp if (pi == NULL) 12295d07b933Sdp continue; 12305d07b933Sdp nr = cands_cache_nranges; 12315d07b933Sdp for (mrange = 0; mrange < nr; mrange++, pi++) { 12325d07b933Sdp pgcntp = pi->pcc_color_free; 12335d07b933Sdp if (pgcntp == NULL) 12345d07b933Sdp continue; 12355d07b933Sdp if ((caddr_t)pgcntp >= kernelheap && 12365d07b933Sdp (caddr_t)pgcntp < ekernelheap) { 12375d07b933Sdp kmem_free(pgcntp, 12385d07b933Sdp colors_per_szc[r] * 12395d07b933Sdp sizeof (pgcnt_t)); 12405d07b933Sdp } 12415d07b933Sdp } 12425d07b933Sdp pi = cands_cache[i * MMU_PAGE_SIZES + r]; 12435d07b933Sdp if ((caddr_t)pi >= kernelheap && 12445d07b933Sdp (caddr_t)pi < ekernelheap) { 12455d07b933Sdp kmem_free(pi, nr * sizeof (pcc_info_t)); 12465d07b933Sdp } 12477c478bd9Sstevel@tonic-gate } 12487c478bd9Sstevel@tonic-gate } 12497c478bd9Sstevel@tonic-gate 12505d07b933Sdp kmem_free(cands_cache, 12515d07b933Sdp sizeof (pcc_info_t *) * NPC_MUTEX * MMU_PAGE_SIZES); 12525d07b933Sdp return (rc); 12537c478bd9Sstevel@tonic-gate } 12547c478bd9Sstevel@tonic-gate 12555d07b933Sdp 12567c478bd9Sstevel@tonic-gate #ifdef DEBUG 12577c478bd9Sstevel@tonic-gate 12587c478bd9Sstevel@tonic-gate /* 12597c478bd9Sstevel@tonic-gate * confirm pp is a large page corresponding to szc 12607c478bd9Sstevel@tonic-gate */ 12617c478bd9Sstevel@tonic-gate void 12627c478bd9Sstevel@tonic-gate chk_lpg(page_t *pp, uchar_t szc) 12637c478bd9Sstevel@tonic-gate { 12647c478bd9Sstevel@tonic-gate spgcnt_t npgs = page_get_pagecnt(pp->p_szc); 12657c478bd9Sstevel@tonic-gate uint_t noreloc; 12667c478bd9Sstevel@tonic-gate 12677c478bd9Sstevel@tonic-gate if (npgs == 1) { 12687c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 12697c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == pp); 12707c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == pp); 12717c478bd9Sstevel@tonic-gate return; 12727c478bd9Sstevel@tonic-gate } 12737c478bd9Sstevel@tonic-gate 12747c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12757c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 12767c478bd9Sstevel@tonic-gate 12777c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pp->p_pagenum, npgs)); 12787c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == (pp->p_next->p_pagenum - 1)); 12797c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev->p_pagenum == (pp->p_pagenum + (npgs - 1))); 12807c478bd9Sstevel@tonic-gate ASSERT(pp->p_prev == (pp + (npgs - 1))); 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate /* 12837c478bd9Sstevel@tonic-gate * Check list of pages. 12847c478bd9Sstevel@tonic-gate */ 12857c478bd9Sstevel@tonic-gate noreloc = PP_ISNORELOC(pp); 12867c478bd9Sstevel@tonic-gate while (npgs--) { 12877c478bd9Sstevel@tonic-gate if (npgs != 0) { 12887c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == pp->p_next->p_pagenum - 1); 12897c478bd9Sstevel@tonic-gate ASSERT(pp->p_next == (pp + 1)); 12907c478bd9Sstevel@tonic-gate } 12917c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 12927c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 12937c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 12947c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpnext == pp || pp->p_vpnext == NULL); 12957c478bd9Sstevel@tonic-gate ASSERT(pp->p_vpprev == pp || pp->p_vpprev == NULL); 12967c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 12977c478bd9Sstevel@tonic-gate ASSERT(PP_ISNORELOC(pp) == noreloc); 12987c478bd9Sstevel@tonic-gate 12997c478bd9Sstevel@tonic-gate pp = pp->p_next; 13007c478bd9Sstevel@tonic-gate } 13017c478bd9Sstevel@tonic-gate } 13027c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 13037c478bd9Sstevel@tonic-gate 13047c478bd9Sstevel@tonic-gate void 13057c478bd9Sstevel@tonic-gate page_freelist_lock(int mnode) 13067c478bd9Sstevel@tonic-gate { 13077c478bd9Sstevel@tonic-gate int i; 13087c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13097c478bd9Sstevel@tonic-gate mutex_enter(FPC_MUTEX(mnode, i)); 13107c478bd9Sstevel@tonic-gate mutex_enter(CPC_MUTEX(mnode, i)); 13117c478bd9Sstevel@tonic-gate } 13127c478bd9Sstevel@tonic-gate } 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate void 13157c478bd9Sstevel@tonic-gate page_freelist_unlock(int mnode) 13167c478bd9Sstevel@tonic-gate { 13177c478bd9Sstevel@tonic-gate int i; 13187c478bd9Sstevel@tonic-gate for (i = 0; i < NPC_MUTEX; i++) { 13197c478bd9Sstevel@tonic-gate mutex_exit(FPC_MUTEX(mnode, i)); 13207c478bd9Sstevel@tonic-gate mutex_exit(CPC_MUTEX(mnode, i)); 13217c478bd9Sstevel@tonic-gate } 13227c478bd9Sstevel@tonic-gate } 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate /* 13257c478bd9Sstevel@tonic-gate * add pp to the specified page list. Defaults to head of the page list 13267c478bd9Sstevel@tonic-gate * unless PG_LIST_TAIL is specified. 13277c478bd9Sstevel@tonic-gate */ 13287c478bd9Sstevel@tonic-gate void 13297c478bd9Sstevel@tonic-gate page_list_add(page_t *pp, int flags) 13307c478bd9Sstevel@tonic-gate { 13317c478bd9Sstevel@tonic-gate page_t **ppp; 13327c478bd9Sstevel@tonic-gate kmutex_t *pcm; 13337c478bd9Sstevel@tonic-gate uint_t bin, mtype; 13347c478bd9Sstevel@tonic-gate int mnode; 13357c478bd9Sstevel@tonic-gate 13367c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 13377c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 13387c478bd9Sstevel@tonic-gate ASSERT(!hat_page_is_mapped(pp)); 13397c478bd9Sstevel@tonic-gate ASSERT(hat_page_getshare(pp) == 0); 13407c478bd9Sstevel@tonic-gate 13417c478bd9Sstevel@tonic-gate /* 13427c478bd9Sstevel@tonic-gate * Large pages should be freed via page_list_add_pages(). 13437c478bd9Sstevel@tonic-gate */ 13447c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 13457c478bd9Sstevel@tonic-gate 13467c478bd9Sstevel@tonic-gate /* 13477c478bd9Sstevel@tonic-gate * Don't need to lock the freelist first here 13487c478bd9Sstevel@tonic-gate * because the page isn't on the freelist yet. 13497c478bd9Sstevel@tonic-gate * This means p_szc can't change on us. 13507c478bd9Sstevel@tonic-gate */ 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 13537c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 13547c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 13557c478bd9Sstevel@tonic-gate 13567c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 13577c478bd9Sstevel@tonic-gate /* 13587c478bd9Sstevel@tonic-gate * PG_LIST_ISINIT is set during system startup (ie. single 13597c478bd9Sstevel@tonic-gate * threaded), add a page to the free list and add to the 13607c478bd9Sstevel@tonic-gate * the free region counters w/o any locking 13617c478bd9Sstevel@tonic-gate */ 13627c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13637c478bd9Sstevel@tonic-gate 13647c478bd9Sstevel@tonic-gate /* inline version of page_add() */ 13657c478bd9Sstevel@tonic-gate if (*ppp != NULL) { 13667c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 13677c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 13687c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 13697c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 13707c478bd9Sstevel@tonic-gate } else 13717c478bd9Sstevel@tonic-gate *ppp = pp; 13727c478bd9Sstevel@tonic-gate 1373affbd3ccSkchow page_ctr_add_internal(mnode, mtype, pp, flags); 1374affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13757c478bd9Sstevel@tonic-gate } else { 13767c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 13777c478bd9Sstevel@tonic-gate 13787c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1379affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[0]); 13807c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 13817c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 13827c478bd9Sstevel@tonic-gate 13837c478bd9Sstevel@tonic-gate } else { 1384affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_cache); 13857c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode); 13867c478bd9Sstevel@tonic-gate ASSERT((pp->p_offset & PAGEOFFSET) == 0); 13877c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 13887c478bd9Sstevel@tonic-gate } 13897c478bd9Sstevel@tonic-gate mutex_enter(pcm); 13907c478bd9Sstevel@tonic-gate page_add(ppp, pp); 13917c478bd9Sstevel@tonic-gate 13927c478bd9Sstevel@tonic-gate if (flags & PG_LIST_TAIL) 13937c478bd9Sstevel@tonic-gate *ppp = (*ppp)->p_next; 13947c478bd9Sstevel@tonic-gate /* 13957c478bd9Sstevel@tonic-gate * Add counters before releasing pcm mutex to avoid a race with 13965d07b933Sdp * page_freelist_coalesce and page_freelist_split. 13977c478bd9Sstevel@tonic-gate */ 1398affbd3ccSkchow page_ctr_add(mnode, mtype, pp, flags); 13997c478bd9Sstevel@tonic-gate mutex_exit(pcm); 14007c478bd9Sstevel@tonic-gate } 14017c478bd9Sstevel@tonic-gate 14027c478bd9Sstevel@tonic-gate 14037c478bd9Sstevel@tonic-gate #if defined(__sparc) 14047c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 14057c478bd9Sstevel@tonic-gate kcage_freemem_add(1); 14067c478bd9Sstevel@tonic-gate } 14077c478bd9Sstevel@tonic-gate #endif 14087c478bd9Sstevel@tonic-gate /* 14097c478bd9Sstevel@tonic-gate * It is up to the caller to unlock the page! 14107c478bd9Sstevel@tonic-gate */ 14117c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp) || (flags & PG_LIST_ISINIT)); 14127c478bd9Sstevel@tonic-gate } 14137c478bd9Sstevel@tonic-gate 14147c478bd9Sstevel@tonic-gate 14157c478bd9Sstevel@tonic-gate #ifdef __sparc 14167c478bd9Sstevel@tonic-gate /* 14177c478bd9Sstevel@tonic-gate * This routine is only used by kcage_init during system startup. 14187c478bd9Sstevel@tonic-gate * It performs the function of page_list_sub/PP_SETNORELOC/page_list_add 14197c478bd9Sstevel@tonic-gate * without the overhead of taking locks and updating counters. 14207c478bd9Sstevel@tonic-gate */ 14217c478bd9Sstevel@tonic-gate void 14227c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 14237c478bd9Sstevel@tonic-gate { 14247c478bd9Sstevel@tonic-gate page_t **ppp; 14257c478bd9Sstevel@tonic-gate uint_t bin; 14267c478bd9Sstevel@tonic-gate int mnode; 14277c478bd9Sstevel@tonic-gate int mtype; 1428e21bae1bSkchow int flags = 0; 14297c478bd9Sstevel@tonic-gate 14307c478bd9Sstevel@tonic-gate /* 14317c478bd9Sstevel@tonic-gate * If this is a large page on the freelist then 14327c478bd9Sstevel@tonic-gate * break it up into smaller pages. 14337c478bd9Sstevel@tonic-gate */ 14347c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) 14357c478bd9Sstevel@tonic-gate page_boot_demote(pp); 14367c478bd9Sstevel@tonic-gate 14377c478bd9Sstevel@tonic-gate /* 14387c478bd9Sstevel@tonic-gate * Get list page is currently on. 14397c478bd9Sstevel@tonic-gate */ 14407c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 14417c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 14427c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14437c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_RELOC); 14447c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 14457c478bd9Sstevel@tonic-gate 14467c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14477c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14487c478bd9Sstevel@tonic-gate flags |= PG_FREE_LIST; 14497c478bd9Sstevel@tonic-gate } else { 14507c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14517c478bd9Sstevel@tonic-gate flags |= PG_CACHE_LIST; 14527c478bd9Sstevel@tonic-gate } 14537c478bd9Sstevel@tonic-gate 14547c478bd9Sstevel@tonic-gate ASSERT(*ppp != NULL); 14557c478bd9Sstevel@tonic-gate 14567c478bd9Sstevel@tonic-gate /* 14577c478bd9Sstevel@tonic-gate * Delete page from current list. 14587c478bd9Sstevel@tonic-gate */ 14597c478bd9Sstevel@tonic-gate if (*ppp == pp) 14607c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 14617c478bd9Sstevel@tonic-gate if (*ppp == pp) { 14627c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 14637c478bd9Sstevel@tonic-gate } else { 14647c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 14657c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 14667c478bd9Sstevel@tonic-gate } 14677c478bd9Sstevel@tonic-gate 14685d07b933Sdp /* 14695d07b933Sdp * Decrement page counters 14705d07b933Sdp */ 14715d07b933Sdp page_ctr_sub_internal(mnode, mtype, pp, flags); 14727c478bd9Sstevel@tonic-gate 14737c478bd9Sstevel@tonic-gate /* 14747c478bd9Sstevel@tonic-gate * Set no reloc for cage initted pages. 14757c478bd9Sstevel@tonic-gate */ 14767c478bd9Sstevel@tonic-gate PP_SETNORELOC(pp); 14777c478bd9Sstevel@tonic-gate 14787c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 14797c478bd9Sstevel@tonic-gate ASSERT(mtype == MTYPE_NORELOC); 14807c478bd9Sstevel@tonic-gate 14817c478bd9Sstevel@tonic-gate /* 14827c478bd9Sstevel@tonic-gate * Get new list for page. 14837c478bd9Sstevel@tonic-gate */ 14847c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 14857c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, 0, bin, mtype); 14867c478bd9Sstevel@tonic-gate } else { 14877c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 14887c478bd9Sstevel@tonic-gate } 14897c478bd9Sstevel@tonic-gate 14907c478bd9Sstevel@tonic-gate /* 14917c478bd9Sstevel@tonic-gate * Insert page on new list. 14927c478bd9Sstevel@tonic-gate */ 14937c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 14947c478bd9Sstevel@tonic-gate *ppp = pp; 14957c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 14967c478bd9Sstevel@tonic-gate } else { 14977c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 14987c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 14997c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 15007c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 15017c478bd9Sstevel@tonic-gate } 15027c478bd9Sstevel@tonic-gate 15035d07b933Sdp /* 15045d07b933Sdp * Increment page counters 15055d07b933Sdp */ 15065d07b933Sdp page_ctr_add_internal(mnode, mtype, pp, flags); 15077c478bd9Sstevel@tonic-gate 15087c478bd9Sstevel@tonic-gate /* 15097c478bd9Sstevel@tonic-gate * Update cage freemem counter 15107c478bd9Sstevel@tonic-gate */ 15117c478bd9Sstevel@tonic-gate atomic_add_long(&kcage_freemem, 1); 15127c478bd9Sstevel@tonic-gate } 15137c478bd9Sstevel@tonic-gate #else /* __sparc */ 15147c478bd9Sstevel@tonic-gate 15157c478bd9Sstevel@tonic-gate /* ARGSUSED */ 15167c478bd9Sstevel@tonic-gate void 15177c478bd9Sstevel@tonic-gate page_list_noreloc_startup(page_t *pp) 15187c478bd9Sstevel@tonic-gate { 15197c478bd9Sstevel@tonic-gate panic("page_list_noreloc_startup: should be here only for sparc"); 15207c478bd9Sstevel@tonic-gate } 15217c478bd9Sstevel@tonic-gate #endif 15227c478bd9Sstevel@tonic-gate 15237c478bd9Sstevel@tonic-gate void 15247c478bd9Sstevel@tonic-gate page_list_add_pages(page_t *pp, int flags) 15257c478bd9Sstevel@tonic-gate { 15267c478bd9Sstevel@tonic-gate kmutex_t *pcm; 15277c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 15287c478bd9Sstevel@tonic-gate uint_t bin, mtype, i; 15297c478bd9Sstevel@tonic-gate int mnode; 15307c478bd9Sstevel@tonic-gate 15317c478bd9Sstevel@tonic-gate /* default to freelist/head */ 15327c478bd9Sstevel@tonic-gate ASSERT((flags & (PG_CACHE_LIST | PG_LIST_TAIL)) == 0); 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 1535affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.pladd_free[pp->p_szc]); 15367c478bd9Sstevel@tonic-gate 15377c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 15387c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 15397c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 15407c478bd9Sstevel@tonic-gate 15417c478bd9Sstevel@tonic-gate if (flags & PG_LIST_ISINIT) { 15427c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == mmu_page_sizes - 1); 15437c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 15447c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 1545affbd3ccSkchow PLCNT_INCR(pp, mnode, mtype, pp->p_szc, flags); 15467c478bd9Sstevel@tonic-gate } else { 15477c478bd9Sstevel@tonic-gate 15487c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 15497c478bd9Sstevel@tonic-gate 15507c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 15517c478bd9Sstevel@tonic-gate 15527c478bd9Sstevel@tonic-gate mutex_enter(pcm); 15537c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 1554affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 15557c478bd9Sstevel@tonic-gate mutex_exit(pcm); 15567c478bd9Sstevel@tonic-gate 15577c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 15587c478bd9Sstevel@tonic-gate #if defined(__sparc) 15597c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) 15607c478bd9Sstevel@tonic-gate kcage_freemem_add(pgcnt); 15617c478bd9Sstevel@tonic-gate #endif 15627c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++, pp++) 15638b464eb8Smec page_unlock_nocapture(pp); 15647c478bd9Sstevel@tonic-gate } 15657c478bd9Sstevel@tonic-gate } 15667c478bd9Sstevel@tonic-gate 15677c478bd9Sstevel@tonic-gate /* 15687c478bd9Sstevel@tonic-gate * During boot, need to demote a large page to base 15697c478bd9Sstevel@tonic-gate * pagesize pages for seg_kmem for use in boot_alloc() 15707c478bd9Sstevel@tonic-gate */ 15717c478bd9Sstevel@tonic-gate void 15727c478bd9Sstevel@tonic-gate page_boot_demote(page_t *pp) 15737c478bd9Sstevel@tonic-gate { 15747c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0); 15757c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15767c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 15777c478bd9Sstevel@tonic-gate 15787c478bd9Sstevel@tonic-gate (void) page_demote(PP_2_MEM_NODE(pp), 15797c478bd9Sstevel@tonic-gate PFN_BASE(pp->p_pagenum, pp->p_szc), pp->p_szc, 0, PC_NO_COLOR, 15807c478bd9Sstevel@tonic-gate PC_FREE); 15817c478bd9Sstevel@tonic-gate 15827c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 15837c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 15847c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 15857c478bd9Sstevel@tonic-gate } 15867c478bd9Sstevel@tonic-gate 15877c478bd9Sstevel@tonic-gate /* 15887c478bd9Sstevel@tonic-gate * Take a particular page off of whatever freelist the page 15897c478bd9Sstevel@tonic-gate * is claimed to be on. 15907c478bd9Sstevel@tonic-gate * 15917c478bd9Sstevel@tonic-gate * NOTE: Only used for PAGESIZE pages. 15927c478bd9Sstevel@tonic-gate */ 15937c478bd9Sstevel@tonic-gate void 15947c478bd9Sstevel@tonic-gate page_list_sub(page_t *pp, int flags) 15957c478bd9Sstevel@tonic-gate { 15967c478bd9Sstevel@tonic-gate int bin; 15977c478bd9Sstevel@tonic-gate uint_t mtype; 15987c478bd9Sstevel@tonic-gate int mnode; 15997c478bd9Sstevel@tonic-gate kmutex_t *pcm; 16007c478bd9Sstevel@tonic-gate page_t **ppp; 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 16037c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16047c478bd9Sstevel@tonic-gate 16057c478bd9Sstevel@tonic-gate /* 16067c478bd9Sstevel@tonic-gate * The p_szc field can only be changed by page_promote() 16077c478bd9Sstevel@tonic-gate * and page_demote(). Only free pages can be promoted and 16087c478bd9Sstevel@tonic-gate * demoted and the free list MUST be locked during these 16097c478bd9Sstevel@tonic-gate * operations. So to prevent a race in page_list_sub() 16107c478bd9Sstevel@tonic-gate * between computing which bin of the freelist lock to 16117c478bd9Sstevel@tonic-gate * grab and actually grabing the lock we check again that 16127c478bd9Sstevel@tonic-gate * the bin we locked is still the correct one. Notice that 16137c478bd9Sstevel@tonic-gate * the p_szc field could have actually changed on us but 16147c478bd9Sstevel@tonic-gate * if the bin happens to still be the same we are safe. 16157c478bd9Sstevel@tonic-gate */ 16167c478bd9Sstevel@tonic-gate try_again: 16177c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 16187c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 16197c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, flags); 16207c478bd9Sstevel@tonic-gate mutex_enter(pcm); 16217c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 16227c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16237c478bd9Sstevel@tonic-gate goto try_again; 16247c478bd9Sstevel@tonic-gate } 16257c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 16267c478bd9Sstevel@tonic-gate 16277c478bd9Sstevel@tonic-gate if (flags & PG_FREE_LIST) { 1628affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[0]); 16297c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16307c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 16317c478bd9Sstevel@tonic-gate } else { 1632affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_cache); 16337c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 16347c478bd9Sstevel@tonic-gate ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 16357c478bd9Sstevel@tonic-gate } 16367c478bd9Sstevel@tonic-gate 16377c478bd9Sstevel@tonic-gate /* 16387c478bd9Sstevel@tonic-gate * Common PAGESIZE case. 16397c478bd9Sstevel@tonic-gate * 16407c478bd9Sstevel@tonic-gate * Note that we locked the freelist. This prevents 16417c478bd9Sstevel@tonic-gate * any page promotion/demotion operations. Therefore 16427c478bd9Sstevel@tonic-gate * the p_szc will not change until we drop pcm mutex. 16437c478bd9Sstevel@tonic-gate */ 16447c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 16457c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 16467c478bd9Sstevel@tonic-gate /* 16477c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16487c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16497c478bd9Sstevel@tonic-gate */ 1650affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 16517c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16527c478bd9Sstevel@tonic-gate 16537c478bd9Sstevel@tonic-gate #if defined(__sparc) 16547c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 16557c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 16567c478bd9Sstevel@tonic-gate } 16577c478bd9Sstevel@tonic-gate #endif 16587c478bd9Sstevel@tonic-gate return; 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate 16617c478bd9Sstevel@tonic-gate /* 16627c478bd9Sstevel@tonic-gate * Large pages on the cache list are not supported. 16637c478bd9Sstevel@tonic-gate */ 16647c478bd9Sstevel@tonic-gate if (flags & PG_CACHE_LIST) 16657c478bd9Sstevel@tonic-gate panic("page_list_sub: large page on cachelist"); 16667c478bd9Sstevel@tonic-gate 16677c478bd9Sstevel@tonic-gate /* 16687c478bd9Sstevel@tonic-gate * Slow but rare. 16697c478bd9Sstevel@tonic-gate * 16707c478bd9Sstevel@tonic-gate * Somebody wants this particular page which is part 16717c478bd9Sstevel@tonic-gate * of a large page. In this case we just demote the page 16727c478bd9Sstevel@tonic-gate * if it's on the freelist. 16737c478bd9Sstevel@tonic-gate * 16747c478bd9Sstevel@tonic-gate * We have to drop pcm before locking the entire freelist. 16757c478bd9Sstevel@tonic-gate * Once we have re-locked the freelist check to make sure 16767c478bd9Sstevel@tonic-gate * the page hasn't already been demoted or completely 16777c478bd9Sstevel@tonic-gate * freed. 16787c478bd9Sstevel@tonic-gate */ 16797c478bd9Sstevel@tonic-gate mutex_exit(pcm); 16807c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 16817c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 16827c478bd9Sstevel@tonic-gate /* 16837c478bd9Sstevel@tonic-gate * Large page is on freelist. 16847c478bd9Sstevel@tonic-gate */ 16857c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, pp->p_szc), 16867c478bd9Sstevel@tonic-gate pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 16877c478bd9Sstevel@tonic-gate } 16887c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 16897c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 16907c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 16917c478bd9Sstevel@tonic-gate 16927c478bd9Sstevel@tonic-gate /* 16937c478bd9Sstevel@tonic-gate * Subtract counters before releasing pcm mutex 16947c478bd9Sstevel@tonic-gate * to avoid race with page_freelist_coalesce. 16957c478bd9Sstevel@tonic-gate */ 16967c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 16977c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 16987c478bd9Sstevel@tonic-gate ppp = &PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype); 16997c478bd9Sstevel@tonic-gate 17007c478bd9Sstevel@tonic-gate page_sub(ppp, pp); 1701affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, flags); 17027c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17037c478bd9Sstevel@tonic-gate 17047c478bd9Sstevel@tonic-gate #if defined(__sparc) 17057c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17067c478bd9Sstevel@tonic-gate kcage_freemem_sub(1); 17077c478bd9Sstevel@tonic-gate } 17087c478bd9Sstevel@tonic-gate #endif 17097c478bd9Sstevel@tonic-gate } 17107c478bd9Sstevel@tonic-gate 17117c478bd9Sstevel@tonic-gate void 17127c478bd9Sstevel@tonic-gate page_list_sub_pages(page_t *pp, uint_t szc) 17137c478bd9Sstevel@tonic-gate { 17147c478bd9Sstevel@tonic-gate kmutex_t *pcm; 17157c478bd9Sstevel@tonic-gate uint_t bin, mtype; 17167c478bd9Sstevel@tonic-gate int mnode; 17177c478bd9Sstevel@tonic-gate 17187c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 17197c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17207c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17217c478bd9Sstevel@tonic-gate 17227c478bd9Sstevel@tonic-gate /* 17237c478bd9Sstevel@tonic-gate * See comment in page_list_sub(). 17247c478bd9Sstevel@tonic-gate */ 17257c478bd9Sstevel@tonic-gate try_again: 17267c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17277c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 17287c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 17297c478bd9Sstevel@tonic-gate mutex_enter(pcm); 17307c478bd9Sstevel@tonic-gate if (PP_2_BIN(pp) != bin) { 17317c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17327c478bd9Sstevel@tonic-gate goto try_again; 17337c478bd9Sstevel@tonic-gate } 17347c478bd9Sstevel@tonic-gate 17357c478bd9Sstevel@tonic-gate /* 17367c478bd9Sstevel@tonic-gate * If we're called with a page larger than szc or it got 17377c478bd9Sstevel@tonic-gate * promoted above szc before we locked the freelist then 17387c478bd9Sstevel@tonic-gate * drop pcm and re-lock entire freelist. If page still larger 17397c478bd9Sstevel@tonic-gate * than szc then demote it. 17407c478bd9Sstevel@tonic-gate */ 17417c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 17427c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17437c478bd9Sstevel@tonic-gate pcm = NULL; 17447c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 17457c478bd9Sstevel@tonic-gate if (pp->p_szc > szc) { 1746affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szcbig); 17477c478bd9Sstevel@tonic-gate (void) page_demote(mnode, 17487c478bd9Sstevel@tonic-gate PFN_BASE(pp->p_pagenum, pp->p_szc), 17497c478bd9Sstevel@tonic-gate pp->p_szc, szc, PC_NO_COLOR, PC_FREE); 17507c478bd9Sstevel@tonic-gate } 17517c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 17527c478bd9Sstevel@tonic-gate } 17537c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 17547c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 17557c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc <= szc); 17567c478bd9Sstevel@tonic-gate ASSERT(pp == PP_PAGEROOT(pp)); 17577c478bd9Sstevel@tonic-gate 1758affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsub_free[pp->p_szc]); 1759affbd3ccSkchow 17607c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 17617c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 17627c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17637c478bd9Sstevel@tonic-gate CHK_LPG(pp, pp->p_szc); 17647c478bd9Sstevel@tonic-gate } else { 1765affbd3ccSkchow VM_STAT_ADD(vmm_vmstats.plsubpages_szc0); 17667c478bd9Sstevel@tonic-gate page_sub(&PAGE_FREELISTS(mnode, pp->p_szc, bin, mtype), pp); 17677c478bd9Sstevel@tonic-gate } 1768affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate if (pcm != NULL) { 17717c478bd9Sstevel@tonic-gate mutex_exit(pcm); 17727c478bd9Sstevel@tonic-gate } else { 17737c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 17747c478bd9Sstevel@tonic-gate } 17757c478bd9Sstevel@tonic-gate 17767c478bd9Sstevel@tonic-gate #if defined(__sparc) 17777c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 17787c478bd9Sstevel@tonic-gate pgcnt_t pgcnt; 17797c478bd9Sstevel@tonic-gate 17807c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(pp->p_szc); 17817c478bd9Sstevel@tonic-gate kcage_freemem_sub(pgcnt); 17827c478bd9Sstevel@tonic-gate } 17837c478bd9Sstevel@tonic-gate #endif 17847c478bd9Sstevel@tonic-gate } 17857c478bd9Sstevel@tonic-gate 17867c478bd9Sstevel@tonic-gate /* 17877c478bd9Sstevel@tonic-gate * Add the page to the front of a linked list of pages 17887c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 17897c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 17907c478bd9Sstevel@tonic-gate */ 17917c478bd9Sstevel@tonic-gate void 17927c478bd9Sstevel@tonic-gate mach_page_add(page_t **ppp, page_t *pp) 17937c478bd9Sstevel@tonic-gate { 17947c478bd9Sstevel@tonic-gate if (*ppp == NULL) { 17957c478bd9Sstevel@tonic-gate pp->p_next = pp->p_prev = pp; 17967c478bd9Sstevel@tonic-gate } else { 17977c478bd9Sstevel@tonic-gate pp->p_next = *ppp; 17987c478bd9Sstevel@tonic-gate pp->p_prev = (*ppp)->p_prev; 17997c478bd9Sstevel@tonic-gate (*ppp)->p_prev = pp; 18007c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp; 18017c478bd9Sstevel@tonic-gate } 18027c478bd9Sstevel@tonic-gate *ppp = pp; 18037c478bd9Sstevel@tonic-gate } 18047c478bd9Sstevel@tonic-gate 18057c478bd9Sstevel@tonic-gate /* 18067c478bd9Sstevel@tonic-gate * Remove this page from a linked list of pages 18077c478bd9Sstevel@tonic-gate * using the p_next & p_prev pointers for the list. 18087c478bd9Sstevel@tonic-gate * 18097c478bd9Sstevel@tonic-gate * The caller is responsible for protecting the list pointers. 18107c478bd9Sstevel@tonic-gate */ 18117c478bd9Sstevel@tonic-gate void 18127c478bd9Sstevel@tonic-gate mach_page_sub(page_t **ppp, page_t *pp) 18137c478bd9Sstevel@tonic-gate { 18147c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 18157c478bd9Sstevel@tonic-gate 18167c478bd9Sstevel@tonic-gate if (*ppp == NULL || pp == NULL) 18177c478bd9Sstevel@tonic-gate panic("mach_page_sub"); 18187c478bd9Sstevel@tonic-gate 18197c478bd9Sstevel@tonic-gate if (*ppp == pp) 18207c478bd9Sstevel@tonic-gate *ppp = pp->p_next; /* go to next page */ 18217c478bd9Sstevel@tonic-gate 18227c478bd9Sstevel@tonic-gate if (*ppp == pp) 18237c478bd9Sstevel@tonic-gate *ppp = NULL; /* page list is gone */ 18247c478bd9Sstevel@tonic-gate else { 18257c478bd9Sstevel@tonic-gate pp->p_prev->p_next = pp->p_next; 18267c478bd9Sstevel@tonic-gate pp->p_next->p_prev = pp->p_prev; 18277c478bd9Sstevel@tonic-gate } 18287c478bd9Sstevel@tonic-gate pp->p_prev = pp->p_next = pp; /* make pp a list of one */ 18297c478bd9Sstevel@tonic-gate } 18307c478bd9Sstevel@tonic-gate 18317c478bd9Sstevel@tonic-gate /* 18327c478bd9Sstevel@tonic-gate * Routine fsflush uses to gradually coalesce the free list into larger pages. 18337c478bd9Sstevel@tonic-gate */ 18347c478bd9Sstevel@tonic-gate void 18357c478bd9Sstevel@tonic-gate page_promote_size(page_t *pp, uint_t cur_szc) 18367c478bd9Sstevel@tonic-gate { 18377c478bd9Sstevel@tonic-gate pfn_t pfn; 18387c478bd9Sstevel@tonic-gate int mnode; 18397c478bd9Sstevel@tonic-gate int idx; 18407c478bd9Sstevel@tonic-gate int new_szc = cur_szc + 1; 18417c478bd9Sstevel@tonic-gate int full = FULL_REGION_CNT(new_szc); 18427c478bd9Sstevel@tonic-gate 18437c478bd9Sstevel@tonic-gate pfn = page_pptonum(pp); 18447c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(pfn); 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 18477c478bd9Sstevel@tonic-gate 18487c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, new_szc, pfn); 18497c478bd9Sstevel@tonic-gate if (PAGE_COUNTERS(mnode, new_szc, idx) == full) 18505d07b933Sdp (void) page_promote(mnode, pfn, new_szc, PC_FREE, PC_MTYPE_ANY); 18517c478bd9Sstevel@tonic-gate 18527c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 18537c478bd9Sstevel@tonic-gate } 18547c478bd9Sstevel@tonic-gate 18557c478bd9Sstevel@tonic-gate static uint_t page_promote_err; 18567c478bd9Sstevel@tonic-gate static uint_t page_promote_noreloc_err; 18577c478bd9Sstevel@tonic-gate 18587c478bd9Sstevel@tonic-gate /* 18597c478bd9Sstevel@tonic-gate * Create a single larger page (of szc new_szc) from smaller contiguous pages 18607c478bd9Sstevel@tonic-gate * for the given mnode starting at pfnum. Pages involved are on the freelist 18617c478bd9Sstevel@tonic-gate * before the call and may be returned to the caller if requested, otherwise 18627c478bd9Sstevel@tonic-gate * they will be placed back on the freelist. 18637c478bd9Sstevel@tonic-gate * If flags is PC_ALLOC, then the large page will be returned to the user in 18647c478bd9Sstevel@tonic-gate * a state which is consistent with a page being taken off the freelist. If 18657c478bd9Sstevel@tonic-gate * we failed to lock the new large page, then we will return NULL to the 18667c478bd9Sstevel@tonic-gate * caller and put the large page on the freelist instead. 18677c478bd9Sstevel@tonic-gate * If flags is PC_FREE, then the large page will be placed on the freelist, 18687c478bd9Sstevel@tonic-gate * and NULL will be returned. 18697c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 18707c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 18717c478bd9Sstevel@tonic-gate * 18727c478bd9Sstevel@tonic-gate * RFE: For performance pass in pp instead of pfnum so 18737c478bd9Sstevel@tonic-gate * we can avoid excessive calls to page_numtopp_nolock(). 18747c478bd9Sstevel@tonic-gate * This would depend on an assumption that all contiguous 18757c478bd9Sstevel@tonic-gate * pages are in the same memseg so we can just add/dec 18767c478bd9Sstevel@tonic-gate * our pp. 18777c478bd9Sstevel@tonic-gate * 18787c478bd9Sstevel@tonic-gate * Lock ordering: 18797c478bd9Sstevel@tonic-gate * 18807c478bd9Sstevel@tonic-gate * There is a potential but rare deadlock situation 18817c478bd9Sstevel@tonic-gate * for page promotion and demotion operations. The problem 18827c478bd9Sstevel@tonic-gate * is there are two paths into the freelist manager and 18837c478bd9Sstevel@tonic-gate * they have different lock orders: 18847c478bd9Sstevel@tonic-gate * 18857c478bd9Sstevel@tonic-gate * page_create() 18867c478bd9Sstevel@tonic-gate * lock freelist 18877c478bd9Sstevel@tonic-gate * page_lock(EXCL) 18887c478bd9Sstevel@tonic-gate * unlock freelist 18897c478bd9Sstevel@tonic-gate * return 18907c478bd9Sstevel@tonic-gate * caller drops page_lock 18917c478bd9Sstevel@tonic-gate * 18927c478bd9Sstevel@tonic-gate * page_free() and page_reclaim() 18937c478bd9Sstevel@tonic-gate * caller grabs page_lock(EXCL) 18947c478bd9Sstevel@tonic-gate * 18957c478bd9Sstevel@tonic-gate * lock freelist 18967c478bd9Sstevel@tonic-gate * unlock freelist 18977c478bd9Sstevel@tonic-gate * drop page_lock 18987c478bd9Sstevel@tonic-gate * 18997c478bd9Sstevel@tonic-gate * What prevents a thread in page_create() from deadlocking 19007c478bd9Sstevel@tonic-gate * with a thread freeing or reclaiming the same page is the 19017c478bd9Sstevel@tonic-gate * page_trylock() in page_get_freelist(). If the trylock fails 19027c478bd9Sstevel@tonic-gate * it skips the page. 19037c478bd9Sstevel@tonic-gate * 19047c478bd9Sstevel@tonic-gate * The lock ordering for promotion and demotion is the same as 19057c478bd9Sstevel@tonic-gate * for page_create(). Since the same deadlock could occur during 19067c478bd9Sstevel@tonic-gate * page promotion and freeing or reclaiming of a page on the 19077c478bd9Sstevel@tonic-gate * cache list we might have to fail the operation and undo what 19087c478bd9Sstevel@tonic-gate * have done so far. Again this is rare. 19097c478bd9Sstevel@tonic-gate */ 19107c478bd9Sstevel@tonic-gate page_t * 19115d07b933Sdp page_promote(int mnode, pfn_t pfnum, uchar_t new_szc, int flags, int mtype) 19127c478bd9Sstevel@tonic-gate { 19137c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *tpp, *start_pp; 19147c478bd9Sstevel@tonic-gate pgcnt_t new_npgs, npgs; 19157c478bd9Sstevel@tonic-gate uint_t bin; 19167c478bd9Sstevel@tonic-gate pgcnt_t tmpnpgs, pages_left; 19177c478bd9Sstevel@tonic-gate uint_t noreloc; 19187c478bd9Sstevel@tonic-gate int which_list; 19197c478bd9Sstevel@tonic-gate ulong_t index; 19207c478bd9Sstevel@tonic-gate kmutex_t *phm; 19217c478bd9Sstevel@tonic-gate 19227c478bd9Sstevel@tonic-gate /* 19237c478bd9Sstevel@tonic-gate * General algorithm: 19247c478bd9Sstevel@tonic-gate * Find the starting page 19257c478bd9Sstevel@tonic-gate * Walk each page struct removing it from the freelist, 19267c478bd9Sstevel@tonic-gate * and linking it to all the other pages removed. 19277c478bd9Sstevel@tonic-gate * Once all pages are off the freelist, 19287c478bd9Sstevel@tonic-gate * walk the list, modifying p_szc to new_szc and what 19297c478bd9Sstevel@tonic-gate * ever other info needs to be done to create a large free page. 19307c478bd9Sstevel@tonic-gate * According to the flags, either return the page or put it 19317c478bd9Sstevel@tonic-gate * on the freelist. 19327c478bd9Sstevel@tonic-gate */ 19337c478bd9Sstevel@tonic-gate 19347c478bd9Sstevel@tonic-gate start_pp = page_numtopp_nolock(pfnum); 19357c478bd9Sstevel@tonic-gate ASSERT(start_pp && (start_pp->p_pagenum == pfnum)); 19367c478bd9Sstevel@tonic-gate new_npgs = page_get_pagecnt(new_szc); 19377c478bd9Sstevel@tonic-gate ASSERT(IS_P2ALIGNED(pfnum, new_npgs)); 19387c478bd9Sstevel@tonic-gate 19395d07b933Sdp /* don't return page of the wrong mtype */ 19405d07b933Sdp if (mtype != PC_MTYPE_ANY && mtype != PP_2_MTYPE(start_pp)) 19415d07b933Sdp return (NULL); 19425d07b933Sdp 19437c478bd9Sstevel@tonic-gate /* 19447c478bd9Sstevel@tonic-gate * Loop through smaller pages to confirm that all pages 19457c478bd9Sstevel@tonic-gate * give the same result for PP_ISNORELOC(). 19467c478bd9Sstevel@tonic-gate * We can check this reliably here as the protocol for setting 19477c478bd9Sstevel@tonic-gate * P_NORELOC requires pages to be taken off the free list first. 19487c478bd9Sstevel@tonic-gate */ 19495d07b933Sdp noreloc = PP_ISNORELOC(start_pp); 19505d07b933Sdp for (pp = start_pp + new_npgs; --pp > start_pp; ) { 19515d07b933Sdp if (noreloc != PP_ISNORELOC(pp)) { 19527c478bd9Sstevel@tonic-gate page_promote_noreloc_err++; 19537c478bd9Sstevel@tonic-gate page_promote_err++; 19547c478bd9Sstevel@tonic-gate return (NULL); 19557c478bd9Sstevel@tonic-gate } 19567c478bd9Sstevel@tonic-gate } 19577c478bd9Sstevel@tonic-gate 19587c478bd9Sstevel@tonic-gate pages_left = new_npgs; 19597c478bd9Sstevel@tonic-gate pplist = NULL; 19607c478bd9Sstevel@tonic-gate pp = start_pp; 19617c478bd9Sstevel@tonic-gate 19627c478bd9Sstevel@tonic-gate /* Loop around coalescing the smaller pages into a big page. */ 19637c478bd9Sstevel@tonic-gate while (pages_left) { 19647c478bd9Sstevel@tonic-gate /* 19657c478bd9Sstevel@tonic-gate * Remove from the freelist. 19667c478bd9Sstevel@tonic-gate */ 19677c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 19687c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 19697c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 19707c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 19717c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 19727c478bd9Sstevel@tonic-gate 19737c478bd9Sstevel@tonic-gate /* 19747c478bd9Sstevel@tonic-gate * PG_FREE_LIST 19757c478bd9Sstevel@tonic-gate */ 19767c478bd9Sstevel@tonic-gate if (pp->p_szc) { 19777c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, 19787c478bd9Sstevel@tonic-gate pp->p_szc, bin, mtype), pp); 19797c478bd9Sstevel@tonic-gate } else { 19807c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_FREELISTS(mnode, 0, 19817c478bd9Sstevel@tonic-gate bin, mtype), pp); 19827c478bd9Sstevel@tonic-gate } 19837c478bd9Sstevel@tonic-gate which_list = PG_FREE_LIST; 19847c478bd9Sstevel@tonic-gate } else { 19857c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 19867c478bd9Sstevel@tonic-gate 19877c478bd9Sstevel@tonic-gate /* 19887c478bd9Sstevel@tonic-gate * PG_CACHE_LIST 19897c478bd9Sstevel@tonic-gate * 19907c478bd9Sstevel@tonic-gate * Since this page comes from the 19917c478bd9Sstevel@tonic-gate * cachelist, we must destroy the 19927c478bd9Sstevel@tonic-gate * vnode association. 19937c478bd9Sstevel@tonic-gate */ 19947c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 19957c478bd9Sstevel@tonic-gate goto fail_promote; 19967c478bd9Sstevel@tonic-gate } 19977c478bd9Sstevel@tonic-gate 19987c478bd9Sstevel@tonic-gate /* 19997c478bd9Sstevel@tonic-gate * We need to be careful not to deadlock 20007c478bd9Sstevel@tonic-gate * with another thread in page_lookup(). 20017c478bd9Sstevel@tonic-gate * The page_lookup() thread could be holding 20027c478bd9Sstevel@tonic-gate * the same phm that we need if the two 20037c478bd9Sstevel@tonic-gate * pages happen to hash to the same phm lock. 20047c478bd9Sstevel@tonic-gate * At this point we have locked the entire 20057c478bd9Sstevel@tonic-gate * freelist and page_lookup() could be trying 20067c478bd9Sstevel@tonic-gate * to grab a freelist lock. 20077c478bd9Sstevel@tonic-gate */ 20087c478bd9Sstevel@tonic-gate index = PAGE_HASH_FUNC(pp->p_vnode, pp->p_offset); 20097c478bd9Sstevel@tonic-gate phm = PAGE_HASH_MUTEX(index); 20107c478bd9Sstevel@tonic-gate if (!mutex_tryenter(phm)) { 20118b464eb8Smec page_unlock_nocapture(pp); 20127c478bd9Sstevel@tonic-gate goto fail_promote; 20137c478bd9Sstevel@tonic-gate } 20147c478bd9Sstevel@tonic-gate 20157c478bd9Sstevel@tonic-gate mach_page_sub(&PAGE_CACHELISTS(mnode, bin, mtype), pp); 20167c478bd9Sstevel@tonic-gate page_hashout(pp, phm); 20177c478bd9Sstevel@tonic-gate mutex_exit(phm); 20187c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 20198b464eb8Smec page_unlock_nocapture(pp); 20207c478bd9Sstevel@tonic-gate which_list = PG_CACHE_LIST; 20217c478bd9Sstevel@tonic-gate } 2022affbd3ccSkchow page_ctr_sub(mnode, mtype, pp, which_list); 20237c478bd9Sstevel@tonic-gate 20247c478bd9Sstevel@tonic-gate /* 20257c478bd9Sstevel@tonic-gate * Concatenate the smaller page(s) onto 20267c478bd9Sstevel@tonic-gate * the large page list. 20277c478bd9Sstevel@tonic-gate */ 20287c478bd9Sstevel@tonic-gate tmpnpgs = npgs = page_get_pagecnt(pp->p_szc); 20297c478bd9Sstevel@tonic-gate pages_left -= npgs; 20307c478bd9Sstevel@tonic-gate tpp = pp; 20317c478bd9Sstevel@tonic-gate while (npgs--) { 20327c478bd9Sstevel@tonic-gate tpp->p_szc = new_szc; 20337c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 20347c478bd9Sstevel@tonic-gate } 20357c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 20367c478bd9Sstevel@tonic-gate pp += tmpnpgs; 20377c478bd9Sstevel@tonic-gate } 20387c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 20397c478bd9Sstevel@tonic-gate 20407c478bd9Sstevel@tonic-gate /* 20417c478bd9Sstevel@tonic-gate * return the page to the user if requested 20427c478bd9Sstevel@tonic-gate * in the properly locked state. 20437c478bd9Sstevel@tonic-gate */ 20447c478bd9Sstevel@tonic-gate if (flags == PC_ALLOC && (page_trylock_cons(pplist, SE_EXCL))) { 20457c478bd9Sstevel@tonic-gate return (pplist); 20467c478bd9Sstevel@tonic-gate } 20477c478bd9Sstevel@tonic-gate 20487c478bd9Sstevel@tonic-gate /* 20497c478bd9Sstevel@tonic-gate * Otherwise place the new large page on the freelist 20507c478bd9Sstevel@tonic-gate */ 20517c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 20527c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pplist); 20537c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 20547c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, bin, mtype), pplist); 20557c478bd9Sstevel@tonic-gate 2056affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, PG_FREE_LIST); 20577c478bd9Sstevel@tonic-gate return (NULL); 20587c478bd9Sstevel@tonic-gate 20597c478bd9Sstevel@tonic-gate fail_promote: 20607c478bd9Sstevel@tonic-gate /* 20617c478bd9Sstevel@tonic-gate * A thread must have still been freeing or 20627c478bd9Sstevel@tonic-gate * reclaiming the page on the cachelist. 20637c478bd9Sstevel@tonic-gate * To prevent a deadlock undo what we have 20647c478bd9Sstevel@tonic-gate * done sofar and return failure. This 20657c478bd9Sstevel@tonic-gate * situation can only happen while promoting 20667c478bd9Sstevel@tonic-gate * PAGESIZE pages. 20677c478bd9Sstevel@tonic-gate */ 20687c478bd9Sstevel@tonic-gate page_promote_err++; 20697c478bd9Sstevel@tonic-gate while (pplist) { 20707c478bd9Sstevel@tonic-gate pp = pplist; 20717c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 20727c478bd9Sstevel@tonic-gate pp->p_szc = 0; 20737c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 20747c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 20757c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, mtype), pp); 2076affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 20777c478bd9Sstevel@tonic-gate } 20787c478bd9Sstevel@tonic-gate return (NULL); 20797c478bd9Sstevel@tonic-gate 20807c478bd9Sstevel@tonic-gate } 20817c478bd9Sstevel@tonic-gate 20827c478bd9Sstevel@tonic-gate /* 20837c478bd9Sstevel@tonic-gate * Break up a large page into smaller size pages. 20847c478bd9Sstevel@tonic-gate * Pages involved are on the freelist before the call and may 20857c478bd9Sstevel@tonic-gate * be returned to the caller if requested, otherwise they will 20867c478bd9Sstevel@tonic-gate * be placed back on the freelist. 20877c478bd9Sstevel@tonic-gate * The caller is responsible for locking the freelist as well as any other 20887c478bd9Sstevel@tonic-gate * accounting which needs to be done for a returned page. 20897c478bd9Sstevel@tonic-gate * If flags is not PC_ALLOC, the color argument is ignored, and thus 20907c478bd9Sstevel@tonic-gate * technically, any value may be passed in but PC_NO_COLOR is the standard 20917c478bd9Sstevel@tonic-gate * which should be followed for clarity's sake. 20927c478bd9Sstevel@tonic-gate */ 20937c478bd9Sstevel@tonic-gate page_t * 20947c478bd9Sstevel@tonic-gate page_demote(int mnode, pfn_t pfnum, uchar_t cur_szc, uchar_t new_szc, 20957c478bd9Sstevel@tonic-gate int color, int flags) 20967c478bd9Sstevel@tonic-gate { 20977c478bd9Sstevel@tonic-gate page_t *pp, *pplist, *npplist; 20987c478bd9Sstevel@tonic-gate pgcnt_t npgs, n; 20997c478bd9Sstevel@tonic-gate uint_t bin; 21007c478bd9Sstevel@tonic-gate uint_t mtype; 21017c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate ASSERT(cur_szc != 0); 21047c478bd9Sstevel@tonic-gate ASSERT(new_szc < cur_szc); 21057c478bd9Sstevel@tonic-gate 21067c478bd9Sstevel@tonic-gate pplist = page_numtopp_nolock(pfnum); 21077c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 21087c478bd9Sstevel@tonic-gate 21097c478bd9Sstevel@tonic-gate ASSERT(pplist->p_szc == cur_szc); 21107c478bd9Sstevel@tonic-gate 21117c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 21127c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pplist)); 21137c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pplist); 21147c478bd9Sstevel@tonic-gate page_vpsub(&PAGE_FREELISTS(mnode, cur_szc, bin, mtype), pplist); 21157c478bd9Sstevel@tonic-gate 21167c478bd9Sstevel@tonic-gate CHK_LPG(pplist, cur_szc); 2117affbd3ccSkchow page_ctr_sub(mnode, mtype, pplist, PG_FREE_LIST); 21187c478bd9Sstevel@tonic-gate 21197c478bd9Sstevel@tonic-gate /* 21207c478bd9Sstevel@tonic-gate * Number of PAGESIZE pages for smaller new_szc 21217c478bd9Sstevel@tonic-gate * page. 21227c478bd9Sstevel@tonic-gate */ 21237c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(new_szc); 21247c478bd9Sstevel@tonic-gate 21257c478bd9Sstevel@tonic-gate while (pplist) { 21267c478bd9Sstevel@tonic-gate pp = pplist; 21277c478bd9Sstevel@tonic-gate 21287c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21297c478bd9Sstevel@tonic-gate 21307c478bd9Sstevel@tonic-gate /* 21317c478bd9Sstevel@tonic-gate * We either break it up into PAGESIZE pages or larger. 21327c478bd9Sstevel@tonic-gate */ 21337c478bd9Sstevel@tonic-gate if (npgs == 1) { /* PAGESIZE case */ 21347c478bd9Sstevel@tonic-gate mach_page_sub(&pplist, pp); 21357c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21367c478bd9Sstevel@tonic-gate ASSERT(new_szc == 0); 21377c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 21387c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21397c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pp); 21407c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 21417c478bd9Sstevel@tonic-gate (ret_pp == NULL) && 21427c478bd9Sstevel@tonic-gate page_trylock_cons(pp, SE_EXCL)) { 21437c478bd9Sstevel@tonic-gate ret_pp = pp; 21447c478bd9Sstevel@tonic-gate } else { 21457c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21467c478bd9Sstevel@tonic-gate mach_page_add(&PAGE_FREELISTS(mnode, 0, bin, 21477c478bd9Sstevel@tonic-gate mtype), pp); 2148affbd3ccSkchow page_ctr_add(mnode, mtype, pp, PG_FREE_LIST); 21497c478bd9Sstevel@tonic-gate } 21507c478bd9Sstevel@tonic-gate } else { 21517c478bd9Sstevel@tonic-gate 21527c478bd9Sstevel@tonic-gate /* 21537c478bd9Sstevel@tonic-gate * Break down into smaller lists of pages. 21547c478bd9Sstevel@tonic-gate */ 21557c478bd9Sstevel@tonic-gate page_list_break(&pplist, &npplist, npgs); 21567c478bd9Sstevel@tonic-gate 21577c478bd9Sstevel@tonic-gate pp = pplist; 21587c478bd9Sstevel@tonic-gate n = npgs; 21597c478bd9Sstevel@tonic-gate while (n--) { 21607c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == cur_szc); 21617c478bd9Sstevel@tonic-gate pp->p_szc = new_szc; 21627c478bd9Sstevel@tonic-gate pp = pp->p_next; 21637c478bd9Sstevel@tonic-gate } 21647c478bd9Sstevel@tonic-gate 21657c478bd9Sstevel@tonic-gate CHK_LPG(pplist, new_szc); 21667c478bd9Sstevel@tonic-gate 21677c478bd9Sstevel@tonic-gate bin = PP_2_BIN(pplist); 21687c478bd9Sstevel@tonic-gate ASSERT(mnode == PP_2_MEM_NODE(pp)); 21697c478bd9Sstevel@tonic-gate if ((bin == color) && (flags == PC_ALLOC) && 21707c478bd9Sstevel@tonic-gate (ret_pp == NULL) && 21717c478bd9Sstevel@tonic-gate page_trylock_cons(pp, SE_EXCL)) { 21727c478bd9Sstevel@tonic-gate ret_pp = pp; 21737c478bd9Sstevel@tonic-gate } else { 21747c478bd9Sstevel@tonic-gate mtype = PP_2_MTYPE(pp); 21757c478bd9Sstevel@tonic-gate page_vpadd(&PAGE_FREELISTS(mnode, new_szc, 21767c478bd9Sstevel@tonic-gate bin, mtype), pplist); 21777c478bd9Sstevel@tonic-gate 2178affbd3ccSkchow page_ctr_add(mnode, mtype, pplist, 2179affbd3ccSkchow PG_FREE_LIST); 21807c478bd9Sstevel@tonic-gate } 21817c478bd9Sstevel@tonic-gate pplist = npplist; 21827c478bd9Sstevel@tonic-gate } 21837c478bd9Sstevel@tonic-gate } 21847c478bd9Sstevel@tonic-gate return (ret_pp); 21857c478bd9Sstevel@tonic-gate } 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate int mpss_coalesce_disable = 0; 21887c478bd9Sstevel@tonic-gate 21897c478bd9Sstevel@tonic-gate /* 21907c478bd9Sstevel@tonic-gate * Coalesce free pages into a page of the given szc and color if possible. 21917c478bd9Sstevel@tonic-gate * Return the pointer to the page created, otherwise, return NULL. 21925d07b933Sdp * 21935d07b933Sdp * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 21947c478bd9Sstevel@tonic-gate */ 21955d07b933Sdp page_t * 21965d07b933Sdp page_freelist_coalesce(int mnode, uchar_t szc, uint_t color, uint_t ceq_mask, 21975d07b933Sdp int mtype, pfn_t pfnhi) 21987c478bd9Sstevel@tonic-gate { 21995d07b933Sdp int r = szc; /* region size */ 22005d07b933Sdp int mrange; 22015d07b933Sdp uint_t full, bin, color_mask, wrap = 0; 22025d07b933Sdp pfn_t pfnum, lo, hi; 22035d07b933Sdp size_t len, idx, idx0; 22045d07b933Sdp pgcnt_t cands = 0, szcpgcnt = page_get_pagecnt(szc); 22057c478bd9Sstevel@tonic-gate page_t *ret_pp; 2206ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 22075d07b933Sdp #if defined(__sparc) 22085d07b933Sdp pfn_t pfnum0, nlo, nhi; 22095d07b933Sdp #endif 22107c478bd9Sstevel@tonic-gate 22117c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 22125d07b933Sdp ASSERT(szc < MMU_PAGE_SIZES); 22135d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[szc][0]); 22147c478bd9Sstevel@tonic-gate return (NULL); 22157c478bd9Sstevel@tonic-gate } 22167c478bd9Sstevel@tonic-gate 22175d07b933Sdp ASSERT(szc < mmu_page_sizes); 22185d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(szc) - 1; 22195d07b933Sdp ASSERT(ceq_mask <= color_mask); 22205d07b933Sdp ASSERT(color <= color_mask); 22215d07b933Sdp color &= ceq_mask; 22225d07b933Sdp 22235d07b933Sdp /* Prevent page_counters dynamic memory from being freed */ 22245d07b933Sdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 22255d07b933Sdp 22265d07b933Sdp mrange = MTYPE_2_MRANGE(mnode, mtype); 22275d07b933Sdp ASSERT(mrange < mnode_nranges[mnode]); 22285d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce[r][mrange]); 22295d07b933Sdp 22305d07b933Sdp /* get pfn range for mtype */ 22315d07b933Sdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 22325d07b933Sdp #if defined(__sparc) 22335d07b933Sdp lo = PAGE_COUNTERS_BASE(mnode, r); 22345d07b933Sdp hi = IDX_TO_PNUM(mnode, r, len); 22355d07b933Sdp #else 22365d07b933Sdp MNODETYPE_2_PFN(mnode, mtype, lo, hi); 22375d07b933Sdp hi++; 22385d07b933Sdp #endif 22395d07b933Sdp 22405d07b933Sdp /* use lower limit if given */ 22415d07b933Sdp if (pfnhi != PFNNULL && pfnhi < hi) 22425d07b933Sdp hi = pfnhi; 22435d07b933Sdp 22445d07b933Sdp /* round to szcpgcnt boundaries */ 22455d07b933Sdp lo = P2ROUNDUP(lo, szcpgcnt); 2246*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 2247*b779d3e0Sdp if (lo == (pfn_t)-1) { 2248*b779d3e0Sdp rw_exit(&page_ctrs_rwlock[mnode]); 2249*b779d3e0Sdp return (NULL); 2250*b779d3e0Sdp } 22515d07b933Sdp hi = hi & ~(szcpgcnt - 1); 22525d07b933Sdp 22535d07b933Sdp /* set lo to the closest pfn of the right color */ 2254ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ color) & ceq_mask) || 2255ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(lo) != mnode)) { 2256ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, color, ceq_mask, color_mask, 2257ce8eb11aSdp &it); 22585d07b933Sdp } 22595d07b933Sdp 22605d07b933Sdp if (hi <= lo) { 22615d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 22627c478bd9Sstevel@tonic-gate return (NULL); 22637c478bd9Sstevel@tonic-gate } 22645d07b933Sdp 22657c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 22667c478bd9Sstevel@tonic-gate 22675d07b933Sdp /* calculate the number of page candidates and initial search index */ 22685d07b933Sdp bin = color; 22695d07b933Sdp idx0 = (size_t)(-1); 22705d07b933Sdp do { 22715d07b933Sdp pgcnt_t acand; 22725d07b933Sdp 22735d07b933Sdp PGCTRS_CANDS_GETVALUECOLOR(mnode, mrange, r, bin, acand); 22745d07b933Sdp if (acand) { 22755d07b933Sdp idx = PAGE_COUNTERS_CURRENT_COLOR(mnode, 22765d07b933Sdp r, bin, mrange); 22775d07b933Sdp idx0 = MIN(idx0, idx); 22785d07b933Sdp cands += acand; 22795d07b933Sdp } 22805d07b933Sdp bin = ADD_MASKED(bin, 1, ceq_mask, color_mask); 22815d07b933Sdp } while (bin != color); 22825d07b933Sdp 22835d07b933Sdp if (cands == 0) { 22845d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_cands_skip[r][mrange]); 22855d07b933Sdp rw_exit(&page_ctrs_rwlock[mnode]); 22865d07b933Sdp return (NULL); 22875d07b933Sdp } 22885d07b933Sdp 22895d07b933Sdp pfnum = IDX_TO_PNUM(mnode, r, idx0); 22905d07b933Sdp if (pfnum < lo || pfnum >= hi) { 22915d07b933Sdp pfnum = lo; 2292ce8eb11aSdp } else { 2293*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2294ce8eb11aSdp if (pfnum == (pfn_t)-1) { 2295ce8eb11aSdp pfnum = lo; 2296*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2297ce8eb11aSdp ASSERT(pfnum != (pfn_t)-1); 2298ce8eb11aSdp } else if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & ceq_mask || 2299ce8eb11aSdp (interleaved_mnodes && PFN_2_MEM_NODE(pfnum) != mnode)) { 2300ce8eb11aSdp /* invalid color, get the closest correct pfn */ 2301ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2302ce8eb11aSdp color_mask, &it); 2303ce8eb11aSdp if (pfnum >= hi) { 2304ce8eb11aSdp pfnum = lo; 2305*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 2306ce8eb11aSdp } 2307ce8eb11aSdp } 23085d07b933Sdp } 23095d07b933Sdp 23105d07b933Sdp /* set starting index */ 23115d07b933Sdp idx0 = PNUM_TO_IDX(mnode, r, pfnum); 23125d07b933Sdp ASSERT(idx0 < len); 23135d07b933Sdp 23145d07b933Sdp #if defined(__sparc) 23155d07b933Sdp pfnum0 = pfnum; /* page corresponding to idx0 */ 23165d07b933Sdp nhi = 0; /* search kcage ranges */ 23175d07b933Sdp #endif 23185d07b933Sdp 23195d07b933Sdp for (idx = idx0; wrap == 0 || (idx < idx0 && wrap < 2); ) { 23205d07b933Sdp 23215d07b933Sdp #if defined(__sparc) 23225d07b933Sdp /* 23235d07b933Sdp * Find lowest intersection of kcage ranges and mnode. 23245d07b933Sdp * MTYPE_NORELOC means look in the cage, otherwise outside. 23255d07b933Sdp */ 23265d07b933Sdp if (nhi <= pfnum) { 23275d07b933Sdp if (kcage_next_range(mtype == MTYPE_NORELOC, pfnum, 23285d07b933Sdp (wrap == 0 ? hi : pfnum0), &nlo, &nhi)) 23295d07b933Sdp goto wrapit; 23305d07b933Sdp 23315d07b933Sdp /* jump to the next page in the range */ 23325d07b933Sdp if (pfnum < nlo) { 23335d07b933Sdp pfnum = P2ROUNDUP(nlo, szcpgcnt); 2334*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 23355d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 23365d07b933Sdp if (idx >= len || pfnum >= hi) 23375d07b933Sdp goto wrapit; 2338ce8eb11aSdp if ((PFN_2_COLOR(pfnum, szc, &it) ^ color) & 23395d07b933Sdp ceq_mask) 23405d07b933Sdp goto next; 2341ce8eb11aSdp if (interleaved_mnodes && 2342ce8eb11aSdp PFN_2_MEM_NODE(pfnum) != mnode) 2343ce8eb11aSdp goto next; 23447c478bd9Sstevel@tonic-gate } 23455d07b933Sdp } 23465d07b933Sdp #endif 23475d07b933Sdp 23485d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) != full) 23495d07b933Sdp goto next; 23505d07b933Sdp 23515d07b933Sdp /* 23525d07b933Sdp * RFE: For performance maybe we can do something less 23535d07b933Sdp * brutal than locking the entire freelist. So far 23545d07b933Sdp * this doesn't seem to be a performance problem? 23555d07b933Sdp */ 23565d07b933Sdp page_freelist_lock(mnode); 23575d07b933Sdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 23585d07b933Sdp ret_pp = 23595d07b933Sdp page_promote(mnode, pfnum, r, PC_ALLOC, mtype); 23607c478bd9Sstevel@tonic-gate if (ret_pp != NULL) { 23615d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfc_coalok[r][mrange]); 23625d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, 2363ce8eb11aSdp PFN_2_COLOR(pfnum, szc, &it), mrange) = idx; 23647c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 23657c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 23667c478bd9Sstevel@tonic-gate #if defined(__sparc) 23677c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 23687c478bd9Sstevel@tonic-gate pgcnt_t npgs; 23697c478bd9Sstevel@tonic-gate 23707c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(ret_pp->p_szc); 23717c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 23727c478bd9Sstevel@tonic-gate } 23737c478bd9Sstevel@tonic-gate #endif 23747c478bd9Sstevel@tonic-gate return (ret_pp); 23757c478bd9Sstevel@tonic-gate } 23765d07b933Sdp } else { 23775d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_changed[r][mrange]); 23785d07b933Sdp } 23795d07b933Sdp 23805d07b933Sdp page_freelist_unlock(mnode); 23815d07b933Sdp /* 23825d07b933Sdp * No point looking for another page if we've 23835d07b933Sdp * already tried all of the ones that 23845d07b933Sdp * page_ctr_cands indicated. Stash off where we left 23855d07b933Sdp * off. 23865d07b933Sdp * Note: this is not exact since we don't hold the 23875d07b933Sdp * page_freelist_locks before we initially get the 23885d07b933Sdp * value of cands for performance reasons, but should 23895d07b933Sdp * be a decent approximation. 23905d07b933Sdp */ 23915d07b933Sdp if (--cands == 0) { 23925d07b933Sdp PAGE_COUNTERS_CURRENT_COLOR(mnode, r, color, mrange) = 23935d07b933Sdp idx; 23945d07b933Sdp break; 23955d07b933Sdp } 23965d07b933Sdp next: 23975d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfnum, szc, color, ceq_mask, 2398ce8eb11aSdp color_mask, &it); 23995d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 24005d07b933Sdp if (idx >= len || pfnum >= hi) { 24015d07b933Sdp wrapit: 24025d07b933Sdp pfnum = lo; 2403*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(pfnum, mnode, szc, &it); 24045d07b933Sdp idx = PNUM_TO_IDX(mnode, r, pfnum); 24055d07b933Sdp wrap++; 24065d07b933Sdp #if defined(__sparc) 24075d07b933Sdp nhi = 0; /* search kcage ranges */ 24085d07b933Sdp #endif 24097c478bd9Sstevel@tonic-gate } 24107c478bd9Sstevel@tonic-gate } 24115d07b933Sdp 24127c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 24135d07b933Sdp VM_STAT_ADD(vmm_vmstats.page_ctrs_failed[r][mrange]); 24147c478bd9Sstevel@tonic-gate return (NULL); 24157c478bd9Sstevel@tonic-gate } 24167c478bd9Sstevel@tonic-gate 24177c478bd9Sstevel@tonic-gate /* 24187c478bd9Sstevel@tonic-gate * For the given mnode, promote as many small pages to large pages as possible. 2419ce8eb11aSdp * mnode can be -1, which means do them all 24207c478bd9Sstevel@tonic-gate */ 24217c478bd9Sstevel@tonic-gate void 24227c478bd9Sstevel@tonic-gate page_freelist_coalesce_all(int mnode) 24237c478bd9Sstevel@tonic-gate { 24247c478bd9Sstevel@tonic-gate int r; /* region size */ 24257c478bd9Sstevel@tonic-gate int idx, full; 24267c478bd9Sstevel@tonic-gate size_t len; 2427ce8eb11aSdp int doall = interleaved_mnodes || mnode < 0; 2428ce8eb11aSdp int mlo = doall ? 0 : mnode; 2429ce8eb11aSdp int mhi = doall ? max_mem_nodes : (mnode + 1); 24307c478bd9Sstevel@tonic-gate 24317c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.page_ctrs_coalesce_all); 24327c478bd9Sstevel@tonic-gate 24337c478bd9Sstevel@tonic-gate if (mpss_coalesce_disable) { 24347c478bd9Sstevel@tonic-gate return; 24357c478bd9Sstevel@tonic-gate } 24367c478bd9Sstevel@tonic-gate 24377c478bd9Sstevel@tonic-gate /* 24387c478bd9Sstevel@tonic-gate * Lock the entire freelist and coalesce what we can. 24397c478bd9Sstevel@tonic-gate * 24407c478bd9Sstevel@tonic-gate * Always promote to the largest page possible 24417c478bd9Sstevel@tonic-gate * first to reduce the number of page promotions. 24427c478bd9Sstevel@tonic-gate */ 2443ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2444ce8eb11aSdp rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 2445ce8eb11aSdp page_freelist_lock(mnode); 2446ce8eb11aSdp } 24477c478bd9Sstevel@tonic-gate for (r = mmu_page_sizes - 1; r > 0; r--) { 2448ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2449ce8eb11aSdp pgcnt_t cands = 0; 2450ce8eb11aSdp int mrange, nranges = mnode_nranges[mnode]; 24517c478bd9Sstevel@tonic-gate 2452ce8eb11aSdp for (mrange = 0; mrange < nranges; mrange++) { 2453ce8eb11aSdp PGCTRS_CANDS_GETVALUE(mnode, mrange, r, cands); 2454ce8eb11aSdp if (cands != 0) 2455ce8eb11aSdp break; 2456ce8eb11aSdp } 2457ce8eb11aSdp if (cands == 0) { 2458ce8eb11aSdp VM_STAT_ADD(vmm_vmstats. 2459ce8eb11aSdp page_ctrs_cands_skip_all); 2460ce8eb11aSdp continue; 2461ce8eb11aSdp } 24627c478bd9Sstevel@tonic-gate 2463ce8eb11aSdp full = FULL_REGION_CNT(r); 2464ce8eb11aSdp len = PAGE_COUNTERS_ENTRIES(mnode, r); 2465ce8eb11aSdp 2466ce8eb11aSdp for (idx = 0; idx < len; idx++) { 2467ce8eb11aSdp if (PAGE_COUNTERS(mnode, r, idx) == full) { 2468ce8eb11aSdp pfn_t pfnum = 2469ce8eb11aSdp IDX_TO_PNUM(mnode, r, idx); 2470ce8eb11aSdp int tmnode = interleaved_mnodes ? 2471ce8eb11aSdp PFN_2_MEM_NODE(pfnum) : mnode; 2472ce8eb11aSdp 2473ce8eb11aSdp ASSERT(pfnum >= 2474ce8eb11aSdp mem_node_config[tmnode].physbase && 2475ce8eb11aSdp pfnum < 2476ce8eb11aSdp mem_node_config[tmnode].physmax); 2477ce8eb11aSdp 2478ce8eb11aSdp (void) page_promote(tmnode, 2479ce8eb11aSdp pfnum, r, PC_FREE, PC_MTYPE_ANY); 2480ce8eb11aSdp } 24817c478bd9Sstevel@tonic-gate } 2482ce8eb11aSdp /* shared hpm_counters covers all mnodes, so we quit */ 2483ce8eb11aSdp if (interleaved_mnodes) 2484ce8eb11aSdp break; 24857c478bd9Sstevel@tonic-gate } 24867c478bd9Sstevel@tonic-gate } 2487ce8eb11aSdp for (mnode = mlo; mnode < mhi; mnode++) { 2488ce8eb11aSdp page_freelist_unlock(mnode); 2489ce8eb11aSdp rw_exit(&page_ctrs_rwlock[mnode]); 2490ce8eb11aSdp } 24917c478bd9Sstevel@tonic-gate } 24927c478bd9Sstevel@tonic-gate 24937c478bd9Sstevel@tonic-gate /* 24947c478bd9Sstevel@tonic-gate * This is where all polices for moving pages around 24957c478bd9Sstevel@tonic-gate * to different page size free lists is implemented. 24967c478bd9Sstevel@tonic-gate * Returns 1 on success, 0 on failure. 24977c478bd9Sstevel@tonic-gate * 24987c478bd9Sstevel@tonic-gate * So far these are the priorities for this algorithm in descending 24997c478bd9Sstevel@tonic-gate * order: 25007c478bd9Sstevel@tonic-gate * 25017c478bd9Sstevel@tonic-gate * 1) When servicing a request try to do so with a free page 25027c478bd9Sstevel@tonic-gate * from next size up. Helps defer fragmentation as long 25037c478bd9Sstevel@tonic-gate * as possible. 25047c478bd9Sstevel@tonic-gate * 25057c478bd9Sstevel@tonic-gate * 2) Page coalesce on demand. Only when a freelist 25067c478bd9Sstevel@tonic-gate * larger than PAGESIZE is empty and step 1 25077c478bd9Sstevel@tonic-gate * will not work since all larger size lists are 25087c478bd9Sstevel@tonic-gate * also empty. 25097c478bd9Sstevel@tonic-gate * 25107c478bd9Sstevel@tonic-gate * If pfnhi is non-zero, search for large page with pfn range less than pfnhi. 25117c478bd9Sstevel@tonic-gate */ 25125d07b933Sdp 25137c478bd9Sstevel@tonic-gate page_t * 25145d07b933Sdp page_freelist_split(uchar_t szc, uint_t color, int mnode, int mtype, 25155d07b933Sdp pfn_t pfnhi, page_list_walker_t *plw) 25167c478bd9Sstevel@tonic-gate { 25177c478bd9Sstevel@tonic-gate uchar_t nszc = szc + 1; 25185d07b933Sdp uint_t bin, sbin, bin_prev; 25197c478bd9Sstevel@tonic-gate page_t *pp, *firstpp; 25207c478bd9Sstevel@tonic-gate page_t *ret_pp = NULL; 25215d07b933Sdp uint_t color_mask; 25227c478bd9Sstevel@tonic-gate 25235d07b933Sdp if (nszc == mmu_page_sizes) 25245d07b933Sdp return (NULL); 25257c478bd9Sstevel@tonic-gate 25265d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25275d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nszc) - 1; 25285d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(szc, color); 25295d07b933Sdp bin_prev = (plw->plw_bin_split_prev == color) ? INVALID_COLOR : 25305d07b933Sdp PAGE_GET_NSZ_COLOR(szc, plw->plw_bin_split_prev); 25315d07b933Sdp 25325d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_req[szc]); 25337c478bd9Sstevel@tonic-gate /* 25345d07b933Sdp * First try to break up a larger page to fill current size freelist. 25357c478bd9Sstevel@tonic-gate */ 25365d07b933Sdp while (plw->plw_bins[nszc] != 0) { 25375d07b933Sdp 25385d07b933Sdp ASSERT(nszc < mmu_page_sizes); 25395d07b933Sdp 25407c478bd9Sstevel@tonic-gate /* 25417c478bd9Sstevel@tonic-gate * If page found then demote it. 25427c478bd9Sstevel@tonic-gate */ 25437c478bd9Sstevel@tonic-gate if (PAGE_FREELISTS(mnode, nszc, bin, mtype)) { 25447c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 25457c478bd9Sstevel@tonic-gate firstpp = pp = PAGE_FREELISTS(mnode, nszc, bin, mtype); 25467c478bd9Sstevel@tonic-gate 25477c478bd9Sstevel@tonic-gate /* 25487c478bd9Sstevel@tonic-gate * If pfnhi is not PFNNULL, look for large page below 25497c478bd9Sstevel@tonic-gate * pfnhi. PFNNULL signifies no pfn requirement. 25507c478bd9Sstevel@tonic-gate */ 25517c478bd9Sstevel@tonic-gate if (pfnhi != PFNNULL && pp->p_pagenum >= pfnhi) { 25527c478bd9Sstevel@tonic-gate do { 25537c478bd9Sstevel@tonic-gate pp = pp->p_vpnext; 25547c478bd9Sstevel@tonic-gate if (pp == firstpp) { 25557c478bd9Sstevel@tonic-gate pp = NULL; 25567c478bd9Sstevel@tonic-gate break; 25577c478bd9Sstevel@tonic-gate } 25587c478bd9Sstevel@tonic-gate } while (pp->p_pagenum >= pfnhi); 25597c478bd9Sstevel@tonic-gate } 25607c478bd9Sstevel@tonic-gate if (pp) { 25615d07b933Sdp uint_t ccolor = page_correct_color(szc, nszc, 25625d07b933Sdp color, bin, plw->plw_ceq_mask[szc]); 25635d07b933Sdp 25647c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == nszc); 25655d07b933Sdp VM_STAT_ADD(vmm_vmstats.pfs_demote[nszc]); 25667c478bd9Sstevel@tonic-gate ret_pp = page_demote(mnode, pp->p_pagenum, 25675d07b933Sdp pp->p_szc, szc, ccolor, PC_ALLOC); 25687c478bd9Sstevel@tonic-gate if (ret_pp) { 25697c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 25707c478bd9Sstevel@tonic-gate #if defined(__sparc) 25717c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(ret_pp)) { 25727c478bd9Sstevel@tonic-gate pgcnt_t npgs; 25737c478bd9Sstevel@tonic-gate 25747c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt( 25757c478bd9Sstevel@tonic-gate ret_pp->p_szc); 25767c478bd9Sstevel@tonic-gate kcage_freemem_sub(npgs); 25777c478bd9Sstevel@tonic-gate } 25787c478bd9Sstevel@tonic-gate #endif 25797c478bd9Sstevel@tonic-gate return (ret_pp); 25807c478bd9Sstevel@tonic-gate } 25817c478bd9Sstevel@tonic-gate } 25827c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 25837c478bd9Sstevel@tonic-gate } 25847c478bd9Sstevel@tonic-gate 25855d07b933Sdp /* loop through next size bins */ 25865d07b933Sdp bin = ADD_MASKED(bin, 1, plw->plw_ceq_mask[nszc], color_mask); 25875d07b933Sdp plw->plw_bins[nszc]--; 25885d07b933Sdp 25895d07b933Sdp if (bin == sbin) { 25905d07b933Sdp uchar_t nnszc = nszc + 1; 25915d07b933Sdp 25925d07b933Sdp /* we are done with this page size - check next */ 25935d07b933Sdp if (plw->plw_bins[nnszc] == 0) 25945d07b933Sdp /* we have already checked next size bins */ 25955d07b933Sdp break; 25965d07b933Sdp 25975d07b933Sdp bin = sbin = PAGE_GET_NSZ_COLOR(nszc, bin); 25985d07b933Sdp if (bin_prev != INVALID_COLOR) { 25995d07b933Sdp bin_prev = PAGE_GET_NSZ_COLOR(nszc, bin_prev); 26005d07b933Sdp if (!((bin ^ bin_prev) & 26015d07b933Sdp plw->plw_ceq_mask[nnszc])) 26025d07b933Sdp break; 26035d07b933Sdp } 26045d07b933Sdp ASSERT(nnszc < mmu_page_sizes); 26055d07b933Sdp color_mask = PAGE_GET_PAGECOLORS(nnszc) - 1; 26065d07b933Sdp nszc = nnszc; 26075d07b933Sdp ASSERT(nszc < mmu_page_sizes); 26085d07b933Sdp } 26097c478bd9Sstevel@tonic-gate } 26107c478bd9Sstevel@tonic-gate 26117c478bd9Sstevel@tonic-gate return (ret_pp); 26127c478bd9Sstevel@tonic-gate } 26137c478bd9Sstevel@tonic-gate 26147c478bd9Sstevel@tonic-gate /* 26157c478bd9Sstevel@tonic-gate * Helper routine used only by the freelist code to lock 26167c478bd9Sstevel@tonic-gate * a page. If the page is a large page then it succeeds in 26177c478bd9Sstevel@tonic-gate * locking all the constituent pages or none at all. 26187c478bd9Sstevel@tonic-gate * Returns 1 on sucess, 0 on failure. 26197c478bd9Sstevel@tonic-gate */ 26207c478bd9Sstevel@tonic-gate static int 26217c478bd9Sstevel@tonic-gate page_trylock_cons(page_t *pp, se_t se) 26227c478bd9Sstevel@tonic-gate { 26237c478bd9Sstevel@tonic-gate page_t *tpp, *first_pp = pp; 26247c478bd9Sstevel@tonic-gate 26257c478bd9Sstevel@tonic-gate /* 26267c478bd9Sstevel@tonic-gate * Fail if can't lock first or only page. 26277c478bd9Sstevel@tonic-gate */ 26287c478bd9Sstevel@tonic-gate if (!page_trylock(pp, se)) { 26297c478bd9Sstevel@tonic-gate return (0); 26307c478bd9Sstevel@tonic-gate } 26317c478bd9Sstevel@tonic-gate 26327c478bd9Sstevel@tonic-gate /* 26337c478bd9Sstevel@tonic-gate * PAGESIZE: common case. 26347c478bd9Sstevel@tonic-gate */ 26357c478bd9Sstevel@tonic-gate if (pp->p_szc == 0) { 26367c478bd9Sstevel@tonic-gate return (1); 26377c478bd9Sstevel@tonic-gate } 26387c478bd9Sstevel@tonic-gate 26397c478bd9Sstevel@tonic-gate /* 26407c478bd9Sstevel@tonic-gate * Large page case. 26417c478bd9Sstevel@tonic-gate */ 26427c478bd9Sstevel@tonic-gate tpp = pp->p_next; 26437c478bd9Sstevel@tonic-gate while (tpp != pp) { 26447c478bd9Sstevel@tonic-gate if (!page_trylock(tpp, se)) { 26457c478bd9Sstevel@tonic-gate /* 26468b464eb8Smec * On failure unlock what we have locked so far. 26478b464eb8Smec * We want to avoid attempting to capture these 26488b464eb8Smec * pages as the pcm mutex may be held which could 26498b464eb8Smec * lead to a recursive mutex panic. 26507c478bd9Sstevel@tonic-gate */ 26517c478bd9Sstevel@tonic-gate while (first_pp != tpp) { 26528b464eb8Smec page_unlock_nocapture(first_pp); 26537c478bd9Sstevel@tonic-gate first_pp = first_pp->p_next; 26547c478bd9Sstevel@tonic-gate } 26557c478bd9Sstevel@tonic-gate return (0); 26567c478bd9Sstevel@tonic-gate } 26577c478bd9Sstevel@tonic-gate tpp = tpp->p_next; 26587c478bd9Sstevel@tonic-gate } 26597c478bd9Sstevel@tonic-gate return (1); 26607c478bd9Sstevel@tonic-gate } 26617c478bd9Sstevel@tonic-gate 26625d07b933Sdp /* 26635d07b933Sdp * init context for walking page lists 26645d07b933Sdp * Called when a page of the given szc in unavailable. Sets markers 26655d07b933Sdp * for the beginning of the search to detect when search has 26665d07b933Sdp * completed a full cycle. Sets flags for splitting larger pages 26675d07b933Sdp * and coalescing smaller pages. Page walking procedes until a page 26685d07b933Sdp * of the desired equivalent color is found. 26695d07b933Sdp */ 26705d07b933Sdp void 26715d07b933Sdp page_list_walk_init(uchar_t szc, uint_t flags, uint_t bin, int can_split, 26725d07b933Sdp int use_ceq, page_list_walker_t *plw) 26737c478bd9Sstevel@tonic-gate { 26745d07b933Sdp uint_t nszc, ceq_mask, colors; 26755d07b933Sdp uchar_t ceq = use_ceq ? colorequivszc[szc] : 0; 26767c478bd9Sstevel@tonic-gate 26777c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 26785d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 26797c478bd9Sstevel@tonic-gate 26805d07b933Sdp plw->plw_colors = colors; 26815d07b933Sdp plw->plw_color_mask = colors - 1; 26825d07b933Sdp plw->plw_bin_marker = plw->plw_bin0 = bin; 26835d07b933Sdp plw->plw_bin_split_prev = bin; 26845d07b933Sdp plw->plw_bin_step = (szc == 0) ? vac_colors : 1; 26857c478bd9Sstevel@tonic-gate 26865d07b933Sdp /* 26875d07b933Sdp * if vac aliasing is possible make sure lower order color 26885d07b933Sdp * bits are never ignored 26895d07b933Sdp */ 26905d07b933Sdp if (vac_colors > 1) 26915d07b933Sdp ceq &= 0xf0; 26927c478bd9Sstevel@tonic-gate 26937c478bd9Sstevel@tonic-gate /* 26945d07b933Sdp * calculate the number of non-equivalent colors and 26955d07b933Sdp * color equivalency mask 26967c478bd9Sstevel@tonic-gate */ 26975d07b933Sdp plw->plw_ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 26985d07b933Sdp ASSERT(szc > 0 || plw->plw_ceq_dif >= vac_colors); 26995d07b933Sdp ASSERT(plw->plw_ceq_dif > 0); 27005d07b933Sdp plw->plw_ceq_mask[szc] = (plw->plw_ceq_dif - 1) << (ceq & 0xf); 27017c478bd9Sstevel@tonic-gate 27025d07b933Sdp if (flags & PG_MATCH_COLOR) { 27035d07b933Sdp if (cpu_page_colors < 0) { 27045d07b933Sdp /* 27055d07b933Sdp * this is a heterogeneous machine with different CPUs 27065d07b933Sdp * having different size e$ (not supported for ni2/rock 27075d07b933Sdp */ 27085d07b933Sdp uint_t cpucolors = CPUSETSIZE() >> PAGE_GET_SHIFT(szc); 27095d07b933Sdp cpucolors = MAX(cpucolors, 1); 27105d07b933Sdp ceq_mask = plw->plw_color_mask & (cpucolors - 1); 27115d07b933Sdp plw->plw_ceq_mask[szc] = 27125d07b933Sdp MIN(ceq_mask, plw->plw_ceq_mask[szc]); 27135d07b933Sdp } 27145d07b933Sdp plw->plw_ceq_dif = 1; 27155d07b933Sdp } 27167c478bd9Sstevel@tonic-gate 27175d07b933Sdp /* we can split pages in the freelist, but not the cachelist */ 27185d07b933Sdp if (can_split) { 2719ce8eb11aSdp plw->plw_do_split = (szc + 1 < mmu_page_sizes) ? 1 : 0; 27207c478bd9Sstevel@tonic-gate 2721ce8eb11aSdp /* set next szc color masks and number of free list bins */ 2722ce8eb11aSdp for (nszc = szc + 1; nszc < mmu_page_sizes; nszc++, szc++) { 2723ce8eb11aSdp plw->plw_ceq_mask[nszc] = PAGE_GET_NSZ_MASK(szc, 2724ce8eb11aSdp plw->plw_ceq_mask[szc]); 2725ce8eb11aSdp plw->plw_bins[nszc] = PAGE_GET_PAGECOLORS(nszc); 2726ce8eb11aSdp } 2727ce8eb11aSdp plw->plw_ceq_mask[nszc] = INVALID_MASK; 2728ce8eb11aSdp plw->plw_bins[nszc] = 0; 27297c478bd9Sstevel@tonic-gate 27305d07b933Sdp } else { 2731ce8eb11aSdp ASSERT(szc == 0); 2732ce8eb11aSdp plw->plw_do_split = 0; 2733ce8eb11aSdp plw->plw_bins[1] = 0; 2734ce8eb11aSdp plw->plw_ceq_mask[1] = INVALID_MASK; 27357c478bd9Sstevel@tonic-gate } 27365d07b933Sdp } 27377c478bd9Sstevel@tonic-gate 27385d07b933Sdp /* 27395d07b933Sdp * set mark to flag where next split should occur 27405d07b933Sdp */ 27415d07b933Sdp #define PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw) { \ 27425d07b933Sdp uint_t bin_nsz = PAGE_GET_NSZ_COLOR(szc, bin); \ 27435d07b933Sdp uint_t bin0_nsz = PAGE_GET_NSZ_COLOR(szc, plw->plw_bin0); \ 27445d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[nszc] & plw->plw_color_mask; \ 27455d07b933Sdp plw->plw_split_next = \ 27465d07b933Sdp INC_MASKED(bin_nsz, neq_mask, plw->plw_color_mask); \ 27475d07b933Sdp if (!((plw->plw_split_next ^ bin0_nsz) & plw->plw_ceq_mask[nszc])) { \ 27485d07b933Sdp plw->plw_split_next = \ 27495d07b933Sdp INC_MASKED(plw->plw_split_next, \ 27505d07b933Sdp neq_mask, plw->plw_color_mask); \ 27515d07b933Sdp } \ 27525d07b933Sdp } 27537c478bd9Sstevel@tonic-gate 27545d07b933Sdp uint_t 27555d07b933Sdp page_list_walk_next_bin(uchar_t szc, uint_t bin, page_list_walker_t *plw) 27565d07b933Sdp { 27575d07b933Sdp uint_t neq_mask = ~plw->plw_ceq_mask[szc] & plw->plw_color_mask; 27585d07b933Sdp uint_t bin0_nsz, nbin_nsz, nbin0, nbin; 27595d07b933Sdp uchar_t nszc = szc + 1; 27605d07b933Sdp 27615d07b933Sdp nbin = ADD_MASKED(bin, 27625d07b933Sdp plw->plw_bin_step, neq_mask, plw->plw_color_mask); 27635d07b933Sdp 27645d07b933Sdp if (plw->plw_do_split) { 27655d07b933Sdp plw->plw_bin_split_prev = bin; 27665d07b933Sdp PAGE_SET_NEXT_SPLIT_MARKER(szc, nszc, bin, plw); 27675d07b933Sdp plw->plw_do_split = 0; 27685d07b933Sdp } 27695d07b933Sdp 27705d07b933Sdp if (szc == 0) { 27715d07b933Sdp if (plw->plw_count != 0 || plw->plw_ceq_dif == vac_colors) { 27725d07b933Sdp if (nbin == plw->plw_bin0 && 27735d07b933Sdp (vac_colors == 1 || nbin != plw->plw_bin_marker)) { 27745d07b933Sdp nbin = ADD_MASKED(nbin, plw->plw_bin_step, 27755d07b933Sdp neq_mask, plw->plw_color_mask); 27765d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 27775d07b933Sdp } 27785d07b933Sdp 27795d07b933Sdp if (vac_colors > 1 && nbin == plw->plw_bin_marker) { 27805d07b933Sdp plw->plw_bin_marker = 27815d07b933Sdp nbin = INC_MASKED(nbin, neq_mask, 2782ce8eb11aSdp plw->plw_color_mask); 27835d07b933Sdp plw->plw_bin_split_prev = plw->plw_bin0; 27845d07b933Sdp /* 27855d07b933Sdp * large pages all have the same vac color 27865d07b933Sdp * so by now we should be done with next 27875d07b933Sdp * size page splitting process 27885d07b933Sdp */ 27895d07b933Sdp ASSERT(plw->plw_bins[1] == 0); 27905d07b933Sdp plw->plw_do_split = 0; 27915d07b933Sdp return (nbin); 27925d07b933Sdp } 27935d07b933Sdp 27945d07b933Sdp } else { 27955d07b933Sdp uint_t bin_jump = (vac_colors == 1) ? 27965d07b933Sdp (BIN_STEP & ~3) - (plw->plw_bin0 & 3) : BIN_STEP; 27975d07b933Sdp 27985d07b933Sdp bin_jump &= ~(vac_colors - 1); 27995d07b933Sdp 28005d07b933Sdp nbin0 = ADD_MASKED(plw->plw_bin0, bin_jump, neq_mask, 28015d07b933Sdp plw->plw_color_mask); 28025d07b933Sdp 28035d07b933Sdp if ((nbin0 ^ plw->plw_bin0) & plw->plw_ceq_mask[szc]) { 28045d07b933Sdp 28055d07b933Sdp plw->plw_bin_marker = nbin = nbin0; 28065d07b933Sdp 28075d07b933Sdp if (plw->plw_bins[nszc] != 0) { 28085d07b933Sdp /* 28095d07b933Sdp * check if next page size bin is the 28105d07b933Sdp * same as the next page size bin for 28115d07b933Sdp * bin0 28125d07b933Sdp */ 28135d07b933Sdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, 28145d07b933Sdp nbin); 28155d07b933Sdp bin0_nsz = PAGE_GET_NSZ_COLOR(szc, 28165d07b933Sdp plw->plw_bin0); 28175d07b933Sdp 28185d07b933Sdp if ((bin0_nsz ^ nbin_nsz) & 28195d07b933Sdp plw->plw_ceq_mask[nszc]) 28205d07b933Sdp plw->plw_do_split = 1; 28215d07b933Sdp } 28225d07b933Sdp return (nbin); 28235d07b933Sdp } 28245d07b933Sdp } 28255d07b933Sdp } 28265d07b933Sdp 28275d07b933Sdp if (plw->plw_bins[nszc] != 0) { 2828ce8eb11aSdp nbin_nsz = PAGE_GET_NSZ_COLOR(szc, nbin); 2829ce8eb11aSdp if (!((plw->plw_split_next ^ nbin_nsz) & 2830ce8eb11aSdp plw->plw_ceq_mask[nszc])) 2831ce8eb11aSdp plw->plw_do_split = 1; 28325d07b933Sdp } 28335d07b933Sdp 28345d07b933Sdp return (nbin); 28355d07b933Sdp } 28365d07b933Sdp 28375d07b933Sdp page_t * 28385d07b933Sdp page_get_mnode_freelist(int mnode, uint_t bin, int mtype, uchar_t szc, 28395d07b933Sdp uint_t flags) 28405d07b933Sdp { 28415d07b933Sdp kmutex_t *pcm; 28425d07b933Sdp page_t *pp, *first_pp; 28435d07b933Sdp uint_t sbin; 28445d07b933Sdp int plw_initialized; 28455d07b933Sdp page_list_walker_t plw; 28465d07b933Sdp 28475d07b933Sdp ASSERT(szc < mmu_page_sizes); 28485d07b933Sdp 28495d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_alloc[szc]); 28505d07b933Sdp 28515d07b933Sdp MTYPE_START(mnode, mtype, flags); 28525d07b933Sdp if (mtype < 0) { /* mnode does not have memory in mtype range */ 28535d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocempty[szc]); 28545d07b933Sdp return (NULL); 28555d07b933Sdp } 28565d07b933Sdp try_again: 28575d07b933Sdp 28585d07b933Sdp plw_initialized = 0; 28595d07b933Sdp plw.plw_ceq_dif = 1; 28607c478bd9Sstevel@tonic-gate 28617c478bd9Sstevel@tonic-gate /* 28627c478bd9Sstevel@tonic-gate * Only hold one freelist lock at a time, that way we 28637c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 28647c478bd9Sstevel@tonic-gate * ordering. 28657c478bd9Sstevel@tonic-gate */ 28665d07b933Sdp for (plw.plw_count = 0; 28675d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 28685d07b933Sdp sbin = bin; 28695d07b933Sdp do { 28705d07b933Sdp if (!PAGE_FREELISTS(mnode, szc, bin, mtype)) 28715d07b933Sdp goto bin_empty_1; 28725d07b933Sdp 28737c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_FREE_LIST); 28747c478bd9Sstevel@tonic-gate mutex_enter(pcm); 28757c478bd9Sstevel@tonic-gate pp = PAGE_FREELISTS(mnode, szc, bin, mtype); 28765d07b933Sdp if (pp == NULL) 28775d07b933Sdp goto bin_empty_0; 28785d07b933Sdp 28795d07b933Sdp /* 28805d07b933Sdp * These were set before the page 28815d07b933Sdp * was put on the free list, 28825d07b933Sdp * they must still be set. 28835d07b933Sdp */ 28845d07b933Sdp ASSERT(PP_ISFREE(pp)); 28855d07b933Sdp ASSERT(PP_ISAGED(pp)); 28865d07b933Sdp ASSERT(pp->p_vnode == NULL); 28875d07b933Sdp ASSERT(pp->p_hash == NULL); 28885d07b933Sdp ASSERT(pp->p_offset == (u_offset_t)-1); 28895d07b933Sdp ASSERT(pp->p_szc == szc); 28905d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 28915d07b933Sdp 28925d07b933Sdp /* 28935d07b933Sdp * Walk down the hash chain. 28945d07b933Sdp * 8k pages are linked on p_next 28955d07b933Sdp * and p_prev fields. Large pages 28965d07b933Sdp * are a contiguous group of 28975d07b933Sdp * constituent pages linked together 28985d07b933Sdp * on their p_next and p_prev fields. 28995d07b933Sdp * The large pages are linked together 29005d07b933Sdp * on the hash chain using p_vpnext 29015d07b933Sdp * p_vpprev of the base constituent 29025d07b933Sdp * page of each large page. 29035d07b933Sdp */ 29045d07b933Sdp first_pp = pp; 29055d07b933Sdp while (!page_trylock_cons(pp, SE_EXCL)) { 29065d07b933Sdp if (szc == 0) { 29075d07b933Sdp pp = pp->p_next; 29085d07b933Sdp } else { 29095d07b933Sdp pp = pp->p_vpnext; 29105d07b933Sdp } 29115d07b933Sdp 29127c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 29137c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 29147c478bd9Sstevel@tonic-gate ASSERT(pp->p_vnode == NULL); 29157c478bd9Sstevel@tonic-gate ASSERT(pp->p_hash == NULL); 29167c478bd9Sstevel@tonic-gate ASSERT(pp->p_offset == (u_offset_t)-1); 29177c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 29187c478bd9Sstevel@tonic-gate ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 29197c478bd9Sstevel@tonic-gate 29205d07b933Sdp if (pp == first_pp) 29215d07b933Sdp goto bin_empty_0; 29225d07b933Sdp } 29237c478bd9Sstevel@tonic-gate 29245d07b933Sdp ASSERT(pp != NULL); 29255d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 29265d07b933Sdp ASSERT(pp->p_szc == szc); 29275d07b933Sdp if (szc == 0) { 29285d07b933Sdp page_sub(&PAGE_FREELISTS(mnode, 29295d07b933Sdp szc, bin, mtype), pp); 29305d07b933Sdp } else { 29315d07b933Sdp page_vpsub(&PAGE_FREELISTS(mnode, 29325d07b933Sdp szc, bin, mtype), pp); 29335d07b933Sdp CHK_LPG(pp, szc); 29345d07b933Sdp } 29355d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_FREE_LIST); 29367c478bd9Sstevel@tonic-gate 29375d07b933Sdp if ((PP_ISFREE(pp) == 0) || (PP_ISAGED(pp) == 0)) 29385d07b933Sdp panic("free page is not. pp %p", (void *)pp); 29395d07b933Sdp mutex_exit(pcm); 29407c478bd9Sstevel@tonic-gate 29417c478bd9Sstevel@tonic-gate #if defined(__sparc) 29425d07b933Sdp ASSERT(!kcage_on || PP_ISNORELOC(pp) || 29435d07b933Sdp (flags & PG_NORELOC) == 0); 29447c478bd9Sstevel@tonic-gate 29455d07b933Sdp if (PP_ISNORELOC(pp)) 29465d07b933Sdp kcage_freemem_sub(page_get_pagecnt(szc)); 29477c478bd9Sstevel@tonic-gate #endif 29485d07b933Sdp VM_STAT_ADD(vmm_vmstats.pgmf_allocok[szc]); 29495d07b933Sdp return (pp); 29507c478bd9Sstevel@tonic-gate 29515d07b933Sdp bin_empty_0: 29525d07b933Sdp mutex_exit(pcm); 29535d07b933Sdp bin_empty_1: 29545d07b933Sdp if (plw_initialized == 0) { 29555d07b933Sdp page_list_walk_init(szc, flags, bin, 1, 1, 29565d07b933Sdp &plw); 29575d07b933Sdp plw_initialized = 1; 29585d07b933Sdp ASSERT(plw.plw_colors <= 29595d07b933Sdp PAGE_GET_PAGECOLORS(szc)); 29605d07b933Sdp ASSERT(plw.plw_colors > 0); 29615d07b933Sdp ASSERT((plw.plw_colors & 29625d07b933Sdp (plw.plw_colors - 1)) == 0); 29635d07b933Sdp ASSERT(bin < plw.plw_colors); 29645d07b933Sdp ASSERT(plw.plw_ceq_mask[szc] < plw.plw_colors); 29657c478bd9Sstevel@tonic-gate } 29665d07b933Sdp /* calculate the next bin with equivalent color */ 29675d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 29685d07b933Sdp plw.plw_ceq_mask[szc], plw.plw_color_mask); 29695d07b933Sdp } while (sbin != bin); 29707c478bd9Sstevel@tonic-gate 29717c478bd9Sstevel@tonic-gate /* 29725d07b933Sdp * color bins are all empty if color match. Try and 29735d07b933Sdp * satisfy the request by breaking up or coalescing 29745d07b933Sdp * pages from a different size freelist of the correct 29755d07b933Sdp * color that satisfies the ORIGINAL color requested. 29765d07b933Sdp * If that fails then try pages of the same size but 29775d07b933Sdp * different colors assuming we are not called with 29787c478bd9Sstevel@tonic-gate * PG_MATCH_COLOR. 29797c478bd9Sstevel@tonic-gate */ 29805d07b933Sdp if (plw.plw_do_split && 29815d07b933Sdp (pp = page_freelist_split(szc, bin, mnode, 2982ce8eb11aSdp mtype, PFNNULL, &plw)) != NULL) 2983ce8eb11aSdp return (pp); 29847c478bd9Sstevel@tonic-gate 29855d07b933Sdp if (szc > 0 && (pp = page_freelist_coalesce(mnode, szc, 29865d07b933Sdp bin, plw.plw_ceq_mask[szc], mtype, PFNNULL)) != NULL) 29875d07b933Sdp return (pp); 29887c478bd9Sstevel@tonic-gate 29895d07b933Sdp if (plw.plw_ceq_dif > 1) 29905d07b933Sdp bin = page_list_walk_next_bin(szc, bin, &plw); 29917c478bd9Sstevel@tonic-gate } 29927c478bd9Sstevel@tonic-gate 2993affbd3ccSkchow /* if allowed, cycle through additional mtypes */ 2994affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 2995affbd3ccSkchow if (mtype >= 0) 29965d07b933Sdp goto try_again; 2997affbd3ccSkchow 29987c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmf_allocfailed[szc]); 29997c478bd9Sstevel@tonic-gate 30007c478bd9Sstevel@tonic-gate return (NULL); 30017c478bd9Sstevel@tonic-gate } 30027c478bd9Sstevel@tonic-gate 30037c478bd9Sstevel@tonic-gate /* 30047c478bd9Sstevel@tonic-gate * Returns the count of free pages for 'pp' with size code 'szc'. 30057c478bd9Sstevel@tonic-gate * Note: This function does not return an exact value as the page freelist 30067c478bd9Sstevel@tonic-gate * locks are not held and thus the values in the page_counters may be 30077c478bd9Sstevel@tonic-gate * changing as we walk through the data. 30087c478bd9Sstevel@tonic-gate */ 30097c478bd9Sstevel@tonic-gate static int 30107c478bd9Sstevel@tonic-gate page_freecnt(int mnode, page_t *pp, uchar_t szc) 30117c478bd9Sstevel@tonic-gate { 30127c478bd9Sstevel@tonic-gate pgcnt_t pgfree; 30137c478bd9Sstevel@tonic-gate pgcnt_t cnt; 30147c478bd9Sstevel@tonic-gate ssize_t r = szc; /* region size */ 30157c478bd9Sstevel@tonic-gate ssize_t idx; 30167c478bd9Sstevel@tonic-gate int i; 30177c478bd9Sstevel@tonic-gate int full, range; 30187c478bd9Sstevel@tonic-gate 30197c478bd9Sstevel@tonic-gate /* Make sure pagenum passed in is aligned properly */ 30207c478bd9Sstevel@tonic-gate ASSERT((pp->p_pagenum & (PNUM_SIZE(szc) - 1)) == 0); 30217c478bd9Sstevel@tonic-gate ASSERT(szc > 0); 30227c478bd9Sstevel@tonic-gate 30237c478bd9Sstevel@tonic-gate /* Prevent page_counters dynamic memory from being freed */ 30247c478bd9Sstevel@tonic-gate rw_enter(&page_ctrs_rwlock[mnode], RW_READER); 30257c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30267c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30277c478bd9Sstevel@tonic-gate pgfree = cnt << PNUM_SHIFT(r - 1); 30287c478bd9Sstevel@tonic-gate range = FULL_REGION_CNT(szc); 30297c478bd9Sstevel@tonic-gate 30307c478bd9Sstevel@tonic-gate /* Check for completely full region */ 30317c478bd9Sstevel@tonic-gate if (cnt == range) { 30327c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30337c478bd9Sstevel@tonic-gate return (pgfree); 30347c478bd9Sstevel@tonic-gate } 30357c478bd9Sstevel@tonic-gate 30367c478bd9Sstevel@tonic-gate while (--r > 0) { 30377c478bd9Sstevel@tonic-gate idx = PNUM_TO_IDX(mnode, r, pp->p_pagenum); 30387c478bd9Sstevel@tonic-gate full = FULL_REGION_CNT(r); 30397c478bd9Sstevel@tonic-gate for (i = 0; i < range; i++, idx++) { 30407c478bd9Sstevel@tonic-gate cnt = PAGE_COUNTERS(mnode, r, idx); 30417c478bd9Sstevel@tonic-gate /* 30427c478bd9Sstevel@tonic-gate * If cnt here is full, that means we have already 30437c478bd9Sstevel@tonic-gate * accounted for these pages earlier. 30447c478bd9Sstevel@tonic-gate */ 30457c478bd9Sstevel@tonic-gate if (cnt != full) { 30467c478bd9Sstevel@tonic-gate pgfree += (cnt << PNUM_SHIFT(r - 1)); 30477c478bd9Sstevel@tonic-gate } 30487c478bd9Sstevel@tonic-gate } 30497c478bd9Sstevel@tonic-gate range *= full; 30507c478bd9Sstevel@tonic-gate } 30517c478bd9Sstevel@tonic-gate rw_exit(&page_ctrs_rwlock[mnode]); 30527c478bd9Sstevel@tonic-gate return (pgfree); 30537c478bd9Sstevel@tonic-gate } 30547c478bd9Sstevel@tonic-gate 30557c478bd9Sstevel@tonic-gate /* 30567c478bd9Sstevel@tonic-gate * Called from page_geti_contig_pages to exclusively lock constituent pages 30577c478bd9Sstevel@tonic-gate * starting from 'spp' for page size code 'szc'. 30587c478bd9Sstevel@tonic-gate * 30597c478bd9Sstevel@tonic-gate * If 'ptcpthreshold' is set, the number of free pages needed in the 'szc' 30607c478bd9Sstevel@tonic-gate * region needs to be greater than or equal to the threshold. 30617c478bd9Sstevel@tonic-gate */ 30627c478bd9Sstevel@tonic-gate static int 30637c478bd9Sstevel@tonic-gate page_trylock_contig_pages(int mnode, page_t *spp, uchar_t szc, int flags) 30647c478bd9Sstevel@tonic-gate { 30657c478bd9Sstevel@tonic-gate pgcnt_t pgcnt = PNUM_SIZE(szc); 30667c478bd9Sstevel@tonic-gate pgcnt_t pgfree, i; 30677c478bd9Sstevel@tonic-gate page_t *pp; 30687c478bd9Sstevel@tonic-gate 30697c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcp[szc]); 30707c478bd9Sstevel@tonic-gate 30717c478bd9Sstevel@tonic-gate 30727c478bd9Sstevel@tonic-gate if ((ptcpthreshold == 0) || (flags & PGI_PGCPHIPRI)) 30737c478bd9Sstevel@tonic-gate goto skipptcpcheck; 30747c478bd9Sstevel@tonic-gate /* 30757c478bd9Sstevel@tonic-gate * check if there are sufficient free pages available before attempting 30767c478bd9Sstevel@tonic-gate * to trylock. Count is approximate as page counters can change. 30777c478bd9Sstevel@tonic-gate */ 30787c478bd9Sstevel@tonic-gate pgfree = page_freecnt(mnode, spp, szc); 30797c478bd9Sstevel@tonic-gate 30807c478bd9Sstevel@tonic-gate /* attempt to trylock if there are sufficient already free pages */ 30817c478bd9Sstevel@tonic-gate if (pgfree < pgcnt/ptcpthreshold) { 30827c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfreethresh[szc]); 30837c478bd9Sstevel@tonic-gate return (0); 30847c478bd9Sstevel@tonic-gate } 30857c478bd9Sstevel@tonic-gate 30867c478bd9Sstevel@tonic-gate skipptcpcheck: 30877c478bd9Sstevel@tonic-gate 30887c478bd9Sstevel@tonic-gate for (i = 0; i < pgcnt; i++) { 30897c478bd9Sstevel@tonic-gate pp = &spp[i]; 30907c478bd9Sstevel@tonic-gate if (!page_trylock(pp, SE_EXCL)) { 30917c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailexcl[szc]); 30927c478bd9Sstevel@tonic-gate while (--i != (pgcnt_t)-1) { 30937c478bd9Sstevel@tonic-gate pp = &spp[i]; 30947c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 30958b464eb8Smec page_unlock_nocapture(pp); 30967c478bd9Sstevel@tonic-gate } 30977c478bd9Sstevel@tonic-gate return (0); 30987c478bd9Sstevel@tonic-gate } 30997c478bd9Sstevel@tonic-gate ASSERT(spp[i].p_pagenum == spp->p_pagenum + i); 31007c478bd9Sstevel@tonic-gate if ((pp->p_szc > szc || (szc && pp->p_szc == szc)) && 31017c478bd9Sstevel@tonic-gate !PP_ISFREE(pp)) { 31027c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailszc[szc]); 31037c478bd9Sstevel@tonic-gate ASSERT(i == 0); 31048b464eb8Smec page_unlock_nocapture(pp); 31057c478bd9Sstevel@tonic-gate return (0); 31067c478bd9Sstevel@tonic-gate } 31077c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(pp)) { 31087c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpfailcage[szc]); 31097c478bd9Sstevel@tonic-gate while (i != (pgcnt_t)-1) { 31107c478bd9Sstevel@tonic-gate pp = &spp[i]; 31117c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31128b464eb8Smec page_unlock_nocapture(pp); 31137c478bd9Sstevel@tonic-gate i--; 31147c478bd9Sstevel@tonic-gate } 31157c478bd9Sstevel@tonic-gate return (0); 31167c478bd9Sstevel@tonic-gate } 31177c478bd9Sstevel@tonic-gate } 31187c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.ptcpok[szc]); 31197c478bd9Sstevel@tonic-gate return (1); 31207c478bd9Sstevel@tonic-gate } 31217c478bd9Sstevel@tonic-gate 31227c478bd9Sstevel@tonic-gate /* 31237c478bd9Sstevel@tonic-gate * Claim large page pointed to by 'pp'. 'pp' is the starting set 31247c478bd9Sstevel@tonic-gate * of 'szc' constituent pages that had been locked exclusively previously. 31257c478bd9Sstevel@tonic-gate * Will attempt to relocate constituent pages in use. 31267c478bd9Sstevel@tonic-gate */ 31277c478bd9Sstevel@tonic-gate static page_t * 31287c478bd9Sstevel@tonic-gate page_claim_contig_pages(page_t *pp, uchar_t szc, int flags) 31297c478bd9Sstevel@tonic-gate { 31307c478bd9Sstevel@tonic-gate spgcnt_t pgcnt, npgs, i; 31317c478bd9Sstevel@tonic-gate page_t *targpp, *rpp, *hpp; 31327c478bd9Sstevel@tonic-gate page_t *replpp = NULL; 31337c478bd9Sstevel@tonic-gate page_t *pplist = NULL; 31347c478bd9Sstevel@tonic-gate 31357c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 31367c478bd9Sstevel@tonic-gate 31377c478bd9Sstevel@tonic-gate pgcnt = page_get_pagecnt(szc); 31387c478bd9Sstevel@tonic-gate while (pgcnt) { 31397c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 31407c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(pp)); 31417c478bd9Sstevel@tonic-gate if (PP_ISFREE(pp)) { 31427c478bd9Sstevel@tonic-gate /* 31437c478bd9Sstevel@tonic-gate * If this is a PG_FREE_LIST page then its 31447c478bd9Sstevel@tonic-gate * size code can change underneath us due to 31457c478bd9Sstevel@tonic-gate * page promotion or demotion. As an optimzation 31467c478bd9Sstevel@tonic-gate * use page_list_sub_pages() instead of 31477c478bd9Sstevel@tonic-gate * page_list_sub(). 31487c478bd9Sstevel@tonic-gate */ 31497c478bd9Sstevel@tonic-gate if (PP_ISAGED(pp)) { 31507c478bd9Sstevel@tonic-gate page_list_sub_pages(pp, szc); 31517c478bd9Sstevel@tonic-gate if (pp->p_szc == szc) { 31527c478bd9Sstevel@tonic-gate return (pp); 31537c478bd9Sstevel@tonic-gate } 31547c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc < szc); 31557c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31567c478bd9Sstevel@tonic-gate hpp = pp; 31577c478bd9Sstevel@tonic-gate for (i = 0; i < npgs; i++, pp++) { 31587c478bd9Sstevel@tonic-gate pp->p_szc = szc; 31597c478bd9Sstevel@tonic-gate } 31607c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &hpp); 31617c478bd9Sstevel@tonic-gate pgcnt -= npgs; 31627c478bd9Sstevel@tonic-gate continue; 31637c478bd9Sstevel@tonic-gate } 31647c478bd9Sstevel@tonic-gate ASSERT(!PP_ISAGED(pp)); 31657c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 31667c478bd9Sstevel@tonic-gate page_list_sub(pp, PG_CACHE_LIST); 31677c478bd9Sstevel@tonic-gate page_hashout(pp, NULL); 31687c478bd9Sstevel@tonic-gate PP_SETAGED(pp); 31697c478bd9Sstevel@tonic-gate pp->p_szc = szc; 31707c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &pp); 31717c478bd9Sstevel@tonic-gate pp++; 31727c478bd9Sstevel@tonic-gate pgcnt--; 31737c478bd9Sstevel@tonic-gate continue; 31747c478bd9Sstevel@tonic-gate } 31757c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31767c478bd9Sstevel@tonic-gate 31777c478bd9Sstevel@tonic-gate /* 31787c478bd9Sstevel@tonic-gate * page_create_wait freemem accounting done by caller of 31797c478bd9Sstevel@tonic-gate * page_get_freelist and not necessary to call it prior to 31807c478bd9Sstevel@tonic-gate * calling page_get_replacement_page. 31817c478bd9Sstevel@tonic-gate * 31827c478bd9Sstevel@tonic-gate * page_get_replacement_page can call page_get_contig_pages 31837c478bd9Sstevel@tonic-gate * to acquire a large page (szc > 0); the replacement must be 31847c478bd9Sstevel@tonic-gate * smaller than the contig page size to avoid looping or 31857c478bd9Sstevel@tonic-gate * szc == 0 and PGI_PGCPSZC0 is set. 31867c478bd9Sstevel@tonic-gate */ 31877c478bd9Sstevel@tonic-gate if (pp->p_szc < szc || (szc == 0 && (flags & PGI_PGCPSZC0))) { 31887c478bd9Sstevel@tonic-gate replpp = page_get_replacement_page(pp, NULL, 0); 31897c478bd9Sstevel@tonic-gate if (replpp) { 31907c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(pp->p_szc); 31917c478bd9Sstevel@tonic-gate ASSERT(npgs <= pgcnt); 31927c478bd9Sstevel@tonic-gate targpp = pp; 31937c478bd9Sstevel@tonic-gate } 31947c478bd9Sstevel@tonic-gate } 31957c478bd9Sstevel@tonic-gate 31967c478bd9Sstevel@tonic-gate /* 31977c478bd9Sstevel@tonic-gate * If replacement is NULL or do_page_relocate fails, fail 31987c478bd9Sstevel@tonic-gate * coalescing of pages. 31997c478bd9Sstevel@tonic-gate */ 32007c478bd9Sstevel@tonic-gate if (replpp == NULL || (do_page_relocate(&targpp, &replpp, 0, 32017c478bd9Sstevel@tonic-gate &npgs, NULL) != 0)) { 32027c478bd9Sstevel@tonic-gate /* 32037c478bd9Sstevel@tonic-gate * Unlock un-processed target list 32047c478bd9Sstevel@tonic-gate */ 32057c478bd9Sstevel@tonic-gate while (pgcnt--) { 32067c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32078b464eb8Smec page_unlock_nocapture(pp); 32087c478bd9Sstevel@tonic-gate pp++; 32097c478bd9Sstevel@tonic-gate } 32107c478bd9Sstevel@tonic-gate /* 32117c478bd9Sstevel@tonic-gate * Free the processed target list. 32127c478bd9Sstevel@tonic-gate */ 32137c478bd9Sstevel@tonic-gate while (pplist) { 32147c478bd9Sstevel@tonic-gate pp = pplist; 32157c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 32167c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(pp)); 32177c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == szc); 32187c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 32197c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(pp)); 32207c478bd9Sstevel@tonic-gate pp->p_szc = 0; 32217c478bd9Sstevel@tonic-gate page_list_add(pp, PG_FREE_LIST | PG_LIST_TAIL); 32228b464eb8Smec page_unlock_nocapture(pp); 32237c478bd9Sstevel@tonic-gate } 32247c478bd9Sstevel@tonic-gate 32257c478bd9Sstevel@tonic-gate if (replpp != NULL) 32267c478bd9Sstevel@tonic-gate page_free_replacement_page(replpp); 32277c478bd9Sstevel@tonic-gate 32287c478bd9Sstevel@tonic-gate return (NULL); 32297c478bd9Sstevel@tonic-gate } 32307c478bd9Sstevel@tonic-gate ASSERT(pp == targpp); 32317c478bd9Sstevel@tonic-gate 32327c478bd9Sstevel@tonic-gate /* LINTED */ 32337c478bd9Sstevel@tonic-gate ASSERT(hpp = pp); /* That's right, it's an assignment */ 32347c478bd9Sstevel@tonic-gate 32357c478bd9Sstevel@tonic-gate pp += npgs; 32367c478bd9Sstevel@tonic-gate pgcnt -= npgs; 32377c478bd9Sstevel@tonic-gate 32387c478bd9Sstevel@tonic-gate while (npgs--) { 32397c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(targpp)); 32407c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(targpp)); 32417c478bd9Sstevel@tonic-gate ASSERT(!PP_ISNORELOC(targpp)); 32427c478bd9Sstevel@tonic-gate PP_SETFREE(targpp); 32437c478bd9Sstevel@tonic-gate ASSERT(PP_ISAGED(targpp)); 32447c478bd9Sstevel@tonic-gate ASSERT(targpp->p_szc < szc || (szc == 0 && 32457c478bd9Sstevel@tonic-gate (flags & PGI_PGCPSZC0))); 32467c478bd9Sstevel@tonic-gate targpp->p_szc = szc; 32477c478bd9Sstevel@tonic-gate targpp = targpp->p_next; 32487c478bd9Sstevel@tonic-gate 32497c478bd9Sstevel@tonic-gate rpp = replpp; 32507c478bd9Sstevel@tonic-gate ASSERT(rpp != NULL); 32517c478bd9Sstevel@tonic-gate page_sub(&replpp, rpp); 32527c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(rpp)); 32537c478bd9Sstevel@tonic-gate ASSERT(!PP_ISFREE(rpp)); 32548b464eb8Smec page_unlock_nocapture(rpp); 32557c478bd9Sstevel@tonic-gate } 32567c478bd9Sstevel@tonic-gate ASSERT(targpp == hpp); 32577c478bd9Sstevel@tonic-gate ASSERT(replpp == NULL); 32587c478bd9Sstevel@tonic-gate page_list_concat(&pplist, &targpp); 32597c478bd9Sstevel@tonic-gate } 32607c478bd9Sstevel@tonic-gate CHK_LPG(pplist, szc); 32617c478bd9Sstevel@tonic-gate return (pplist); 32627c478bd9Sstevel@tonic-gate } 32637c478bd9Sstevel@tonic-gate 32647c478bd9Sstevel@tonic-gate /* 32657c478bd9Sstevel@tonic-gate * Trim kernel cage from pfnlo-pfnhi and store result in lo-hi. Return code 32667c478bd9Sstevel@tonic-gate * of 0 means nothing left after trim. 32677c478bd9Sstevel@tonic-gate */ 32687c478bd9Sstevel@tonic-gate int 32697c478bd9Sstevel@tonic-gate trimkcage(struct memseg *mseg, pfn_t *lo, pfn_t *hi, pfn_t pfnlo, pfn_t pfnhi) 32707c478bd9Sstevel@tonic-gate { 32717c478bd9Sstevel@tonic-gate pfn_t kcagepfn; 32727c478bd9Sstevel@tonic-gate int decr; 32737c478bd9Sstevel@tonic-gate int rc = 0; 32747c478bd9Sstevel@tonic-gate 32757c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->pages)) { 32767c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1) == 0) { 32777c478bd9Sstevel@tonic-gate 32787c478bd9Sstevel@tonic-gate /* lower part of this mseg inside kernel cage */ 32797c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 32807c478bd9Sstevel@tonic-gate 32817c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 32827c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 32837c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 32847c478bd9Sstevel@tonic-gate ASSERT(decr == 0); 328578b03d3aSkchow *lo = MAX(kcagepfn, pfnlo); 328678b03d3aSkchow *hi = MIN(pfnhi, (mseg->pages_end - 1)); 32877c478bd9Sstevel@tonic-gate rc = 1; 32887c478bd9Sstevel@tonic-gate } 32897c478bd9Sstevel@tonic-gate } 32907c478bd9Sstevel@tonic-gate /* else entire mseg in the cage */ 32917c478bd9Sstevel@tonic-gate } else { 32927c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(mseg->epages - 1)) { 32937c478bd9Sstevel@tonic-gate 32947c478bd9Sstevel@tonic-gate /* upper part of this mseg inside kernel cage */ 32957c478bd9Sstevel@tonic-gate decr = kcage_current_pfn(&kcagepfn); 32967c478bd9Sstevel@tonic-gate 32977c478bd9Sstevel@tonic-gate /* kernel cage may have transitioned past mseg */ 32987c478bd9Sstevel@tonic-gate if (kcagepfn >= mseg->pages_base && 32997c478bd9Sstevel@tonic-gate kcagepfn < mseg->pages_end) { 33007c478bd9Sstevel@tonic-gate ASSERT(decr); 330178b03d3aSkchow *hi = MIN(kcagepfn, pfnhi); 33027c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 33037c478bd9Sstevel@tonic-gate rc = 1; 33047c478bd9Sstevel@tonic-gate } 33057c478bd9Sstevel@tonic-gate } else { 33067c478bd9Sstevel@tonic-gate /* entire mseg outside of kernel cage */ 33077c478bd9Sstevel@tonic-gate *lo = MAX(pfnlo, mseg->pages_base); 33087c478bd9Sstevel@tonic-gate *hi = MIN(pfnhi, (mseg->pages_end - 1)); 33097c478bd9Sstevel@tonic-gate rc = 1; 33107c478bd9Sstevel@tonic-gate } 33117c478bd9Sstevel@tonic-gate } 33127c478bd9Sstevel@tonic-gate return (rc); 33137c478bd9Sstevel@tonic-gate } 33147c478bd9Sstevel@tonic-gate 33157c478bd9Sstevel@tonic-gate /* 33165d07b933Sdp * called from page_get_contig_pages to search 'pfnlo' thru 'pfnhi' to claim a 33177c478bd9Sstevel@tonic-gate * page with size code 'szc'. Claiming such a page requires acquiring 33187c478bd9Sstevel@tonic-gate * exclusive locks on all constituent pages (page_trylock_contig_pages), 33197c478bd9Sstevel@tonic-gate * relocating pages in use and concatenating these constituent pages into a 33207c478bd9Sstevel@tonic-gate * large page. 33217c478bd9Sstevel@tonic-gate * 33225d07b933Sdp * The page lists do not have such a large page and page_freelist_split has 33237c478bd9Sstevel@tonic-gate * already failed to demote larger pages and/or coalesce smaller free pages. 33247c478bd9Sstevel@tonic-gate * 33257c478bd9Sstevel@tonic-gate * 'flags' may specify PG_COLOR_MATCH which would limit the search of large 33267c478bd9Sstevel@tonic-gate * pages with the same color as 'bin'. 33277c478bd9Sstevel@tonic-gate * 33287c478bd9Sstevel@tonic-gate * 'pfnflag' specifies the subset of the pfn range to search. 33297c478bd9Sstevel@tonic-gate */ 33307c478bd9Sstevel@tonic-gate 33317c478bd9Sstevel@tonic-gate static page_t * 33327c478bd9Sstevel@tonic-gate page_geti_contig_pages(int mnode, uint_t bin, uchar_t szc, int flags, 333383f9b804Skchow pfn_t pfnlo, pfn_t pfnhi, pgcnt_t pfnflag) 33347c478bd9Sstevel@tonic-gate { 33357c478bd9Sstevel@tonic-gate struct memseg *mseg; 33367c478bd9Sstevel@tonic-gate pgcnt_t szcpgcnt = page_get_pagecnt(szc); 33377c478bd9Sstevel@tonic-gate pgcnt_t szcpgmask = szcpgcnt - 1; 33387c478bd9Sstevel@tonic-gate pfn_t randpfn; 33397c478bd9Sstevel@tonic-gate page_t *pp, *randpp, *endpp; 33405d07b933Sdp uint_t colors, ceq_mask; 33415d07b933Sdp /* LINTED : set but not used in function */ 33425d07b933Sdp uint_t color_mask; 33437c478bd9Sstevel@tonic-gate pfn_t hi, lo; 33447c478bd9Sstevel@tonic-gate uint_t skip; 3345ce8eb11aSdp MEM_NODE_ITERATOR_DECL(it); 33467c478bd9Sstevel@tonic-gate 33477c478bd9Sstevel@tonic-gate ASSERT(szc != 0 || (flags & PGI_PGCPSZC0)); 33487c478bd9Sstevel@tonic-gate 334902bc52beSkchow pfnlo = P2ROUNDUP(pfnlo, szcpgcnt); 335002bc52beSkchow 335102bc52beSkchow if ((pfnhi - pfnlo) + 1 < szcpgcnt || pfnlo >= pfnhi) 33527c478bd9Sstevel@tonic-gate return (NULL); 33537c478bd9Sstevel@tonic-gate 33547c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 33557c478bd9Sstevel@tonic-gate 33565d07b933Sdp colors = PAGE_GET_PAGECOLORS(szc); 33575d07b933Sdp color_mask = colors - 1; 33585d07b933Sdp if ((colors > 1) && (flags & PG_MATCH_COLOR)) { 33595d07b933Sdp uchar_t ceq = colorequivszc[szc]; 33605d07b933Sdp uint_t ceq_dif = colors >> ((ceq >> 4) + (ceq & 0xf)); 33615d07b933Sdp 33625d07b933Sdp ASSERT(ceq_dif > 0); 33635d07b933Sdp ceq_mask = (ceq_dif - 1) << (ceq & 0xf); 33645d07b933Sdp } else { 33655d07b933Sdp ceq_mask = 0; 33665d07b933Sdp } 33677c478bd9Sstevel@tonic-gate 33687c478bd9Sstevel@tonic-gate ASSERT(bin < colors); 33697c478bd9Sstevel@tonic-gate 33705d07b933Sdp /* clear "non-significant" color bits */ 33715d07b933Sdp bin &= ceq_mask; 33725d07b933Sdp 33737c478bd9Sstevel@tonic-gate /* 33747c478bd9Sstevel@tonic-gate * trim the pfn range to search based on pfnflag. pfnflag is set 33757c478bd9Sstevel@tonic-gate * when there have been previous page_get_contig_page failures to 33767c478bd9Sstevel@tonic-gate * limit the search. 33777c478bd9Sstevel@tonic-gate * 33787c478bd9Sstevel@tonic-gate * The high bit in pfnflag specifies the number of 'slots' in the 33797c478bd9Sstevel@tonic-gate * pfn range and the remainder of pfnflag specifies which slot. 33807c478bd9Sstevel@tonic-gate * For example, a value of 1010b would mean the second slot of 33817c478bd9Sstevel@tonic-gate * the pfn range that has been divided into 8 slots. 33827c478bd9Sstevel@tonic-gate */ 33837c478bd9Sstevel@tonic-gate if (pfnflag > 1) { 33847c478bd9Sstevel@tonic-gate int slots = 1 << (highbit(pfnflag) - 1); 33857c478bd9Sstevel@tonic-gate int slotid = pfnflag & (slots - 1); 33867c478bd9Sstevel@tonic-gate pgcnt_t szcpages; 33877c478bd9Sstevel@tonic-gate int slotlen; 33887c478bd9Sstevel@tonic-gate 338902bc52beSkchow pfnhi = P2ALIGN((pfnhi + 1), szcpgcnt) - 1; 33907c478bd9Sstevel@tonic-gate szcpages = ((pfnhi - pfnlo) + 1) / szcpgcnt; 33917c478bd9Sstevel@tonic-gate slotlen = howmany(szcpages, slots); 339202bc52beSkchow /* skip if 'slotid' slot is empty */ 339302bc52beSkchow if (slotid * slotlen >= szcpages) 339402bc52beSkchow return (NULL); 33957c478bd9Sstevel@tonic-gate pfnlo = pfnlo + (((slotid * slotlen) % szcpages) * szcpgcnt); 33967c478bd9Sstevel@tonic-gate ASSERT(pfnlo < pfnhi); 33977c478bd9Sstevel@tonic-gate if (pfnhi > pfnlo + (slotlen * szcpgcnt)) 339802bc52beSkchow pfnhi = pfnlo + (slotlen * szcpgcnt) - 1; 33997c478bd9Sstevel@tonic-gate } 34007c478bd9Sstevel@tonic-gate 34017c478bd9Sstevel@tonic-gate memsegs_lock(0); 34027c478bd9Sstevel@tonic-gate 34037c478bd9Sstevel@tonic-gate /* 34047c478bd9Sstevel@tonic-gate * loop through memsegs to look for contig page candidates 34057c478bd9Sstevel@tonic-gate */ 34067c478bd9Sstevel@tonic-gate 34077c478bd9Sstevel@tonic-gate for (mseg = memsegs; mseg != NULL; mseg = mseg->next) { 34087c478bd9Sstevel@tonic-gate if (pfnhi < mseg->pages_base || pfnlo >= mseg->pages_end) { 34097c478bd9Sstevel@tonic-gate /* no overlap */ 34107c478bd9Sstevel@tonic-gate continue; 34117c478bd9Sstevel@tonic-gate } 34127c478bd9Sstevel@tonic-gate 34137c478bd9Sstevel@tonic-gate if (mseg->pages_end - mseg->pages_base < szcpgcnt) 34147c478bd9Sstevel@tonic-gate /* mseg too small */ 34157c478bd9Sstevel@tonic-gate continue; 34167c478bd9Sstevel@tonic-gate 341778b03d3aSkchow /* 341878b03d3aSkchow * trim off kernel cage pages from pfn range and check for 341978b03d3aSkchow * a trimmed pfn range returned that does not span the 342078b03d3aSkchow * desired large page size. 342178b03d3aSkchow */ 34227c478bd9Sstevel@tonic-gate if (kcage_on) { 342378b03d3aSkchow if (trimkcage(mseg, &lo, &hi, pfnlo, pfnhi) == 0 || 3424d94d1888Skchow lo >= hi || ((hi - lo) + 1) < szcpgcnt) 34257c478bd9Sstevel@tonic-gate continue; 34267c478bd9Sstevel@tonic-gate } else { 34277c478bd9Sstevel@tonic-gate lo = MAX(pfnlo, mseg->pages_base); 34287c478bd9Sstevel@tonic-gate hi = MIN(pfnhi, (mseg->pages_end - 1)); 34297c478bd9Sstevel@tonic-gate } 34307c478bd9Sstevel@tonic-gate 34317c478bd9Sstevel@tonic-gate /* round to szcpgcnt boundaries */ 34327c478bd9Sstevel@tonic-gate lo = P2ROUNDUP(lo, szcpgcnt); 343302bc52beSkchow 3434*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 343502bc52beSkchow hi = P2ALIGN((hi + 1), szcpgcnt) - 1; 34367c478bd9Sstevel@tonic-gate 34377c478bd9Sstevel@tonic-gate if (hi <= lo) 34387c478bd9Sstevel@tonic-gate continue; 34397c478bd9Sstevel@tonic-gate 34407c478bd9Sstevel@tonic-gate /* 34417c478bd9Sstevel@tonic-gate * set lo to point to the pfn for the desired bin. Large 34427c478bd9Sstevel@tonic-gate * page sizes may only have a single page color 34437c478bd9Sstevel@tonic-gate */ 34445d07b933Sdp skip = szcpgcnt; 3445ce8eb11aSdp if (ceq_mask > 0 || interleaved_mnodes) { 34465d07b933Sdp /* set lo to point at appropriate color */ 3447ce8eb11aSdp if (((PFN_2_COLOR(lo, szc, &it) ^ bin) & ceq_mask) || 3448ce8eb11aSdp (interleaved_mnodes && 3449ce8eb11aSdp PFN_2_MEM_NODE(lo) != mnode)) { 3450ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(lo, szc, bin, ceq_mask, 3451ce8eb11aSdp color_mask, &it); 3452ce8eb11aSdp } 34535d07b933Sdp if (hi <= lo) 34545d07b933Sdp /* mseg cannot satisfy color request */ 34555d07b933Sdp continue; 34567c478bd9Sstevel@tonic-gate } 34577c478bd9Sstevel@tonic-gate 34587c478bd9Sstevel@tonic-gate /* randomly choose a point between lo and hi to begin search */ 34597c478bd9Sstevel@tonic-gate 34607c478bd9Sstevel@tonic-gate randpfn = (pfn_t)GETTICK(); 34617c478bd9Sstevel@tonic-gate randpfn = ((randpfn % (hi - lo)) + lo) & ~(skip - 1); 3462*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, szc, &it); 3463*b779d3e0Sdp if (ceq_mask || interleaved_mnodes || randpfn == (pfn_t)-1) { 3464a7c3ca36Sdp if (randpfn != (pfn_t)-1) { 3465ce8eb11aSdp PAGE_NEXT_PFN_FOR_COLOR(randpfn, szc, bin, 3466ce8eb11aSdp ceq_mask, color_mask, &it); 3467a7c3ca36Sdp } 3468ce8eb11aSdp if (randpfn >= hi) { 3469ce8eb11aSdp randpfn = lo; 3470*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(randpfn, mnode, szc, 3471*b779d3e0Sdp &it); 3472ce8eb11aSdp } 34735d07b933Sdp } 34747c478bd9Sstevel@tonic-gate randpp = mseg->pages + (randpfn - mseg->pages_base); 34757c478bd9Sstevel@tonic-gate 34767c478bd9Sstevel@tonic-gate ASSERT(randpp->p_pagenum == randpfn); 34777c478bd9Sstevel@tonic-gate 34787c478bd9Sstevel@tonic-gate pp = randpp; 347902bc52beSkchow endpp = mseg->pages + (hi - mseg->pages_base) + 1; 34807c478bd9Sstevel@tonic-gate 34817c478bd9Sstevel@tonic-gate ASSERT(randpp + szcpgcnt <= endpp); 34827c478bd9Sstevel@tonic-gate 34837c478bd9Sstevel@tonic-gate do { 34847c478bd9Sstevel@tonic-gate ASSERT(!(pp->p_pagenum & szcpgmask)); 34855d07b933Sdp ASSERT(((PP_2_BIN(pp) ^ bin) & ceq_mask) == 0); 34865d07b933Sdp 34877c478bd9Sstevel@tonic-gate if (page_trylock_contig_pages(mnode, pp, szc, flags)) { 34887c478bd9Sstevel@tonic-gate /* pages unlocked by page_claim on failure */ 34897c478bd9Sstevel@tonic-gate if (page_claim_contig_pages(pp, szc, flags)) { 34907c478bd9Sstevel@tonic-gate memsegs_unlock(0); 34917c478bd9Sstevel@tonic-gate return (pp); 34927c478bd9Sstevel@tonic-gate } 34937c478bd9Sstevel@tonic-gate } 34947c478bd9Sstevel@tonic-gate 3495ce8eb11aSdp if (ceq_mask == 0 && !interleaved_mnodes) { 34965d07b933Sdp pp += skip; 34975d07b933Sdp } else { 34985d07b933Sdp pfn_t pfn = pp->p_pagenum; 34995d07b933Sdp 35005d07b933Sdp PAGE_NEXT_PFN_FOR_COLOR(pfn, szc, bin, 3501ce8eb11aSdp ceq_mask, color_mask, &it); 3502ce8eb11aSdp if (pfn == (pfn_t)-1) { 3503ce8eb11aSdp pp = endpp; 3504ce8eb11aSdp } else { 3505ce8eb11aSdp pp = mseg->pages + 3506ce8eb11aSdp (pfn - mseg->pages_base); 3507ce8eb11aSdp } 35085d07b933Sdp } 35097c478bd9Sstevel@tonic-gate if (pp >= endpp) { 35107c478bd9Sstevel@tonic-gate /* start from the beginning */ 3511*b779d3e0Sdp MEM_NODE_ITERATOR_INIT(lo, mnode, szc, &it); 35127c478bd9Sstevel@tonic-gate pp = mseg->pages + (lo - mseg->pages_base); 35137c478bd9Sstevel@tonic-gate ASSERT(pp->p_pagenum == lo); 35147c478bd9Sstevel@tonic-gate ASSERT(pp + szcpgcnt <= endpp); 35157c478bd9Sstevel@tonic-gate } 35167c478bd9Sstevel@tonic-gate } while (pp != randpp); 35177c478bd9Sstevel@tonic-gate } 35187c478bd9Sstevel@tonic-gate memsegs_unlock(0); 35197c478bd9Sstevel@tonic-gate return (NULL); 35207c478bd9Sstevel@tonic-gate } 35217c478bd9Sstevel@tonic-gate 35227c478bd9Sstevel@tonic-gate 35237c478bd9Sstevel@tonic-gate /* 35247c478bd9Sstevel@tonic-gate * controlling routine that searches through physical memory in an attempt to 35257c478bd9Sstevel@tonic-gate * claim a large page based on the input parameters. 35267c478bd9Sstevel@tonic-gate * on the page free lists. 35277c478bd9Sstevel@tonic-gate * 35287c478bd9Sstevel@tonic-gate * calls page_geti_contig_pages with an initial pfn range from the mnode 35297c478bd9Sstevel@tonic-gate * and mtype. page_geti_contig_pages will trim off the parts of the pfn range 35307c478bd9Sstevel@tonic-gate * that overlaps with the kernel cage or does not match the requested page 35317c478bd9Sstevel@tonic-gate * color if PG_MATCH_COLOR is set. Since this search is very expensive, 35327c478bd9Sstevel@tonic-gate * page_geti_contig_pages may further limit the search range based on 35337c478bd9Sstevel@tonic-gate * previous failure counts (pgcpfailcnt[]). 35347c478bd9Sstevel@tonic-gate * 35357c478bd9Sstevel@tonic-gate * for PGI_PGCPSZC0 requests, page_get_contig_pages will relocate a base 35367c478bd9Sstevel@tonic-gate * pagesize page that satisfies mtype. 35377c478bd9Sstevel@tonic-gate */ 35387c478bd9Sstevel@tonic-gate page_t * 35397c478bd9Sstevel@tonic-gate page_get_contig_pages(int mnode, uint_t bin, int mtype, uchar_t szc, 35407c478bd9Sstevel@tonic-gate uint_t flags) 35417c478bd9Sstevel@tonic-gate { 35427c478bd9Sstevel@tonic-gate pfn_t pfnlo, pfnhi; /* contig pages pfn range */ 35437c478bd9Sstevel@tonic-gate page_t *pp; 354483f9b804Skchow pgcnt_t pfnflag = 0; /* no limit on search if 0 */ 35457c478bd9Sstevel@tonic-gate 35467c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_alloc[szc]); 35477c478bd9Sstevel@tonic-gate 35480b5aa17bSmec /* no allocations from cage */ 35490b5aa17bSmec flags |= PGI_NOCAGE; 35500b5aa17bSmec 35517c478bd9Sstevel@tonic-gate /* LINTED */ 35527c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 35537c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 35547c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocempty[szc]); 35557c478bd9Sstevel@tonic-gate return (NULL); 35567c478bd9Sstevel@tonic-gate } 35577c478bd9Sstevel@tonic-gate 35587c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 35597c478bd9Sstevel@tonic-gate 35607c478bd9Sstevel@tonic-gate /* do not limit search and ignore color if hi pri */ 35617c478bd9Sstevel@tonic-gate 35627c478bd9Sstevel@tonic-gate if (pgcplimitsearch && ((flags & PGI_PGCPHIPRI) == 0)) 35637c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 35647c478bd9Sstevel@tonic-gate 35657c478bd9Sstevel@tonic-gate /* remove color match to improve chances */ 35667c478bd9Sstevel@tonic-gate 35677c478bd9Sstevel@tonic-gate if (flags & PGI_PGCPHIPRI || pfnflag) 35687c478bd9Sstevel@tonic-gate flags &= ~PG_MATCH_COLOR; 35697c478bd9Sstevel@tonic-gate 35707c478bd9Sstevel@tonic-gate do { 35717c478bd9Sstevel@tonic-gate /* get pfn range based on mnode and mtype */ 35727c478bd9Sstevel@tonic-gate MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi); 35737c478bd9Sstevel@tonic-gate 35747c478bd9Sstevel@tonic-gate ASSERT(pfnhi >= pfnlo); 35757c478bd9Sstevel@tonic-gate 35767c478bd9Sstevel@tonic-gate pp = page_geti_contig_pages(mnode, bin, szc, flags, 35777c478bd9Sstevel@tonic-gate pfnlo, pfnhi, pfnflag); 35787c478bd9Sstevel@tonic-gate 35797c478bd9Sstevel@tonic-gate if (pp != NULL) { 35807c478bd9Sstevel@tonic-gate pfnflag = pgcpfailcnt[szc]; 35817c478bd9Sstevel@tonic-gate if (pfnflag) { 35827c478bd9Sstevel@tonic-gate /* double the search size */ 35837c478bd9Sstevel@tonic-gate pgcpfailcnt[szc] = pfnflag >> 1; 35847c478bd9Sstevel@tonic-gate } 35857c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocok[szc]); 35867c478bd9Sstevel@tonic-gate return (pp); 35877c478bd9Sstevel@tonic-gate } 3588affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 3589affbd3ccSkchow } while (mtype >= 0); 35907c478bd9Sstevel@tonic-gate 35917c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgcp_allocfailed[szc]); 35927c478bd9Sstevel@tonic-gate return (NULL); 35937c478bd9Sstevel@tonic-gate } 35947c478bd9Sstevel@tonic-gate 359578b03d3aSkchow #if defined(__i386) || defined(__amd64) 359678b03d3aSkchow /* 359778b03d3aSkchow * Determine the likelihood of finding/coalescing a szc page. 359878b03d3aSkchow * Return 0 if the likelihood is small otherwise return 1. 359978b03d3aSkchow * 360078b03d3aSkchow * For now, be conservative and check only 1g pages and return 0 360178b03d3aSkchow * if there had been previous coalescing failures and the szc pages 360278b03d3aSkchow * needed to satisfy request would exhaust most of freemem. 360378b03d3aSkchow */ 360478b03d3aSkchow int 360578b03d3aSkchow page_chk_freelist(uint_t szc) 360678b03d3aSkchow { 360778b03d3aSkchow pgcnt_t pgcnt; 360878b03d3aSkchow 360978b03d3aSkchow if (szc <= 1) 361078b03d3aSkchow return (1); 361178b03d3aSkchow 361278b03d3aSkchow pgcnt = page_get_pagecnt(szc); 361378b03d3aSkchow if (pgcpfailcnt[szc] && pgcnt + throttlefree >= freemem) { 361478b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_deny[szc]); 361578b03d3aSkchow return (0); 361678b03d3aSkchow } 361778b03d3aSkchow VM_STAT_ADD(vmm_vmstats.pcf_allow[szc]); 361878b03d3aSkchow return (1); 361978b03d3aSkchow } 362078b03d3aSkchow #endif 36217c478bd9Sstevel@tonic-gate 36227c478bd9Sstevel@tonic-gate /* 36237c478bd9Sstevel@tonic-gate * Find the `best' page on the freelist for this (vp,off) (as,vaddr) pair. 36247c478bd9Sstevel@tonic-gate * 36257c478bd9Sstevel@tonic-gate * Does its own locking and accounting. 36267c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 36277c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 36287c478bd9Sstevel@tonic-gate * 36297c478bd9Sstevel@tonic-gate * Finds a page, removes it, THEN locks it. 36307c478bd9Sstevel@tonic-gate */ 36317c478bd9Sstevel@tonic-gate 36327c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 36337c478bd9Sstevel@tonic-gate page_t * 36347c478bd9Sstevel@tonic-gate page_get_freelist(struct vnode *vp, u_offset_t off, struct seg *seg, 36357c478bd9Sstevel@tonic-gate caddr_t vaddr, size_t size, uint_t flags, struct lgrp *lgrp) 36367c478bd9Sstevel@tonic-gate { 36377c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 36387c478bd9Sstevel@tonic-gate page_t *pp = NULL; 36397c478bd9Sstevel@tonic-gate ulong_t bin; 36407c478bd9Sstevel@tonic-gate uchar_t szc; 36417c478bd9Sstevel@tonic-gate int mnode; 36427c478bd9Sstevel@tonic-gate int mtype; 36437c478bd9Sstevel@tonic-gate page_t *(*page_get_func)(int, uint_t, int, uchar_t, uint_t); 36447c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 36457c478bd9Sstevel@tonic-gate 36467c478bd9Sstevel@tonic-gate page_get_func = page_get_mnode_freelist; 36477c478bd9Sstevel@tonic-gate 36487c478bd9Sstevel@tonic-gate /* 36497c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or passed a freed lgrp 36507c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 36517c478bd9Sstevel@tonic-gate */ 36527c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 36537c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 36547c478bd9Sstevel@tonic-gate 36557c478bd9Sstevel@tonic-gate if (kcage_on) { 36567c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC)) == PG_NORELOC && 36577c478bd9Sstevel@tonic-gate kcage_freemem < kcage_throttlefree + btop(size) && 36587c478bd9Sstevel@tonic-gate curthread != kcage_cageout_thread) { 36597c478bd9Sstevel@tonic-gate /* 36607c478bd9Sstevel@tonic-gate * Set a "reserve" of kcage_throttlefree pages for 36617c478bd9Sstevel@tonic-gate * PG_PANIC and cageout thread allocations. 36627c478bd9Sstevel@tonic-gate * 36637c478bd9Sstevel@tonic-gate * Everybody else has to serialize in 36647c478bd9Sstevel@tonic-gate * page_create_get_something() to get a cage page, so 36657c478bd9Sstevel@tonic-gate * that we don't deadlock cageout! 36667c478bd9Sstevel@tonic-gate */ 36677c478bd9Sstevel@tonic-gate return (NULL); 36687c478bd9Sstevel@tonic-gate } 36697c478bd9Sstevel@tonic-gate } else { 36707c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 36717c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 36727c478bd9Sstevel@tonic-gate } 36737c478bd9Sstevel@tonic-gate 36747c478bd9Sstevel@tonic-gate /* LINTED */ 367507ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, size); 36767c478bd9Sstevel@tonic-gate 36777c478bd9Sstevel@tonic-gate /* 36787c478bd9Sstevel@tonic-gate * Convert size to page size code. 36797c478bd9Sstevel@tonic-gate */ 36807c478bd9Sstevel@tonic-gate if ((szc = page_szc(size)) == (uchar_t)-1) 36817c478bd9Sstevel@tonic-gate panic("page_get_freelist: illegal page size request"); 36827c478bd9Sstevel@tonic-gate ASSERT(szc < mmu_page_sizes); 36837c478bd9Sstevel@tonic-gate 36847c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_alloc[szc]); 36857c478bd9Sstevel@tonic-gate 36867c478bd9Sstevel@tonic-gate /* LINTED */ 36875d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, szc); 36887c478bd9Sstevel@tonic-gate 36895d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(szc)); 36907c478bd9Sstevel@tonic-gate 36917c478bd9Sstevel@tonic-gate /* 36927c478bd9Sstevel@tonic-gate * Try to get a local page first, but try remote if we can't 36937c478bd9Sstevel@tonic-gate * get a page of the right color. 36947c478bd9Sstevel@tonic-gate */ 36957c478bd9Sstevel@tonic-gate pgretry: 36967c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 36977c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 36987c478bd9Sstevel@tonic-gate pp = page_get_func(mnode, bin, mtype, szc, flags); 36997c478bd9Sstevel@tonic-gate if (pp != NULL) { 37007c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocok[szc]); 37017c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 37027c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 37037c478bd9Sstevel@tonic-gate int, mnode, 37047c478bd9Sstevel@tonic-gate ulong_t, bin, 37057c478bd9Sstevel@tonic-gate uint_t, flags); 37067c478bd9Sstevel@tonic-gate return (pp); 37077c478bd9Sstevel@tonic-gate } 37087c478bd9Sstevel@tonic-gate } 37097c478bd9Sstevel@tonic-gate ASSERT(pp == NULL); 37107c478bd9Sstevel@tonic-gate 37117c478bd9Sstevel@tonic-gate /* 37127c478bd9Sstevel@tonic-gate * for non-SZC0 PAGESIZE requests, check cachelist before checking 37137c478bd9Sstevel@tonic-gate * remote free lists. Caller expected to call page_get_cachelist which 37147c478bd9Sstevel@tonic-gate * will check local cache lists and remote free lists. 37157c478bd9Sstevel@tonic-gate */ 37167c478bd9Sstevel@tonic-gate if (szc == 0 && ((flags & PGI_PGCPSZC0) == 0)) { 37177c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocdeferred); 37187c478bd9Sstevel@tonic-gate return (NULL); 37197c478bd9Sstevel@tonic-gate } 37207c478bd9Sstevel@tonic-gate 37217c478bd9Sstevel@tonic-gate ASSERT(szc > 0 || (flags & PGI_PGCPSZC0)); 37227c478bd9Sstevel@tonic-gate 37237c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 37247c478bd9Sstevel@tonic-gate 37252cb27123Saguzovsk if (!(flags & PG_LOCAL)) { 37262cb27123Saguzovsk /* 37272cb27123Saguzovsk * Try to get a non-local freelist page. 37282cb27123Saguzovsk */ 37292cb27123Saguzovsk LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 37302cb27123Saguzovsk while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 37312cb27123Saguzovsk pp = page_get_func(mnode, bin, mtype, szc, flags); 37322cb27123Saguzovsk if (pp != NULL) { 37332cb27123Saguzovsk DTRACE_PROBE4(page__get, 37342cb27123Saguzovsk lgrp_t *, lgrp, 37352cb27123Saguzovsk int, mnode, 37362cb27123Saguzovsk ulong_t, bin, 37372cb27123Saguzovsk uint_t, flags); 37382cb27123Saguzovsk VM_STAT_ADD(vmm_vmstats.pgf_allocokrem[szc]); 37392cb27123Saguzovsk return (pp); 37402cb27123Saguzovsk } 37417c478bd9Sstevel@tonic-gate } 37422cb27123Saguzovsk ASSERT(pp == NULL); 37437c478bd9Sstevel@tonic-gate } 37447c478bd9Sstevel@tonic-gate 37457c478bd9Sstevel@tonic-gate /* 37467c478bd9Sstevel@tonic-gate * when the cage is off chances are page_get_contig_pages() will fail 37477c478bd9Sstevel@tonic-gate * to lock a large page chunk therefore when the cage is off it's not 37487c478bd9Sstevel@tonic-gate * called by default. this can be changed via /etc/system. 37497c478bd9Sstevel@tonic-gate * 37507c478bd9Sstevel@tonic-gate * page_get_contig_pages() also called to acquire a base pagesize page 37517c478bd9Sstevel@tonic-gate * for page_create_get_something(). 37527c478bd9Sstevel@tonic-gate */ 37537c478bd9Sstevel@tonic-gate if (!(flags & PG_NORELOC) && (pg_contig_disable == 0) && 37547c478bd9Sstevel@tonic-gate (kcage_on || pg_lpgcreate_nocage || szc == 0) && 37557c478bd9Sstevel@tonic-gate (page_get_func != page_get_contig_pages)) { 37567c478bd9Sstevel@tonic-gate 37577c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocretry[szc]); 37587c478bd9Sstevel@tonic-gate page_get_func = page_get_contig_pages; 37597c478bd9Sstevel@tonic-gate goto pgretry; 37607c478bd9Sstevel@tonic-gate } 37617c478bd9Sstevel@tonic-gate 37622cb27123Saguzovsk if (!(flags & PG_LOCAL) && pgcplimitsearch && 37632cb27123Saguzovsk page_get_func == page_get_contig_pages) 376483f9b804Skchow SETPGCPFAILCNT(szc); 37657c478bd9Sstevel@tonic-gate 37667c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgf_allocfailed[szc]); 37677c478bd9Sstevel@tonic-gate return (NULL); 37687c478bd9Sstevel@tonic-gate } 37697c478bd9Sstevel@tonic-gate 37707c478bd9Sstevel@tonic-gate /* 37717c478bd9Sstevel@tonic-gate * Find the `best' page on the cachelist for this (vp,off) (as,vaddr) pair. 37727c478bd9Sstevel@tonic-gate * 37737c478bd9Sstevel@tonic-gate * Does its own locking. 37747c478bd9Sstevel@tonic-gate * If PG_MATCH_COLOR is set, then NULL will be returned if there are no 37757c478bd9Sstevel@tonic-gate * pages of the proper color even if there are pages of a different color. 37767c478bd9Sstevel@tonic-gate * Otherwise, scan the bins for ones with pages. For each bin with pages, 37777c478bd9Sstevel@tonic-gate * try to lock one of them. If no page can be locked, try the 37787c478bd9Sstevel@tonic-gate * next bin. Return NULL if a page can not be found and locked. 37797c478bd9Sstevel@tonic-gate * 37807c478bd9Sstevel@tonic-gate * Finds a pages, trys to lock it, then removes it. 37817c478bd9Sstevel@tonic-gate */ 37827c478bd9Sstevel@tonic-gate 37837c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 37847c478bd9Sstevel@tonic-gate page_t * 37857c478bd9Sstevel@tonic-gate page_get_cachelist(struct vnode *vp, u_offset_t off, struct seg *seg, 37867c478bd9Sstevel@tonic-gate caddr_t vaddr, uint_t flags, struct lgrp *lgrp) 37877c478bd9Sstevel@tonic-gate { 37887c478bd9Sstevel@tonic-gate page_t *pp; 37897c478bd9Sstevel@tonic-gate struct as *as = seg->s_as; 37907c478bd9Sstevel@tonic-gate ulong_t bin; 37917c478bd9Sstevel@tonic-gate /*LINTED*/ 37927c478bd9Sstevel@tonic-gate int mnode; 37937c478bd9Sstevel@tonic-gate int mtype; 37947c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 37957c478bd9Sstevel@tonic-gate 37967c478bd9Sstevel@tonic-gate /* 37977c478bd9Sstevel@tonic-gate * If we aren't passed a specific lgroup, or pasased a freed lgrp 37987c478bd9Sstevel@tonic-gate * assume we wish to allocate near to the current thread's home. 37997c478bd9Sstevel@tonic-gate */ 38007c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 38017c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 38027c478bd9Sstevel@tonic-gate 38037c478bd9Sstevel@tonic-gate if (!kcage_on) { 38047c478bd9Sstevel@tonic-gate flags &= ~PG_NORELOC; 38057c478bd9Sstevel@tonic-gate flags |= PGI_NOCAGE; 38067c478bd9Sstevel@tonic-gate } 38077c478bd9Sstevel@tonic-gate 38087c478bd9Sstevel@tonic-gate if ((flags & (PG_NORELOC | PG_PANIC | PG_PUSHPAGE)) == PG_NORELOC && 38097c478bd9Sstevel@tonic-gate kcage_freemem <= kcage_throttlefree) { 38107c478bd9Sstevel@tonic-gate /* 38117c478bd9Sstevel@tonic-gate * Reserve kcage_throttlefree pages for critical kernel 38127c478bd9Sstevel@tonic-gate * threads. 38137c478bd9Sstevel@tonic-gate * 38147c478bd9Sstevel@tonic-gate * Everybody else has to go to page_create_get_something() 38157c478bd9Sstevel@tonic-gate * to get a cage page, so we don't deadlock cageout. 38167c478bd9Sstevel@tonic-gate */ 38177c478bd9Sstevel@tonic-gate return (NULL); 38187c478bd9Sstevel@tonic-gate } 38197c478bd9Sstevel@tonic-gate 38207c478bd9Sstevel@tonic-gate /* LINTED */ 38215d07b933Sdp AS_2_BIN(as, seg, vp, vaddr, bin, 0); 38227c478bd9Sstevel@tonic-gate 38235d07b933Sdp ASSERT(bin < PAGE_GET_PAGECOLORS(0)); 38247c478bd9Sstevel@tonic-gate 38257c478bd9Sstevel@tonic-gate /* LINTED */ 382607ad560dSkchow MTYPE_INIT(mtype, vp, vaddr, flags, MMU_PAGESIZE); 38277c478bd9Sstevel@tonic-gate 38287c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_alloc); 38297c478bd9Sstevel@tonic-gate 38307c478bd9Sstevel@tonic-gate /* 38317c478bd9Sstevel@tonic-gate * Try local cachelists first 38327c478bd9Sstevel@tonic-gate */ 38337c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, LGRP_SRCH_LOCAL); 38347c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38357c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 38367c478bd9Sstevel@tonic-gate if (pp != NULL) { 38377c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocok); 38387c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38397c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38407c478bd9Sstevel@tonic-gate int, mnode, 38417c478bd9Sstevel@tonic-gate ulong_t, bin, 38427c478bd9Sstevel@tonic-gate uint_t, flags); 38437c478bd9Sstevel@tonic-gate return (pp); 38447c478bd9Sstevel@tonic-gate } 38457c478bd9Sstevel@tonic-gate } 38467c478bd9Sstevel@tonic-gate 38477c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ALLOC_FAIL, 1); 38487c478bd9Sstevel@tonic-gate 38497c478bd9Sstevel@tonic-gate /* 38507c478bd9Sstevel@tonic-gate * Try freelists/cachelists that are farther away 38517c478bd9Sstevel@tonic-gate * This is our only chance to allocate remote pages for PAGESIZE 38527c478bd9Sstevel@tonic-gate * requests. 38537c478bd9Sstevel@tonic-gate */ 38547c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_UPGRADE(lgrp_cookie); 38557c478bd9Sstevel@tonic-gate while ((mnode = lgrp_memnode_choose(&lgrp_cookie)) >= 0) { 38567c478bd9Sstevel@tonic-gate pp = page_get_mnode_freelist(mnode, bin, mtype, 38577c478bd9Sstevel@tonic-gate 0, flags); 38587c478bd9Sstevel@tonic-gate if (pp != NULL) { 38597c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokdeferred); 38607c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38617c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38627c478bd9Sstevel@tonic-gate int, mnode, 38637c478bd9Sstevel@tonic-gate ulong_t, bin, 38647c478bd9Sstevel@tonic-gate uint_t, flags); 38657c478bd9Sstevel@tonic-gate return (pp); 38667c478bd9Sstevel@tonic-gate } 38677c478bd9Sstevel@tonic-gate pp = page_get_mnode_cachelist(bin, flags, mnode, mtype); 38687c478bd9Sstevel@tonic-gate if (pp != NULL) { 38697c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocokrem); 38707c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 38717c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 38727c478bd9Sstevel@tonic-gate int, mnode, 38737c478bd9Sstevel@tonic-gate ulong_t, bin, 38747c478bd9Sstevel@tonic-gate uint_t, flags); 38757c478bd9Sstevel@tonic-gate return (pp); 38767c478bd9Sstevel@tonic-gate } 38777c478bd9Sstevel@tonic-gate } 38787c478bd9Sstevel@tonic-gate 38797c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgc_allocfailed); 38807c478bd9Sstevel@tonic-gate return (NULL); 38817c478bd9Sstevel@tonic-gate } 38827c478bd9Sstevel@tonic-gate 38837c478bd9Sstevel@tonic-gate page_t * 38847c478bd9Sstevel@tonic-gate page_get_mnode_cachelist(uint_t bin, uint_t flags, int mnode, int mtype) 38857c478bd9Sstevel@tonic-gate { 38865d07b933Sdp kmutex_t *pcm; 38875d07b933Sdp page_t *pp, *first_pp; 38885d07b933Sdp uint_t sbin; 38895d07b933Sdp int plw_initialized; 38905d07b933Sdp page_list_walker_t plw; 38917c478bd9Sstevel@tonic-gate 38927c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_alloc); 38937c478bd9Sstevel@tonic-gate 38947c478bd9Sstevel@tonic-gate /* LINTED */ 38957c478bd9Sstevel@tonic-gate MTYPE_START(mnode, mtype, flags); 38967c478bd9Sstevel@tonic-gate if (mtype < 0) { /* mnode does not have memory in mtype range */ 38977c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocempty); 38987c478bd9Sstevel@tonic-gate return (NULL); 38997c478bd9Sstevel@tonic-gate } 39007c478bd9Sstevel@tonic-gate 39015d07b933Sdp try_again: 39027c478bd9Sstevel@tonic-gate 39035d07b933Sdp plw_initialized = 0; 39045d07b933Sdp plw.plw_ceq_dif = 1; 39057c478bd9Sstevel@tonic-gate 39067c478bd9Sstevel@tonic-gate /* 39077c478bd9Sstevel@tonic-gate * Only hold one cachelist lock at a time, that way we 39087c478bd9Sstevel@tonic-gate * can start anywhere and not have to worry about lock 39097c478bd9Sstevel@tonic-gate * ordering. 39107c478bd9Sstevel@tonic-gate */ 39117c478bd9Sstevel@tonic-gate 39125d07b933Sdp for (plw.plw_count = 0; 39135d07b933Sdp plw.plw_count < plw.plw_ceq_dif; plw.plw_count++) { 39145d07b933Sdp sbin = bin; 39155d07b933Sdp do { 39165d07b933Sdp 39175d07b933Sdp if (!PAGE_CACHELISTS(mnode, bin, mtype)) 39185d07b933Sdp goto bin_empty_1; 39197c478bd9Sstevel@tonic-gate pcm = PC_BIN_MUTEX(mnode, bin, PG_CACHE_LIST); 39207c478bd9Sstevel@tonic-gate mutex_enter(pcm); 39217c478bd9Sstevel@tonic-gate pp = PAGE_CACHELISTS(mnode, bin, mtype); 39225d07b933Sdp if (pp == NULL) 39235d07b933Sdp goto bin_empty_0; 39247c478bd9Sstevel@tonic-gate 39255d07b933Sdp first_pp = pp; 39265d07b933Sdp ASSERT(pp->p_vnode); 39275d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39285d07b933Sdp ASSERT(pp->p_szc == 0); 39295d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == mnode); 39305d07b933Sdp while (!page_trylock(pp, SE_EXCL)) { 39315d07b933Sdp pp = pp->p_next; 39325d07b933Sdp ASSERT(pp->p_szc == 0); 39335d07b933Sdp if (pp == first_pp) { 39347c478bd9Sstevel@tonic-gate /* 39355d07b933Sdp * We have searched the complete list! 39365d07b933Sdp * And all of them (might only be one) 39375d07b933Sdp * are locked. This can happen since 39385d07b933Sdp * these pages can also be found via 39395d07b933Sdp * the hash list. When found via the 39405d07b933Sdp * hash list, they are locked first, 39415d07b933Sdp * then removed. We give up to let the 39425d07b933Sdp * other thread run. 39437c478bd9Sstevel@tonic-gate */ 39445d07b933Sdp pp = NULL; 39455d07b933Sdp break; 39467c478bd9Sstevel@tonic-gate } 39475d07b933Sdp ASSERT(pp->p_vnode); 39485d07b933Sdp ASSERT(PP_ISFREE(pp)); 39495d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39505d07b933Sdp ASSERT(PFN_2_MEM_NODE(pp->p_pagenum) == 39515d07b933Sdp mnode); 39527c478bd9Sstevel@tonic-gate } 39537c478bd9Sstevel@tonic-gate 39545d07b933Sdp if (pp) { 39555d07b933Sdp page_t **ppp; 39565d07b933Sdp /* 39575d07b933Sdp * Found and locked a page. 39585d07b933Sdp * Pull it off the list. 39595d07b933Sdp */ 39605d07b933Sdp ASSERT(mtype == PP_2_MTYPE(pp)); 39615d07b933Sdp ppp = &PAGE_CACHELISTS(mnode, bin, mtype); 39625d07b933Sdp page_sub(ppp, pp); 39635d07b933Sdp /* 39645d07b933Sdp * Subtract counters before releasing pcm mutex 39655d07b933Sdp * to avoid a race with page_freelist_coalesce 39665d07b933Sdp * and page_freelist_split. 39675d07b933Sdp */ 39685d07b933Sdp page_ctr_sub(mnode, mtype, pp, PG_CACHE_LIST); 39695d07b933Sdp mutex_exit(pcm); 39705d07b933Sdp ASSERT(pp->p_vnode); 39715d07b933Sdp ASSERT(PP_ISAGED(pp) == 0); 39725d07b933Sdp #if defined(__sparc) 39735d07b933Sdp ASSERT(!kcage_on || 39745d07b933Sdp (flags & PG_NORELOC) == 0 || 39755d07b933Sdp PP_ISNORELOC(pp)); 39765d07b933Sdp if (PP_ISNORELOC(pp)) { 39775d07b933Sdp kcage_freemem_sub(1); 39787c478bd9Sstevel@tonic-gate } 39795d07b933Sdp #endif 39805d07b933Sdp VM_STAT_ADD(vmm_vmstats. pgmc_allocok); 39815d07b933Sdp return (pp); 39827c478bd9Sstevel@tonic-gate } 39835d07b933Sdp bin_empty_0: 39845d07b933Sdp mutex_exit(pcm); 39855d07b933Sdp bin_empty_1: 39865d07b933Sdp if (plw_initialized == 0) { 39875d07b933Sdp page_list_walk_init(0, flags, bin, 0, 1, &plw); 39885d07b933Sdp plw_initialized = 1; 39897c478bd9Sstevel@tonic-gate } 39905d07b933Sdp /* calculate the next bin with equivalent color */ 39915d07b933Sdp bin = ADD_MASKED(bin, plw.plw_bin_step, 39925d07b933Sdp plw.plw_ceq_mask[0], plw.plw_color_mask); 39935d07b933Sdp } while (sbin != bin); 39947c478bd9Sstevel@tonic-gate 39955d07b933Sdp if (plw.plw_ceq_dif > 1) 39965d07b933Sdp bin = page_list_walk_next_bin(0, bin, &plw); 39977c478bd9Sstevel@tonic-gate } 39987c478bd9Sstevel@tonic-gate 3999affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); 4000affbd3ccSkchow if (mtype >= 0) 40015d07b933Sdp goto try_again; 4002affbd3ccSkchow 40037c478bd9Sstevel@tonic-gate VM_STAT_ADD(vmm_vmstats.pgmc_allocfailed); 40047c478bd9Sstevel@tonic-gate return (NULL); 40057c478bd9Sstevel@tonic-gate } 40067c478bd9Sstevel@tonic-gate 40077c478bd9Sstevel@tonic-gate #ifdef DEBUG 40087c478bd9Sstevel@tonic-gate #define REPL_PAGE_STATS 40097c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 40107c478bd9Sstevel@tonic-gate 40117c478bd9Sstevel@tonic-gate #ifdef REPL_PAGE_STATS 40127c478bd9Sstevel@tonic-gate struct repl_page_stats { 40137c478bd9Sstevel@tonic-gate uint_t ngets; 40147c478bd9Sstevel@tonic-gate uint_t ngets_noreloc; 40157c478bd9Sstevel@tonic-gate uint_t npgr_noreloc; 40167c478bd9Sstevel@tonic-gate uint_t nnopage_first; 40177c478bd9Sstevel@tonic-gate uint_t nnopage; 40187c478bd9Sstevel@tonic-gate uint_t nhashout; 40197c478bd9Sstevel@tonic-gate uint_t nnofree; 40207c478bd9Sstevel@tonic-gate uint_t nnext_pp; 40217c478bd9Sstevel@tonic-gate } repl_page_stats; 40227c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) atomic_add_32(&repl_page_stats.v, 1) 40237c478bd9Sstevel@tonic-gate #else /* REPL_PAGE_STATS */ 40247c478bd9Sstevel@tonic-gate #define REPL_STAT_INCR(v) 40257c478bd9Sstevel@tonic-gate #endif /* REPL_PAGE_STATS */ 40267c478bd9Sstevel@tonic-gate 40277c478bd9Sstevel@tonic-gate int pgrppgcp; 40287c478bd9Sstevel@tonic-gate 40297c478bd9Sstevel@tonic-gate /* 40307c478bd9Sstevel@tonic-gate * The freemem accounting must be done by the caller. 40317c478bd9Sstevel@tonic-gate * First we try to get a replacement page of the same size as like_pp, 40327c478bd9Sstevel@tonic-gate * if that is not possible, then we just get a set of discontiguous 40337c478bd9Sstevel@tonic-gate * PAGESIZE pages. 40347c478bd9Sstevel@tonic-gate */ 40357c478bd9Sstevel@tonic-gate page_t * 40362dae3fb5Sjjc page_get_replacement_page(page_t *orig_like_pp, struct lgrp *lgrp_target, 40377c478bd9Sstevel@tonic-gate uint_t pgrflags) 40387c478bd9Sstevel@tonic-gate { 40397c478bd9Sstevel@tonic-gate page_t *like_pp; 40407c478bd9Sstevel@tonic-gate page_t *pp, *pplist; 40417c478bd9Sstevel@tonic-gate page_t *pl = NULL; 40427c478bd9Sstevel@tonic-gate ulong_t bin; 40437c478bd9Sstevel@tonic-gate int mnode, page_mnode; 40447c478bd9Sstevel@tonic-gate int szc; 40457c478bd9Sstevel@tonic-gate spgcnt_t npgs, pg_cnt; 40467c478bd9Sstevel@tonic-gate pfn_t pfnum; 40477c478bd9Sstevel@tonic-gate int mtype; 40487c478bd9Sstevel@tonic-gate int flags = 0; 40497c478bd9Sstevel@tonic-gate lgrp_mnode_cookie_t lgrp_cookie; 40502dae3fb5Sjjc lgrp_t *lgrp; 40517c478bd9Sstevel@tonic-gate 40527c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets); 40537c478bd9Sstevel@tonic-gate like_pp = orig_like_pp; 40547c478bd9Sstevel@tonic-gate ASSERT(PAGE_EXCL(like_pp)); 40557c478bd9Sstevel@tonic-gate 40567c478bd9Sstevel@tonic-gate szc = like_pp->p_szc; 40577c478bd9Sstevel@tonic-gate npgs = page_get_pagecnt(szc); 40587c478bd9Sstevel@tonic-gate /* 40597c478bd9Sstevel@tonic-gate * Now we reset like_pp to the base page_t. 40607c478bd9Sstevel@tonic-gate * That way, we won't walk past the end of this 'szc' page. 40617c478bd9Sstevel@tonic-gate */ 40627c478bd9Sstevel@tonic-gate pfnum = PFN_BASE(like_pp->p_pagenum, szc); 40637c478bd9Sstevel@tonic-gate like_pp = page_numtopp_nolock(pfnum); 40647c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == szc); 40657c478bd9Sstevel@tonic-gate 40667c478bd9Sstevel@tonic-gate if (PP_ISNORELOC(like_pp)) { 40677c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 40687c478bd9Sstevel@tonic-gate REPL_STAT_INCR(ngets_noreloc); 40697c478bd9Sstevel@tonic-gate flags = PGI_RELOCONLY; 40707c478bd9Sstevel@tonic-gate } else if (pgrflags & PGR_NORELOC) { 40717c478bd9Sstevel@tonic-gate ASSERT(kcage_on); 40727c478bd9Sstevel@tonic-gate REPL_STAT_INCR(npgr_noreloc); 40737c478bd9Sstevel@tonic-gate flags = PG_NORELOC; 40747c478bd9Sstevel@tonic-gate } 40757c478bd9Sstevel@tonic-gate 40767c478bd9Sstevel@tonic-gate /* 40777c478bd9Sstevel@tonic-gate * Kernel pages must always be replaced with the same size 40787c478bd9Sstevel@tonic-gate * pages, since we cannot properly handle demotion of kernel 40797c478bd9Sstevel@tonic-gate * pages. 40807c478bd9Sstevel@tonic-gate */ 4081ad23a2dbSjohansen if (PP_ISKAS(like_pp)) 40827c478bd9Sstevel@tonic-gate pgrflags |= PGR_SAMESZC; 40837c478bd9Sstevel@tonic-gate 40847c478bd9Sstevel@tonic-gate /* LINTED */ 408507ad560dSkchow MTYPE_PGR_INIT(mtype, flags, like_pp, page_mnode, npgs); 40867c478bd9Sstevel@tonic-gate 40877c478bd9Sstevel@tonic-gate while (npgs) { 40887c478bd9Sstevel@tonic-gate pplist = NULL; 40897c478bd9Sstevel@tonic-gate for (;;) { 40907c478bd9Sstevel@tonic-gate pg_cnt = page_get_pagecnt(szc); 40917c478bd9Sstevel@tonic-gate bin = PP_2_BIN(like_pp); 40927c478bd9Sstevel@tonic-gate ASSERT(like_pp->p_szc == orig_like_pp->p_szc); 40937c478bd9Sstevel@tonic-gate ASSERT(pg_cnt <= npgs); 40947c478bd9Sstevel@tonic-gate 40957c478bd9Sstevel@tonic-gate /* 40967c478bd9Sstevel@tonic-gate * If an lgroup was specified, try to get the 40977c478bd9Sstevel@tonic-gate * page from that lgroup. 40982dae3fb5Sjjc * NOTE: Must be careful with code below because 40992dae3fb5Sjjc * lgroup may disappear and reappear since there 41002dae3fb5Sjjc * is no locking for lgroup here. 41017c478bd9Sstevel@tonic-gate */ 41022dae3fb5Sjjc if (LGRP_EXISTS(lgrp_target)) { 41032dae3fb5Sjjc /* 41042dae3fb5Sjjc * Keep local variable for lgroup separate 41052dae3fb5Sjjc * from lgroup argument since this code should 41062dae3fb5Sjjc * only be exercised when lgroup argument 41072dae3fb5Sjjc * exists.... 41082dae3fb5Sjjc */ 41092dae3fb5Sjjc lgrp = lgrp_target; 41102dae3fb5Sjjc 41117c478bd9Sstevel@tonic-gate /* Try the lgroup's freelists first */ 41127c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41137c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41147c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41157c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41167c478bd9Sstevel@tonic-gate != -1) { 4117ce8eb11aSdp pplist = 4118ce8eb11aSdp page_get_mnode_freelist(mnode, bin, 4119ce8eb11aSdp mtype, szc, flags); 41207c478bd9Sstevel@tonic-gate } 41217c478bd9Sstevel@tonic-gate 41227c478bd9Sstevel@tonic-gate /* 41237c478bd9Sstevel@tonic-gate * Now try it's cachelists if this is a 41247c478bd9Sstevel@tonic-gate * small page. Don't need to do it for 41257c478bd9Sstevel@tonic-gate * larger ones since page_freelist_coalesce() 41267c478bd9Sstevel@tonic-gate * already failed. 41277c478bd9Sstevel@tonic-gate */ 41287c478bd9Sstevel@tonic-gate if (pplist != NULL || szc != 0) 41297c478bd9Sstevel@tonic-gate break; 41307c478bd9Sstevel@tonic-gate 41317c478bd9Sstevel@tonic-gate /* Now try it's cachelists */ 41327c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41337c478bd9Sstevel@tonic-gate LGRP_SRCH_LOCAL); 41347c478bd9Sstevel@tonic-gate 41357c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 41367c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41377c478bd9Sstevel@tonic-gate != -1) { 4138ce8eb11aSdp pplist = 4139ce8eb11aSdp page_get_mnode_cachelist(bin, flags, 4140ce8eb11aSdp mnode, mtype); 41417c478bd9Sstevel@tonic-gate } 41427c478bd9Sstevel@tonic-gate if (pplist != NULL) { 41437c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 41447c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 41457c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 41467c478bd9Sstevel@tonic-gate break; 41477c478bd9Sstevel@tonic-gate } 41487c478bd9Sstevel@tonic-gate /* Done looking in this lgroup. Bail out. */ 41497c478bd9Sstevel@tonic-gate break; 41507c478bd9Sstevel@tonic-gate } 41517c478bd9Sstevel@tonic-gate 41527c478bd9Sstevel@tonic-gate /* 41532dae3fb5Sjjc * No lgroup was specified (or lgroup was removed by 41542dae3fb5Sjjc * DR, so just try to get the page as close to 41552dae3fb5Sjjc * like_pp's mnode as possible. 41567c478bd9Sstevel@tonic-gate * First try the local freelist... 41577c478bd9Sstevel@tonic-gate */ 41587c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(like_pp); 41597c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, bin, 41607c478bd9Sstevel@tonic-gate mtype, szc, flags); 41617c478bd9Sstevel@tonic-gate if (pplist != NULL) 41627c478bd9Sstevel@tonic-gate break; 41637c478bd9Sstevel@tonic-gate 41647c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnofree); 41657c478bd9Sstevel@tonic-gate 41667c478bd9Sstevel@tonic-gate /* 41677c478bd9Sstevel@tonic-gate * ...then the local cachelist. Don't need to do it for 41687c478bd9Sstevel@tonic-gate * larger pages cause page_freelist_coalesce() already 41697c478bd9Sstevel@tonic-gate * failed there anyway. 41707c478bd9Sstevel@tonic-gate */ 41717c478bd9Sstevel@tonic-gate if (szc == 0) { 41727c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, flags, 41737c478bd9Sstevel@tonic-gate mnode, mtype); 41747c478bd9Sstevel@tonic-gate if (pplist != NULL) { 41757c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 41767c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 41777c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 41787c478bd9Sstevel@tonic-gate break; 41797c478bd9Sstevel@tonic-gate } 41807c478bd9Sstevel@tonic-gate } 41817c478bd9Sstevel@tonic-gate 41827c478bd9Sstevel@tonic-gate /* Now try remote freelists */ 41837c478bd9Sstevel@tonic-gate page_mnode = mnode; 41847c478bd9Sstevel@tonic-gate lgrp = 41857c478bd9Sstevel@tonic-gate lgrp_hand_to_lgrp(MEM_NODE_2_LGRPHAND(page_mnode)); 41867c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 41877c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 41887c478bd9Sstevel@tonic-gate while (pplist == NULL && 41897c478bd9Sstevel@tonic-gate (mnode = lgrp_memnode_choose(&lgrp_cookie)) 41907c478bd9Sstevel@tonic-gate != -1) { 41917c478bd9Sstevel@tonic-gate /* 41927c478bd9Sstevel@tonic-gate * Skip local mnode. 41937c478bd9Sstevel@tonic-gate */ 41947c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 41957c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 41967c478bd9Sstevel@tonic-gate continue; 41977c478bd9Sstevel@tonic-gate 41987c478bd9Sstevel@tonic-gate pplist = page_get_mnode_freelist(mnode, 41997c478bd9Sstevel@tonic-gate bin, mtype, szc, flags); 42007c478bd9Sstevel@tonic-gate } 42017c478bd9Sstevel@tonic-gate 42027c478bd9Sstevel@tonic-gate if (pplist != NULL) 42037c478bd9Sstevel@tonic-gate break; 42047c478bd9Sstevel@tonic-gate 42057c478bd9Sstevel@tonic-gate 42067c478bd9Sstevel@tonic-gate /* Now try remote cachelists */ 42077c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42087c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42097c478bd9Sstevel@tonic-gate while (pplist == NULL && szc == 0) { 42107c478bd9Sstevel@tonic-gate mnode = lgrp_memnode_choose(&lgrp_cookie); 42117c478bd9Sstevel@tonic-gate if (mnode == -1) 42127c478bd9Sstevel@tonic-gate break; 42137c478bd9Sstevel@tonic-gate /* 42147c478bd9Sstevel@tonic-gate * Skip local mnode. 42157c478bd9Sstevel@tonic-gate */ 42167c478bd9Sstevel@tonic-gate if ((mnode == page_mnode) || 42177c478bd9Sstevel@tonic-gate (mem_node_config[mnode].exists == 0)) 42187c478bd9Sstevel@tonic-gate continue; 42197c478bd9Sstevel@tonic-gate 42207c478bd9Sstevel@tonic-gate pplist = page_get_mnode_cachelist(bin, 42217c478bd9Sstevel@tonic-gate flags, mnode, mtype); 42227c478bd9Sstevel@tonic-gate 42237c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42247c478bd9Sstevel@tonic-gate page_hashout(pplist, NULL); 42257c478bd9Sstevel@tonic-gate PP_SETAGED(pplist); 42267c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nhashout); 42277c478bd9Sstevel@tonic-gate break; 42287c478bd9Sstevel@tonic-gate } 42297c478bd9Sstevel@tonic-gate } 42307c478bd9Sstevel@tonic-gate 42317c478bd9Sstevel@tonic-gate /* 42327c478bd9Sstevel@tonic-gate * Break out of while loop under the following cases: 42337c478bd9Sstevel@tonic-gate * - If we successfully got a page. 42347c478bd9Sstevel@tonic-gate * - If pgrflags specified only returning a specific 42357c478bd9Sstevel@tonic-gate * page size and we could not find that page size. 42367c478bd9Sstevel@tonic-gate * - If we could not satisfy the request with PAGESIZE 42377c478bd9Sstevel@tonic-gate * or larger pages. 42387c478bd9Sstevel@tonic-gate */ 42397c478bd9Sstevel@tonic-gate if (pplist != NULL || szc == 0) 42407c478bd9Sstevel@tonic-gate break; 42417c478bd9Sstevel@tonic-gate 42427c478bd9Sstevel@tonic-gate if ((pgrflags & PGR_SAMESZC) || pgrppgcp) { 42437c478bd9Sstevel@tonic-gate /* try to find contig page */ 42447c478bd9Sstevel@tonic-gate 42457c478bd9Sstevel@tonic-gate LGRP_MNODE_COOKIE_INIT(lgrp_cookie, lgrp, 42467c478bd9Sstevel@tonic-gate LGRP_SRCH_HIER); 42477c478bd9Sstevel@tonic-gate 42487c478bd9Sstevel@tonic-gate while ((pplist == NULL) && 42497c478bd9Sstevel@tonic-gate (mnode = 4250ce8eb11aSdp lgrp_memnode_choose(&lgrp_cookie)) 42517c478bd9Sstevel@tonic-gate != -1) { 42527c478bd9Sstevel@tonic-gate pplist = page_get_contig_pages( 4253ce8eb11aSdp mnode, bin, mtype, szc, 4254ce8eb11aSdp flags | PGI_PGCPHIPRI); 42557c478bd9Sstevel@tonic-gate } 42567c478bd9Sstevel@tonic-gate break; 42577c478bd9Sstevel@tonic-gate } 42587c478bd9Sstevel@tonic-gate 42597c478bd9Sstevel@tonic-gate /* 42607c478bd9Sstevel@tonic-gate * The correct thing to do here is try the next 42617c478bd9Sstevel@tonic-gate * page size down using szc--. Due to a bug 42627c478bd9Sstevel@tonic-gate * with the processing of HAT_RELOAD_SHARE 42637c478bd9Sstevel@tonic-gate * where the sfmmu_ttecnt arrays of all 42647c478bd9Sstevel@tonic-gate * hats sharing an ISM segment don't get updated, 42657c478bd9Sstevel@tonic-gate * using intermediate size pages for relocation 42667c478bd9Sstevel@tonic-gate * can lead to continuous page faults. 42677c478bd9Sstevel@tonic-gate */ 42687c478bd9Sstevel@tonic-gate szc = 0; 42697c478bd9Sstevel@tonic-gate } 42707c478bd9Sstevel@tonic-gate 42717c478bd9Sstevel@tonic-gate if (pplist != NULL) { 42727c478bd9Sstevel@tonic-gate DTRACE_PROBE4(page__get, 42737c478bd9Sstevel@tonic-gate lgrp_t *, lgrp, 42747c478bd9Sstevel@tonic-gate int, mnode, 42757c478bd9Sstevel@tonic-gate ulong_t, bin, 42767c478bd9Sstevel@tonic-gate uint_t, flags); 42777c478bd9Sstevel@tonic-gate 42787c478bd9Sstevel@tonic-gate while (pplist != NULL && pg_cnt--) { 42797c478bd9Sstevel@tonic-gate ASSERT(pplist != NULL); 42807c478bd9Sstevel@tonic-gate pp = pplist; 42817c478bd9Sstevel@tonic-gate page_sub(&pplist, pp); 42827c478bd9Sstevel@tonic-gate PP_CLRFREE(pp); 42837c478bd9Sstevel@tonic-gate PP_CLRAGED(pp); 42847c478bd9Sstevel@tonic-gate page_list_concat(&pl, &pp); 42857c478bd9Sstevel@tonic-gate npgs--; 42867c478bd9Sstevel@tonic-gate like_pp = like_pp + 1; 42877c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnext_pp); 42887c478bd9Sstevel@tonic-gate } 42897c478bd9Sstevel@tonic-gate ASSERT(pg_cnt == 0); 42907c478bd9Sstevel@tonic-gate } else { 42917c478bd9Sstevel@tonic-gate break; 42927c478bd9Sstevel@tonic-gate } 42937c478bd9Sstevel@tonic-gate } 42947c478bd9Sstevel@tonic-gate 42957c478bd9Sstevel@tonic-gate if (npgs) { 42967c478bd9Sstevel@tonic-gate /* 42977c478bd9Sstevel@tonic-gate * We were unable to allocate the necessary number 42987c478bd9Sstevel@tonic-gate * of pages. 42997c478bd9Sstevel@tonic-gate * We need to free up any pl. 43007c478bd9Sstevel@tonic-gate */ 43017c478bd9Sstevel@tonic-gate REPL_STAT_INCR(nnopage); 43027c478bd9Sstevel@tonic-gate page_free_replacement_page(pl); 43037c478bd9Sstevel@tonic-gate return (NULL); 43047c478bd9Sstevel@tonic-gate } else { 43057c478bd9Sstevel@tonic-gate return (pl); 43067c478bd9Sstevel@tonic-gate } 43077c478bd9Sstevel@tonic-gate } 43087c478bd9Sstevel@tonic-gate 43097c478bd9Sstevel@tonic-gate /* 43107c478bd9Sstevel@tonic-gate * demote a free large page to it's constituent pages 43117c478bd9Sstevel@tonic-gate */ 43127c478bd9Sstevel@tonic-gate void 43137c478bd9Sstevel@tonic-gate page_demote_free_pages(page_t *pp) 43147c478bd9Sstevel@tonic-gate { 43157c478bd9Sstevel@tonic-gate 43167c478bd9Sstevel@tonic-gate int mnode; 43177c478bd9Sstevel@tonic-gate 43187c478bd9Sstevel@tonic-gate ASSERT(pp != NULL); 43197c478bd9Sstevel@tonic-gate ASSERT(PAGE_LOCKED(pp)); 43207c478bd9Sstevel@tonic-gate ASSERT(PP_ISFREE(pp)); 43217c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc != 0 && pp->p_szc < mmu_page_sizes); 43227c478bd9Sstevel@tonic-gate 43237c478bd9Sstevel@tonic-gate mnode = PP_2_MEM_NODE(pp); 43247c478bd9Sstevel@tonic-gate page_freelist_lock(mnode); 43257c478bd9Sstevel@tonic-gate if (pp->p_szc != 0) { 43267c478bd9Sstevel@tonic-gate (void) page_demote(mnode, PFN_BASE(pp->p_pagenum, 43277c478bd9Sstevel@tonic-gate pp->p_szc), pp->p_szc, 0, PC_NO_COLOR, PC_FREE); 43287c478bd9Sstevel@tonic-gate } 43297c478bd9Sstevel@tonic-gate page_freelist_unlock(mnode); 43307c478bd9Sstevel@tonic-gate ASSERT(pp->p_szc == 0); 43317c478bd9Sstevel@tonic-gate } 4332932dc8e5Sdp 4333932dc8e5Sdp /* 4334932dc8e5Sdp * Factor in colorequiv to check additional 'equivalent' bins. 4335932dc8e5Sdp * colorequiv may be set in /etc/system 4336932dc8e5Sdp */ 4337932dc8e5Sdp void 4338932dc8e5Sdp page_set_colorequiv_arr(void) 4339932dc8e5Sdp { 4340932dc8e5Sdp if (colorequiv > 1) { 4341932dc8e5Sdp int i; 4342fe70c9cfSdp uint_t sv_a = lowbit(colorequiv) - 1; 4343932dc8e5Sdp 4344fe70c9cfSdp if (sv_a > 15) 4345fe70c9cfSdp sv_a = 15; 4346932dc8e5Sdp 4347932dc8e5Sdp for (i = 0; i < MMU_PAGE_SIZES; i++) { 4348fe70c9cfSdp uint_t colors; 4349fe70c9cfSdp uint_t a = sv_a; 4350932dc8e5Sdp 4351932dc8e5Sdp if ((colors = hw_page_array[i].hp_colors) <= 1) { 4352932dc8e5Sdp continue; 4353932dc8e5Sdp } 4354932dc8e5Sdp while ((colors >> a) == 0) 4355932dc8e5Sdp a--; 4356932dc8e5Sdp if ((a << 4) > colorequivszc[i]) { 4357932dc8e5Sdp colorequivszc[i] = (a << 4); 4358932dc8e5Sdp } 4359932dc8e5Sdp } 4360932dc8e5Sdp } 4361932dc8e5Sdp } 4362