17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 516eca74fSstans * Common Development and Distribution License (the "License"). 616eca74fSstans * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22b52a336eSPavel Tatashin * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved. 23d3b5f563SJohn Levon * Copyright 2019 Joyent, Inc. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* 277c478bd9Sstevel@tonic-gate * UNIX machine dependent virtual memory support. 287c478bd9Sstevel@tonic-gate */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #ifndef _VM_DEP_H 317c478bd9Sstevel@tonic-gate #define _VM_DEP_H 327c478bd9Sstevel@tonic-gate 337c478bd9Sstevel@tonic-gate #ifdef __cplusplus 347c478bd9Sstevel@tonic-gate extern "C" { 357c478bd9Sstevel@tonic-gate #endif 367c478bd9Sstevel@tonic-gate 377c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 387c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 397c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate #define GETTICK() gettick() 4282c9d5baSVijay Balakrishna, SG-RPE 43b52a336eSPavel Tatashin /* tick value that should be used for random values */ 44b52a336eSPavel Tatashin extern u_longlong_t randtick(void); 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate /* 477c478bd9Sstevel@tonic-gate * Per page size free lists. Allocated dynamically. 487c478bd9Sstevel@tonic-gate */ 497c478bd9Sstevel@tonic-gate #define MAX_MEM_TYPES 2 /* 0 = reloc, 1 = noreloc */ 507c478bd9Sstevel@tonic-gate #define MTYPE_RELOC 0 517c478bd9Sstevel@tonic-gate #define MTYPE_NORELOC 1 527c478bd9Sstevel@tonic-gate 537c478bd9Sstevel@tonic-gate #define PP_2_MTYPE(pp) (PP_ISNORELOC(pp) ? MTYPE_NORELOC : MTYPE_RELOC) 547c478bd9Sstevel@tonic-gate 5507ad560dSkchow #define MTYPE_INIT(mtype, vp, vaddr, flags, pgsz) \ 567c478bd9Sstevel@tonic-gate mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC; 577c478bd9Sstevel@tonic-gate 587c478bd9Sstevel@tonic-gate /* mtype init for page_get_replacement_page */ 59d3b5f563SJohn Levon #define MTYPE_PGR_INIT(mtype, flags, pp, pgcnt) \ 607c478bd9Sstevel@tonic-gate mtype = (flags & PG_NORELOC) ? MTYPE_NORELOC : MTYPE_RELOC; 617c478bd9Sstevel@tonic-gate 627c478bd9Sstevel@tonic-gate #define MNODETYPE_2_PFN(mnode, mtype, pfnlo, pfnhi) \ 637c478bd9Sstevel@tonic-gate pfnlo = mem_node_config[mnode].physbase; \ 647c478bd9Sstevel@tonic-gate pfnhi = mem_node_config[mnode].physmax; 657c478bd9Sstevel@tonic-gate 665d07b933Sdp /* 675d07b933Sdp * candidate counters in vm_pagelist.c are indexed by color and range 685d07b933Sdp */ 695d07b933Sdp #define MAX_MNODE_MRANGES MAX_MEM_TYPES 705d07b933Sdp #define MNODE_RANGE_CNT(mnode) MAX_MNODE_MRANGES 715d07b933Sdp #define MNODE_MAX_MRANGE(mnode) (MAX_MEM_TYPES - 1) 725d07b933Sdp #define MTYPE_2_MRANGE(mnode, mtype) (mtype) 735d07b933Sdp 747c478bd9Sstevel@tonic-gate /* 757c478bd9Sstevel@tonic-gate * Internal PG_ flags. 767c478bd9Sstevel@tonic-gate */ 777c478bd9Sstevel@tonic-gate #define PGI_RELOCONLY 0x10000 /* acts in the opposite sense to PG_NORELOC */ 787c478bd9Sstevel@tonic-gate #define PGI_NOCAGE 0x20000 /* indicates Cage is disabled */ 797c478bd9Sstevel@tonic-gate #define PGI_PGCPHIPRI 0x40000 /* page_get_contig_page priority allocation */ 807c478bd9Sstevel@tonic-gate #define PGI_PGCPSZC0 0x80000 /* relocate base pagesize page */ 817c478bd9Sstevel@tonic-gate 827c478bd9Sstevel@tonic-gate /* 837c478bd9Sstevel@tonic-gate * PGI mtype flags - should not overlap PGI flags 847c478bd9Sstevel@tonic-gate */ 857c478bd9Sstevel@tonic-gate #define PGI_MT_RANGE 0x1000000 /* mtype range */ 867c478bd9Sstevel@tonic-gate #define PGI_MT_NEXT 0x2000000 /* get next mtype */ 877c478bd9Sstevel@tonic-gate 88d94ffb28Sjmcp extern page_t ***page_freelists[MMU_PAGE_SIZES][MAX_MEM_TYPES]; 897c478bd9Sstevel@tonic-gate extern page_t ***page_cachelists[MAX_MEM_TYPES]; 907c478bd9Sstevel@tonic-gate 91d94ffb28Sjmcp #define PAGE_FREELISTS(mnode, szc, color, mtype) \ 92d94ffb28Sjmcp (*(page_freelists[szc][mtype][mnode] + (color))) 93d94ffb28Sjmcp 947c478bd9Sstevel@tonic-gate #define PAGE_CACHELISTS(mnode, color, mtype) \ 957c478bd9Sstevel@tonic-gate (*(page_cachelists[mtype][mnode] + (color))) 967c478bd9Sstevel@tonic-gate 977c478bd9Sstevel@tonic-gate /* 987c478bd9Sstevel@tonic-gate * There are 'page_colors' colors/bins. Spread them out under a 997c478bd9Sstevel@tonic-gate * couple of locks. There are mutexes for both the page freelist 1007c478bd9Sstevel@tonic-gate * and the page cachelist. We want enough locks to make contention 1017c478bd9Sstevel@tonic-gate * reasonable, but not too many -- otherwise page_freelist_lock() gets 1027c478bd9Sstevel@tonic-gate * so expensive that it becomes the bottleneck! 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate #define NPC_MUTEX 16 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate extern kmutex_t *fpc_mutex[NPC_MUTEX]; 1077c478bd9Sstevel@tonic-gate extern kmutex_t *cpc_mutex[NPC_MUTEX]; 1087c478bd9Sstevel@tonic-gate 109ce8eb11aSdp /* 110ce8eb11aSdp * Iterator provides the info needed to convert RA to PA. 111ce8eb11aSdp * MEM_NODE_ITERATOR_INIT() should be called before 112ce8eb11aSdp * PAGE_NEXT_PFN_FOR_COLOR() if pfn was not obtained via a previous 113ce8eb11aSdp * PAGE_NEXT_PFN_FOR_COLOR() call. Iterator caches color 2 hash 114ce8eb11aSdp * translations requiring initializer call if color or ceq_mask changes, 115ce8eb11aSdp * even if pfn doesn't. MEM_NODE_ITERATOR_INIT() must also be called before 116ce8eb11aSdp * PFN_2_COLOR() that uses a valid iterator argument. 1179853d9e8SJason Beloro * 1189853d9e8SJason Beloro * plat_mem_node_iterator_init() starts from last mblock in continuation 1199853d9e8SJason Beloro * case which may be invalid because memory DR. To detect this situation 1209853d9e8SJason Beloro * mi_genid is checked against mpo_genid which is incremented after a 1219853d9e8SJason Beloro * memory DR operation. See also plat_slice_add()/plat_slice_del(). 122ce8eb11aSdp */ 123ce8eb11aSdp #ifdef sun4v 124ce8eb11aSdp 125ce8eb11aSdp typedef struct mem_node_iterator { 126ce8eb11aSdp uint_t mi_mnode; /* mnode in which to iterate */ 127ce8eb11aSdp int mi_init; /* set to 1 when first init */ 1289853d9e8SJason Beloro int mi_genid; /* set/checked against mpo_genid */ 129ce8eb11aSdp int mi_last_mblock; /* last mblock visited */ 130ce8eb11aSdp uint_t mi_hash_ceq_mask; /* cached copy of ceq_mask */ 131ce8eb11aSdp uint_t mi_hash_color; /* cached copy of color */ 132ce8eb11aSdp uint_t mi_mnode_mask; /* number of mask bits */ 133ce8eb11aSdp uint_t mi_mnode_pfn_shift; /* mnode position in pfn */ 134ce8eb11aSdp pfn_t mi_mblock_base; /* first valid pfn in current mblock */ 135ce8eb11aSdp pfn_t mi_mblock_end; /* last valid pfn in current mblock */ 136ce8eb11aSdp pfn_t mi_ra_to_pa; /* ra adjustment for current mblock */ 137ce8eb11aSdp pfn_t mi_mnode_pfn_mask; /* mask to obtain mnode id bits */ 138ce8eb11aSdp } mem_node_iterator_t; 139ce8eb11aSdp 140ce8eb11aSdp #define MEM_NODE_ITERATOR_DECL(it) \ 141ce8eb11aSdp mem_node_iterator_t it 142b779d3e0Sdp #define MEM_NODE_ITERATOR_INIT(pfn, mnode, szc, it) \ 143b779d3e0Sdp (pfn) = plat_mem_node_iterator_init((pfn), (mnode), (szc), (it), 1) 144ce8eb11aSdp 145b779d3e0Sdp extern pfn_t plat_mem_node_iterator_init(pfn_t, int, uchar_t, 146ce8eb11aSdp mem_node_iterator_t *, int); 147ce8eb11aSdp extern pfn_t plat_rapfn_to_papfn(pfn_t); 148ce8eb11aSdp extern int interleaved_mnodes; 149ce8eb11aSdp 150ce8eb11aSdp #else /* sun4v */ 151ce8eb11aSdp 152ce8eb11aSdp #define MEM_NODE_ITERATOR_DECL(it) \ 153ce8eb11aSdp void *it = NULL 154b779d3e0Sdp #define MEM_NODE_ITERATOR_INIT(pfn, mnode, szc, it) 155ce8eb11aSdp 156ce8eb11aSdp #endif /* sun4v */ 157ce8eb11aSdp 158ce8eb11aSdp /* 159ce8eb11aSdp * Return the mnode limits so that hpc_counters length and base 160ce8eb11aSdp * index can be determined. When interleaved_mnodes is set, we 161ce8eb11aSdp * create an array only for the first mnode that exists. All other 162ce8eb11aSdp * mnodes will share the array in this case. 163ce8eb11aSdp * If interleaved_mnodes is not set, simply return the limits for 164ce8eb11aSdp * the given mnode. 165ce8eb11aSdp */ 166ce8eb11aSdp #define HPM_COUNTERS_LIMITS(mnode, physbase, physmax, first) \ 167ce8eb11aSdp if (!interleaved_mnodes) { \ 168ce8eb11aSdp (physbase) = mem_node_config[(mnode)].physbase; \ 169ce8eb11aSdp (physmax) = mem_node_config[(mnode)].physmax; \ 170ce8eb11aSdp (first) = (mnode); \ 171ce8eb11aSdp } else if ((first) < 0) { \ 172ce8eb11aSdp mem_node_max_range(&(physbase), &(physmax)); \ 173ce8eb11aSdp (first) = (mnode); \ 174ce8eb11aSdp } 175ce8eb11aSdp 176ce8eb11aSdp #define PAGE_CTRS_WRITE_LOCK(mnode) \ 177ce8eb11aSdp if (!interleaved_mnodes) { \ 178ce8eb11aSdp rw_enter(&page_ctrs_rwlock[(mnode)], RW_WRITER); \ 179ce8eb11aSdp page_freelist_lock(mnode); \ 180ce8eb11aSdp } else { \ 181ce8eb11aSdp /* changing shared hpm_counters */ \ 182ce8eb11aSdp int _i; \ 183ce8eb11aSdp for (_i = 0; _i < max_mem_nodes; _i++) { \ 184ce8eb11aSdp rw_enter(&page_ctrs_rwlock[_i], RW_WRITER); \ 185ce8eb11aSdp page_freelist_lock(_i); \ 186ce8eb11aSdp } \ 187ce8eb11aSdp } 188ce8eb11aSdp 189ce8eb11aSdp #define PAGE_CTRS_WRITE_UNLOCK(mnode) \ 190ce8eb11aSdp if (!interleaved_mnodes) { \ 191ce8eb11aSdp page_freelist_unlock(mnode); \ 192ce8eb11aSdp rw_exit(&page_ctrs_rwlock[(mnode)]); \ 193ce8eb11aSdp } else { \ 194ce8eb11aSdp int _i; \ 195ce8eb11aSdp for (_i = 0; _i < max_mem_nodes; _i++) { \ 196ce8eb11aSdp page_freelist_unlock(_i); \ 197ce8eb11aSdp rw_exit(&page_ctrs_rwlock[_i]); \ 198ce8eb11aSdp } \ 199ce8eb11aSdp } 200ce8eb11aSdp 2015d07b933Sdp /* 2025d07b933Sdp * cpu specific color conversion functions 2035d07b933Sdp */ 2045d07b933Sdp extern uint_t page_get_nsz_color_mask_cpu(uchar_t, uint_t); 2055d07b933Sdp #pragma weak page_get_nsz_color_mask_cpu 2065d07b933Sdp 2075d07b933Sdp extern uint_t page_get_nsz_color_cpu(uchar_t, uint_t); 2085d07b933Sdp #pragma weak page_get_nsz_color_cpu 2095d07b933Sdp 2105d07b933Sdp extern uint_t page_get_color_shift_cpu(uchar_t, uchar_t); 2115d07b933Sdp #pragma weak page_get_color_shift_cpu 2125d07b933Sdp 213ce8eb11aSdp extern uint_t page_convert_color_cpu(uint_t, uchar_t, uchar_t); 214ce8eb11aSdp #pragma weak page_convert_color_cpu 215ce8eb11aSdp 2165d07b933Sdp extern pfn_t page_next_pfn_for_color_cpu(pfn_t, 217ce8eb11aSdp uchar_t, uint_t, uint_t, uint_t, void *); 2185d07b933Sdp #pragma weak page_next_pfn_for_color_cpu 2195d07b933Sdp 220ce8eb11aSdp extern uint_t page_pfn_2_color_cpu(pfn_t, uchar_t, void *); 2215d07b933Sdp #pragma weak page_pfn_2_color_cpu 2225d07b933Sdp 2235d07b933Sdp #define PAGE_GET_COLOR_SHIFT(szc, nszc) \ 2245d07b933Sdp ((&page_get_color_shift_cpu != NULL) ? \ 2255d07b933Sdp page_get_color_shift_cpu(szc, nszc) : \ 2265d07b933Sdp (hw_page_array[(nszc)].hp_shift - \ 2275d07b933Sdp hw_page_array[(szc)].hp_shift)) 2285d07b933Sdp 229ce8eb11aSdp #define PAGE_CONVERT_COLOR(ncolor, szc, nszc) \ 230ce8eb11aSdp ((&page_convert_color_cpu != NULL) ? \ 231ce8eb11aSdp page_convert_color_cpu(ncolor, szc, nszc) : \ 232ce8eb11aSdp ((ncolor) << PAGE_GET_COLOR_SHIFT((szc), (nszc)))) 233ce8eb11aSdp 234ce8eb11aSdp #define PFN_2_COLOR(pfn, szc, it) \ 2355d07b933Sdp ((&page_pfn_2_color_cpu != NULL) ? \ 236ce8eb11aSdp page_pfn_2_color_cpu(pfn, szc, it) : \ 2375d07b933Sdp ((pfn & (hw_page_array[0].hp_colors - 1)) >> \ 2385d07b933Sdp (hw_page_array[szc].hp_shift - \ 2395d07b933Sdp hw_page_array[0].hp_shift))) 2405d07b933Sdp 2415d07b933Sdp #define PNUM_SIZE(szc) \ 2425d07b933Sdp (hw_page_array[(szc)].hp_pgcnt) 2435d07b933Sdp #define PNUM_SHIFT(szc) \ 2445d07b933Sdp (hw_page_array[(szc)].hp_shift - hw_page_array[0].hp_shift) 2455d07b933Sdp #define PAGE_GET_SHIFT(szc) \ 2465d07b933Sdp (hw_page_array[(szc)].hp_shift) 2475d07b933Sdp #define PAGE_GET_PAGECOLORS(szc) \ 2485d07b933Sdp (hw_page_array[(szc)].hp_colors) 2495d07b933Sdp 2505d07b933Sdp /* 2515d07b933Sdp * This macro calculates the next sequential pfn with the specified 2525d07b933Sdp * color using color equivalency mask 2535d07b933Sdp */ 254a7c3ca36Sdp #define PAGE_NEXT_PFN_FOR_COLOR(pfn, szc, color, ceq_mask, color_mask, it) \ 255a7c3ca36Sdp { \ 256a7c3ca36Sdp ASSERT(((color) & ~(ceq_mask)) == 0); \ 257a7c3ca36Sdp if (&page_next_pfn_for_color_cpu == NULL) { \ 258a7c3ca36Sdp uint_t pfn_shift = PAGE_BSZS_SHIFT(szc); \ 259a7c3ca36Sdp pfn_t spfn = pfn >> pfn_shift; \ 260a7c3ca36Sdp pfn_t stride = (ceq_mask) + 1; \ 261a7c3ca36Sdp ASSERT((((ceq_mask) + 1) & (ceq_mask)) == 0); \ 262a7c3ca36Sdp if (((spfn ^ (color)) & (ceq_mask)) == 0) { \ 263a7c3ca36Sdp pfn += stride << pfn_shift; \ 264a7c3ca36Sdp } else { \ 265a7c3ca36Sdp pfn = (spfn & ~(pfn_t)(ceq_mask)) | (color); \ 266a7c3ca36Sdp pfn = (pfn > spfn ? pfn : pfn + stride) << \ 267a7c3ca36Sdp pfn_shift; \ 268a7c3ca36Sdp } \ 269a7c3ca36Sdp } else { \ 270a7c3ca36Sdp pfn = page_next_pfn_for_color_cpu(pfn, szc, color, \ 271a7c3ca36Sdp ceq_mask, color_mask, it); \ 272a7c3ca36Sdp } \ 2735d07b933Sdp } 2745d07b933Sdp 2755d07b933Sdp /* get the color equivalency mask for the next szc */ 2765d07b933Sdp #define PAGE_GET_NSZ_MASK(szc, mask) \ 2775d07b933Sdp ((&page_get_nsz_color_mask_cpu == NULL) ? \ 2785d07b933Sdp ((mask) >> (PAGE_GET_SHIFT((szc) + 1) - PAGE_GET_SHIFT(szc))) : \ 2795d07b933Sdp page_get_nsz_color_mask_cpu(szc, mask)) 2805d07b933Sdp 2815d07b933Sdp /* get the color of the next szc */ 2825d07b933Sdp #define PAGE_GET_NSZ_COLOR(szc, color) \ 2835d07b933Sdp ((&page_get_nsz_color_cpu == NULL) ? \ 2845d07b933Sdp ((color) >> (PAGE_GET_SHIFT((szc) + 1) - PAGE_GET_SHIFT(szc))) : \ 2855d07b933Sdp page_get_nsz_color_cpu(szc, color)) 2865d07b933Sdp 2877c478bd9Sstevel@tonic-gate /* Find the bin for the given page if it was of size szc */ 288ce8eb11aSdp #define PP_2_BIN_SZC(pp, szc) (PFN_2_COLOR(pp->p_pagenum, szc, (void *)(-1))) 2897c478bd9Sstevel@tonic-gate 2907c478bd9Sstevel@tonic-gate #define PP_2_BIN(pp) (PP_2_BIN_SZC(pp, pp->p_szc)) 2917c478bd9Sstevel@tonic-gate 2927c478bd9Sstevel@tonic-gate #define PP_2_MEM_NODE(pp) (PFN_2_MEM_NODE(pp->p_pagenum)) 2937c478bd9Sstevel@tonic-gate 294d94ffb28Sjmcp #define PC_BIN_MUTEX(mnode, bin, flags) ((flags & PG_FREE_LIST) ? \ 295d94ffb28Sjmcp &fpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode] : \ 2967c478bd9Sstevel@tonic-gate &cpc_mutex[(bin) & (NPC_MUTEX - 1)][mnode]) 2977c478bd9Sstevel@tonic-gate 2987c478bd9Sstevel@tonic-gate #define FPC_MUTEX(mnode, i) (&fpc_mutex[i][mnode]) 2997c478bd9Sstevel@tonic-gate #define CPC_MUTEX(mnode, i) (&cpc_mutex[i][mnode]) 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate #define PFN_BASE(pfnum, szc) (pfnum & ~((1 << PAGE_BSZS_SHIFT(szc)) - 1)) 3027c478bd9Sstevel@tonic-gate 3035d07b933Sdp /* 3045d07b933Sdp * this structure is used for walking free page lists 3055d07b933Sdp * controls when to split large pages into smaller pages, 3065d07b933Sdp * and when to coalesce smaller pages into larger pages 3075d07b933Sdp */ 3085d07b933Sdp typedef struct page_list_walker { 3095d07b933Sdp uint_t plw_colors; /* num of colors for szc */ 3105d07b933Sdp uint_t plw_color_mask; /* colors-1 */ 3115d07b933Sdp uint_t plw_bin_step; /* next bin: 1 or 2 */ 3125d07b933Sdp uint_t plw_count; /* loop count */ 3135d07b933Sdp uint_t plw_bin0; /* starting bin */ 3145d07b933Sdp uint_t plw_bin_marker; /* bin after initial jump */ 3155d07b933Sdp uint_t plw_bin_split_prev; /* last bin we tried to split */ 3165d07b933Sdp uint_t plw_do_split; /* set if OK to split */ 3175d07b933Sdp uint_t plw_split_next; /* next bin to split */ 3185d07b933Sdp uint_t plw_ceq_dif; /* number of different color groups */ 3195d07b933Sdp /* to check */ 3205d07b933Sdp uint_t plw_ceq_mask[MMU_PAGE_SIZES + 1]; /* color equiv mask */ 3215d07b933Sdp uint_t plw_bins[MMU_PAGE_SIZES + 1]; /* num of bins */ 3225d07b933Sdp } page_list_walker_t; 3235d07b933Sdp 3245d07b933Sdp void page_list_walk_init(uchar_t szc, uint_t flags, uint_t bin, 3255d07b933Sdp int can_split, int use_ceq, page_list_walker_t *plw); 3265d07b933Sdp 3277c478bd9Sstevel@tonic-gate typedef char hpmctr_t; 3287c478bd9Sstevel@tonic-gate 3297c478bd9Sstevel@tonic-gate #ifdef DEBUG 3307c478bd9Sstevel@tonic-gate #define CHK_LPG(pp, szc) chk_lpg(pp, szc) 3317c478bd9Sstevel@tonic-gate extern void chk_lpg(page_t *, uchar_t); 3327c478bd9Sstevel@tonic-gate #else 3337c478bd9Sstevel@tonic-gate #define CHK_LPG(pp, szc) 3347c478bd9Sstevel@tonic-gate #endif 3357c478bd9Sstevel@tonic-gate 336affbd3ccSkchow /* 337affbd3ccSkchow * page list count per mnode and type. 338affbd3ccSkchow */ 3397c478bd9Sstevel@tonic-gate typedef struct { 340affbd3ccSkchow pgcnt_t plc_mt_pgmax; /* max page cnt */ 341affbd3ccSkchow pgcnt_t plc_mt_clpgcnt; /* cache list cnt */ 342affbd3ccSkchow pgcnt_t plc_mt_flpgcnt; /* free list cnt - small pages */ 343affbd3ccSkchow pgcnt_t plc_mt_lgpgcnt; /* free list cnt - large pages */ 344affbd3ccSkchow #ifdef DEBUG 3457c478bd9Sstevel@tonic-gate struct { 346affbd3ccSkchow pgcnt_t plc_mts_pgcnt; /* per page size count */ 347affbd3ccSkchow int plc_mts_colors; 348affbd3ccSkchow pgcnt_t *plc_mtsc_pgcnt; /* per color bin count */ 349affbd3ccSkchow } plc_mts[MMU_PAGE_SIZES]; 350affbd3ccSkchow #endif 351affbd3ccSkchow } plcnt_t[MAX_MEM_NODES][MAX_MEM_TYPES]; 3527c478bd9Sstevel@tonic-gate 353affbd3ccSkchow #ifdef DEBUG 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate #define PLCNT_SZ(ctrs_sz) { \ 3567c478bd9Sstevel@tonic-gate int szc; \ 3575d07b933Sdp for (szc = 0; szc < mmu_page_sizes; szc++) { \ 3587c478bd9Sstevel@tonic-gate int colors = page_get_pagecolors(szc); \ 3597c478bd9Sstevel@tonic-gate ctrs_sz += (max_mem_nodes * MAX_MEM_TYPES * \ 3607c478bd9Sstevel@tonic-gate colors * sizeof (pgcnt_t)); \ 3617c478bd9Sstevel@tonic-gate } \ 3627c478bd9Sstevel@tonic-gate } 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate #define PLCNT_INIT(base) { \ 3657c478bd9Sstevel@tonic-gate int mn, mt, szc, colors; \ 3667c478bd9Sstevel@tonic-gate for (szc = 0; szc < mmu_page_sizes; szc++) { \ 3677c478bd9Sstevel@tonic-gate colors = page_get_pagecolors(szc); \ 3687c478bd9Sstevel@tonic-gate for (mn = 0; mn < max_mem_nodes; mn++) { \ 3697c478bd9Sstevel@tonic-gate for (mt = 0; mt < MAX_MEM_TYPES; mt++) { \ 37007ad560dSkchow plcnt[mn][mt].plc_mts[szc]. \ 3717c478bd9Sstevel@tonic-gate plc_mts_colors = colors; \ 37207ad560dSkchow plcnt[mn][mt].plc_mts[szc]. \ 3737c478bd9Sstevel@tonic-gate plc_mtsc_pgcnt = (pgcnt_t *)base; \ 3747c478bd9Sstevel@tonic-gate base += (colors * sizeof (pgcnt_t)); \ 3757c478bd9Sstevel@tonic-gate } \ 3767c478bd9Sstevel@tonic-gate } \ 3777c478bd9Sstevel@tonic-gate } \ 3787c478bd9Sstevel@tonic-gate } 3797c478bd9Sstevel@tonic-gate 380affbd3ccSkchow #define PLCNT_DO(pp, mn, mtype, szc, cnt, flags) { \ 3817c478bd9Sstevel@tonic-gate int bin = PP_2_BIN(pp); \ 3827c478bd9Sstevel@tonic-gate if (flags & PG_CACHE_LIST) \ 383affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt); \ 384affbd3ccSkchow else if (szc) \ 385affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt); \ 386affbd3ccSkchow else \ 387affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt); \ 388affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mts[szc].plc_mts_pgcnt, \ 389affbd3ccSkchow cnt); \ 390affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mts[szc]. \ 3917c478bd9Sstevel@tonic-gate plc_mtsc_pgcnt[bin], cnt); \ 3927c478bd9Sstevel@tonic-gate } 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate #else 3957c478bd9Sstevel@tonic-gate 3967c478bd9Sstevel@tonic-gate #define PLCNT_SZ(ctrs_sz) 3977c478bd9Sstevel@tonic-gate 3987c478bd9Sstevel@tonic-gate #define PLCNT_INIT(base) 3997c478bd9Sstevel@tonic-gate 400affbd3ccSkchow /* PG_FREE_LIST may not be explicitly set in flags for large pages */ 401affbd3ccSkchow 402affbd3ccSkchow #define PLCNT_DO(pp, mn, mtype, szc, cnt, flags) { \ 403affbd3ccSkchow if (flags & PG_CACHE_LIST) \ 404affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mt_clpgcnt, cnt); \ 405affbd3ccSkchow else if (szc) \ 406affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mt_lgpgcnt, cnt); \ 407affbd3ccSkchow else \ 408affbd3ccSkchow atomic_add_long(&plcnt[mn][mtype].plc_mt_flpgcnt, cnt); \ 409affbd3ccSkchow } 410affbd3ccSkchow 411affbd3ccSkchow #endif 412affbd3ccSkchow 413affbd3ccSkchow #define PLCNT_INCR(pp, mn, mtype, szc, flags) { \ 4147c478bd9Sstevel@tonic-gate long cnt = (1 << PAGE_BSZS_SHIFT(szc)); \ 415affbd3ccSkchow PLCNT_DO(pp, mn, mtype, szc, cnt, flags); \ 4167c478bd9Sstevel@tonic-gate } 4177c478bd9Sstevel@tonic-gate 418affbd3ccSkchow #define PLCNT_DECR(pp, mn, mtype, szc, flags) { \ 419*4cf53d50SToomas Soome long cnt = ((ULONG_MAX) << PAGE_BSZS_SHIFT(szc)); \ 420affbd3ccSkchow PLCNT_DO(pp, mn, mtype, szc, cnt, flags); \ 4217c478bd9Sstevel@tonic-gate } 4227c478bd9Sstevel@tonic-gate 423affbd3ccSkchow /* 424affbd3ccSkchow * macros to update page list max counts - done when pages transferred 425e21bae1bSkchow * from RELOC to NORELOC mtype (kcage_init or kcage_assimilate_page). 426affbd3ccSkchow */ 427e21bae1bSkchow 428e21bae1bSkchow #define PLCNT_XFER_NORELOC(pp) { \ 429e21bae1bSkchow long cnt = (1 << PAGE_BSZS_SHIFT((pp)->p_szc)); \ 430e21bae1bSkchow int mn = PP_2_MEM_NODE(pp); \ 431e21bae1bSkchow atomic_add_long(&plcnt[mn][MTYPE_NORELOC].plc_mt_pgmax, cnt); \ 432e21bae1bSkchow atomic_add_long(&plcnt[mn][MTYPE_RELOC].plc_mt_pgmax, -cnt); \ 433affbd3ccSkchow } 434affbd3ccSkchow 435e21bae1bSkchow /* 436e21bae1bSkchow * macro to modify the page list max counts when memory is added to 437e21bae1bSkchow * the page lists during startup (add_physmem) or during a DR operation 438e21bae1bSkchow * when memory is added (kphysm_add_memory_dynamic) or deleted 439e21bae1bSkchow * (kphysm_del_cleanup). 440e21bae1bSkchow */ 441ce8eb11aSdp #define PLCNT_MODIFY_MAX(pfn, cnt) { \ 442ce8eb11aSdp spgcnt_t _cnt = (spgcnt_t)(cnt); \ 443ce8eb11aSdp pgcnt_t _acnt = ABS(_cnt); \ 444ce8eb11aSdp int _mn; \ 445ce8eb11aSdp pgcnt_t _np; \ 446ce8eb11aSdp if (&plat_mem_node_intersect_range != NULL) { \ 447ce8eb11aSdp for (_mn = 0; _mn < max_mem_nodes; _mn++) { \ 448ce8eb11aSdp plat_mem_node_intersect_range((pfn), _acnt, _mn, &_np);\ 449ce8eb11aSdp if (_np == 0) \ 450ce8eb11aSdp continue; \ 451ce8eb11aSdp atomic_add_long(&plcnt[_mn][MTYPE_RELOC].plc_mt_pgmax, \ 452ce8eb11aSdp (_cnt < 0) ? -_np : _np); \ 453ce8eb11aSdp } \ 454ce8eb11aSdp } else { \ 455ce8eb11aSdp pfn_t _pfn = (pfn); \ 456ce8eb11aSdp pfn_t _endpfn = _pfn + _acnt; \ 457ce8eb11aSdp while (_pfn < _endpfn) { \ 458ce8eb11aSdp _mn = PFN_2_MEM_NODE(_pfn); \ 459ce8eb11aSdp _np = MIN(_endpfn, mem_node_config[_mn].physmax + 1) - \ 460ce8eb11aSdp _pfn; \ 461ce8eb11aSdp _pfn += _np; \ 462ce8eb11aSdp atomic_add_long(&plcnt[_mn][MTYPE_RELOC].plc_mt_pgmax, \ 463ce8eb11aSdp (_cnt < 0) ? -_np : _np); \ 464ce8eb11aSdp } \ 465ce8eb11aSdp } \ 466affbd3ccSkchow } 467affbd3ccSkchow 4689853d9e8SJason Beloro /* 4699853d9e8SJason Beloro * macro to call page_ctrs_adjust() when memory is added 4709853d9e8SJason Beloro * during a DR operation. 4719853d9e8SJason Beloro */ 4729853d9e8SJason Beloro #define PAGE_CTRS_ADJUST(pfn, cnt, rv) { \ 4739853d9e8SJason Beloro spgcnt_t _cnt = (spgcnt_t)(cnt); \ 4749853d9e8SJason Beloro int _mn; \ 4759853d9e8SJason Beloro pgcnt_t _np; \ 476584b574aSToomas Soome rv = 0; \ 4779853d9e8SJason Beloro if (&plat_mem_node_intersect_range != NULL) { \ 4789853d9e8SJason Beloro for (_mn = 0; _mn < max_mem_nodes; _mn++) { \ 4799853d9e8SJason Beloro plat_mem_node_intersect_range((pfn), _cnt, _mn, &_np); \ 4809853d9e8SJason Beloro if (_np == 0) \ 4819853d9e8SJason Beloro continue; \ 4829853d9e8SJason Beloro if ((rv = page_ctrs_adjust(_mn)) != 0) \ 4839853d9e8SJason Beloro break; \ 4849853d9e8SJason Beloro } \ 4859853d9e8SJason Beloro } else { \ 4869853d9e8SJason Beloro pfn_t _pfn = (pfn); \ 4879853d9e8SJason Beloro pfn_t _endpfn = _pfn + _cnt; \ 4889853d9e8SJason Beloro while (_pfn < _endpfn) { \ 4899853d9e8SJason Beloro _mn = PFN_2_MEM_NODE(_pfn); \ 4909853d9e8SJason Beloro _np = MIN(_endpfn, mem_node_config[_mn].physmax + 1) - \ 4919853d9e8SJason Beloro _pfn; \ 4929853d9e8SJason Beloro _pfn += _np; \ 4939853d9e8SJason Beloro if ((rv = page_ctrs_adjust(_mn)) != 0) \ 4949853d9e8SJason Beloro break; \ 4959853d9e8SJason Beloro } \ 4969853d9e8SJason Beloro } \ 4979853d9e8SJason Beloro } 4989853d9e8SJason Beloro 499affbd3ccSkchow extern plcnt_t plcnt; 500affbd3ccSkchow 501affbd3ccSkchow #define MNODE_PGCNT(mn) \ 502affbd3ccSkchow (plcnt[mn][MTYPE_RELOC].plc_mt_clpgcnt + \ 503affbd3ccSkchow plcnt[mn][MTYPE_NORELOC].plc_mt_clpgcnt + \ 504affbd3ccSkchow plcnt[mn][MTYPE_RELOC].plc_mt_flpgcnt + \ 505affbd3ccSkchow plcnt[mn][MTYPE_NORELOC].plc_mt_flpgcnt + \ 506affbd3ccSkchow plcnt[mn][MTYPE_RELOC].plc_mt_lgpgcnt + \ 507affbd3ccSkchow plcnt[mn][MTYPE_NORELOC].plc_mt_lgpgcnt) 508affbd3ccSkchow 509affbd3ccSkchow #define MNODETYPE_PGCNT(mn, mtype) \ 510affbd3ccSkchow (plcnt[mn][mtype].plc_mt_clpgcnt + \ 511affbd3ccSkchow plcnt[mn][mtype].plc_mt_flpgcnt + \ 512affbd3ccSkchow plcnt[mn][mtype].plc_mt_lgpgcnt) 513affbd3ccSkchow 514affbd3ccSkchow /* 515affbd3ccSkchow * macros to loop through the mtype range - MTYPE_START returns -1 in 516affbd3ccSkchow * mtype if no pages in mnode/mtype and possibly NEXT mtype. 517affbd3ccSkchow */ 518affbd3ccSkchow #define MTYPE_START(mnode, mtype, flags) { \ 519affbd3ccSkchow if (plcnt[mnode][mtype].plc_mt_pgmax == 0) { \ 520102033aaSdp ASSERT(mtype == MTYPE_RELOC || \ 521102033aaSdp MNODETYPE_PGCNT(mnode, mtype) == 0 || \ 522102033aaSdp plcnt[mnode][mtype].plc_mt_pgmax != 0); \ 523affbd3ccSkchow MTYPE_NEXT(mnode, mtype, flags); \ 524affbd3ccSkchow } \ 525affbd3ccSkchow } 526affbd3ccSkchow 527affbd3ccSkchow /* 528affbd3ccSkchow * if allocation from the RELOC pool failed and there is sufficient cage 529affbd3ccSkchow * memory, attempt to allocate from the NORELOC pool. 530affbd3ccSkchow */ 531584b574aSToomas Soome #define MTYPE_NEXT(mnode, mtype, flags) { \ 532affbd3ccSkchow if (!(flags & (PG_NORELOC | PGI_NOCAGE | PGI_RELOCONLY)) && \ 533affbd3ccSkchow (kcage_freemem >= kcage_lotsfree)) { \ 53416eca74fSstans if (plcnt[mnode][MTYPE_NORELOC].plc_mt_pgmax == 0) { \ 535102033aaSdp ASSERT(MNODETYPE_PGCNT(mnode, MTYPE_NORELOC) == 0 || \ 536102033aaSdp plcnt[mnode][MTYPE_NORELOC].plc_mt_pgmax != 0); \ 537affbd3ccSkchow mtype = -1; \ 538affbd3ccSkchow } else { \ 539affbd3ccSkchow mtype = MTYPE_NORELOC; \ 540affbd3ccSkchow flags |= PG_NORELOC; \ 541affbd3ccSkchow } \ 542affbd3ccSkchow } else { \ 543affbd3ccSkchow mtype = -1; \ 544affbd3ccSkchow } \ 545affbd3ccSkchow } 5467c478bd9Sstevel@tonic-gate 5477c478bd9Sstevel@tonic-gate /* 5487c478bd9Sstevel@tonic-gate * get the ecache setsize for the current cpu. 5497c478bd9Sstevel@tonic-gate */ 5507c478bd9Sstevel@tonic-gate #define CPUSETSIZE() (cpunodes[CPU->cpu_id].ecache_setsize) 5517c478bd9Sstevel@tonic-gate 552affbd3ccSkchow extern struct cpu cpu0; 553affbd3ccSkchow #define CPU0 &cpu0 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate #define PAGE_BSZS_SHIFT(szc) TTE_BSZS_SHIFT(szc) 5567c478bd9Sstevel@tonic-gate /* 5577c478bd9Sstevel@tonic-gate * For sfmmu each larger page is 8 times the size of the previous 5587c478bd9Sstevel@tonic-gate * size page. 5597c478bd9Sstevel@tonic-gate */ 5607c478bd9Sstevel@tonic-gate #define FULL_REGION_CNT(rg_szc) (8) 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate /* 5637c478bd9Sstevel@tonic-gate * The counter base must be per page_counter element to prevent 5647c478bd9Sstevel@tonic-gate * races when re-indexing, and the base page size element should 5657c478bd9Sstevel@tonic-gate * be aligned on a boundary of the given region size. 5667c478bd9Sstevel@tonic-gate * 5677c478bd9Sstevel@tonic-gate * We also round up the number of pages spanned by the counters 5687c478bd9Sstevel@tonic-gate * for a given region to PC_BASE_ALIGN in certain situations to simplify 5697c478bd9Sstevel@tonic-gate * the coding for some non-performance critical routines. 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate #define PC_BASE_ALIGN ((pfn_t)1 << PAGE_BSZS_SHIFT(mmu_page_sizes-1)) 5727c478bd9Sstevel@tonic-gate #define PC_BASE_ALIGN_MASK (PC_BASE_ALIGN - 1) 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate extern int ecache_alignsize; 5757c478bd9Sstevel@tonic-gate #define L2CACHE_ALIGN ecache_alignsize 5766061ce8aSkchow #define L2CACHE_ALIGN_MAX 512 5777c478bd9Sstevel@tonic-gate 57885f58038Sdp extern int update_proc_pgcolorbase_after_fork; 5797c478bd9Sstevel@tonic-gate extern int consistent_coloring; 5807c478bd9Sstevel@tonic-gate extern uint_t vac_colors_mask; 5817c478bd9Sstevel@tonic-gate extern int vac_size; 5827c478bd9Sstevel@tonic-gate extern int vac_shift; 5837c478bd9Sstevel@tonic-gate 584bb121940Sdp /* 585bb121940Sdp * Kernel mem segment in 64-bit space 586bb121940Sdp */ 587bb121940Sdp extern caddr_t kmem64_base, kmem64_end, kmem64_aligned_end; 588bb121940Sdp extern int kmem64_alignsize, kmem64_szc; 589bb121940Sdp extern uint64_t kmem64_pabase; 590bb121940Sdp extern int max_bootlp_tteszc; 591bb121940Sdp 5927c478bd9Sstevel@tonic-gate /* 593ec25b48fSsusans * Maximum and default values for user heap, stack, private and shared 594ec25b48fSsusans * anonymous memory, and user text and initialized data. 595ec25b48fSsusans * 596ec25b48fSsusans * Initial values are defined in architecture specific mach_vm_dep.c file. 597ec25b48fSsusans * Used by map_pgsz*() routines. 598ec25b48fSsusans */ 599ec25b48fSsusans extern size_t max_uheap_lpsize; 600ec25b48fSsusans extern size_t default_uheap_lpsize; 601ec25b48fSsusans extern size_t max_ustack_lpsize; 602ec25b48fSsusans extern size_t default_ustack_lpsize; 603ec25b48fSsusans extern size_t max_privmap_lpsize; 604ec25b48fSsusans extern size_t max_uidata_lpsize; 605ec25b48fSsusans extern size_t max_utext_lpsize; 606ec25b48fSsusans extern size_t max_shm_lpsize; 607ec25b48fSsusans 608ec25b48fSsusans /* 609ec25b48fSsusans * For adjusting the default lpsize, for DTLB-limited page sizes. 610ec25b48fSsusans */ 611ec25b48fSsusans extern void adjust_data_maxlpsize(size_t ismpagesize); 612ec25b48fSsusans 613ec25b48fSsusans /* 614ec25b48fSsusans * Sanity control. Don't use large pages regardless of user 615ec25b48fSsusans * settings if there's less than priv or shm_lpg_min_physmem memory installed. 616ec25b48fSsusans * The units for this variable are 8K pages. 6177c478bd9Sstevel@tonic-gate */ 618ec25b48fSsusans extern pgcnt_t privm_lpg_min_physmem; 619ec25b48fSsusans extern pgcnt_t shm_lpg_min_physmem; 6207c478bd9Sstevel@tonic-gate 6217c478bd9Sstevel@tonic-gate /* 6227c478bd9Sstevel@tonic-gate * AS_2_BIN macro controls the page coloring policy. 6237c478bd9Sstevel@tonic-gate * 0 (default) uses various vaddr bits 6247c478bd9Sstevel@tonic-gate * 1 virtual=paddr 6257c478bd9Sstevel@tonic-gate * 2 bin hopping 6267c478bd9Sstevel@tonic-gate */ 627d94ffb28Sjmcp #define AS_2_BIN(as, seg, vp, addr, bin, szc) \ 6287c478bd9Sstevel@tonic-gate switch (consistent_coloring) { \ 6297c478bd9Sstevel@tonic-gate default: \ 6307c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, \ 6317c478bd9Sstevel@tonic-gate "AS_2_BIN: bad consistent coloring value"); \ 6327c478bd9Sstevel@tonic-gate /* assume default algorithm -> continue */ \ 633*4cf53d50SToomas Soome /* FALLTHROUGH */ \ 6347c478bd9Sstevel@tonic-gate case 0: { \ 6357c478bd9Sstevel@tonic-gate uint32_t ndx, new; \ 6367c478bd9Sstevel@tonic-gate int slew = 0; \ 6375d07b933Sdp pfn_t pfn; \ 6387c478bd9Sstevel@tonic-gate \ 6397c478bd9Sstevel@tonic-gate if (vp != NULL && IS_SWAPVP(vp) && \ 6405d07b933Sdp seg->s_ops == &segvn_ops) \ 6417c478bd9Sstevel@tonic-gate slew = as_color_bin(as); \ 6427c478bd9Sstevel@tonic-gate \ 6435d07b933Sdp pfn = ((uintptr_t)addr >> MMU_PAGESHIFT) + \ 6447c478bd9Sstevel@tonic-gate (((uintptr_t)addr >> page_coloring_shift) << \ 6455d07b933Sdp (vac_shift - MMU_PAGESHIFT)); \ 646102033aaSdp if ((szc) == 0 || &page_pfn_2_color_cpu == NULL) { \ 6475d07b933Sdp pfn += slew; \ 648ce8eb11aSdp bin = PFN_2_COLOR(pfn, szc, NULL); \ 6495d07b933Sdp } else { \ 650ce8eb11aSdp bin = PFN_2_COLOR(pfn, szc, NULL); \ 6515d07b933Sdp bin += slew >> (vac_shift - MMU_PAGESHIFT); \ 6525d07b933Sdp bin &= hw_page_array[(szc)].hp_colors - 1; \ 6535d07b933Sdp } \ 6547c478bd9Sstevel@tonic-gate break; \ 6557c478bd9Sstevel@tonic-gate } \ 6567c478bd9Sstevel@tonic-gate case 1: \ 657ce8eb11aSdp bin = PFN_2_COLOR(((uintptr_t)addr >> MMU_PAGESHIFT), \ 658ce8eb11aSdp szc, NULL); \ 6597c478bd9Sstevel@tonic-gate break; \ 6607c478bd9Sstevel@tonic-gate case 2: { \ 6617c478bd9Sstevel@tonic-gate int cnt = as_color_bin(as); \ 6625d07b933Sdp uint_t color_mask = page_get_pagecolors(0) - 1; \ 6635d07b933Sdp \ 6647c478bd9Sstevel@tonic-gate /* make sure physical color aligns with vac color */ \ 6657c478bd9Sstevel@tonic-gate while ((cnt & vac_colors_mask) != \ 6667c478bd9Sstevel@tonic-gate addr_to_vcolor(addr)) { \ 6677c478bd9Sstevel@tonic-gate cnt++; \ 6687c478bd9Sstevel@tonic-gate } \ 6695d07b933Sdp bin = cnt = cnt & color_mask; \ 6705d07b933Sdp bin >>= PAGE_GET_COLOR_SHIFT(0, szc); \ 6717c478bd9Sstevel@tonic-gate /* update per as page coloring fields */ \ 6725d07b933Sdp cnt = (cnt + 1) & color_mask; \ 6735d07b933Sdp if (cnt == (as_color_start(as) & color_mask)) { \ 6747c478bd9Sstevel@tonic-gate cnt = as_color_start(as) = as_color_start(as) + \ 6757c478bd9Sstevel@tonic-gate PGCLR_LOOPFACTOR; \ 6767c478bd9Sstevel@tonic-gate } \ 6775d07b933Sdp as_color_bin(as) = cnt & color_mask; \ 6787c478bd9Sstevel@tonic-gate break; \ 6797c478bd9Sstevel@tonic-gate } \ 6807c478bd9Sstevel@tonic-gate } \ 6815d07b933Sdp ASSERT(bin < page_get_pagecolors(szc)); 6827c478bd9Sstevel@tonic-gate 683affbd3ccSkchow /* 684affbd3ccSkchow * cpu private vm data - accessed thru CPU->cpu_vm_data 685affbd3ccSkchow * vc_pnum_memseg: tracks last memseg visited in page_numtopp_nolock() 686affbd3ccSkchow * vc_pnext_memseg: tracks last memseg visited in page_nextn() 687affbd3ccSkchow * vc_kmptr: unaligned kmem pointer for this vm_cpu_data_t 6886061ce8aSkchow * vc_kmsize: orignal kmem size for this vm_cpu_data_t 689affbd3ccSkchow */ 690affbd3ccSkchow 691affbd3ccSkchow typedef struct { 692affbd3ccSkchow struct memseg *vc_pnum_memseg; 693affbd3ccSkchow struct memseg *vc_pnext_memseg; 694affbd3ccSkchow void *vc_kmptr; 6956061ce8aSkchow size_t vc_kmsize; 696affbd3ccSkchow } vm_cpu_data_t; 697affbd3ccSkchow 698affbd3ccSkchow /* allocation size to ensure vm_cpu_data_t resides in its own cache line */ 699affbd3ccSkchow #define VM_CPU_DATA_PADSIZE \ 700affbd3ccSkchow (P2ROUNDUP(sizeof (vm_cpu_data_t), L2CACHE_ALIGN_MAX)) 701affbd3ccSkchow 7027c478bd9Sstevel@tonic-gate /* 7037c478bd9Sstevel@tonic-gate * Function to get an ecache color bin: F(as, cnt, vcolor). 7047c478bd9Sstevel@tonic-gate * the goal of this function is to: 7057c478bd9Sstevel@tonic-gate * - to spread a processes' physical pages across the entire ecache to 7067c478bd9Sstevel@tonic-gate * maximize its use. 7077c478bd9Sstevel@tonic-gate * - to minimize vac flushes caused when we reuse a physical page on a 7087c478bd9Sstevel@tonic-gate * different vac color than it was previously used. 7097c478bd9Sstevel@tonic-gate * - to prevent all processes to use the same exact colors and trash each 7107c478bd9Sstevel@tonic-gate * other. 7117c478bd9Sstevel@tonic-gate * 7127c478bd9Sstevel@tonic-gate * cnt is a bin ptr kept on a per as basis. As we page_create we increment 7137c478bd9Sstevel@tonic-gate * the ptr so we spread out the physical pages to cover the entire ecache. 7147c478bd9Sstevel@tonic-gate * The virtual color is made a subset of the physical color in order to 7157c478bd9Sstevel@tonic-gate * in minimize virtual cache flushing. 7167c478bd9Sstevel@tonic-gate * We add in the as to spread out different as. This happens when we 7177c478bd9Sstevel@tonic-gate * initialize the start count value. 7187c478bd9Sstevel@tonic-gate * sizeof(struct as) is 60 so we shift by 3 to get into the bit range 7197c478bd9Sstevel@tonic-gate * that will tend to change. For example, on spitfire based machines 7207c478bd9Sstevel@tonic-gate * (vcshft == 1) contigous as are spread bu ~6 bins. 7217c478bd9Sstevel@tonic-gate * vcshft provides for proper virtual color alignment. 7227c478bd9Sstevel@tonic-gate * In theory cnt should be updated using cas only but if we are off by one 7237c478bd9Sstevel@tonic-gate * or 2 it is no big deal. 7247c478bd9Sstevel@tonic-gate * We also keep a start value which is used to randomize on what bin we 7257c478bd9Sstevel@tonic-gate * start counting when it is time to start another loop. This avoids 7267c478bd9Sstevel@tonic-gate * contigous allocations of ecache size to point to the same bin. 7277c478bd9Sstevel@tonic-gate * Why 3? Seems work ok. Better than 7 or anything larger. 7287c478bd9Sstevel@tonic-gate */ 7297c478bd9Sstevel@tonic-gate #define PGCLR_LOOPFACTOR 3 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate /* 7327c478bd9Sstevel@tonic-gate * When a bin is empty, and we can't satisfy a color request correctly, 7337c478bd9Sstevel@tonic-gate * we scan. If we assume that the programs have reasonable spatial 7347c478bd9Sstevel@tonic-gate * behavior, then it will not be a good idea to use the adjacent color. 7357c478bd9Sstevel@tonic-gate * Using the adjacent color would result in virtually adjacent addresses 7367c478bd9Sstevel@tonic-gate * mapping into the same spot in the cache. So, if we stumble across 7377c478bd9Sstevel@tonic-gate * an empty bin, skip a bunch before looking. After the first skip, 7387c478bd9Sstevel@tonic-gate * then just look one bin at a time so we don't miss our cache on 7397c478bd9Sstevel@tonic-gate * every look. Be sure to check every bin. Page_create() will panic 7407c478bd9Sstevel@tonic-gate * if we miss a page. 7417c478bd9Sstevel@tonic-gate * 7427c478bd9Sstevel@tonic-gate * This also explains the `<=' in the for loops in both page_get_freelist() 7437c478bd9Sstevel@tonic-gate * and page_get_cachelist(). Since we checked the target bin, skipped 7447c478bd9Sstevel@tonic-gate * a bunch, then continued one a time, we wind up checking the target bin 7457c478bd9Sstevel@tonic-gate * twice to make sure we get all of them bins. 7467c478bd9Sstevel@tonic-gate */ 7477c478bd9Sstevel@tonic-gate #define BIN_STEP 20 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate #ifdef VM_STATS 7507c478bd9Sstevel@tonic-gate struct vmm_vmstats_str { 751d94ffb28Sjmcp ulong_t pgf_alloc[MMU_PAGE_SIZES]; /* page_get_freelist */ 752d94ffb28Sjmcp ulong_t pgf_allocok[MMU_PAGE_SIZES]; 753d94ffb28Sjmcp ulong_t pgf_allocokrem[MMU_PAGE_SIZES]; 754d94ffb28Sjmcp ulong_t pgf_allocfailed[MMU_PAGE_SIZES]; 755d94ffb28Sjmcp ulong_t pgf_allocdeferred; 756d94ffb28Sjmcp ulong_t pgf_allocretry[MMU_PAGE_SIZES]; 757affbd3ccSkchow ulong_t pgc_alloc; /* page_get_cachelist */ 7587c478bd9Sstevel@tonic-gate ulong_t pgc_allocok; 7597c478bd9Sstevel@tonic-gate ulong_t pgc_allocokrem; 760d94ffb28Sjmcp ulong_t pgc_allocokdeferred; 7617c478bd9Sstevel@tonic-gate ulong_t pgc_allocfailed; 762affbd3ccSkchow ulong_t pgcp_alloc[MMU_PAGE_SIZES]; /* page_get_contig_pages */ 7637c478bd9Sstevel@tonic-gate ulong_t pgcp_allocfailed[MMU_PAGE_SIZES]; 7647c478bd9Sstevel@tonic-gate ulong_t pgcp_allocempty[MMU_PAGE_SIZES]; 7657c478bd9Sstevel@tonic-gate ulong_t pgcp_allocok[MMU_PAGE_SIZES]; 766affbd3ccSkchow ulong_t ptcp[MMU_PAGE_SIZES]; /* page_trylock_contig_pages */ 7677c478bd9Sstevel@tonic-gate ulong_t ptcpfreethresh[MMU_PAGE_SIZES]; 7687c478bd9Sstevel@tonic-gate ulong_t ptcpfailexcl[MMU_PAGE_SIZES]; 7697c478bd9Sstevel@tonic-gate ulong_t ptcpfailszc[MMU_PAGE_SIZES]; 7707c478bd9Sstevel@tonic-gate ulong_t ptcpfailcage[MMU_PAGE_SIZES]; 7717c478bd9Sstevel@tonic-gate ulong_t ptcpok[MMU_PAGE_SIZES]; 772affbd3ccSkchow ulong_t pgmf_alloc[MMU_PAGE_SIZES]; /* page_get_mnode_freelist */ 7737c478bd9Sstevel@tonic-gate ulong_t pgmf_allocfailed[MMU_PAGE_SIZES]; 7747c478bd9Sstevel@tonic-gate ulong_t pgmf_allocempty[MMU_PAGE_SIZES]; 7757c478bd9Sstevel@tonic-gate ulong_t pgmf_allocok[MMU_PAGE_SIZES]; 776affbd3ccSkchow ulong_t pgmc_alloc; /* page_get_mnode_cachelist */ 7777c478bd9Sstevel@tonic-gate ulong_t pgmc_allocfailed; 7787c478bd9Sstevel@tonic-gate ulong_t pgmc_allocempty; 7797c478bd9Sstevel@tonic-gate ulong_t pgmc_allocok; 780affbd3ccSkchow ulong_t pladd_free[MMU_PAGE_SIZES]; /* page_list_add/sub */ 781affbd3ccSkchow ulong_t plsub_free[MMU_PAGE_SIZES]; 782affbd3ccSkchow ulong_t pladd_cache; 783affbd3ccSkchow ulong_t plsub_cache; 784affbd3ccSkchow ulong_t plsubpages_szcbig; 785affbd3ccSkchow ulong_t plsubpages_szc0; 7865d07b933Sdp ulong_t pfs_req[MMU_PAGE_SIZES]; /* page_freelist_split */ 7875d07b933Sdp ulong_t pfs_demote[MMU_PAGE_SIZES]; 7885d07b933Sdp ulong_t pfc_coalok[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 789d94ffb28Sjmcp ulong_t ppr_reloc[MMU_PAGE_SIZES]; /* page_relocate */ 790d94ffb28Sjmcp ulong_t ppr_relocok[MMU_PAGE_SIZES]; 7917c478bd9Sstevel@tonic-gate ulong_t ppr_relocnoroot[MMU_PAGE_SIZES]; 7927c478bd9Sstevel@tonic-gate ulong_t ppr_reloc_replnoroot[MMU_PAGE_SIZES]; 7937c478bd9Sstevel@tonic-gate ulong_t ppr_relocnolock[MMU_PAGE_SIZES]; 7947c478bd9Sstevel@tonic-gate ulong_t ppr_relocnomem[MMU_PAGE_SIZES]; 7957c478bd9Sstevel@tonic-gate ulong_t ppr_krelocfail[MMU_PAGE_SIZES]; 7968b464eb8Smec ulong_t ppr_copyfail; 7975d07b933Sdp /* page coalesce counter */ 798d94ffb28Sjmcp ulong_t page_ctrs_coalesce[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 7995d07b933Sdp /* candidates useful */ 800d94ffb28Sjmcp ulong_t page_ctrs_cands_skip[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 8015d07b933Sdp /* ctrs changed after locking */ 802d94ffb28Sjmcp ulong_t page_ctrs_changed[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 8035d07b933Sdp /* page_freelist_coalesce failed */ 804d94ffb28Sjmcp ulong_t page_ctrs_failed[MMU_PAGE_SIZES][MAX_MNODE_MRANGES]; 805d94ffb28Sjmcp ulong_t page_ctrs_coalesce_all; /* page coalesce all counter */ 806d94ffb28Sjmcp ulong_t page_ctrs_cands_skip_all; /* candidates useful for all func */ 8077c478bd9Sstevel@tonic-gate }; 8087c478bd9Sstevel@tonic-gate extern struct vmm_vmstats_str vmm_vmstats; 8097c478bd9Sstevel@tonic-gate #endif /* VM_STATS */ 8107c478bd9Sstevel@tonic-gate 8117c478bd9Sstevel@tonic-gate /* 8127c478bd9Sstevel@tonic-gate * Used to hold off page relocations into the cage until OBP has completed 8137c478bd9Sstevel@tonic-gate * its boot-time handoff of its resources to the kernel. 8147c478bd9Sstevel@tonic-gate */ 8157c478bd9Sstevel@tonic-gate extern int page_relocate_ready; 8167c478bd9Sstevel@tonic-gate 8177c478bd9Sstevel@tonic-gate /* 8187c478bd9Sstevel@tonic-gate * cpu/mmu-dependent vm variables may be reset at bootup. 8197c478bd9Sstevel@tonic-gate */ 8207c478bd9Sstevel@tonic-gate extern uint_t mmu_page_sizes; 8217c478bd9Sstevel@tonic-gate extern uint_t max_mmu_page_sizes; 8227c478bd9Sstevel@tonic-gate extern uint_t mmu_hashcnt; 8237c478bd9Sstevel@tonic-gate extern uint_t max_mmu_hashcnt; 8247c478bd9Sstevel@tonic-gate extern size_t mmu_ism_pagesize; 8257c478bd9Sstevel@tonic-gate extern int mmu_exported_pagesize_mask; 8267c478bd9Sstevel@tonic-gate extern uint_t mmu_exported_page_sizes; 8277c478bd9Sstevel@tonic-gate extern uint_t szc_2_userszc[]; 8287c478bd9Sstevel@tonic-gate extern uint_t userszc_2_szc[]; 8297c478bd9Sstevel@tonic-gate 83002bc52beSkchow #define mmu_legacy_page_sizes mmu_exported_page_sizes 8317c478bd9Sstevel@tonic-gate #define USERSZC_2_SZC(userszc) (userszc_2_szc[userszc]) 8327c478bd9Sstevel@tonic-gate #define SZC_2_USERSZC(szc) (szc_2_userszc[szc]) 8337c478bd9Sstevel@tonic-gate 8347c478bd9Sstevel@tonic-gate /* 8357c478bd9Sstevel@tonic-gate * Platform specific page routines 8367c478bd9Sstevel@tonic-gate */ 8377c478bd9Sstevel@tonic-gate extern void mach_page_add(page_t **, page_t *); 8387c478bd9Sstevel@tonic-gate extern void mach_page_sub(page_t **, page_t *); 8397c478bd9Sstevel@tonic-gate extern uint_t page_get_pagecolors(uint_t); 8407c478bd9Sstevel@tonic-gate extern void ppcopy_kernel__relocatable(page_t *, page_t *); 8417c478bd9Sstevel@tonic-gate #define ppcopy_kernel(p1, p2) ppcopy_kernel__relocatable(p1, p2) 8427c478bd9Sstevel@tonic-gate 8437c478bd9Sstevel@tonic-gate /* 8447c478bd9Sstevel@tonic-gate * platform specific large pages for kernel heap support 8457c478bd9Sstevel@tonic-gate */ 8467c478bd9Sstevel@tonic-gate extern size_t get_segkmem_lpsize(size_t lpsize); 8477c478bd9Sstevel@tonic-gate extern size_t mmu_get_kernel_lpsize(size_t lpsize); 8487c478bd9Sstevel@tonic-gate extern void mmu_init_kernel_pgsz(struct hat *hat); 8497c478bd9Sstevel@tonic-gate extern void mmu_init_kcontext(); 8507c478bd9Sstevel@tonic-gate extern uint64_t kcontextreg; 8517c478bd9Sstevel@tonic-gate 8521e45ea5aSblakej /* 8531e45ea5aSblakej * Nucleus data page allocator routines 8541e45ea5aSblakej */ 8551e45ea5aSblakej extern void ndata_alloc_init(struct memlist *, uintptr_t, uintptr_t); 8561e45ea5aSblakej extern void *ndata_alloc(struct memlist *, size_t, size_t); 8571e45ea5aSblakej extern void *ndata_extra_base(struct memlist *, size_t, caddr_t); 8581e45ea5aSblakej extern size_t ndata_maxsize(struct memlist *); 8591e45ea5aSblakej extern size_t ndata_spare(struct memlist *, size_t, size_t); 8601e45ea5aSblakej 8617c478bd9Sstevel@tonic-gate #ifdef __cplusplus 8627c478bd9Sstevel@tonic-gate } 8637c478bd9Sstevel@tonic-gate #endif 8647c478bd9Sstevel@tonic-gate 8657c478bd9Sstevel@tonic-gate #endif /* _VM_DEP_H */ 866