1*7c478bd9Sstevel@tonic-gate /* 2*7c478bd9Sstevel@tonic-gate * CDDL HEADER START 3*7c478bd9Sstevel@tonic-gate * 4*7c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*7c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*7c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*7c478bd9Sstevel@tonic-gate * with the License. 8*7c478bd9Sstevel@tonic-gate * 9*7c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*7c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*7c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 12*7c478bd9Sstevel@tonic-gate * and limitations under the License. 13*7c478bd9Sstevel@tonic-gate * 14*7c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*7c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*7c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*7c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*7c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*7c478bd9Sstevel@tonic-gate * 20*7c478bd9Sstevel@tonic-gate * CDDL HEADER END 21*7c478bd9Sstevel@tonic-gate */ 22*7c478bd9Sstevel@tonic-gate /* 23*7c478bd9Sstevel@tonic-gate * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24*7c478bd9Sstevel@tonic-gate * Use is subject to license terms. 25*7c478bd9Sstevel@tonic-gate */ 26*7c478bd9Sstevel@tonic-gate 27*7c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*7c478bd9Sstevel@tonic-gate 29*7c478bd9Sstevel@tonic-gate #include <sys/systm.h> 30*7c478bd9Sstevel@tonic-gate #include <sys/platform_module.h> 31*7c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 32*7c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 33*7c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 34*7c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 35*7c478bd9Sstevel@tonic-gate 36*7c478bd9Sstevel@tonic-gate int max_mem_nodes = 1; /* max memory nodes on this system */ 37*7c478bd9Sstevel@tonic-gate 38*7c478bd9Sstevel@tonic-gate struct mem_node_conf mem_node_config[MAX_MEM_NODES]; 39*7c478bd9Sstevel@tonic-gate int mem_node_pfn_shift; 40*7c478bd9Sstevel@tonic-gate /* 41*7c478bd9Sstevel@tonic-gate * num_memnodes should be updated atomically and always >= 42*7c478bd9Sstevel@tonic-gate * the number of bits in memnodes_mask or the algorithm may fail. 43*7c478bd9Sstevel@tonic-gate */ 44*7c478bd9Sstevel@tonic-gate uint16_t num_memnodes; 45*7c478bd9Sstevel@tonic-gate mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */ 46*7c478bd9Sstevel@tonic-gate 47*7c478bd9Sstevel@tonic-gate /* 48*7c478bd9Sstevel@tonic-gate * If set, mem_node_physalign should be a power of two, and 49*7c478bd9Sstevel@tonic-gate * should reflect the minimum address alignment of each node. 50*7c478bd9Sstevel@tonic-gate */ 51*7c478bd9Sstevel@tonic-gate uint64_t mem_node_physalign; 52*7c478bd9Sstevel@tonic-gate 53*7c478bd9Sstevel@tonic-gate /* 54*7c478bd9Sstevel@tonic-gate * Platform hooks we will need. 55*7c478bd9Sstevel@tonic-gate */ 56*7c478bd9Sstevel@tonic-gate 57*7c478bd9Sstevel@tonic-gate #pragma weak plat_build_mem_nodes 58*7c478bd9Sstevel@tonic-gate #pragma weak plat_slice_add 59*7c478bd9Sstevel@tonic-gate #pragma weak plat_slice_del 60*7c478bd9Sstevel@tonic-gate 61*7c478bd9Sstevel@tonic-gate /* 62*7c478bd9Sstevel@tonic-gate * Adjust the memnode config after a DR operation. 63*7c478bd9Sstevel@tonic-gate * 64*7c478bd9Sstevel@tonic-gate * It is rather tricky to do these updates since we can't 65*7c478bd9Sstevel@tonic-gate * protect the memnode structures with locks, so we must 66*7c478bd9Sstevel@tonic-gate * be mindful of the order in which updates and reads to 67*7c478bd9Sstevel@tonic-gate * these values can occur. 68*7c478bd9Sstevel@tonic-gate */ 69*7c478bd9Sstevel@tonic-gate void 70*7c478bd9Sstevel@tonic-gate mem_node_add_slice(pfn_t start, pfn_t end) 71*7c478bd9Sstevel@tonic-gate { 72*7c478bd9Sstevel@tonic-gate int mnode; 73*7c478bd9Sstevel@tonic-gate mnodeset_t newmask, oldmask; 74*7c478bd9Sstevel@tonic-gate 75*7c478bd9Sstevel@tonic-gate /* 76*7c478bd9Sstevel@tonic-gate * DR will pass us the first pfn that is allocatable. 77*7c478bd9Sstevel@tonic-gate * We need to round down to get the real start of 78*7c478bd9Sstevel@tonic-gate * the slice. 79*7c478bd9Sstevel@tonic-gate */ 80*7c478bd9Sstevel@tonic-gate if (mem_node_physalign) { 81*7c478bd9Sstevel@tonic-gate start &= ~(btop(mem_node_physalign) - 1); 82*7c478bd9Sstevel@tonic-gate end = roundup(end, btop(mem_node_physalign)) - 1; 83*7c478bd9Sstevel@tonic-gate } 84*7c478bd9Sstevel@tonic-gate 85*7c478bd9Sstevel@tonic-gate if (&plat_slice_add) 86*7c478bd9Sstevel@tonic-gate plat_slice_add(start, end); 87*7c478bd9Sstevel@tonic-gate 88*7c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(start); 89*7c478bd9Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 90*7c478bd9Sstevel@tonic-gate 91*7c478bd9Sstevel@tonic-gate if (cas32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) { 92*7c478bd9Sstevel@tonic-gate /* 93*7c478bd9Sstevel@tonic-gate * Add slice to existing node. 94*7c478bd9Sstevel@tonic-gate */ 95*7c478bd9Sstevel@tonic-gate if (start < mem_node_config[mnode].physbase) 96*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = start; 97*7c478bd9Sstevel@tonic-gate if (end > mem_node_config[mnode].physmax) 98*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = end; 99*7c478bd9Sstevel@tonic-gate } else { 100*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = start; 101*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = end; 102*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].cursize = 0; 103*7c478bd9Sstevel@tonic-gate atomic_add_16(&num_memnodes, 1); 104*7c478bd9Sstevel@tonic-gate do { 105*7c478bd9Sstevel@tonic-gate oldmask = memnodes_mask; 106*7c478bd9Sstevel@tonic-gate newmask = memnodes_mask | (1ull << mnode); 107*7c478bd9Sstevel@tonic-gate } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask); 108*7c478bd9Sstevel@tonic-gate } 109*7c478bd9Sstevel@tonic-gate /* 110*7c478bd9Sstevel@tonic-gate * Let the common lgrp framework know about the new memory 111*7c478bd9Sstevel@tonic-gate */ 112*7c478bd9Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode)); 113*7c478bd9Sstevel@tonic-gate } 114*7c478bd9Sstevel@tonic-gate 115*7c478bd9Sstevel@tonic-gate /* ARGSUSED */ 116*7c478bd9Sstevel@tonic-gate void 117*7c478bd9Sstevel@tonic-gate mem_node_pre_del_slice(pfn_t start, pfn_t end) 118*7c478bd9Sstevel@tonic-gate { 119*7c478bd9Sstevel@tonic-gate int mnode = PFN_2_MEM_NODE(start); 120*7c478bd9Sstevel@tonic-gate 121*7c478bd9Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 122*7c478bd9Sstevel@tonic-gate ASSERT(mem_node_config[mnode].exists == 1); 123*7c478bd9Sstevel@tonic-gate } 124*7c478bd9Sstevel@tonic-gate 125*7c478bd9Sstevel@tonic-gate /* 126*7c478bd9Sstevel@tonic-gate * Remove a PFN range from a memnode. On some platforms, 127*7c478bd9Sstevel@tonic-gate * the memnode will be created with physbase at the first 128*7c478bd9Sstevel@tonic-gate * allocatable PFN, but later deleted with the MC slice 129*7c478bd9Sstevel@tonic-gate * base address converted to a PFN, in which case we need 130*7c478bd9Sstevel@tonic-gate * to assume physbase and up. 131*7c478bd9Sstevel@tonic-gate */ 132*7c478bd9Sstevel@tonic-gate void 133*7c478bd9Sstevel@tonic-gate mem_node_post_del_slice(pfn_t start, pfn_t end, int cancelled) 134*7c478bd9Sstevel@tonic-gate { 135*7c478bd9Sstevel@tonic-gate int mnode; 136*7c478bd9Sstevel@tonic-gate pgcnt_t delta_pgcnt, node_size; 137*7c478bd9Sstevel@tonic-gate mnodeset_t omask, nmask; 138*7c478bd9Sstevel@tonic-gate 139*7c478bd9Sstevel@tonic-gate if (mem_node_physalign) { 140*7c478bd9Sstevel@tonic-gate start &= ~(btop(mem_node_physalign) - 1); 141*7c478bd9Sstevel@tonic-gate end = roundup(end, btop(mem_node_physalign)) - 1; 142*7c478bd9Sstevel@tonic-gate } 143*7c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(start); 144*7c478bd9Sstevel@tonic-gate 145*7c478bd9Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 146*7c478bd9Sstevel@tonic-gate ASSERT(mem_node_config[mnode].exists == 1); 147*7c478bd9Sstevel@tonic-gate 148*7c478bd9Sstevel@tonic-gate if (!cancelled) { 149*7c478bd9Sstevel@tonic-gate delta_pgcnt = end - start; 150*7c478bd9Sstevel@tonic-gate node_size = mem_node_config[mnode].physmax - 151*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase; 152*7c478bd9Sstevel@tonic-gate 153*7c478bd9Sstevel@tonic-gate if (node_size > delta_pgcnt) { 154*7c478bd9Sstevel@tonic-gate /* 155*7c478bd9Sstevel@tonic-gate * Subtract the slice from the memnode. 156*7c478bd9Sstevel@tonic-gate */ 157*7c478bd9Sstevel@tonic-gate if (start <= mem_node_config[mnode].physbase) 158*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = end + 1; 159*7c478bd9Sstevel@tonic-gate ASSERT(end <= mem_node_config[mnode].physmax); 160*7c478bd9Sstevel@tonic-gate if (end == mem_node_config[mnode].physmax) 161*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = start - 1; 162*7c478bd9Sstevel@tonic-gate } else { 163*7c478bd9Sstevel@tonic-gate 164*7c478bd9Sstevel@tonic-gate /* 165*7c478bd9Sstevel@tonic-gate * Let the common lgrp framework know the mnode is 166*7c478bd9Sstevel@tonic-gate * leaving 167*7c478bd9Sstevel@tonic-gate */ 168*7c478bd9Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_MEM_DEL, mnode, 169*7c478bd9Sstevel@tonic-gate MEM_NODE_2_LGRPHAND(mnode)); 170*7c478bd9Sstevel@tonic-gate 171*7c478bd9Sstevel@tonic-gate /* 172*7c478bd9Sstevel@tonic-gate * Delete the whole node. 173*7c478bd9Sstevel@tonic-gate */ 174*7c478bd9Sstevel@tonic-gate ASSERT(mem_node_config[mnode].cursize == 0); 175*7c478bd9Sstevel@tonic-gate do { 176*7c478bd9Sstevel@tonic-gate omask = memnodes_mask; 177*7c478bd9Sstevel@tonic-gate nmask = omask & ~(1ull << mnode); 178*7c478bd9Sstevel@tonic-gate } while (cas64(&memnodes_mask, omask, nmask) != omask); 179*7c478bd9Sstevel@tonic-gate atomic_add_16(&num_memnodes, -1); 180*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].exists = 0; 181*7c478bd9Sstevel@tonic-gate } 182*7c478bd9Sstevel@tonic-gate 183*7c478bd9Sstevel@tonic-gate if (&plat_slice_del) 184*7c478bd9Sstevel@tonic-gate plat_slice_del(start, end); 185*7c478bd9Sstevel@tonic-gate } 186*7c478bd9Sstevel@tonic-gate } 187*7c478bd9Sstevel@tonic-gate 188*7c478bd9Sstevel@tonic-gate void 189*7c478bd9Sstevel@tonic-gate startup_build_mem_nodes(u_longlong_t *list, size_t nelems) 190*7c478bd9Sstevel@tonic-gate { 191*7c478bd9Sstevel@tonic-gate size_t elem; 192*7c478bd9Sstevel@tonic-gate pfn_t basepfn; 193*7c478bd9Sstevel@tonic-gate pgcnt_t npgs; 194*7c478bd9Sstevel@tonic-gate 195*7c478bd9Sstevel@tonic-gate /* LINTED: ASSERT will always true or false */ 196*7c478bd9Sstevel@tonic-gate ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes); 197*7c478bd9Sstevel@tonic-gate 198*7c478bd9Sstevel@tonic-gate if (&plat_build_mem_nodes) { 199*7c478bd9Sstevel@tonic-gate plat_build_mem_nodes(list, nelems); 200*7c478bd9Sstevel@tonic-gate } else { 201*7c478bd9Sstevel@tonic-gate /* 202*7c478bd9Sstevel@tonic-gate * Boot install lists are arranged <addr, len>, ... 203*7c478bd9Sstevel@tonic-gate */ 204*7c478bd9Sstevel@tonic-gate for (elem = 0; elem < nelems; elem += 2) { 205*7c478bd9Sstevel@tonic-gate basepfn = btop(list[elem]); 206*7c478bd9Sstevel@tonic-gate npgs = btop(list[elem+1]); 207*7c478bd9Sstevel@tonic-gate mem_node_add_slice(basepfn, basepfn + npgs - 1); 208*7c478bd9Sstevel@tonic-gate } 209*7c478bd9Sstevel@tonic-gate mem_node_physalign = 0; 210*7c478bd9Sstevel@tonic-gate mem_node_pfn_shift = 0; 211*7c478bd9Sstevel@tonic-gate } 212*7c478bd9Sstevel@tonic-gate } 213*7c478bd9Sstevel@tonic-gate 214*7c478bd9Sstevel@tonic-gate /* 215*7c478bd9Sstevel@tonic-gate * Allocate an unassigned memnode. 216*7c478bd9Sstevel@tonic-gate */ 217*7c478bd9Sstevel@tonic-gate int 218*7c478bd9Sstevel@tonic-gate mem_node_alloc() 219*7c478bd9Sstevel@tonic-gate { 220*7c478bd9Sstevel@tonic-gate int mnode; 221*7c478bd9Sstevel@tonic-gate mnodeset_t newmask, oldmask; 222*7c478bd9Sstevel@tonic-gate 223*7c478bd9Sstevel@tonic-gate /* 224*7c478bd9Sstevel@tonic-gate * Find an unused memnode. Update it atomically to prevent 225*7c478bd9Sstevel@tonic-gate * a first time memnode creation race. 226*7c478bd9Sstevel@tonic-gate */ 227*7c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 228*7c478bd9Sstevel@tonic-gate if (cas32((uint32_t *)&mem_node_config[mnode].exists, 229*7c478bd9Sstevel@tonic-gate 0, 1) == 0) 230*7c478bd9Sstevel@tonic-gate break; 231*7c478bd9Sstevel@tonic-gate 232*7c478bd9Sstevel@tonic-gate if (mnode >= max_mem_nodes) 233*7c478bd9Sstevel@tonic-gate panic("Out of free memnodes\n"); 234*7c478bd9Sstevel@tonic-gate 235*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = (uint64_t)-1; 236*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = 0; 237*7c478bd9Sstevel@tonic-gate mem_node_config[mnode].cursize = 0; 238*7c478bd9Sstevel@tonic-gate atomic_add_16(&num_memnodes, 1); 239*7c478bd9Sstevel@tonic-gate do { 240*7c478bd9Sstevel@tonic-gate oldmask = memnodes_mask; 241*7c478bd9Sstevel@tonic-gate newmask = memnodes_mask | (1ull << mnode); 242*7c478bd9Sstevel@tonic-gate } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask); 243*7c478bd9Sstevel@tonic-gate 244*7c478bd9Sstevel@tonic-gate return (mnode); 245*7c478bd9Sstevel@tonic-gate } 246*7c478bd9Sstevel@tonic-gate 247*7c478bd9Sstevel@tonic-gate /* 248*7c478bd9Sstevel@tonic-gate * Find the intersection between a memnode and a memlist 249*7c478bd9Sstevel@tonic-gate * and returns the number of pages that overlap. 250*7c478bd9Sstevel@tonic-gate * 251*7c478bd9Sstevel@tonic-gate * Assumes the list is protected from DR operations by 252*7c478bd9Sstevel@tonic-gate * the memlist lock. 253*7c478bd9Sstevel@tonic-gate */ 254*7c478bd9Sstevel@tonic-gate pgcnt_t 255*7c478bd9Sstevel@tonic-gate mem_node_memlist_pages(int mnode, struct memlist *mlist) 256*7c478bd9Sstevel@tonic-gate { 257*7c478bd9Sstevel@tonic-gate pfn_t base, end; 258*7c478bd9Sstevel@tonic-gate pfn_t cur_base, cur_end; 259*7c478bd9Sstevel@tonic-gate pgcnt_t npgs; 260*7c478bd9Sstevel@tonic-gate struct memlist *pmem; 261*7c478bd9Sstevel@tonic-gate 262*7c478bd9Sstevel@tonic-gate base = mem_node_config[mnode].physbase; 263*7c478bd9Sstevel@tonic-gate end = mem_node_config[mnode].physmax; 264*7c478bd9Sstevel@tonic-gate npgs = 0; 265*7c478bd9Sstevel@tonic-gate 266*7c478bd9Sstevel@tonic-gate memlist_read_lock(); 267*7c478bd9Sstevel@tonic-gate 268*7c478bd9Sstevel@tonic-gate for (pmem = mlist; pmem; pmem = pmem->next) { 269*7c478bd9Sstevel@tonic-gate cur_base = btop(pmem->address); 270*7c478bd9Sstevel@tonic-gate cur_end = cur_base + btop(pmem->size) - 1; 271*7c478bd9Sstevel@tonic-gate if (end <= cur_base || base >= cur_end) 272*7c478bd9Sstevel@tonic-gate continue; 273*7c478bd9Sstevel@tonic-gate npgs = npgs + (MIN(cur_end, end) - 274*7c478bd9Sstevel@tonic-gate MAX(cur_base, base)) + 1; 275*7c478bd9Sstevel@tonic-gate } 276*7c478bd9Sstevel@tonic-gate 277*7c478bd9Sstevel@tonic-gate memlist_read_unlock(); 278*7c478bd9Sstevel@tonic-gate 279*7c478bd9Sstevel@tonic-gate return (npgs); 280*7c478bd9Sstevel@tonic-gate } 281