17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ce8eb11aSdp * Common Development and Distribution License (the "License"). 6ce8eb11aSdp * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22ce8eb11aSdp * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/systm.h> 297c478bd9Sstevel@tonic-gate #include <sys/platform_module.h> 307c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 317c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 327c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 337c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 34affbd3ccSkchow #include <vm/vm_dep.h> 357c478bd9Sstevel@tonic-gate 367c478bd9Sstevel@tonic-gate int max_mem_nodes = 1; /* max memory nodes on this system */ 377c478bd9Sstevel@tonic-gate 387c478bd9Sstevel@tonic-gate struct mem_node_conf mem_node_config[MAX_MEM_NODES]; 397c478bd9Sstevel@tonic-gate int mem_node_pfn_shift; 407c478bd9Sstevel@tonic-gate /* 417c478bd9Sstevel@tonic-gate * num_memnodes should be updated atomically and always >= 427c478bd9Sstevel@tonic-gate * the number of bits in memnodes_mask or the algorithm may fail. 437c478bd9Sstevel@tonic-gate */ 447c478bd9Sstevel@tonic-gate uint16_t num_memnodes; 457c478bd9Sstevel@tonic-gate mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */ 467c478bd9Sstevel@tonic-gate 477c478bd9Sstevel@tonic-gate /* 487c478bd9Sstevel@tonic-gate * If set, mem_node_physalign should be a power of two, and 497c478bd9Sstevel@tonic-gate * should reflect the minimum address alignment of each node. 507c478bd9Sstevel@tonic-gate */ 517c478bd9Sstevel@tonic-gate uint64_t mem_node_physalign; 527c478bd9Sstevel@tonic-gate 537c478bd9Sstevel@tonic-gate /* 547c478bd9Sstevel@tonic-gate * Platform hooks we will need. 557c478bd9Sstevel@tonic-gate */ 567c478bd9Sstevel@tonic-gate 577c478bd9Sstevel@tonic-gate #pragma weak plat_build_mem_nodes 587c478bd9Sstevel@tonic-gate #pragma weak plat_slice_add 597c478bd9Sstevel@tonic-gate #pragma weak plat_slice_del 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate /* 627c478bd9Sstevel@tonic-gate * Adjust the memnode config after a DR operation. 637c478bd9Sstevel@tonic-gate * 647c478bd9Sstevel@tonic-gate * It is rather tricky to do these updates since we can't 657c478bd9Sstevel@tonic-gate * protect the memnode structures with locks, so we must 667c478bd9Sstevel@tonic-gate * be mindful of the order in which updates and reads to 677c478bd9Sstevel@tonic-gate * these values can occur. 687c478bd9Sstevel@tonic-gate */ 697c478bd9Sstevel@tonic-gate void 707c478bd9Sstevel@tonic-gate mem_node_add_slice(pfn_t start, pfn_t end) 717c478bd9Sstevel@tonic-gate { 727c478bd9Sstevel@tonic-gate int mnode; 737c478bd9Sstevel@tonic-gate mnodeset_t newmask, oldmask; 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate /* 767c478bd9Sstevel@tonic-gate * DR will pass us the first pfn that is allocatable. 777c478bd9Sstevel@tonic-gate * We need to round down to get the real start of 787c478bd9Sstevel@tonic-gate * the slice. 797c478bd9Sstevel@tonic-gate */ 807c478bd9Sstevel@tonic-gate if (mem_node_physalign) { 817c478bd9Sstevel@tonic-gate start &= ~(btop(mem_node_physalign) - 1); 827c478bd9Sstevel@tonic-gate end = roundup(end, btop(mem_node_physalign)) - 1; 837c478bd9Sstevel@tonic-gate } 847c478bd9Sstevel@tonic-gate 85ce8eb11aSdp if (&plat_slice_add != NULL) 867c478bd9Sstevel@tonic-gate plat_slice_add(start, end); 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(start); 897c478bd9Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate if (cas32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) { 927c478bd9Sstevel@tonic-gate /* 937c478bd9Sstevel@tonic-gate * Add slice to existing node. 947c478bd9Sstevel@tonic-gate */ 957c478bd9Sstevel@tonic-gate if (start < mem_node_config[mnode].physbase) 967c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = start; 977c478bd9Sstevel@tonic-gate if (end > mem_node_config[mnode].physmax) 987c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = end; 997c478bd9Sstevel@tonic-gate } else { 1007c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = start; 1017c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = end; 1027c478bd9Sstevel@tonic-gate atomic_add_16(&num_memnodes, 1); 1037c478bd9Sstevel@tonic-gate do { 1047c478bd9Sstevel@tonic-gate oldmask = memnodes_mask; 1057c478bd9Sstevel@tonic-gate newmask = memnodes_mask | (1ull << mnode); 1067c478bd9Sstevel@tonic-gate } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask); 1077c478bd9Sstevel@tonic-gate } 1087c478bd9Sstevel@tonic-gate /* 1097c478bd9Sstevel@tonic-gate * Let the common lgrp framework know about the new memory 1107c478bd9Sstevel@tonic-gate */ 1117c478bd9Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode)); 1127c478bd9Sstevel@tonic-gate } 1137c478bd9Sstevel@tonic-gate 1147c478bd9Sstevel@tonic-gate /* ARGSUSED */ 1157c478bd9Sstevel@tonic-gate void 1167c478bd9Sstevel@tonic-gate mem_node_pre_del_slice(pfn_t start, pfn_t end) 1177c478bd9Sstevel@tonic-gate { 1187c478bd9Sstevel@tonic-gate int mnode = PFN_2_MEM_NODE(start); 1197c478bd9Sstevel@tonic-gate 1207c478bd9Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 1217c478bd9Sstevel@tonic-gate ASSERT(mem_node_config[mnode].exists == 1); 1227c478bd9Sstevel@tonic-gate } 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate /* 1257c478bd9Sstevel@tonic-gate * Remove a PFN range from a memnode. On some platforms, 1267c478bd9Sstevel@tonic-gate * the memnode will be created with physbase at the first 1277c478bd9Sstevel@tonic-gate * allocatable PFN, but later deleted with the MC slice 1287c478bd9Sstevel@tonic-gate * base address converted to a PFN, in which case we need 1297c478bd9Sstevel@tonic-gate * to assume physbase and up. 1307c478bd9Sstevel@tonic-gate */ 1317c478bd9Sstevel@tonic-gate void 1327c478bd9Sstevel@tonic-gate mem_node_post_del_slice(pfn_t start, pfn_t end, int cancelled) 1337c478bd9Sstevel@tonic-gate { 1347c478bd9Sstevel@tonic-gate int mnode; 1357c478bd9Sstevel@tonic-gate pgcnt_t delta_pgcnt, node_size; 1367c478bd9Sstevel@tonic-gate mnodeset_t omask, nmask; 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate if (mem_node_physalign) { 1397c478bd9Sstevel@tonic-gate start &= ~(btop(mem_node_physalign) - 1); 1407c478bd9Sstevel@tonic-gate end = roundup(end, btop(mem_node_physalign)) - 1; 1417c478bd9Sstevel@tonic-gate } 1427c478bd9Sstevel@tonic-gate mnode = PFN_2_MEM_NODE(start); 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate ASSERT(mnode < max_mem_nodes); 1457c478bd9Sstevel@tonic-gate ASSERT(mem_node_config[mnode].exists == 1); 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate if (!cancelled) { 1487c478bd9Sstevel@tonic-gate delta_pgcnt = end - start; 1497c478bd9Sstevel@tonic-gate node_size = mem_node_config[mnode].physmax - 150ce8eb11aSdp mem_node_config[mnode].physbase; 1517c478bd9Sstevel@tonic-gate 1527c478bd9Sstevel@tonic-gate if (node_size > delta_pgcnt) { 1537c478bd9Sstevel@tonic-gate /* 1547c478bd9Sstevel@tonic-gate * Subtract the slice from the memnode. 1557c478bd9Sstevel@tonic-gate */ 1567c478bd9Sstevel@tonic-gate if (start <= mem_node_config[mnode].physbase) 1577c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = end + 1; 1587c478bd9Sstevel@tonic-gate ASSERT(end <= mem_node_config[mnode].physmax); 1597c478bd9Sstevel@tonic-gate if (end == mem_node_config[mnode].physmax) 1607c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = start - 1; 1617c478bd9Sstevel@tonic-gate } else { 1627c478bd9Sstevel@tonic-gate 1637c478bd9Sstevel@tonic-gate /* 1647c478bd9Sstevel@tonic-gate * Let the common lgrp framework know the mnode is 1657c478bd9Sstevel@tonic-gate * leaving 1667c478bd9Sstevel@tonic-gate */ 1677c478bd9Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_MEM_DEL, mnode, 1687c478bd9Sstevel@tonic-gate MEM_NODE_2_LGRPHAND(mnode)); 1697c478bd9Sstevel@tonic-gate 1707c478bd9Sstevel@tonic-gate /* 1717c478bd9Sstevel@tonic-gate * Delete the whole node. 1727c478bd9Sstevel@tonic-gate */ 173affbd3ccSkchow ASSERT(MNODE_PGCNT(mnode) == 0); 1747c478bd9Sstevel@tonic-gate do { 1757c478bd9Sstevel@tonic-gate omask = memnodes_mask; 1767c478bd9Sstevel@tonic-gate nmask = omask & ~(1ull << mnode); 1777c478bd9Sstevel@tonic-gate } while (cas64(&memnodes_mask, omask, nmask) != omask); 1787c478bd9Sstevel@tonic-gate atomic_add_16(&num_memnodes, -1); 1797c478bd9Sstevel@tonic-gate mem_node_config[mnode].exists = 0; 1807c478bd9Sstevel@tonic-gate } 1817c478bd9Sstevel@tonic-gate 182ce8eb11aSdp if (&plat_slice_del != NULL) 1837c478bd9Sstevel@tonic-gate plat_slice_del(start, end); 1847c478bd9Sstevel@tonic-gate } 1857c478bd9Sstevel@tonic-gate } 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate void 188*986fd29aSsetje startup_build_mem_nodes(prom_memlist_t *list, size_t nelems) 1897c478bd9Sstevel@tonic-gate { 1907c478bd9Sstevel@tonic-gate size_t elem; 1917c478bd9Sstevel@tonic-gate pfn_t basepfn; 1927c478bd9Sstevel@tonic-gate pgcnt_t npgs; 1937c478bd9Sstevel@tonic-gate 1947c478bd9Sstevel@tonic-gate /* LINTED: ASSERT will always true or false */ 1957c478bd9Sstevel@tonic-gate ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes); 1967c478bd9Sstevel@tonic-gate 197ce8eb11aSdp if (&plat_build_mem_nodes != NULL) { 1987c478bd9Sstevel@tonic-gate plat_build_mem_nodes(list, nelems); 1997c478bd9Sstevel@tonic-gate } else { 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * Boot install lists are arranged <addr, len>, ... 2027c478bd9Sstevel@tonic-gate */ 203*986fd29aSsetje for (elem = 0; elem < nelems; list++, elem++) { 204*986fd29aSsetje basepfn = btop(list->addr); 205*986fd29aSsetje npgs = btop(list->size); 2067c478bd9Sstevel@tonic-gate mem_node_add_slice(basepfn, basepfn + npgs - 1); 2077c478bd9Sstevel@tonic-gate } 2087c478bd9Sstevel@tonic-gate } 2097c478bd9Sstevel@tonic-gate } 2107c478bd9Sstevel@tonic-gate 2117c478bd9Sstevel@tonic-gate /* 2127c478bd9Sstevel@tonic-gate * Allocate an unassigned memnode. 2137c478bd9Sstevel@tonic-gate */ 2147c478bd9Sstevel@tonic-gate int 2157c478bd9Sstevel@tonic-gate mem_node_alloc() 2167c478bd9Sstevel@tonic-gate { 2177c478bd9Sstevel@tonic-gate int mnode; 2187c478bd9Sstevel@tonic-gate mnodeset_t newmask, oldmask; 2197c478bd9Sstevel@tonic-gate 2207c478bd9Sstevel@tonic-gate /* 2217c478bd9Sstevel@tonic-gate * Find an unused memnode. Update it atomically to prevent 2227c478bd9Sstevel@tonic-gate * a first time memnode creation race. 2237c478bd9Sstevel@tonic-gate */ 2247c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 2257c478bd9Sstevel@tonic-gate if (cas32((uint32_t *)&mem_node_config[mnode].exists, 226ce8eb11aSdp 0, 1) == 0) 2277c478bd9Sstevel@tonic-gate break; 2287c478bd9Sstevel@tonic-gate 2297c478bd9Sstevel@tonic-gate if (mnode >= max_mem_nodes) 2307c478bd9Sstevel@tonic-gate panic("Out of free memnodes\n"); 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate mem_node_config[mnode].physbase = (uint64_t)-1; 2337c478bd9Sstevel@tonic-gate mem_node_config[mnode].physmax = 0; 2347c478bd9Sstevel@tonic-gate atomic_add_16(&num_memnodes, 1); 2357c478bd9Sstevel@tonic-gate do { 2367c478bd9Sstevel@tonic-gate oldmask = memnodes_mask; 2377c478bd9Sstevel@tonic-gate newmask = memnodes_mask | (1ull << mnode); 2387c478bd9Sstevel@tonic-gate } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask); 2397c478bd9Sstevel@tonic-gate 2407c478bd9Sstevel@tonic-gate return (mnode); 2417c478bd9Sstevel@tonic-gate } 2427c478bd9Sstevel@tonic-gate 2437c478bd9Sstevel@tonic-gate /* 2447c478bd9Sstevel@tonic-gate * Find the intersection between a memnode and a memlist 2457c478bd9Sstevel@tonic-gate * and returns the number of pages that overlap. 2467c478bd9Sstevel@tonic-gate * 247ce8eb11aSdp * Grab the memlist lock to protect the list from DR operations. 2487c478bd9Sstevel@tonic-gate */ 2497c478bd9Sstevel@tonic-gate pgcnt_t 2507c478bd9Sstevel@tonic-gate mem_node_memlist_pages(int mnode, struct memlist *mlist) 2517c478bd9Sstevel@tonic-gate { 2527c478bd9Sstevel@tonic-gate pfn_t base, end; 2537c478bd9Sstevel@tonic-gate pfn_t cur_base, cur_end; 254ce8eb11aSdp pgcnt_t npgs = 0; 255ce8eb11aSdp pgcnt_t pages; 2567c478bd9Sstevel@tonic-gate struct memlist *pmem; 2577c478bd9Sstevel@tonic-gate 258ce8eb11aSdp if (&plat_mem_node_intersect_range != NULL) { 259ce8eb11aSdp memlist_read_lock(); 260ce8eb11aSdp 261ce8eb11aSdp for (pmem = mlist; pmem; pmem = pmem->next) { 262ce8eb11aSdp plat_mem_node_intersect_range(btop(pmem->address), 263ce8eb11aSdp btop(pmem->size), mnode, &pages); 264ce8eb11aSdp npgs += pages; 265ce8eb11aSdp } 266ce8eb11aSdp 267ce8eb11aSdp memlist_read_unlock(); 268ce8eb11aSdp return (npgs); 269ce8eb11aSdp } 270ce8eb11aSdp 2717c478bd9Sstevel@tonic-gate base = mem_node_config[mnode].physbase; 2727c478bd9Sstevel@tonic-gate end = mem_node_config[mnode].physmax; 2737c478bd9Sstevel@tonic-gate 2747c478bd9Sstevel@tonic-gate memlist_read_lock(); 2757c478bd9Sstevel@tonic-gate 2767c478bd9Sstevel@tonic-gate for (pmem = mlist; pmem; pmem = pmem->next) { 2777c478bd9Sstevel@tonic-gate cur_base = btop(pmem->address); 2787c478bd9Sstevel@tonic-gate cur_end = cur_base + btop(pmem->size) - 1; 279ce8eb11aSdp if (end < cur_base || base > cur_end) 2807c478bd9Sstevel@tonic-gate continue; 2817c478bd9Sstevel@tonic-gate npgs = npgs + (MIN(cur_end, end) - 2827c478bd9Sstevel@tonic-gate MAX(cur_base, base)) + 1; 2837c478bd9Sstevel@tonic-gate } 2847c478bd9Sstevel@tonic-gate 2857c478bd9Sstevel@tonic-gate memlist_read_unlock(); 2867c478bd9Sstevel@tonic-gate 2877c478bd9Sstevel@tonic-gate return (npgs); 2887c478bd9Sstevel@tonic-gate } 289ce8eb11aSdp 290ce8eb11aSdp /* 291ce8eb11aSdp * Find MIN(physbase) and MAX(physmax) over all mnodes 292ce8eb11aSdp * 293ce8eb11aSdp * Called during startup and DR to find hpm_counters limits when 294ce8eb11aSdp * interleaved_mnodes is set. 295ce8eb11aSdp * NOTE: there is a race condition with DR if it tries to change more than 296ce8eb11aSdp * one mnode in parallel. Sizing shared hpm_counters depends on finding the 297ce8eb11aSdp * min(physbase) and max(physmax) across all mnodes. Therefore, the caller of 298ce8eb11aSdp * page_ctrs_adjust must ensure that mem_node_config does not change while it 299ce8eb11aSdp * is running. 300ce8eb11aSdp */ 301ce8eb11aSdp void 302ce8eb11aSdp mem_node_max_range(pfn_t *basep, pfn_t *maxp) 303ce8eb11aSdp { 304ce8eb11aSdp int mnode; 305ce8eb11aSdp pfn_t max = 0; 306ce8eb11aSdp pfn_t base = (pfn_t)-1; 307ce8eb11aSdp 308ce8eb11aSdp for (mnode = 0; mnode < max_mem_nodes; mnode++) { 309ce8eb11aSdp if (mem_node_config[mnode].exists == 0) 310ce8eb11aSdp continue; 311ce8eb11aSdp if (max < mem_node_config[mnode].physmax) 312ce8eb11aSdp max = mem_node_config[mnode].physmax; 313ce8eb11aSdp if (base > mem_node_config[mnode].physbase) 314ce8eb11aSdp base = mem_node_config[mnode].physbase; 315ce8eb11aSdp } 316ce8eb11aSdp ASSERT(base != (pfn_t)-1 && max != 0); 317ce8eb11aSdp *basep = base; 318ce8eb11aSdp *maxp = max; 319ce8eb11aSdp } 320