xref: /illumos-gate/usr/src/uts/sun4/os/memnode.c (revision 1a5e258f)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ce8eb11aSdp  * Common Development and Distribution License (the "License").
6ce8eb11aSdp  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2256f33205SJonathan Adams  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #include <sys/systm.h>
277c478bd9Sstevel@tonic-gate #include <sys/platform_module.h>
287c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
297c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
307c478bd9Sstevel@tonic-gate #include <sys/memlist.h>
317c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
32affbd3ccSkchow #include <vm/vm_dep.h>
337c478bd9Sstevel@tonic-gate 
347c478bd9Sstevel@tonic-gate int max_mem_nodes = 1;		/* max memory nodes on this system */
357c478bd9Sstevel@tonic-gate 
367c478bd9Sstevel@tonic-gate struct mem_node_conf mem_node_config[MAX_MEM_NODES];
377c478bd9Sstevel@tonic-gate int mem_node_pfn_shift;
387c478bd9Sstevel@tonic-gate /*
397c478bd9Sstevel@tonic-gate  * num_memnodes should be updated atomically and always >=
407c478bd9Sstevel@tonic-gate  * the number of bits in memnodes_mask or the algorithm may fail.
417c478bd9Sstevel@tonic-gate  */
427c478bd9Sstevel@tonic-gate uint16_t num_memnodes;
437c478bd9Sstevel@tonic-gate mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate /*
467c478bd9Sstevel@tonic-gate  * If set, mem_node_physalign should be a power of two, and
477c478bd9Sstevel@tonic-gate  * should reflect the minimum address alignment of each node.
487c478bd9Sstevel@tonic-gate  */
497c478bd9Sstevel@tonic-gate uint64_t mem_node_physalign;
507c478bd9Sstevel@tonic-gate 
517c478bd9Sstevel@tonic-gate /*
527c478bd9Sstevel@tonic-gate  * Platform hooks we will need.
537c478bd9Sstevel@tonic-gate  */
547c478bd9Sstevel@tonic-gate 
557c478bd9Sstevel@tonic-gate #pragma weak plat_build_mem_nodes
567c478bd9Sstevel@tonic-gate #pragma weak plat_slice_add
577c478bd9Sstevel@tonic-gate #pragma weak plat_slice_del
587c478bd9Sstevel@tonic-gate 
597c478bd9Sstevel@tonic-gate /*
607c478bd9Sstevel@tonic-gate  * Adjust the memnode config after a DR operation.
617c478bd9Sstevel@tonic-gate  *
627c478bd9Sstevel@tonic-gate  * It is rather tricky to do these updates since we can't
637c478bd9Sstevel@tonic-gate  * protect the memnode structures with locks, so we must
647c478bd9Sstevel@tonic-gate  * be mindful of the order in which updates and reads to
657c478bd9Sstevel@tonic-gate  * these values can occur.
667c478bd9Sstevel@tonic-gate  */
677c478bd9Sstevel@tonic-gate void
mem_node_add_slice(pfn_t start,pfn_t end)687c478bd9Sstevel@tonic-gate mem_node_add_slice(pfn_t start, pfn_t end)
697c478bd9Sstevel@tonic-gate {
707c478bd9Sstevel@tonic-gate 	int mnode;
717c478bd9Sstevel@tonic-gate 	mnodeset_t newmask, oldmask;
727c478bd9Sstevel@tonic-gate 
737c478bd9Sstevel@tonic-gate 	/*
747c478bd9Sstevel@tonic-gate 	 * DR will pass us the first pfn that is allocatable.
757c478bd9Sstevel@tonic-gate 	 * We need to round down to get the real start of
767c478bd9Sstevel@tonic-gate 	 * the slice.
777c478bd9Sstevel@tonic-gate 	 */
787c478bd9Sstevel@tonic-gate 	if (mem_node_physalign) {
797c478bd9Sstevel@tonic-gate 		start &= ~(btop(mem_node_physalign) - 1);
807c478bd9Sstevel@tonic-gate 		end = roundup(end, btop(mem_node_physalign)) - 1;
817c478bd9Sstevel@tonic-gate 	}
827c478bd9Sstevel@tonic-gate 
837c478bd9Sstevel@tonic-gate 	mnode = PFN_2_MEM_NODE(start);
847c478bd9Sstevel@tonic-gate 	ASSERT(mnode < max_mem_nodes);
857c478bd9Sstevel@tonic-gate 
8675d94465SJosef 'Jeff' Sipek 	if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
877c478bd9Sstevel@tonic-gate 		/*
887c478bd9Sstevel@tonic-gate 		 * Add slice to existing node.
897c478bd9Sstevel@tonic-gate 		 */
907c478bd9Sstevel@tonic-gate 		if (start < mem_node_config[mnode].physbase)
917c478bd9Sstevel@tonic-gate 			mem_node_config[mnode].physbase = start;
927c478bd9Sstevel@tonic-gate 		if (end > mem_node_config[mnode].physmax)
937c478bd9Sstevel@tonic-gate 			mem_node_config[mnode].physmax = end;
947c478bd9Sstevel@tonic-gate 	} else {
957c478bd9Sstevel@tonic-gate 		mem_node_config[mnode].physbase = start;
967c478bd9Sstevel@tonic-gate 		mem_node_config[mnode].physmax = end;
97*1a5e258fSJosef 'Jeff' Sipek 		atomic_inc_16(&num_memnodes);
987c478bd9Sstevel@tonic-gate 		do {
997c478bd9Sstevel@tonic-gate 			oldmask = memnodes_mask;
1007c478bd9Sstevel@tonic-gate 			newmask = memnodes_mask | (1ull << mnode);
10175d94465SJosef 'Jeff' Sipek 		} while (atomic_cas_64(&memnodes_mask, oldmask, newmask) !=
10275d94465SJosef 'Jeff' Sipek 			 oldmask);
1037c478bd9Sstevel@tonic-gate 	}
1047c478bd9Sstevel@tonic-gate 	/*
1057c478bd9Sstevel@tonic-gate 	 * Let the common lgrp framework know about the new memory
1067c478bd9Sstevel@tonic-gate 	 */
1077c478bd9Sstevel@tonic-gate 	lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
1087c478bd9Sstevel@tonic-gate }
1097c478bd9Sstevel@tonic-gate 
1107c478bd9Sstevel@tonic-gate /*
1117c478bd9Sstevel@tonic-gate  * Remove a PFN range from a memnode.  On some platforms,
1127c478bd9Sstevel@tonic-gate  * the memnode will be created with physbase at the first
1137c478bd9Sstevel@tonic-gate  * allocatable PFN, but later deleted with the MC slice
1147c478bd9Sstevel@tonic-gate  * base address converted to a PFN, in which case we need
1157c478bd9Sstevel@tonic-gate  * to assume physbase and up.
1167c478bd9Sstevel@tonic-gate  */
1177c478bd9Sstevel@tonic-gate void
mem_node_del_slice(pfn_t start,pfn_t end)1189853d9e8SJason Beloro mem_node_del_slice(pfn_t start, pfn_t end)
1197c478bd9Sstevel@tonic-gate {
1207c478bd9Sstevel@tonic-gate 	int mnode;
1217c478bd9Sstevel@tonic-gate 	pgcnt_t delta_pgcnt, node_size;
1227c478bd9Sstevel@tonic-gate 	mnodeset_t omask, nmask;
1237c478bd9Sstevel@tonic-gate 
1247c478bd9Sstevel@tonic-gate 	if (mem_node_physalign) {
1257c478bd9Sstevel@tonic-gate 		start &= ~(btop(mem_node_physalign) - 1);
1267c478bd9Sstevel@tonic-gate 		end = roundup(end, btop(mem_node_physalign)) - 1;
1277c478bd9Sstevel@tonic-gate 	}
1287c478bd9Sstevel@tonic-gate 	mnode = PFN_2_MEM_NODE(start);
1297c478bd9Sstevel@tonic-gate 
1307c478bd9Sstevel@tonic-gate 	ASSERT(mnode < max_mem_nodes);
1317c478bd9Sstevel@tonic-gate 	ASSERT(mem_node_config[mnode].exists == 1);
1327c478bd9Sstevel@tonic-gate 
1339853d9e8SJason Beloro 	delta_pgcnt = end - start;
1349853d9e8SJason Beloro 	node_size = mem_node_config[mnode].physmax -
1359853d9e8SJason Beloro 	    mem_node_config[mnode].physbase;
1369853d9e8SJason Beloro 
1379853d9e8SJason Beloro 	if (node_size > delta_pgcnt) {
1389853d9e8SJason Beloro 		/*
1399853d9e8SJason Beloro 		 * Subtract the slice from the memnode.
1409853d9e8SJason Beloro 		 */
1419853d9e8SJason Beloro 		if (start <= mem_node_config[mnode].physbase)
1429853d9e8SJason Beloro 			mem_node_config[mnode].physbase = end + 1;
1439853d9e8SJason Beloro 		ASSERT(end <= mem_node_config[mnode].physmax);
1449853d9e8SJason Beloro 		if (end == mem_node_config[mnode].physmax)
1459853d9e8SJason Beloro 			mem_node_config[mnode].physmax = start - 1;
1469853d9e8SJason Beloro 	} else {
1477c478bd9Sstevel@tonic-gate 
1489853d9e8SJason Beloro 		/*
1499853d9e8SJason Beloro 		 * Let the common lgrp framework know the mnode is
1509853d9e8SJason Beloro 		 * leaving
1519853d9e8SJason Beloro 		 */
1529853d9e8SJason Beloro 		lgrp_config(LGRP_CONFIG_MEM_DEL, mnode,
1539853d9e8SJason Beloro 		    MEM_NODE_2_LGRPHAND(mnode));
1549853d9e8SJason Beloro 
1559853d9e8SJason Beloro 		/*
1569853d9e8SJason Beloro 		 * Delete the whole node.
1579853d9e8SJason Beloro 		 */
1589853d9e8SJason Beloro 		ASSERT(MNODE_PGCNT(mnode) == 0);
1599853d9e8SJason Beloro 		do {
1609853d9e8SJason Beloro 			omask = memnodes_mask;
1619853d9e8SJason Beloro 			nmask = omask & ~(1ull << mnode);
16275d94465SJosef 'Jeff' Sipek 		} while (atomic_cas_64(&memnodes_mask, omask, nmask) != omask);
163*1a5e258fSJosef 'Jeff' Sipek 		atomic_dec_16(&num_memnodes);
1649853d9e8SJason Beloro 		mem_node_config[mnode].exists = 0;
1657c478bd9Sstevel@tonic-gate 	}
1667c478bd9Sstevel@tonic-gate }
1677c478bd9Sstevel@tonic-gate 
1689853d9e8SJason Beloro void
mem_node_add_range(pfn_t start,pfn_t end)1699853d9e8SJason Beloro mem_node_add_range(pfn_t start, pfn_t end)
1709853d9e8SJason Beloro {
1719853d9e8SJason Beloro 	if (&plat_slice_add != NULL)
1729853d9e8SJason Beloro 		plat_slice_add(start, end);
1739853d9e8SJason Beloro 	else
1749853d9e8SJason Beloro 		mem_node_add_slice(start, end);
1759853d9e8SJason Beloro }
1769853d9e8SJason Beloro 
1779853d9e8SJason Beloro void
mem_node_del_range(pfn_t start,pfn_t end)1789853d9e8SJason Beloro mem_node_del_range(pfn_t start, pfn_t end)
1799853d9e8SJason Beloro {
1809853d9e8SJason Beloro 	if (&plat_slice_del != NULL)
1819853d9e8SJason Beloro 		plat_slice_del(start, end);
1829853d9e8SJason Beloro 	else
1839853d9e8SJason Beloro 		mem_node_del_slice(start, end);
1849853d9e8SJason Beloro }
1859853d9e8SJason Beloro 
1867c478bd9Sstevel@tonic-gate void
startup_build_mem_nodes(prom_memlist_t * list,size_t nelems)187986fd29aSsetje startup_build_mem_nodes(prom_memlist_t *list, size_t nelems)
1887c478bd9Sstevel@tonic-gate {
1897c478bd9Sstevel@tonic-gate 	size_t	elem;
1907c478bd9Sstevel@tonic-gate 	pfn_t	basepfn;
1917c478bd9Sstevel@tonic-gate 	pgcnt_t	npgs;
1927c478bd9Sstevel@tonic-gate 
1937c478bd9Sstevel@tonic-gate 	/* LINTED: ASSERT will always true or false */
1947c478bd9Sstevel@tonic-gate 	ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes);
1957c478bd9Sstevel@tonic-gate 
196ce8eb11aSdp 	if (&plat_build_mem_nodes != NULL) {
1977c478bd9Sstevel@tonic-gate 		plat_build_mem_nodes(list, nelems);
1987c478bd9Sstevel@tonic-gate 	} else {
1997c478bd9Sstevel@tonic-gate 		/*
2007c478bd9Sstevel@tonic-gate 		 * Boot install lists are arranged <addr, len>, ...
2017c478bd9Sstevel@tonic-gate 		 */
202986fd29aSsetje 		for (elem = 0; elem < nelems; list++, elem++) {
203986fd29aSsetje 			basepfn = btop(list->addr);
204986fd29aSsetje 			npgs = btop(list->size);
2059853d9e8SJason Beloro 			mem_node_add_range(basepfn, basepfn + npgs - 1);
2067c478bd9Sstevel@tonic-gate 		}
2077c478bd9Sstevel@tonic-gate 	}
2087c478bd9Sstevel@tonic-gate }
2097c478bd9Sstevel@tonic-gate 
2107c478bd9Sstevel@tonic-gate /*
2117c478bd9Sstevel@tonic-gate  * Allocate an unassigned memnode.
2127c478bd9Sstevel@tonic-gate  */
2137c478bd9Sstevel@tonic-gate int
mem_node_alloc()2147c478bd9Sstevel@tonic-gate mem_node_alloc()
2157c478bd9Sstevel@tonic-gate {
2167c478bd9Sstevel@tonic-gate 	int mnode;
2177c478bd9Sstevel@tonic-gate 	mnodeset_t newmask, oldmask;
2187c478bd9Sstevel@tonic-gate 
2197c478bd9Sstevel@tonic-gate 	/*
2207c478bd9Sstevel@tonic-gate 	 * Find an unused memnode.  Update it atomically to prevent
2217c478bd9Sstevel@tonic-gate 	 * a first time memnode creation race.
2227c478bd9Sstevel@tonic-gate 	 */
2237c478bd9Sstevel@tonic-gate 	for (mnode = 0; mnode < max_mem_nodes; mnode++)
22475d94465SJosef 'Jeff' Sipek 		if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists,
225ce8eb11aSdp 		    0, 1) == 0)
2267c478bd9Sstevel@tonic-gate 			break;
2277c478bd9Sstevel@tonic-gate 
2287c478bd9Sstevel@tonic-gate 	if (mnode >= max_mem_nodes)
2297c478bd9Sstevel@tonic-gate 			panic("Out of free memnodes\n");
2307c478bd9Sstevel@tonic-gate 
2317c478bd9Sstevel@tonic-gate 	mem_node_config[mnode].physbase = (uint64_t)-1;
2327c478bd9Sstevel@tonic-gate 	mem_node_config[mnode].physmax = 0;
233*1a5e258fSJosef 'Jeff' Sipek 	atomic_inc_16(&num_memnodes);
2347c478bd9Sstevel@tonic-gate 	do {
2357c478bd9Sstevel@tonic-gate 		oldmask = memnodes_mask;
2367c478bd9Sstevel@tonic-gate 		newmask = memnodes_mask | (1ull << mnode);
23775d94465SJosef 'Jeff' Sipek 	} while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask);
2387c478bd9Sstevel@tonic-gate 
2397c478bd9Sstevel@tonic-gate 	return (mnode);
2407c478bd9Sstevel@tonic-gate }
2417c478bd9Sstevel@tonic-gate 
2427c478bd9Sstevel@tonic-gate /*
2437c478bd9Sstevel@tonic-gate  * Find the intersection between a memnode and a memlist
2447c478bd9Sstevel@tonic-gate  * and returns the number of pages that overlap.
2457c478bd9Sstevel@tonic-gate  *
246ce8eb11aSdp  * Grab the memlist lock to protect the list from DR operations.
2477c478bd9Sstevel@tonic-gate  */
2487c478bd9Sstevel@tonic-gate pgcnt_t
mem_node_memlist_pages(int mnode,struct memlist * mlist)2497c478bd9Sstevel@tonic-gate mem_node_memlist_pages(int mnode, struct memlist *mlist)
2507c478bd9Sstevel@tonic-gate {
2517c478bd9Sstevel@tonic-gate 	pfn_t		base, end;
2527c478bd9Sstevel@tonic-gate 	pfn_t		cur_base, cur_end;
253ce8eb11aSdp 	pgcnt_t		npgs = 0;
254ce8eb11aSdp 	pgcnt_t		pages;
2557c478bd9Sstevel@tonic-gate 	struct memlist	*pmem;
2567c478bd9Sstevel@tonic-gate 
257ce8eb11aSdp 	if (&plat_mem_node_intersect_range != NULL) {
258ce8eb11aSdp 		memlist_read_lock();
259ce8eb11aSdp 
26056f33205SJonathan Adams 		for (pmem = mlist; pmem; pmem = pmem->ml_next) {
26156f33205SJonathan Adams 			plat_mem_node_intersect_range(btop(pmem->ml_address),
26256f33205SJonathan Adams 			    btop(pmem->ml_size), mnode, &pages);
263ce8eb11aSdp 			npgs += pages;
264ce8eb11aSdp 		}
265ce8eb11aSdp 
266ce8eb11aSdp 		memlist_read_unlock();
267ce8eb11aSdp 		return (npgs);
268ce8eb11aSdp 	}
269ce8eb11aSdp 
2707c478bd9Sstevel@tonic-gate 	base = mem_node_config[mnode].physbase;
2717c478bd9Sstevel@tonic-gate 	end = mem_node_config[mnode].physmax;
2727c478bd9Sstevel@tonic-gate 
2737c478bd9Sstevel@tonic-gate 	memlist_read_lock();
2747c478bd9Sstevel@tonic-gate 
27556f33205SJonathan Adams 	for (pmem = mlist; pmem; pmem = pmem->ml_next) {
27656f33205SJonathan Adams 		cur_base = btop(pmem->ml_address);
27756f33205SJonathan Adams 		cur_end = cur_base + btop(pmem->ml_size) - 1;
278ce8eb11aSdp 		if (end < cur_base || base > cur_end)
2797c478bd9Sstevel@tonic-gate 			continue;
2807c478bd9Sstevel@tonic-gate 		npgs = npgs + (MIN(cur_end, end) -
2817c478bd9Sstevel@tonic-gate 		    MAX(cur_base, base)) + 1;
2827c478bd9Sstevel@tonic-gate 	}
2837c478bd9Sstevel@tonic-gate 
2847c478bd9Sstevel@tonic-gate 	memlist_read_unlock();
2857c478bd9Sstevel@tonic-gate 
2867c478bd9Sstevel@tonic-gate 	return (npgs);
2877c478bd9Sstevel@tonic-gate }
288ce8eb11aSdp 
289ce8eb11aSdp /*
290ce8eb11aSdp  * Find MIN(physbase) and MAX(physmax) over all mnodes
291ce8eb11aSdp  *
292ce8eb11aSdp  * Called during startup and DR to find hpm_counters limits when
293ce8eb11aSdp  * interleaved_mnodes is set.
294ce8eb11aSdp  * NOTE: there is a race condition with DR if it tries to change more than
295ce8eb11aSdp  * one mnode in parallel. Sizing shared hpm_counters depends on finding the
296ce8eb11aSdp  * min(physbase) and max(physmax) across all mnodes. Therefore, the caller of
297ce8eb11aSdp  * page_ctrs_adjust must ensure that mem_node_config does not change while it
298ce8eb11aSdp  * is running.
299ce8eb11aSdp  */
300ce8eb11aSdp void
mem_node_max_range(pfn_t * basep,pfn_t * maxp)301ce8eb11aSdp mem_node_max_range(pfn_t *basep, pfn_t *maxp)
302ce8eb11aSdp {
303ce8eb11aSdp 	int mnode;
304ce8eb11aSdp 	pfn_t max = 0;
305ce8eb11aSdp 	pfn_t base = (pfn_t)-1;
306ce8eb11aSdp 
307ce8eb11aSdp 	for (mnode = 0; mnode < max_mem_nodes; mnode++) {
308ce8eb11aSdp 		if (mem_node_config[mnode].exists == 0)
309ce8eb11aSdp 			continue;
310ce8eb11aSdp 		if (max < mem_node_config[mnode].physmax)
311ce8eb11aSdp 			max = mem_node_config[mnode].physmax;
312ce8eb11aSdp 		if (base > mem_node_config[mnode].physbase)
313ce8eb11aSdp 			base = mem_node_config[mnode].physbase;
314ce8eb11aSdp 	}
315ce8eb11aSdp 	ASSERT(base != (pfn_t)-1 && max != 0);
316ce8eb11aSdp 	*basep = base;
317ce8eb11aSdp 	*maxp = max;
318ce8eb11aSdp }
319