17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5c39996a7Sstevel * Common Development and Distribution License (the "License").
6c39996a7Sstevel * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
21c39996a7Sstevel
227c478bd9Sstevel@tonic-gate /*
237417cfdeSKuriakose Kuruvilla * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
247c478bd9Sstevel@tonic-gate */
25a3114836SGerry Liu /*
26a3114836SGerry Liu * Copyright (c) 2010, Intel Corporation.
27a3114836SGerry Liu * All rights reserved.
28a3114836SGerry Liu */
297c478bd9Sstevel@tonic-gate
302e2c009bSjjc /*
312e2c009bSjjc * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS
322e2c009bSjjc * ================================================================
332e2c009bSjjc * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access
342e2c009bSjjc * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of
352e2c009bSjjc * one or more CPUs and some local memory. The CPUs in each node can access
362e2c009bSjjc * the memory in the other nodes but at a higher latency than accessing their
372e2c009bSjjc * local memory. Typically, a system with only one node has Uniform Memory
382e2c009bSjjc * Access (UMA), but it may be possible to have a one node system that has
392e2c009bSjjc * some global memory outside of the node which is higher latency.
402e2c009bSjjc *
412e2c009bSjjc * Module Description
422e2c009bSjjc * ------------------
432e2c009bSjjc * This module provides a platform interface for determining which CPUs and
442e2c009bSjjc * which memory (and how much) are in a NUMA node and how far each node is from
452e2c009bSjjc * each other. The interface is used by the Virtual Memory (VM) system and the
462e2c009bSjjc * common lgroup framework. The VM system uses the plat_*() routines to fill
472e2c009bSjjc * in its memory node (memnode) array with the physical address range spanned
482e2c009bSjjc * by each NUMA node to know which memory belongs to which node, so it can
492e2c009bSjjc * build and manage a physical page free list for each NUMA node and allocate
502e2c009bSjjc * local memory from each node as needed. The common lgroup framework uses the
512e2c009bSjjc * exported lgrp_plat_*() routines to figure out which CPUs and memory belong
522e2c009bSjjc * to each node (leaf lgroup) and how far each node is from each other, so it
532e2c009bSjjc * can build the latency (lgroup) topology for the machine in order to optimize
542e2c009bSjjc * for locality. Also, an lgroup platform handle instead of lgroups are used
552e2c009bSjjc * in the interface with this module, so this module shouldn't need to know
562e2c009bSjjc * anything about lgroups. Instead, it just needs to know which CPUs, memory,
572e2c009bSjjc * etc. are in each NUMA node, how far each node is from each other, and to use
582e2c009bSjjc * a unique lgroup platform handle to refer to each node through the interface.
592e2c009bSjjc *
602e2c009bSjjc * Determining NUMA Configuration
612e2c009bSjjc * ------------------------------
622e2c009bSjjc * By default, this module will try to determine the NUMA configuration of the
632e2c009bSjjc * machine by reading the ACPI System Resource Affinity Table (SRAT) and System
642e2c009bSjjc * Locality Information Table (SLIT). The SRAT contains info to tell which
652e2c009bSjjc * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT
662e2c009bSjjc * is a matrix that gives the distance between each system locality (which is
672e2c009bSjjc * a NUMA node and should correspond to proximity domains in the SRAT). For
682e2c009bSjjc * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer
692e2c009bSjjc * specification.
702e2c009bSjjc *
712e2c009bSjjc * If the SRAT doesn't exist on a system with AMD Opteron processors, we
722e2c009bSjjc * examine registers in PCI configuration space to determine how many nodes are
732e2c009bSjjc * in the system and which CPUs and memory are in each node.
742e2c009bSjjc * do while booting the kernel.
752e2c009bSjjc *
762e2c009bSjjc * NOTE: Using these PCI configuration space registers to determine this
772e2c009bSjjc * locality info is not guaranteed to work or be compatible across all
782e2c009bSjjc * Opteron processor families.
792e2c009bSjjc *
802e2c009bSjjc * If the SLIT does not exist or look right, the kernel will probe to determine
812e2c009bSjjc * the distance between nodes as long as the NUMA CPU and memory configuration
822e2c009bSjjc * has been determined (see lgrp_plat_probe() for details).
832e2c009bSjjc *
842e2c009bSjjc * Data Structures
852e2c009bSjjc * ---------------
862e2c009bSjjc * The main data structures used by this code are the following:
872e2c009bSjjc *
88dae2fa37Sjjc * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by
89dae2fa37Sjjc * CPU ID (only used for SRAT)
902e2c009bSjjc *
912e2c009bSjjc * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and
922e2c009bSjjc * different nodes indexed by node ID
932e2c009bSjjc *
94a3114836SGerry Liu * - lgrp_plat_node_cnt Number of NUMA nodes in system for
95a3114836SGerry Liu * non-DR-capable systems,
96a3114836SGerry Liu * maximum possible number of NUMA nodes
97a3114836SGerry Liu * in system for DR capable systems.
982e2c009bSjjc *
992e2c009bSjjc * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping
1002e2c009bSjjc * table indexed by node ID (only used
1012e2c009bSjjc * for SRAT)
1022e2c009bSjjc *
103a3114836SGerry Liu * - lgrp_plat_memnode_info[] Table with physical address range for
104a3114836SGerry Liu * each memory node indexed by memory node
105a3114836SGerry Liu * ID
1062e2c009bSjjc *
1072e2c009bSjjc * The code is implemented to make the following always be true:
1082e2c009bSjjc *
1092e2c009bSjjc * lgroup platform handle == node ID == memnode ID
1102e2c009bSjjc *
1112e2c009bSjjc * Moreover, it allows for the proximity domain ID to be equal to all of the
1122e2c009bSjjc * above as long as the proximity domains IDs are numbered from 0 to <number of
1132e2c009bSjjc * nodes - 1>. This is done by hashing each proximity domain ID into the range
1142e2c009bSjjc * from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID
1152e2c009bSjjc * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N]
1162e2c009bSjjc * and be assigned node ID N. If the proximity domain IDs aren't numbered
1172e2c009bSjjc * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into
1182e2c009bSjjc * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs
1192e2c009bSjjc * to node IDs. However, the proximity domain IDs may not map to the
1202e2c009bSjjc * equivalent node ID since we want to keep the node IDs numbered from 0 to
1212e2c009bSjjc * <number of nodes - 1> to minimize cost of searching and potentially space.
12281d9ccb6SJonathan Chew *
123a3114836SGerry Liu * With the introduction of support of memory DR operations on x86 platforms,
124a3114836SGerry Liu * things get a little complicated. The addresses of hot-added memory may not
125a3114836SGerry Liu * be continuous with other memory connected to the same lgrp node. In other
126a3114836SGerry Liu * words, memory addresses may get interleaved among lgrp nodes after memory
127a3114836SGerry Liu * DR operations. To work around this limitation, we have extended the
128a3114836SGerry Liu * relationship between lgrp node and memory node from 1:1 map to 1:N map,
129a3114836SGerry Liu * that means there may be multiple memory nodes associated with a lgrp node
130a3114836SGerry Liu * after memory DR operations.
13181d9ccb6SJonathan Chew *
132a3114836SGerry Liu * To minimize the code changes to support memory DR operations, the
133a3114836SGerry Liu * following policies have been adopted.
134a3114836SGerry Liu * 1) On non-DR-capable systems, the relationship among lgroup platform handle,
135a3114836SGerry Liu * node ID and memnode ID is still kept as:
136a3114836SGerry Liu * lgroup platform handle == node ID == memnode ID
137a3114836SGerry Liu * 2) For memory present at boot time on DR capable platforms, the relationship
138a3114836SGerry Liu * is still kept as is.
139a3114836SGerry Liu * lgroup platform handle == node ID == memnode ID
140a3114836SGerry Liu * 3) For hot-added memory, the relationship between lgrp ID and memnode ID have
141a3114836SGerry Liu * been changed from 1:1 map to 1:N map. Memnode IDs [0 - lgrp_plat_node_cnt)
142a3114836SGerry Liu * are reserved for memory present at boot time, and memnode IDs
143a3114836SGerry Liu * [lgrp_plat_node_cnt, max_mem_nodes) are used to dynamically allocate
144a3114836SGerry Liu * memnode ID for hot-added memory.
145a3114836SGerry Liu * 4) All boot code having the assumption "node ID == memnode ID" can live as
146a3114836SGerry Liu * is, that's because node ID is always equal to memnode ID at boot time.
147a3114836SGerry Liu * 5) The lgrp_plat_memnode_info_update(), plat_pfn_to_mem_node() and
148a3114836SGerry Liu * lgrp_plat_mem_size() related logics have been enhanced to deal with
149a3114836SGerry Liu * the 1:N map relationship.
150a3114836SGerry Liu * 6) The latency probing related logics, which have the assumption
151a3114836SGerry Liu * "node ID == memnode ID" and may be called at run time, is disabled if
152a3114836SGerry Liu * memory DR operation is enabled.
1532e2c009bSjjc */
1542e2c009bSjjc
1552e2c009bSjjc
1567c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> /* for {in,out}{b,w,l}() */
157a3114836SGerry Liu #include <sys/atomic.h>
158dae2fa37Sjjc #include <sys/bootconf.h>
1597c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
160f78a91cdSjjc #include <sys/controlregs.h>
1617c478bd9Sstevel@tonic-gate #include <sys/cpupart.h>
1627c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
1637c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
1647c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
1657c478bd9Sstevel@tonic-gate #include <sys/memlist.h>
1667c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
1677c478bd9Sstevel@tonic-gate #include <sys/mman.h>
168a3114836SGerry Liu #include <sys/note.h>
169ef50d8c0Sesaxe #include <sys/pci_cfgspace.h>
170ef50d8c0Sesaxe #include <sys/pci_impl.h>
1717c478bd9Sstevel@tonic-gate #include <sys/param.h>
172fb2f18f8Sesaxe #include <sys/pghw.h>
1737c478bd9Sstevel@tonic-gate #include <sys/promif.h> /* for prom_printf() */
1742e2c009bSjjc #include <sys/sysmacros.h>
1757c478bd9Sstevel@tonic-gate #include <sys/systm.h>
1767c478bd9Sstevel@tonic-gate #include <sys/thread.h>
1777c478bd9Sstevel@tonic-gate #include <sys/types.h>
1787c478bd9Sstevel@tonic-gate #include <sys/var.h>
1797417cfdeSKuriakose Kuruvilla #include <sys/x86_archext.h>
1807c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
1817c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
182affbd3ccSkchow #include <vm/vm_dep.h>
1837c478bd9Sstevel@tonic-gate
184a3114836SGerry Liu #include <sys/acpidev.h>
1857bc870e6SToomas Soome #include <sys/acpi/acpi.h> /* for SRAT, SLIT and MSCT */
1867c478bd9Sstevel@tonic-gate
1877bc870e6SToomas Soome /* from fakebop.c */
1887bc870e6SToomas Soome extern ACPI_TABLE_SRAT *srat_ptr;
1897bc870e6SToomas Soome extern ACPI_TABLE_SLIT *slit_ptr;
1907bc870e6SToomas Soome extern ACPI_TABLE_MSCT *msct_ptr;
1917c478bd9Sstevel@tonic-gate
1927c478bd9Sstevel@tonic-gate #define MAX_NODES 8
1937c478bd9Sstevel@tonic-gate #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1)
1947c478bd9Sstevel@tonic-gate
1952e2c009bSjjc /*
1962e2c009bSjjc * Constants for configuring probing
1972e2c009bSjjc */
1987c478bd9Sstevel@tonic-gate #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */
1997c478bd9Sstevel@tonic-gate #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */
2008949bcd6Sandrei #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */
2017c478bd9Sstevel@tonic-gate
2027c478bd9Sstevel@tonic-gate /*
2032e2c009bSjjc * Flags for probing
2047c478bd9Sstevel@tonic-gate */
2052e2c009bSjjc #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */
2062e2c009bSjjc #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */
2072e2c009bSjjc #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */
2087c478bd9Sstevel@tonic-gate
2097c478bd9Sstevel@tonic-gate /*
21081d9ccb6SJonathan Chew * Hash proximity domain ID into node to domain mapping table "mod" number of
21181d9ccb6SJonathan Chew * nodes to minimize span of entries used and try to have lowest numbered
21281d9ccb6SJonathan Chew * proximity domain be node 0
2137c478bd9Sstevel@tonic-gate */
21481d9ccb6SJonathan Chew #define NODE_DOMAIN_HASH(domain, node_cnt) \
21581d9ccb6SJonathan Chew ((lgrp_plat_prox_domain_min == UINT32_MAX) ? (domain) % node_cnt : \
21681d9ccb6SJonathan Chew ((domain) - lgrp_plat_prox_domain_min) % node_cnt)
2177c478bd9Sstevel@tonic-gate
2187c478bd9Sstevel@tonic-gate /*
219dae2fa37Sjjc * CPU to node ID mapping structure (only used with SRAT)
2207c478bd9Sstevel@tonic-gate */
2212e2c009bSjjc typedef struct cpu_node_map {
2222e2c009bSjjc int exists;
2232e2c009bSjjc uint_t node;
2242e2c009bSjjc uint32_t apicid;
2252e2c009bSjjc uint32_t prox_domain;
2262e2c009bSjjc } cpu_node_map_t;
2277c478bd9Sstevel@tonic-gate
2287c478bd9Sstevel@tonic-gate /*
2292e2c009bSjjc * Latency statistics
2307c478bd9Sstevel@tonic-gate */
2312e2c009bSjjc typedef struct lgrp_plat_latency_stats {
2322e2c009bSjjc hrtime_t latencies[MAX_NODES][MAX_NODES];
2332e2c009bSjjc hrtime_t latency_max;
2342e2c009bSjjc hrtime_t latency_min;
2352e2c009bSjjc } lgrp_plat_latency_stats_t;
2367c478bd9Sstevel@tonic-gate
2377c478bd9Sstevel@tonic-gate /*
2382e2c009bSjjc * Memory configuration for probing
2397c478bd9Sstevel@tonic-gate */
2402e2c009bSjjc typedef struct lgrp_plat_probe_mem_config {
2412e2c009bSjjc size_t probe_memsize; /* how much memory to probe per node */
2422e2c009bSjjc caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */
2432e2c009bSjjc pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */
2442e2c009bSjjc } lgrp_plat_probe_mem_config_t;
2457c478bd9Sstevel@tonic-gate
2467c478bd9Sstevel@tonic-gate /*
2472e2c009bSjjc * Statistics kept for probing
2487c478bd9Sstevel@tonic-gate */
2492e2c009bSjjc typedef struct lgrp_plat_probe_stats {
2502e2c009bSjjc hrtime_t flush_cost;
2512e2c009bSjjc hrtime_t probe_cost;
2522e2c009bSjjc hrtime_t probe_cost_total;
2532e2c009bSjjc hrtime_t probe_error_code;
2542e2c009bSjjc hrtime_t probe_errors[MAX_NODES][MAX_NODES];
2552e2c009bSjjc int probe_suspect[MAX_NODES][MAX_NODES];
2562e2c009bSjjc hrtime_t probe_max[MAX_NODES][MAX_NODES];
2572e2c009bSjjc hrtime_t probe_min[MAX_NODES][MAX_NODES];
2582e2c009bSjjc } lgrp_plat_probe_stats_t;
2597c478bd9Sstevel@tonic-gate
2607c478bd9Sstevel@tonic-gate /*
2612e2c009bSjjc * Node to proximity domain ID mapping structure (only used with SRAT)
2627c478bd9Sstevel@tonic-gate */
2632e2c009bSjjc typedef struct node_domain_map {
2642e2c009bSjjc int exists;
2652e2c009bSjjc uint32_t prox_domain;
2662e2c009bSjjc } node_domain_map_t;
2677c478bd9Sstevel@tonic-gate
2687c478bd9Sstevel@tonic-gate /*
269a3114836SGerry Liu * Node ID and starting and ending page for physical memory in memory node
2707c478bd9Sstevel@tonic-gate */
271a3114836SGerry Liu typedef struct memnode_phys_addr_map {
2722e2c009bSjjc pfn_t start;
2732e2c009bSjjc pfn_t end;
2742e2c009bSjjc int exists;
2752e2c009bSjjc uint32_t prox_domain;
276a3114836SGerry Liu uint32_t device_id;
277a3114836SGerry Liu uint_t lgrphand;
278a3114836SGerry Liu } memnode_phys_addr_map_t;
2797c478bd9Sstevel@tonic-gate
280dae2fa37Sjjc /*
281d821f0f0Sjjc * Number of CPUs for which we got APIC IDs
282dae2fa37Sjjc */
283d821f0f0Sjjc static int lgrp_plat_apic_ncpus = 0;
2847c478bd9Sstevel@tonic-gate
2857c478bd9Sstevel@tonic-gate /*
286d5d7cf4eSJonathan Chew * CPU to node ID mapping table (only used for SRAT) and its max number of
287d5d7cf4eSJonathan Chew * entries
2887c478bd9Sstevel@tonic-gate */
289d5d7cf4eSJonathan Chew static cpu_node_map_t *lgrp_plat_cpu_node = NULL;
290d5d7cf4eSJonathan Chew static uint_t lgrp_plat_cpu_node_nentries = 0;
2917c478bd9Sstevel@tonic-gate
2927c478bd9Sstevel@tonic-gate /*
2932e2c009bSjjc * Latency statistics
2947c478bd9Sstevel@tonic-gate */
2952e2c009bSjjc lgrp_plat_latency_stats_t lgrp_plat_lat_stats;
2967c478bd9Sstevel@tonic-gate
2977c478bd9Sstevel@tonic-gate /*
2982e2c009bSjjc * Whether memory is interleaved across nodes causing MPO to be disabled
2997c478bd9Sstevel@tonic-gate */
3002e2c009bSjjc static int lgrp_plat_mem_intrlv = 0;
3017c478bd9Sstevel@tonic-gate
3027c478bd9Sstevel@tonic-gate /*
3032e2c009bSjjc * Node ID to proximity domain ID mapping table (only used for SRAT)
3047c478bd9Sstevel@tonic-gate */
3052e2c009bSjjc static node_domain_map_t lgrp_plat_node_domain[MAX_NODES];
3067c478bd9Sstevel@tonic-gate
3077c478bd9Sstevel@tonic-gate /*
3082e2c009bSjjc * Physical address range for memory in each node
3097c478bd9Sstevel@tonic-gate */
310a3114836SGerry Liu static memnode_phys_addr_map_t lgrp_plat_memnode_info[MAX_MEM_NODES];
3117c478bd9Sstevel@tonic-gate
3127c478bd9Sstevel@tonic-gate /*
3132e2c009bSjjc * Statistics gotten from probing
3147c478bd9Sstevel@tonic-gate */
3152e2c009bSjjc static lgrp_plat_probe_stats_t lgrp_plat_probe_stats;
3167c478bd9Sstevel@tonic-gate
3177c478bd9Sstevel@tonic-gate /*
3182e2c009bSjjc * Memory configuration for probing
3197c478bd9Sstevel@tonic-gate */
3202e2c009bSjjc static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config;
3217c478bd9Sstevel@tonic-gate
32281d9ccb6SJonathan Chew /*
32381d9ccb6SJonathan Chew * Lowest proximity domain ID seen in ACPI SRAT
32481d9ccb6SJonathan Chew */
32581d9ccb6SJonathan Chew static uint32_t lgrp_plat_prox_domain_min = UINT32_MAX;
32681d9ccb6SJonathan Chew
3277c478bd9Sstevel@tonic-gate /*
3282e2c009bSjjc * Error code from processing ACPI SRAT
3297c478bd9Sstevel@tonic-gate */
3302e2c009bSjjc static int lgrp_plat_srat_error = 0;
3317c478bd9Sstevel@tonic-gate
3327c478bd9Sstevel@tonic-gate /*
3332e2c009bSjjc * Error code from processing ACPI SLIT
3347c478bd9Sstevel@tonic-gate */
3352e2c009bSjjc static int lgrp_plat_slit_error = 0;
3367c478bd9Sstevel@tonic-gate
337a3114836SGerry Liu /*
338a3114836SGerry Liu * Whether lgrp topology has been flattened to 2 levels.
339a3114836SGerry Liu */
340a3114836SGerry Liu static int lgrp_plat_topo_flatten = 0;
341a3114836SGerry Liu
342a3114836SGerry Liu
343a3114836SGerry Liu /*
344a3114836SGerry Liu * Maximum memory node ID in use.
345a3114836SGerry Liu */
346a3114836SGerry Liu static uint_t lgrp_plat_max_mem_node;
347a3114836SGerry Liu
3487c478bd9Sstevel@tonic-gate /*
3492e2c009bSjjc * Allocate lgroup array statically
3507c478bd9Sstevel@tonic-gate */
3512e2c009bSjjc static lgrp_t lgrp_space[NLGRP];
3522e2c009bSjjc static int nlgrps_alloc;
3537c478bd9Sstevel@tonic-gate
3547c478bd9Sstevel@tonic-gate
35581d9ccb6SJonathan Chew /*
35681d9ccb6SJonathan Chew * Enable finding and using minimum proximity domain ID when hashing
35781d9ccb6SJonathan Chew */
35881d9ccb6SJonathan Chew int lgrp_plat_domain_min_enable = 1;
35981d9ccb6SJonathan Chew
3607c478bd9Sstevel@tonic-gate /*
361a3114836SGerry Liu * Maximum possible number of nodes in system
3627c478bd9Sstevel@tonic-gate */
3637c478bd9Sstevel@tonic-gate uint_t lgrp_plat_node_cnt = 1;
3647c478bd9Sstevel@tonic-gate
36581d9ccb6SJonathan Chew /*
36681d9ccb6SJonathan Chew * Enable sorting nodes in ascending order by starting physical address
36781d9ccb6SJonathan Chew */
36881d9ccb6SJonathan Chew int lgrp_plat_node_sort_enable = 1;
36981d9ccb6SJonathan Chew
3707c478bd9Sstevel@tonic-gate /*
3712e2c009bSjjc * Configuration Parameters for Probing
3722e2c009bSjjc * - lgrp_plat_probe_flags Flags to specify enabling probing, probe
3732e2c009bSjjc * operation, etc.
3742e2c009bSjjc * - lgrp_plat_probe_nrounds How many rounds of probing to do
3752e2c009bSjjc * - lgrp_plat_probe_nsamples Number of samples to take when probing each
3762e2c009bSjjc * node
3772e2c009bSjjc * - lgrp_plat_probe_nreads Number of times to read vendor ID from
3782e2c009bSjjc * Northbridge for each probe
3797c478bd9Sstevel@tonic-gate */
3802e2c009bSjjc uint_t lgrp_plat_probe_flags = 0;
3812e2c009bSjjc int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS;
3822e2c009bSjjc int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES;
3832e2c009bSjjc int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS;
3847c478bd9Sstevel@tonic-gate
3857c478bd9Sstevel@tonic-gate /*
386a3114836SGerry Liu * Enable use of ACPI System Resource Affinity Table (SRAT), System
387a3114836SGerry Liu * Locality Information Table (SLIT) and Maximum System Capability Table (MSCT)
3887c478bd9Sstevel@tonic-gate */
3892e2c009bSjjc int lgrp_plat_srat_enable = 1;
3902e2c009bSjjc int lgrp_plat_slit_enable = 1;
391a3114836SGerry Liu int lgrp_plat_msct_enable = 1;
3927c478bd9Sstevel@tonic-gate
39318968004SKit Chow /*
39418968004SKit Chow * mnode_xwa: set to non-zero value to initiate workaround if large pages are
39518968004SKit Chow * found to be crossing memory node boundaries. The workaround will eliminate
39618968004SKit Chow * a base size page at the end of each memory node boundary to ensure that
39718968004SKit Chow * a large page with constituent pages that span more than 1 memory node
39818968004SKit Chow * can never be formed.
39918968004SKit Chow *
40018968004SKit Chow */
40118968004SKit Chow int mnode_xwa = 1;
40218968004SKit Chow
4037c478bd9Sstevel@tonic-gate /*
4042e2c009bSjjc * Static array to hold lgroup statistics
4057c478bd9Sstevel@tonic-gate */
4062e2c009bSjjc struct lgrp_stats lgrp_stats[NLGRP];
4077c478bd9Sstevel@tonic-gate
4087c478bd9Sstevel@tonic-gate
4097c478bd9Sstevel@tonic-gate /*
4102e2c009bSjjc * Forward declarations of platform interface routines
4117c478bd9Sstevel@tonic-gate */
4122e2c009bSjjc void plat_build_mem_nodes(struct memlist *list);
4137c478bd9Sstevel@tonic-gate
414a3114836SGerry Liu int plat_mnode_xcheck(pfn_t pfncnt);
4157c478bd9Sstevel@tonic-gate
4162e2c009bSjjc lgrp_handle_t plat_mem_node_to_lgrphand(int mnode);
4177c478bd9Sstevel@tonic-gate
4182e2c009bSjjc int plat_pfn_to_mem_node(pfn_t pfn);
4197c478bd9Sstevel@tonic-gate
4207c478bd9Sstevel@tonic-gate /*
4212e2c009bSjjc * Forward declarations of lgroup platform interface routines
4227c478bd9Sstevel@tonic-gate */
4232e2c009bSjjc lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid);
4247c478bd9Sstevel@tonic-gate
4252e2c009bSjjc void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg);
4268949bcd6Sandrei
4272e2c009bSjjc lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id);
4287c478bd9Sstevel@tonic-gate
429d5d7cf4eSJonathan Chew void lgrp_plat_init(lgrp_init_stages_t stage);
4307c478bd9Sstevel@tonic-gate
4312e2c009bSjjc int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to);
4327c478bd9Sstevel@tonic-gate
4332e2c009bSjjc int lgrp_plat_max_lgrps(void);
4347c478bd9Sstevel@tonic-gate
4352e2c009bSjjc pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand,
4362e2c009bSjjc lgrp_mem_query_t query);
4377c478bd9Sstevel@tonic-gate
4382e2c009bSjjc lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn);
4397c478bd9Sstevel@tonic-gate
4402e2c009bSjjc void lgrp_plat_probe(void);
4417c478bd9Sstevel@tonic-gate
4422e2c009bSjjc lgrp_handle_t lgrp_plat_root_hand(void);
4437c478bd9Sstevel@tonic-gate
444f78a91cdSjjc
445f78a91cdSjjc /*
4462e2c009bSjjc * Forward declarations of local routines
447f78a91cdSjjc */
4482e2c009bSjjc static int is_opteron(void);
449f78a91cdSjjc
450dae2fa37Sjjc static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain,
451d821f0f0Sjjc int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid,
452d821f0f0Sjjc uint32_t domain);
453dae2fa37Sjjc
454d5d7cf4eSJonathan Chew static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node,
455d5d7cf4eSJonathan Chew int cpu_node_nentries);
4567c478bd9Sstevel@tonic-gate
4572e2c009bSjjc static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain,
458d821f0f0Sjjc int node_cnt, uint32_t domain);
4597c478bd9Sstevel@tonic-gate
460d5d7cf4eSJonathan Chew static void lgrp_plat_get_numa_config(void);
461d5d7cf4eSJonathan Chew
462a3114836SGerry Liu static void lgrp_plat_latency_adjust(memnode_phys_addr_map_t *memnode_info,
4632e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats,
4642e2c009bSjjc lgrp_plat_probe_stats_t *probe_stats);
465f78a91cdSjjc
466a3114836SGerry Liu static int lgrp_plat_latency_verify(memnode_phys_addr_map_t *memnode_info,
4672e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats);
4687c478bd9Sstevel@tonic-gate
469d5d7cf4eSJonathan Chew static void lgrp_plat_main_init(void);
470d5d7cf4eSJonathan Chew
4712e2c009bSjjc static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t);
4727c478bd9Sstevel@tonic-gate
4732e2c009bSjjc static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain,
474d821f0f0Sjjc int node_cnt, uint32_t domain);
4757c478bd9Sstevel@tonic-gate
476a3114836SGerry Liu static int lgrp_plat_memnode_info_update(node_domain_map_t *node_domain,
477a3114836SGerry Liu int node_cnt, memnode_phys_addr_map_t *memnode_info, int memnode_cnt,
478a3114836SGerry Liu uint64_t start, uint64_t end, uint32_t domain, uint32_t device_id);
4797c478bd9Sstevel@tonic-gate
48081d9ccb6SJonathan Chew static void lgrp_plat_node_sort(node_domain_map_t *node_domain,
48181d9ccb6SJonathan Chew int node_cnt, cpu_node_map_t *cpu_node, int cpu_count,
482a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info);
48381d9ccb6SJonathan Chew
4842e2c009bSjjc static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node,
485d5d7cf4eSJonathan Chew int cpu_node_nentries, lgrp_plat_probe_mem_config_t *probe_mem_config,
486d5d7cf4eSJonathan Chew lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats);
4877c478bd9Sstevel@tonic-gate
488d821f0f0Sjjc static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node);
489dae2fa37Sjjc
4907bc870e6SToomas Soome static int lgrp_plat_process_slit(ACPI_TABLE_SLIT *tp,
491a3114836SGerry Liu node_domain_map_t *node_domain, uint_t node_cnt,
492a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info,
493a3114836SGerry Liu lgrp_plat_latency_stats_t *lat_stats);
494a3114836SGerry Liu
495a3114836SGerry Liu static int lgrp_plat_process_sli(uint32_t domain, uchar_t *sli_info,
496a3114836SGerry Liu uint32_t sli_cnt, node_domain_map_t *node_domain, uint_t node_cnt,
497a3114836SGerry Liu lgrp_plat_latency_stats_t *lat_stats);
4982e2c009bSjjc
4997bc870e6SToomas Soome static int lgrp_plat_process_srat(ACPI_TABLE_SRAT *tp, ACPI_TABLE_MSCT *mp,
50081d9ccb6SJonathan Chew uint32_t *prox_domain_min, node_domain_map_t *node_domain,
50181d9ccb6SJonathan Chew cpu_node_map_t *cpu_node, int cpu_count,
502a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info);
5032e2c009bSjjc
504d5d7cf4eSJonathan Chew static void lgrp_plat_release_bootstrap(void);
505d5d7cf4eSJonathan Chew
5067bc870e6SToomas Soome static int lgrp_plat_srat_domains(ACPI_TABLE_SRAT *tp,
50781d9ccb6SJonathan Chew uint32_t *prox_domain_min);
5082e2c009bSjjc
5097bc870e6SToomas Soome static int lgrp_plat_msct_domains(ACPI_TABLE_MSCT *tp,
510a3114836SGerry Liu uint32_t *prox_domain_min);
511a3114836SGerry Liu
512a3114836SGerry Liu static void lgrp_plat_2level_setup(lgrp_plat_latency_stats_t *lat_stats);
5132e2c009bSjjc
5142e2c009bSjjc static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
515a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info);
5162e2c009bSjjc
5172e2c009bSjjc static hrtime_t opt_probe_vendor(int dest_node, int nreads);
5187c478bd9Sstevel@tonic-gate
519472714d6Skchow
520472714d6Skchow /*
5212e2c009bSjjc * PLATFORM INTERFACE ROUTINES
522472714d6Skchow */
5232e2c009bSjjc
5242e2c009bSjjc /*
5252e2c009bSjjc * Configure memory nodes for machines with more than one node (ie NUMA)
5262e2c009bSjjc */
5272e2c009bSjjc void
plat_build_mem_nodes(struct memlist * list)5282e2c009bSjjc plat_build_mem_nodes(struct memlist *list)
529472714d6Skchow {
5302e2c009bSjjc pfn_t cur_start; /* start addr of subrange */
5312e2c009bSjjc pfn_t cur_end; /* end addr of subrange */
5322e2c009bSjjc pfn_t start; /* start addr of whole range */
5332e2c009bSjjc pfn_t end; /* end addr of whole range */
53418968004SKit Chow pgcnt_t endcnt; /* pages to sacrifice */
535472714d6Skchow
5362e2c009bSjjc /*
5372e2c009bSjjc * Boot install lists are arranged <addr, len>, ...
5382e2c009bSjjc */
5392e2c009bSjjc while (list) {
5402e2c009bSjjc int node;
541472714d6Skchow
54256f33205SJonathan Adams start = list->ml_address >> PAGESHIFT;
54356f33205SJonathan Adams end = (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
544472714d6Skchow
5452e2c009bSjjc if (start > physmax) {
54656f33205SJonathan Adams list = list->ml_next;
547472714d6Skchow continue;
548472714d6Skchow }
5492e2c009bSjjc if (end > physmax)
5502e2c009bSjjc end = physmax;
551472714d6Skchow
552472714d6Skchow /*
5532e2c009bSjjc * When there is only one memnode, just add memory to memnode
554472714d6Skchow */
5552e2c009bSjjc if (max_mem_nodes == 1) {
5562e2c009bSjjc mem_node_add_slice(start, end);
55756f33205SJonathan Adams list = list->ml_next;
558472714d6Skchow continue;
559472714d6Skchow }
560472714d6Skchow
5612e2c009bSjjc /*
5622e2c009bSjjc * mem_node_add_slice() expects to get a memory range that
5632e2c009bSjjc * is within one memnode, so need to split any memory range
5642e2c009bSjjc * that spans multiple memnodes into subranges that are each
5652e2c009bSjjc * contained within one memnode when feeding them to
5662e2c009bSjjc * mem_node_add_slice()
5672e2c009bSjjc */
5682e2c009bSjjc cur_start = start;
5692e2c009bSjjc do {
5702e2c009bSjjc node = plat_pfn_to_mem_node(cur_start);
5712e2c009bSjjc
5722e2c009bSjjc /*
5732e2c009bSjjc * Panic if DRAM address map registers or SRAT say
5742e2c009bSjjc * memory in node doesn't exist or address from
5752e2c009bSjjc * boot installed memory list entry isn't in this node.
5762e2c009bSjjc * This shouldn't happen and rest of code can't deal
5772e2c009bSjjc * with this if it does.
5782e2c009bSjjc */
579a3114836SGerry Liu if (node < 0 || node >= lgrp_plat_max_mem_node ||
580a3114836SGerry Liu !lgrp_plat_memnode_info[node].exists ||
581a3114836SGerry Liu cur_start < lgrp_plat_memnode_info[node].start ||
582a3114836SGerry Liu cur_start > lgrp_plat_memnode_info[node].end) {
5832e2c009bSjjc cmn_err(CE_PANIC, "Don't know which memnode "
5842e2c009bSjjc "to add installed memory address 0x%lx\n",
5852e2c009bSjjc cur_start);
5862e2c009bSjjc }
5872e2c009bSjjc
5882e2c009bSjjc /*
5892e2c009bSjjc * End of current subrange should not span memnodes
5902e2c009bSjjc */
5912e2c009bSjjc cur_end = end;
59218968004SKit Chow endcnt = 0;
593a3114836SGerry Liu if (lgrp_plat_memnode_info[node].exists &&
594a3114836SGerry Liu cur_end > lgrp_plat_memnode_info[node].end) {
595a3114836SGerry Liu cur_end = lgrp_plat_memnode_info[node].end;
59618968004SKit Chow if (mnode_xwa > 1) {
59718968004SKit Chow /*
59818968004SKit Chow * sacrifice the last page in each
59918968004SKit Chow * node to eliminate large pages
60018968004SKit Chow * that span more than 1 memory node.
60118968004SKit Chow */
60218968004SKit Chow endcnt = 1;
603bcee7a0bSKit Chow physinstalled--;
60418968004SKit Chow }
60518968004SKit Chow }
6062e2c009bSjjc
60718968004SKit Chow mem_node_add_slice(cur_start, cur_end - endcnt);
6082e2c009bSjjc
6092e2c009bSjjc /*
6102e2c009bSjjc * Next subrange starts after end of current one
6112e2c009bSjjc */
6122e2c009bSjjc cur_start = cur_end + 1;
6132e2c009bSjjc } while (cur_end < end);
6142e2c009bSjjc
61556f33205SJonathan Adams list = list->ml_next;
6162e2c009bSjjc }
6172e2c009bSjjc mem_node_physalign = 0;
6182e2c009bSjjc mem_node_pfn_shift = 0;
6192e2c009bSjjc }
6202e2c009bSjjc
6212e2c009bSjjc
6222e2c009bSjjc /*
6232e2c009bSjjc * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt
6242e2c009bSjjc * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if
6252e2c009bSjjc * a crossing is found and returns 0 otherwise.
6262e2c009bSjjc */
6272e2c009bSjjc int
plat_mnode_xcheck(pfn_t pfncnt)6282e2c009bSjjc plat_mnode_xcheck(pfn_t pfncnt)
6292e2c009bSjjc {
6302e2c009bSjjc int node, prevnode = -1, basenode;
6312e2c009bSjjc pfn_t ea, sa;
6322e2c009bSjjc
633a3114836SGerry Liu for (node = 0; node < lgrp_plat_max_mem_node; node++) {
6342e2c009bSjjc
635a3114836SGerry Liu if (lgrp_plat_memnode_info[node].exists == 0)
6362e2c009bSjjc continue;
6372e2c009bSjjc
6382e2c009bSjjc if (prevnode == -1) {
6392e2c009bSjjc prevnode = node;
6402e2c009bSjjc basenode = node;
6412e2c009bSjjc continue;
6422e2c009bSjjc }
6432e2c009bSjjc
6442e2c009bSjjc /* assume x86 node pfn ranges are in increasing order */
645a3114836SGerry Liu ASSERT(lgrp_plat_memnode_info[node].start >
646a3114836SGerry Liu lgrp_plat_memnode_info[prevnode].end);
6472e2c009bSjjc
6482e2c009bSjjc /*
6492e2c009bSjjc * continue if the starting address of node is not contiguous
6502e2c009bSjjc * with the previous node.
6512e2c009bSjjc */
6522e2c009bSjjc
653a3114836SGerry Liu if (lgrp_plat_memnode_info[node].start !=
654a3114836SGerry Liu (lgrp_plat_memnode_info[prevnode].end + 1)) {
6552e2c009bSjjc basenode = node;
6562e2c009bSjjc prevnode = node;
6572e2c009bSjjc continue;
6582e2c009bSjjc }
6592e2c009bSjjc
6602e2c009bSjjc /* check if the starting address of node is pfncnt aligned */
661a3114836SGerry Liu if ((lgrp_plat_memnode_info[node].start & (pfncnt - 1)) != 0) {
6622e2c009bSjjc
6632e2c009bSjjc /*
6642e2c009bSjjc * at this point, node starts at an unaligned boundary
6652e2c009bSjjc * and is contiguous with the previous node(s) to
6662e2c009bSjjc * basenode. Check if there is an aligned contiguous
667472714d6Skchow * range of length pfncnt that crosses this boundary.
668472714d6Skchow */
669472714d6Skchow
670a3114836SGerry Liu sa = P2ALIGN(lgrp_plat_memnode_info[prevnode].end,
671472714d6Skchow pfncnt);
672a3114836SGerry Liu ea = P2ROUNDUP((lgrp_plat_memnode_info[node].start),
673472714d6Skchow pfncnt);
674472714d6Skchow
675472714d6Skchow ASSERT((ea - sa) == pfncnt);
676a3114836SGerry Liu if (sa >= lgrp_plat_memnode_info[basenode].start &&
677a3114836SGerry Liu ea <= (lgrp_plat_memnode_info[node].end + 1)) {
67818968004SKit Chow /*
67918968004SKit Chow * large page found to cross mnode boundary.
68018968004SKit Chow * Return Failure if workaround not enabled.
68118968004SKit Chow */
68218968004SKit Chow if (mnode_xwa == 0)
68318968004SKit Chow return (1);
68418968004SKit Chow mnode_xwa++;
68518968004SKit Chow }
686472714d6Skchow }
687472714d6Skchow prevnode = node;
688472714d6Skchow }
689472714d6Skchow return (0);
690472714d6Skchow }
691472714d6Skchow
6922e2c009bSjjc
6932e2c009bSjjc lgrp_handle_t
plat_mem_node_to_lgrphand(int mnode)6942e2c009bSjjc plat_mem_node_to_lgrphand(int mnode)
6952e2c009bSjjc {
6962e2c009bSjjc if (max_mem_nodes == 1)
6972e2c009bSjjc return (LGRP_DEFAULT_HANDLE);
6982e2c009bSjjc
699a3114836SGerry Liu ASSERT(0 <= mnode && mnode < lgrp_plat_max_mem_node);
7002e2c009bSjjc
701a3114836SGerry Liu return ((lgrp_handle_t)(lgrp_plat_memnode_info[mnode].lgrphand));
702a3114836SGerry Liu }
7032e2c009bSjjc
7047c478bd9Sstevel@tonic-gate int
plat_pfn_to_mem_node(pfn_t pfn)7057c478bd9Sstevel@tonic-gate plat_pfn_to_mem_node(pfn_t pfn)
7067c478bd9Sstevel@tonic-gate {
7077c478bd9Sstevel@tonic-gate int node;
7087c478bd9Sstevel@tonic-gate
7097c478bd9Sstevel@tonic-gate if (max_mem_nodes == 1)
7107c478bd9Sstevel@tonic-gate return (0);
7117c478bd9Sstevel@tonic-gate
712a3114836SGerry Liu for (node = 0; node < lgrp_plat_max_mem_node; node++) {
713a940d195Sjjc /*
714a940d195Sjjc * Skip nodes with no memory
715a940d195Sjjc */
716a3114836SGerry Liu if (!lgrp_plat_memnode_info[node].exists)
717a940d195Sjjc continue;
718a940d195Sjjc
719a3114836SGerry Liu membar_consumer();
720a3114836SGerry Liu if (pfn >= lgrp_plat_memnode_info[node].start &&
721a3114836SGerry Liu pfn <= lgrp_plat_memnode_info[node].end)
7227c478bd9Sstevel@tonic-gate return (node);
7237c478bd9Sstevel@tonic-gate }
7247c478bd9Sstevel@tonic-gate
7252e2c009bSjjc /*
7262e2c009bSjjc * Didn't find memnode where this PFN lives which should never happen
7272e2c009bSjjc */
728a3114836SGerry Liu ASSERT(node < lgrp_plat_max_mem_node);
7297c478bd9Sstevel@tonic-gate return (-1);
7307c478bd9Sstevel@tonic-gate }
7317c478bd9Sstevel@tonic-gate
7322e2c009bSjjc
7337c478bd9Sstevel@tonic-gate /*
7342e2c009bSjjc * LGROUP PLATFORM INTERFACE ROUTINES
7357c478bd9Sstevel@tonic-gate */
7367c478bd9Sstevel@tonic-gate
7372e2c009bSjjc /*
7382e2c009bSjjc * Allocate additional space for an lgroup.
7392e2c009bSjjc */
7402e2c009bSjjc lgrp_t *
lgrp_plat_alloc(lgrp_id_t lgrpid)7412e2c009bSjjc lgrp_plat_alloc(lgrp_id_t lgrpid)
7422e2c009bSjjc {
7432e2c009bSjjc lgrp_t *lgrp;
7447c478bd9Sstevel@tonic-gate
7452e2c009bSjjc lgrp = &lgrp_space[nlgrps_alloc++];
7462e2c009bSjjc if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP)
7472e2c009bSjjc return (NULL);
7482e2c009bSjjc return (lgrp);
7492e2c009bSjjc }
7507c478bd9Sstevel@tonic-gate
7517c478bd9Sstevel@tonic-gate
7522e2c009bSjjc /*
7532e2c009bSjjc * Platform handling for (re)configuration changes
754a3114836SGerry Liu *
755a3114836SGerry Liu * Mechanism to protect lgrp_plat_cpu_node[] at CPU hotplug:
756a3114836SGerry Liu * 1) Use cpu_lock to synchronize between lgrp_plat_config() and
757a3114836SGerry Liu * lgrp_plat_cpu_to_hand().
758a3114836SGerry Liu * 2) Disable latency probing logic by making sure that the flag
759a3114836SGerry Liu * LGRP_PLAT_PROBE_ENABLE is cleared.
760a3114836SGerry Liu *
761a3114836SGerry Liu * Mechanism to protect lgrp_plat_memnode_info[] at memory hotplug:
762a3114836SGerry Liu * 1) Only inserts into lgrp_plat_memnode_info at memory hotplug, no removal.
763a3114836SGerry Liu * 2) Only expansion to existing entries, no shrinking.
764a3114836SGerry Liu * 3) On writing side, DR framework ensures that lgrp_plat_config() is called
765a3114836SGerry Liu * in single-threaded context. And membar_producer() is used to ensure that
766a3114836SGerry Liu * all changes are visible to other CPUs before setting the "exists" flag.
767a3114836SGerry Liu * 4) On reading side, membar_consumer() after checking the "exists" flag
768a3114836SGerry Liu * ensures that right values are retrieved.
769a3114836SGerry Liu *
770a3114836SGerry Liu * Mechanism to protect lgrp_plat_node_domain[] at hotplug:
771a3114836SGerry Liu * 1) Only insertion into lgrp_plat_node_domain at hotplug, no removal.
772a3114836SGerry Liu * 2) On writing side, it's single-threaded and membar_producer() is used to
773a3114836SGerry Liu * ensure all changes are visible to other CPUs before setting the "exists"
774a3114836SGerry Liu * flag.
775a3114836SGerry Liu * 3) On reading side, membar_consumer() after checking the "exists" flag
776a3114836SGerry Liu * ensures that right values are retrieved.
7772e2c009bSjjc */
7782e2c009bSjjc void
lgrp_plat_config(lgrp_config_flag_t flag,uintptr_t arg)7792e2c009bSjjc lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg)
7802e2c009bSjjc {
781a3114836SGerry Liu #ifdef __xpv
782a3114836SGerry Liu _NOTE(ARGUNUSED(flag, arg));
783a3114836SGerry Liu #else
784a3114836SGerry Liu int rc, node;
785a3114836SGerry Liu cpu_t *cp;
786a3114836SGerry Liu void *hdl = NULL;
787a3114836SGerry Liu uchar_t *sliptr = NULL;
788a3114836SGerry Liu uint32_t domain, apicid, slicnt = 0;
789a3114836SGerry Liu update_membounds_t *mp;
790a3114836SGerry Liu
791a3114836SGerry Liu extern int acpidev_dr_get_cpu_numa_info(cpu_t *, void **, uint32_t *,
792a3114836SGerry Liu uint32_t *, uint32_t *, uchar_t **);
793a3114836SGerry Liu extern void acpidev_dr_free_cpu_numa_info(void *);
794a3114836SGerry Liu
795a3114836SGerry Liu /*
796a3114836SGerry Liu * This interface is used to support CPU/memory DR operations.
797a3114836SGerry Liu * Don't bother here if it's still during boot or only one lgrp node
798a3114836SGerry Liu * is supported.
799a3114836SGerry Liu */
800a3114836SGerry Liu if (!lgrp_topo_initialized || lgrp_plat_node_cnt == 1)
801a3114836SGerry Liu return;
802a3114836SGerry Liu
803a3114836SGerry Liu switch (flag) {
804a3114836SGerry Liu case LGRP_CONFIG_CPU_ADD:
805a3114836SGerry Liu cp = (cpu_t *)arg;
806a3114836SGerry Liu ASSERT(cp != NULL);
807a3114836SGerry Liu ASSERT(MUTEX_HELD(&cpu_lock));
808a3114836SGerry Liu
809a3114836SGerry Liu /* Check whether CPU already exists. */
810a3114836SGerry Liu ASSERT(!lgrp_plat_cpu_node[cp->cpu_id].exists);
811a3114836SGerry Liu if (lgrp_plat_cpu_node[cp->cpu_id].exists) {
812a3114836SGerry Liu cmn_err(CE_WARN,
813a3114836SGerry Liu "!lgrp: CPU(%d) already exists in cpu_node map.",
814a3114836SGerry Liu cp->cpu_id);
815a3114836SGerry Liu break;
816a3114836SGerry Liu }
817a3114836SGerry Liu
818a3114836SGerry Liu /* Query CPU lgrp information. */
819a3114836SGerry Liu rc = acpidev_dr_get_cpu_numa_info(cp, &hdl, &apicid, &domain,
820a3114836SGerry Liu &slicnt, &sliptr);
821a3114836SGerry Liu ASSERT(rc == 0);
822a3114836SGerry Liu if (rc != 0) {
823a3114836SGerry Liu cmn_err(CE_WARN,
824a3114836SGerry Liu "!lgrp: failed to query lgrp info for CPU(%d).",
825a3114836SGerry Liu cp->cpu_id);
826a3114836SGerry Liu break;
827a3114836SGerry Liu }
828a3114836SGerry Liu
829a3114836SGerry Liu /* Update node to proximity domain mapping */
830a3114836SGerry Liu node = lgrp_plat_domain_to_node(lgrp_plat_node_domain,
831a3114836SGerry Liu lgrp_plat_node_cnt, domain);
832a3114836SGerry Liu if (node == -1) {
833a3114836SGerry Liu node = lgrp_plat_node_domain_update(
834a3114836SGerry Liu lgrp_plat_node_domain, lgrp_plat_node_cnt, domain);
835a3114836SGerry Liu ASSERT(node != -1);
836a3114836SGerry Liu if (node == -1) {
837a3114836SGerry Liu acpidev_dr_free_cpu_numa_info(hdl);
838a3114836SGerry Liu cmn_err(CE_WARN, "!lgrp: failed to update "
839a3114836SGerry Liu "node_domain map for domain(%u).", domain);
840a3114836SGerry Liu break;
841a3114836SGerry Liu }
842a3114836SGerry Liu }
843a3114836SGerry Liu
844a3114836SGerry Liu /* Update latency information among lgrps. */
845a3114836SGerry Liu if (slicnt != 0 && sliptr != NULL) {
846a3114836SGerry Liu if (lgrp_plat_process_sli(domain, sliptr, slicnt,
847a3114836SGerry Liu lgrp_plat_node_domain, lgrp_plat_node_cnt,
848a3114836SGerry Liu &lgrp_plat_lat_stats) != 0) {
849a3114836SGerry Liu cmn_err(CE_WARN, "!lgrp: failed to update "
850a3114836SGerry Liu "latency information for domain (%u).",
851a3114836SGerry Liu domain);
852a3114836SGerry Liu }
853a3114836SGerry Liu }
854a3114836SGerry Liu
855a3114836SGerry Liu /* Update CPU to node mapping. */
856a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].prox_domain = domain;
857a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].node = node;
858a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].apicid = apicid;
859a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].exists = 1;
860a3114836SGerry Liu lgrp_plat_apic_ncpus++;
861a3114836SGerry Liu
862a3114836SGerry Liu acpidev_dr_free_cpu_numa_info(hdl);
863a3114836SGerry Liu break;
864a3114836SGerry Liu
865a3114836SGerry Liu case LGRP_CONFIG_CPU_DEL:
866a3114836SGerry Liu cp = (cpu_t *)arg;
867a3114836SGerry Liu ASSERT(cp != NULL);
868a3114836SGerry Liu ASSERT(MUTEX_HELD(&cpu_lock));
869a3114836SGerry Liu
870a3114836SGerry Liu /* Check whether CPU exists. */
871a3114836SGerry Liu ASSERT(lgrp_plat_cpu_node[cp->cpu_id].exists);
872a3114836SGerry Liu if (!lgrp_plat_cpu_node[cp->cpu_id].exists) {
873a3114836SGerry Liu cmn_err(CE_WARN,
874a3114836SGerry Liu "!lgrp: CPU(%d) doesn't exist in cpu_node map.",
875a3114836SGerry Liu cp->cpu_id);
876a3114836SGerry Liu break;
877a3114836SGerry Liu }
878a3114836SGerry Liu
879a3114836SGerry Liu /* Query CPU lgrp information. */
880a3114836SGerry Liu rc = acpidev_dr_get_cpu_numa_info(cp, &hdl, &apicid, &domain,
881a3114836SGerry Liu NULL, NULL);
882a3114836SGerry Liu ASSERT(rc == 0);
883a3114836SGerry Liu if (rc != 0) {
884a3114836SGerry Liu cmn_err(CE_WARN,
885a3114836SGerry Liu "!lgrp: failed to query lgrp info for CPU(%d).",
886a3114836SGerry Liu cp->cpu_id);
887a3114836SGerry Liu break;
888a3114836SGerry Liu }
889a3114836SGerry Liu
890a3114836SGerry Liu /* Update map. */
891a3114836SGerry Liu ASSERT(lgrp_plat_cpu_node[cp->cpu_id].apicid == apicid);
892a3114836SGerry Liu ASSERT(lgrp_plat_cpu_node[cp->cpu_id].prox_domain == domain);
893a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].exists = 0;
894a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].apicid = UINT32_MAX;
895a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].prox_domain = UINT32_MAX;
896a3114836SGerry Liu lgrp_plat_cpu_node[cp->cpu_id].node = UINT_MAX;
897a3114836SGerry Liu lgrp_plat_apic_ncpus--;
898a3114836SGerry Liu
899a3114836SGerry Liu acpidev_dr_free_cpu_numa_info(hdl);
900a3114836SGerry Liu break;
901a3114836SGerry Liu
902a3114836SGerry Liu case LGRP_CONFIG_MEM_ADD:
903a3114836SGerry Liu mp = (update_membounds_t *)arg;
904a3114836SGerry Liu ASSERT(mp != NULL);
905a3114836SGerry Liu
906a3114836SGerry Liu /* Update latency information among lgrps. */
907a3114836SGerry Liu if (mp->u_sli_cnt != 0 && mp->u_sli_ptr != NULL) {
908a3114836SGerry Liu if (lgrp_plat_process_sli(mp->u_domain,
909a3114836SGerry Liu mp->u_sli_ptr, mp->u_sli_cnt,
910a3114836SGerry Liu lgrp_plat_node_domain, lgrp_plat_node_cnt,
911a3114836SGerry Liu &lgrp_plat_lat_stats) != 0) {
912a3114836SGerry Liu cmn_err(CE_WARN, "!lgrp: failed to update "
913a3114836SGerry Liu "latency information for domain (%u).",
914a3114836SGerry Liu domain);
915a3114836SGerry Liu }
916a3114836SGerry Liu }
917a3114836SGerry Liu
918a3114836SGerry Liu if (lgrp_plat_memnode_info_update(lgrp_plat_node_domain,
919a3114836SGerry Liu lgrp_plat_node_cnt, lgrp_plat_memnode_info, max_mem_nodes,
920a3114836SGerry Liu mp->u_base, mp->u_base + mp->u_length,
921a3114836SGerry Liu mp->u_domain, mp->u_device_id) < 0) {
922a3114836SGerry Liu cmn_err(CE_WARN,
923a3114836SGerry Liu "!lgrp: failed to update latency information for "
924a3114836SGerry Liu "memory (0x%" PRIx64 " - 0x%" PRIx64 ").",
925a3114836SGerry Liu mp->u_base, mp->u_base + mp->u_length);
926a3114836SGerry Liu }
927a3114836SGerry Liu break;
928a3114836SGerry Liu
929a3114836SGerry Liu default:
930a3114836SGerry Liu break;
931a3114836SGerry Liu }
932a3114836SGerry Liu #endif /* __xpv */
9332e2c009bSjjc }
9347c478bd9Sstevel@tonic-gate
9357c478bd9Sstevel@tonic-gate
9362e2c009bSjjc /*
9372e2c009bSjjc * Return the platform handle for the lgroup containing the given CPU
9382e2c009bSjjc */
9392e2c009bSjjc lgrp_handle_t
lgrp_plat_cpu_to_hand(processorid_t id)9402e2c009bSjjc lgrp_plat_cpu_to_hand(processorid_t id)
9412e2c009bSjjc {
9422e2c009bSjjc lgrp_handle_t hand;
9437c478bd9Sstevel@tonic-gate
944a3114836SGerry Liu ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock));
945a3114836SGerry Liu
9462e2c009bSjjc if (lgrp_plat_node_cnt == 1)
9472e2c009bSjjc return (LGRP_DEFAULT_HANDLE);
9487c478bd9Sstevel@tonic-gate
9492e2c009bSjjc hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id],
950d5d7cf4eSJonathan Chew lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries);
9517c478bd9Sstevel@tonic-gate
9522e2c009bSjjc ASSERT(hand != (lgrp_handle_t)-1);
9532e2c009bSjjc if (hand == (lgrp_handle_t)-1)
9542e2c009bSjjc return (LGRP_NULL_HANDLE);
9557c478bd9Sstevel@tonic-gate
9562e2c009bSjjc return (hand);
9577c478bd9Sstevel@tonic-gate }
9587c478bd9Sstevel@tonic-gate
9597c478bd9Sstevel@tonic-gate
9607c478bd9Sstevel@tonic-gate /*
9617c478bd9Sstevel@tonic-gate * Platform-specific initialization of lgroups
9627c478bd9Sstevel@tonic-gate */
9637c478bd9Sstevel@tonic-gate void
lgrp_plat_init(lgrp_init_stages_t stage)964d5d7cf4eSJonathan Chew lgrp_plat_init(lgrp_init_stages_t stage)
9657c478bd9Sstevel@tonic-gate {
966843e1988Sjohnlev #if defined(__xpv)
967843e1988Sjohnlev #else /* __xpv */
9682baa66a0SJonathan Chew u_longlong_t value;
969d5d7cf4eSJonathan Chew #endif /* __xpv */
9702baa66a0SJonathan Chew
971d5d7cf4eSJonathan Chew switch (stage) {
972d5d7cf4eSJonathan Chew case LGRP_INIT_STAGE1:
973d5d7cf4eSJonathan Chew #if defined(__xpv)
974d5d7cf4eSJonathan Chew /*
975d5d7cf4eSJonathan Chew * XXPV For now, the hypervisor treats all memory equally.
976d5d7cf4eSJonathan Chew */
9777c478bd9Sstevel@tonic-gate lgrp_plat_node_cnt = max_mem_nodes = 1;
978d5d7cf4eSJonathan Chew #else /* __xpv */
979a3114836SGerry Liu
980d5d7cf4eSJonathan Chew /*
981d5d7cf4eSJonathan Chew * Get boot property for lgroup topology height limit
982d5d7cf4eSJonathan Chew */
983d5d7cf4eSJonathan Chew if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0)
984d5d7cf4eSJonathan Chew (void) lgrp_topo_ht_limit_set((int)value);
9857c478bd9Sstevel@tonic-gate
986d5d7cf4eSJonathan Chew /*
987d5d7cf4eSJonathan Chew * Get boot property for enabling/disabling SRAT
988d5d7cf4eSJonathan Chew */
989d5d7cf4eSJonathan Chew if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0)
990d5d7cf4eSJonathan Chew lgrp_plat_srat_enable = (int)value;
991dae2fa37Sjjc
992d5d7cf4eSJonathan Chew /*
993d5d7cf4eSJonathan Chew * Get boot property for enabling/disabling SLIT
994d5d7cf4eSJonathan Chew */
995d5d7cf4eSJonathan Chew if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0)
996d5d7cf4eSJonathan Chew lgrp_plat_slit_enable = (int)value;
997d821f0f0Sjjc
998a3114836SGerry Liu /*
999a3114836SGerry Liu * Get boot property for enabling/disabling MSCT
1000a3114836SGerry Liu */
1001a3114836SGerry Liu if (bootprop_getval(BP_LGRP_MSCT_ENABLE, &value) == 0)
1002a3114836SGerry Liu lgrp_plat_msct_enable = (int)value;
1003a3114836SGerry Liu
1004d5d7cf4eSJonathan Chew /*
1005d5d7cf4eSJonathan Chew * Initialize as a UMA machine
1006d5d7cf4eSJonathan Chew */
1007d5d7cf4eSJonathan Chew if (lgrp_topo_ht_limit() == 1) {
1008d5d7cf4eSJonathan Chew lgrp_plat_node_cnt = max_mem_nodes = 1;
1009a3114836SGerry Liu lgrp_plat_max_mem_node = 1;
1010d5d7cf4eSJonathan Chew return;
1011d821f0f0Sjjc }
1012f78a91cdSjjc
1013d5d7cf4eSJonathan Chew lgrp_plat_get_numa_config();
1014a3114836SGerry Liu
1015a3114836SGerry Liu /*
1016a3114836SGerry Liu * Each lgrp node needs MAX_MEM_NODES_PER_LGROUP memnodes
1017a3114836SGerry Liu * to support memory DR operations if memory DR is enabled.
1018a3114836SGerry Liu */
1019a3114836SGerry Liu lgrp_plat_max_mem_node = lgrp_plat_node_cnt;
1020a3114836SGerry Liu if (plat_dr_support_memory() && lgrp_plat_node_cnt != 1) {
1021a3114836SGerry Liu max_mem_nodes = MAX_MEM_NODES_PER_LGROUP *
1022a3114836SGerry Liu lgrp_plat_node_cnt;
1023a3114836SGerry Liu ASSERT(max_mem_nodes <= MAX_MEM_NODES);
1024a3114836SGerry Liu }
1025d5d7cf4eSJonathan Chew #endif /* __xpv */
1026d5d7cf4eSJonathan Chew break;
10275b7cf7f0Sjjc
1028d5d7cf4eSJonathan Chew case LGRP_INIT_STAGE3:
1029d5d7cf4eSJonathan Chew lgrp_plat_probe();
1030d5d7cf4eSJonathan Chew lgrp_plat_release_bootstrap();
1031d5d7cf4eSJonathan Chew break;
10327c478bd9Sstevel@tonic-gate
1033d5d7cf4eSJonathan Chew case LGRP_INIT_STAGE4:
1034d5d7cf4eSJonathan Chew lgrp_plat_main_init();
1035d5d7cf4eSJonathan Chew break;
10367c478bd9Sstevel@tonic-gate
1037d5d7cf4eSJonathan Chew default:
1038d5d7cf4eSJonathan Chew break;
10397c478bd9Sstevel@tonic-gate }
10407c478bd9Sstevel@tonic-gate }
10417c478bd9Sstevel@tonic-gate
10427c478bd9Sstevel@tonic-gate
10437c478bd9Sstevel@tonic-gate /*
10442e2c009bSjjc * Return latency between "from" and "to" lgroups
10452e2c009bSjjc *
10462e2c009bSjjc * This latency number can only be used for relative comparison
10472e2c009bSjjc * between lgroups on the running system, cannot be used across platforms,
10482e2c009bSjjc * and may not reflect the actual latency. It is platform and implementation
10492e2c009bSjjc * specific, so platform gets to decide its value. It would be nice if the
10502e2c009bSjjc * number was at least proportional to make comparisons more meaningful though.
10517c478bd9Sstevel@tonic-gate */
10522e2c009bSjjc int
lgrp_plat_latency(lgrp_handle_t from,lgrp_handle_t to)10532e2c009bSjjc lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to)
10542e2c009bSjjc {
10552e2c009bSjjc lgrp_handle_t src, dest;
10562e2c009bSjjc int node;
10577c478bd9Sstevel@tonic-gate
10582e2c009bSjjc if (max_mem_nodes == 1)
10592e2c009bSjjc return (0);
10602e2c009bSjjc
10612e2c009bSjjc /*
10622e2c009bSjjc * Return max latency for root lgroup
10632e2c009bSjjc */
10642e2c009bSjjc if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE)
10652e2c009bSjjc return (lgrp_plat_lat_stats.latency_max);
10662e2c009bSjjc
10672e2c009bSjjc src = from;
10682e2c009bSjjc dest = to;
10692e2c009bSjjc
10702e2c009bSjjc /*
10712e2c009bSjjc * Return 0 for nodes (lgroup platform handles) out of range
10722e2c009bSjjc */
1073*3df2e8b2SRobert Mustacchi if (src >= MAX_NODES || dest >= MAX_NODES)
10742e2c009bSjjc return (0);
10752e2c009bSjjc
10762e2c009bSjjc /*
10772e2c009bSjjc * Probe from current CPU if its lgroup latencies haven't been set yet
1078a3114836SGerry Liu * and we are trying to get latency from current CPU to some node.
1079a3114836SGerry Liu * Avoid probing if CPU/memory DR is enabled.
10802e2c009bSjjc */
1081a3114836SGerry Liu if (lgrp_plat_lat_stats.latencies[src][src] == 0) {
1082a3114836SGerry Liu /*
1083a3114836SGerry Liu * Latency information should be updated by lgrp_plat_config()
1084a3114836SGerry Liu * for DR operations. Something is wrong if reaches here.
1085a3114836SGerry Liu * For safety, flatten lgrp topology to two levels.
1086a3114836SGerry Liu */
1087a3114836SGerry Liu if (plat_dr_support_cpu() || plat_dr_support_memory()) {
1088a3114836SGerry Liu ASSERT(lgrp_plat_lat_stats.latencies[src][src]);
1089a3114836SGerry Liu cmn_err(CE_WARN,
1090a3114836SGerry Liu "lgrp: failed to get latency information, "
1091a3114836SGerry Liu "fall back to two-level topology.");
1092a3114836SGerry Liu lgrp_plat_2level_setup(&lgrp_plat_lat_stats);
1093a3114836SGerry Liu } else {
1094a3114836SGerry Liu node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node,
1095a3114836SGerry Liu lgrp_plat_cpu_node_nentries);
1096a3114836SGerry Liu ASSERT(node >= 0 && node < lgrp_plat_node_cnt);
1097a3114836SGerry Liu if (node == src)
1098a3114836SGerry Liu lgrp_plat_probe();
1099a3114836SGerry Liu }
1100a3114836SGerry Liu }
11012e2c009bSjjc
11022e2c009bSjjc return (lgrp_plat_lat_stats.latencies[src][dest]);
11032e2c009bSjjc }
11047c478bd9Sstevel@tonic-gate
11057c478bd9Sstevel@tonic-gate
11062e2c009bSjjc /*
11072e2c009bSjjc * Return the maximum number of lgrps supported by the platform.
11082e2c009bSjjc * Before lgrp topology is known it returns an estimate based on the number of
1109a3114836SGerry Liu * nodes. Once topology is known it returns:
1110a3114836SGerry Liu * 1) the actual maximim number of lgrps created if CPU/memory DR operations
1111a3114836SGerry Liu * are not suppported.
1112a3114836SGerry Liu * 2) the maximum possible number of lgrps if CPU/memory DR operations are
1113a3114836SGerry Liu * supported.
11142e2c009bSjjc */
11152e2c009bSjjc int
lgrp_plat_max_lgrps(void)11162e2c009bSjjc lgrp_plat_max_lgrps(void)
11172e2c009bSjjc {
1118a3114836SGerry Liu if (!lgrp_topo_initialized || plat_dr_support_cpu() ||
1119a3114836SGerry Liu plat_dr_support_memory()) {
1120a3114836SGerry Liu return (lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1);
1121a3114836SGerry Liu } else {
1122a3114836SGerry Liu return (lgrp_alloc_max + 1);
1123a3114836SGerry Liu }
11242e2c009bSjjc }
11257c478bd9Sstevel@tonic-gate
11267c478bd9Sstevel@tonic-gate
1127a3114836SGerry Liu /*
1128a3114836SGerry Liu * Count number of memory pages (_t) based on mnode id (_n) and query type (_t).
1129a3114836SGerry Liu */
1130a3114836SGerry Liu #define _LGRP_PLAT_MEM_SIZE(_n, _q, _t) \
1131a3114836SGerry Liu if (mem_node_config[_n].exists) { \
1132a3114836SGerry Liu switch (_q) { \
1133a3114836SGerry Liu case LGRP_MEM_SIZE_FREE: \
1134a3114836SGerry Liu _t += MNODE_PGCNT(_n); \
1135a3114836SGerry Liu break; \
1136a3114836SGerry Liu case LGRP_MEM_SIZE_AVAIL: \
1137a3114836SGerry Liu _t += mem_node_memlist_pages(_n, phys_avail); \
1138a3114836SGerry Liu break; \
1139a3114836SGerry Liu case LGRP_MEM_SIZE_INSTALL: \
1140a3114836SGerry Liu _t += mem_node_memlist_pages(_n, phys_install); \
1141a3114836SGerry Liu break; \
1142a3114836SGerry Liu default: \
1143a3114836SGerry Liu break; \
1144a3114836SGerry Liu } \
1145a3114836SGerry Liu }
1146a3114836SGerry Liu
11472e2c009bSjjc /*
11482e2c009bSjjc * Return the number of free pages in an lgroup.
11492e2c009bSjjc *
11502e2c009bSjjc * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize
11512e2c009bSjjc * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the
11522e2c009bSjjc * number of allocatable base pagesize pages corresponding to the
11532e2c009bSjjc * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..)
11542e2c009bSjjc * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical
11552e2c009bSjjc * memory installed, regardless of whether or not it's usable.
11562e2c009bSjjc */
11572e2c009bSjjc pgcnt_t
lgrp_plat_mem_size(lgrp_handle_t plathand,lgrp_mem_query_t query)11582e2c009bSjjc lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query)
11592e2c009bSjjc {
11602e2c009bSjjc int mnode;
11612e2c009bSjjc pgcnt_t npgs = (pgcnt_t)0;
11622e2c009bSjjc extern struct memlist *phys_avail;
11632e2c009bSjjc extern struct memlist *phys_install;
11647c478bd9Sstevel@tonic-gate
11657c478bd9Sstevel@tonic-gate
11662e2c009bSjjc if (plathand == LGRP_DEFAULT_HANDLE)
11672e2c009bSjjc return (lgrp_plat_mem_size_default(plathand, query));
11687c478bd9Sstevel@tonic-gate
11692e2c009bSjjc if (plathand != LGRP_NULL_HANDLE) {
1170a3114836SGerry Liu /* Count memory node present at boot. */
1171a3114836SGerry Liu mnode = (int)plathand;
1172a3114836SGerry Liu ASSERT(mnode < lgrp_plat_node_cnt);
1173a3114836SGerry Liu _LGRP_PLAT_MEM_SIZE(mnode, query, npgs);
1174a3114836SGerry Liu
1175a3114836SGerry Liu /* Count possible hot-added memory nodes. */
1176a3114836SGerry Liu for (mnode = lgrp_plat_node_cnt;
1177a3114836SGerry Liu mnode < lgrp_plat_max_mem_node; mnode++) {
1178a3114836SGerry Liu if (lgrp_plat_memnode_info[mnode].lgrphand == plathand)
1179a3114836SGerry Liu _LGRP_PLAT_MEM_SIZE(mnode, query, npgs);
11807c478bd9Sstevel@tonic-gate }
11817c478bd9Sstevel@tonic-gate }
1182a3114836SGerry Liu
11832e2c009bSjjc return (npgs);
11847c478bd9Sstevel@tonic-gate }
11857c478bd9Sstevel@tonic-gate
11867c478bd9Sstevel@tonic-gate
11877c478bd9Sstevel@tonic-gate /*
11882e2c009bSjjc * Return the platform handle of the lgroup that contains the physical memory
11892e2c009bSjjc * corresponding to the given page frame number
11907c478bd9Sstevel@tonic-gate */
11912e2c009bSjjc lgrp_handle_t
lgrp_plat_pfn_to_hand(pfn_t pfn)11922e2c009bSjjc lgrp_plat_pfn_to_hand(pfn_t pfn)
11937c478bd9Sstevel@tonic-gate {
11942e2c009bSjjc int mnode;
11957c478bd9Sstevel@tonic-gate
11962e2c009bSjjc if (max_mem_nodes == 1)
11972e2c009bSjjc return (LGRP_DEFAULT_HANDLE);
11987c478bd9Sstevel@tonic-gate
11992e2c009bSjjc if (pfn > physmax)
12002e2c009bSjjc return (LGRP_NULL_HANDLE);
12017c478bd9Sstevel@tonic-gate
12022e2c009bSjjc mnode = plat_pfn_to_mem_node(pfn);
12032e2c009bSjjc if (mnode < 0)
12042e2c009bSjjc return (LGRP_NULL_HANDLE);
12057c478bd9Sstevel@tonic-gate
12062e2c009bSjjc return (MEM_NODE_2_LGRPHAND(mnode));
12072e2c009bSjjc }
12082dae3fb5Sjjc
12097c478bd9Sstevel@tonic-gate
12102e2c009bSjjc /*
12112e2c009bSjjc * Probe memory in each node from current CPU to determine latency topology
12122e2c009bSjjc *
12132e2c009bSjjc * The probing code will probe the vendor ID register on the Northbridge of
12142e2c009bSjjc * Opteron processors and probe memory for other processors by default.
12152e2c009bSjjc *
12162e2c009bSjjc * Since probing is inherently error prone, the code takes laps across all the
12172e2c009bSjjc * nodes probing from each node to each of the other nodes some number of
12182e2c009bSjjc * times. Furthermore, each node is probed some number of times before moving
12192e2c009bSjjc * onto the next one during each lap. The minimum latency gotten between nodes
12202e2c009bSjjc * is kept as the latency between the nodes.
12212e2c009bSjjc *
12222e2c009bSjjc * After all that, the probe times are adjusted by normalizing values that are
12232e2c009bSjjc * close to each other and local latencies are made the same. Lastly, the
12242e2c009bSjjc * latencies are verified to make sure that certain conditions are met (eg.
12252e2c009bSjjc * local < remote, latency(a, b) == latency(b, a), etc.).
12262e2c009bSjjc *
12272e2c009bSjjc * If any of the conditions aren't met, the code will export a NUMA
12282e2c009bSjjc * configuration with the local CPUs and memory given by the SRAT or PCI config
12292e2c009bSjjc * space registers and one remote memory latency since it can't tell exactly
12302e2c009bSjjc * how far each node is from each other.
12312e2c009bSjjc */
12322e2c009bSjjc void
lgrp_plat_probe(void)12332e2c009bSjjc lgrp_plat_probe(void)
12342e2c009bSjjc {
12352e2c009bSjjc int from;
12362e2c009bSjjc int i;
12372e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats;
12381ce8847aSJonathan Chew boolean_t probed;
12392e2c009bSjjc hrtime_t probe_time;
12402e2c009bSjjc int to;
12412dae3fb5Sjjc
12422e2c009bSjjc if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
12432e2c009bSjjc max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2)
12442e2c009bSjjc return;
12457c478bd9Sstevel@tonic-gate
1246a3114836SGerry Liu /* SRAT and SLIT should be enabled if DR operations are enabled. */
1247a3114836SGerry Liu if (plat_dr_support_cpu() || plat_dr_support_memory())
1248a3114836SGerry Liu return;
1249a3114836SGerry Liu
12507c478bd9Sstevel@tonic-gate /*
12512e2c009bSjjc * Determine ID of node containing current CPU
12527c478bd9Sstevel@tonic-gate */
1253d5d7cf4eSJonathan Chew from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node,
1254d5d7cf4eSJonathan Chew lgrp_plat_cpu_node_nentries);
12552e2c009bSjjc ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
12562e2c009bSjjc if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error)
12572e2c009bSjjc ASSERT(lgrp_plat_node_domain[from].exists);
12587c478bd9Sstevel@tonic-gate
12597c478bd9Sstevel@tonic-gate /*
12602e2c009bSjjc * Don't need to probe if got times already
12617c478bd9Sstevel@tonic-gate */
12622e2c009bSjjc lat_stats = &lgrp_plat_lat_stats;
12632e2c009bSjjc if (lat_stats->latencies[from][from] != 0)
12642e2c009bSjjc return;
12657c478bd9Sstevel@tonic-gate
12667c478bd9Sstevel@tonic-gate /*
12672e2c009bSjjc * Read vendor ID in Northbridge or read and write page(s)
12682e2c009bSjjc * in each node from current CPU and remember how long it takes,
12692e2c009bSjjc * so we can build latency topology of machine later.
12702e2c009bSjjc * This should approximate the memory latency between each node.
12717c478bd9Sstevel@tonic-gate */
12721ce8847aSJonathan Chew probed = B_FALSE;
12732e2c009bSjjc for (i = 0; i < lgrp_plat_probe_nrounds; i++) {
12742e2c009bSjjc for (to = 0; to < lgrp_plat_node_cnt; to++) {
12752e2c009bSjjc /*
12761ce8847aSJonathan Chew * Get probe time and skip over any nodes that can't be
12771ce8847aSJonathan Chew * probed yet or don't have memory
12782e2c009bSjjc */
12792e2c009bSjjc probe_time = lgrp_plat_probe_time(to,
1280d5d7cf4eSJonathan Chew lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries,
1281d5d7cf4eSJonathan Chew &lgrp_plat_probe_mem_config, &lgrp_plat_lat_stats,
1282d5d7cf4eSJonathan Chew &lgrp_plat_probe_stats);
12832e2c009bSjjc if (probe_time == 0)
12841ce8847aSJonathan Chew continue;
12851ce8847aSJonathan Chew
12861ce8847aSJonathan Chew probed = B_TRUE;
12877c478bd9Sstevel@tonic-gate
12887c478bd9Sstevel@tonic-gate /*
12892e2c009bSjjc * Keep lowest probe time as latency between nodes
12907c478bd9Sstevel@tonic-gate */
12912e2c009bSjjc if (lat_stats->latencies[from][to] == 0 ||
12922e2c009bSjjc probe_time < lat_stats->latencies[from][to])
12932e2c009bSjjc lat_stats->latencies[from][to] = probe_time;
12947c478bd9Sstevel@tonic-gate
12957c478bd9Sstevel@tonic-gate /*
12962e2c009bSjjc * Update overall minimum and maximum probe times
12972e2c009bSjjc * across all nodes
12987c478bd9Sstevel@tonic-gate */
12992e2c009bSjjc if (probe_time < lat_stats->latency_min ||
13002e2c009bSjjc lat_stats->latency_min == -1)
13012e2c009bSjjc lat_stats->latency_min = probe_time;
13022e2c009bSjjc if (probe_time > lat_stats->latency_max)
13032e2c009bSjjc lat_stats->latency_max = probe_time;
13047c478bd9Sstevel@tonic-gate }
13057c478bd9Sstevel@tonic-gate }
13067c478bd9Sstevel@tonic-gate
13071ce8847aSJonathan Chew /*
13081ce8847aSJonathan Chew * Bail out if weren't able to probe any nodes from current CPU
13091ce8847aSJonathan Chew */
13101ce8847aSJonathan Chew if (probed == B_FALSE)
13111ce8847aSJonathan Chew return;
13121ce8847aSJonathan Chew
13137c478bd9Sstevel@tonic-gate /*
13142e2c009bSjjc * - Fix up latencies such that local latencies are same,
13152e2c009bSjjc * latency(i, j) == latency(j, i), etc. (if possible)
13162e2c009bSjjc *
13172e2c009bSjjc * - Verify that latencies look ok
13182e2c009bSjjc *
13192e2c009bSjjc * - Fallback to just optimizing for local and remote if
13202e2c009bSjjc * latencies didn't look right
13217c478bd9Sstevel@tonic-gate */
1322a3114836SGerry Liu lgrp_plat_latency_adjust(lgrp_plat_memnode_info, &lgrp_plat_lat_stats,
13232e2c009bSjjc &lgrp_plat_probe_stats);
13242e2c009bSjjc lgrp_plat_probe_stats.probe_error_code =
1325a3114836SGerry Liu lgrp_plat_latency_verify(lgrp_plat_memnode_info,
13262e2c009bSjjc &lgrp_plat_lat_stats);
13272e2c009bSjjc if (lgrp_plat_probe_stats.probe_error_code)
1328a3114836SGerry Liu lgrp_plat_2level_setup(&lgrp_plat_lat_stats);
13292e2c009bSjjc }
13307c478bd9Sstevel@tonic-gate
13317c478bd9Sstevel@tonic-gate
13327c478bd9Sstevel@tonic-gate /*
13332e2c009bSjjc * Return platform handle for root lgroup
13347c478bd9Sstevel@tonic-gate */
13352e2c009bSjjc lgrp_handle_t
lgrp_plat_root_hand(void)13362e2c009bSjjc lgrp_plat_root_hand(void)
13377c478bd9Sstevel@tonic-gate {
13382e2c009bSjjc return (LGRP_DEFAULT_HANDLE);
13392e2c009bSjjc }
13407c478bd9Sstevel@tonic-gate
13417c478bd9Sstevel@tonic-gate
13422e2c009bSjjc /*
13432e2c009bSjjc * INTERNAL ROUTINES
13442e2c009bSjjc */
13457c478bd9Sstevel@tonic-gate
13467c478bd9Sstevel@tonic-gate
13477c478bd9Sstevel@tonic-gate /*
1348a3114836SGerry Liu * Update CPU to node mapping for given CPU and proximity domain.
1349a3114836SGerry Liu * Return values:
1350584b574aSToomas Soome * - zero for success
1351a3114836SGerry Liu * - positive numbers for warnings
1352a3114836SGerry Liu * - negative numbers for errors
13537c478bd9Sstevel@tonic-gate */
13542e2c009bSjjc static int
lgrp_plat_cpu_node_update(node_domain_map_t * node_domain,int node_cnt,cpu_node_map_t * cpu_node,int nentries,uint32_t apicid,uint32_t domain)1355d821f0f0Sjjc lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt,
1356dae2fa37Sjjc cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain)
13577c478bd9Sstevel@tonic-gate {
13582e2c009bSjjc uint_t i;
13592e2c009bSjjc int node;
13607c478bd9Sstevel@tonic-gate
13617c478bd9Sstevel@tonic-gate /*
13622e2c009bSjjc * Get node number for proximity domain
13637c478bd9Sstevel@tonic-gate */
1364d821f0f0Sjjc node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain);
13652e2c009bSjjc if (node == -1) {
1366d821f0f0Sjjc node = lgrp_plat_node_domain_update(node_domain, node_cnt,
1367d821f0f0Sjjc domain);
13682e2c009bSjjc if (node == -1)
13692e2c009bSjjc return (-1);
13707c478bd9Sstevel@tonic-gate }
13717c478bd9Sstevel@tonic-gate
13727c478bd9Sstevel@tonic-gate /*
1373dae2fa37Sjjc * Search for entry with given APIC ID and fill in its node and
1374dae2fa37Sjjc * proximity domain IDs (if they haven't been set already)
13757c478bd9Sstevel@tonic-gate */
1376dae2fa37Sjjc for (i = 0; i < nentries; i++) {
1377dae2fa37Sjjc /*
1378dae2fa37Sjjc * Skip nonexistent entries and ones without matching APIC ID
1379dae2fa37Sjjc */
1380dae2fa37Sjjc if (!cpu_node[i].exists || cpu_node[i].apicid != apicid)
1381dae2fa37Sjjc continue;
1382dae2fa37Sjjc
1383dae2fa37Sjjc /*
1384dae2fa37Sjjc * Just return if entry completely and correctly filled in
1385dae2fa37Sjjc * already
1386dae2fa37Sjjc */
1387dae2fa37Sjjc if (cpu_node[i].prox_domain == domain &&
1388dae2fa37Sjjc cpu_node[i].node == node)
1389dae2fa37Sjjc return (1);
1390dae2fa37Sjjc
1391a3114836SGerry Liu /*
1392a3114836SGerry Liu * It's invalid to have more than one entry with the same
1393a3114836SGerry Liu * local APIC ID in SRAT table.
1394a3114836SGerry Liu */
1395a3114836SGerry Liu if (cpu_node[i].node != UINT_MAX)
1396a3114836SGerry Liu return (-2);
1397a3114836SGerry Liu
1398dae2fa37Sjjc /*
1399dae2fa37Sjjc * Fill in node and proximity domain IDs
1400dae2fa37Sjjc */
1401dae2fa37Sjjc cpu_node[i].prox_domain = domain;
1402dae2fa37Sjjc cpu_node[i].node = node;
1403dae2fa37Sjjc
1404d5d7cf4eSJonathan Chew return (0);
1405d5d7cf4eSJonathan Chew }
1406d5d7cf4eSJonathan Chew
1407d5d7cf4eSJonathan Chew /*
1408a3114836SGerry Liu * It's possible that an apicid doesn't exist in the cpu_node map due
1409a3114836SGerry Liu * to user limits number of CPUs powered on at boot by specifying the
1410a3114836SGerry Liu * boot_ncpus kernel option.
1411d5d7cf4eSJonathan Chew */
1412a3114836SGerry Liu return (2);
1413d5d7cf4eSJonathan Chew }
1414d5d7cf4eSJonathan Chew
1415d5d7cf4eSJonathan Chew
1416d5d7cf4eSJonathan Chew /*
1417d5d7cf4eSJonathan Chew * Get node ID for given CPU
1418d5d7cf4eSJonathan Chew */
1419d5d7cf4eSJonathan Chew static int
lgrp_plat_cpu_to_node(cpu_t * cp,cpu_node_map_t * cpu_node,int cpu_node_nentries)1420d5d7cf4eSJonathan Chew lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node,
1421d5d7cf4eSJonathan Chew int cpu_node_nentries)
1422d5d7cf4eSJonathan Chew {
1423d5d7cf4eSJonathan Chew processorid_t cpuid;
1424d5d7cf4eSJonathan Chew
1425d5d7cf4eSJonathan Chew if (cp == NULL)
1426d5d7cf4eSJonathan Chew return (-1);
1427d5d7cf4eSJonathan Chew
1428d5d7cf4eSJonathan Chew cpuid = cp->cpu_id;
1429d5d7cf4eSJonathan Chew if (cpuid < 0 || cpuid >= max_ncpus)
1430d5d7cf4eSJonathan Chew return (-1);
1431d5d7cf4eSJonathan Chew
1432d5d7cf4eSJonathan Chew /*
1433d5d7cf4eSJonathan Chew * SRAT doesn't exist, isn't enabled, or there was an error processing
14348031591dSSrihari Venkatesan * it, so return node ID for Opteron and -1 otherwise.
1435d5d7cf4eSJonathan Chew */
1436d5d7cf4eSJonathan Chew if (srat_ptr == NULL || !lgrp_plat_srat_enable ||
1437d5d7cf4eSJonathan Chew lgrp_plat_srat_error) {
1438d5d7cf4eSJonathan Chew if (is_opteron())
14398031591dSSrihari Venkatesan return (pg_plat_hw_instance_id(cp, PGHW_PROCNODE));
1440d5d7cf4eSJonathan Chew return (-1);
1441d5d7cf4eSJonathan Chew }
1442d5d7cf4eSJonathan Chew
1443d5d7cf4eSJonathan Chew /*
1444d5d7cf4eSJonathan Chew * Return -1 when CPU to node ID mapping entry doesn't exist for given
1445d5d7cf4eSJonathan Chew * CPU
1446d5d7cf4eSJonathan Chew */
1447d5d7cf4eSJonathan Chew if (cpuid >= cpu_node_nentries || !cpu_node[cpuid].exists)
1448d5d7cf4eSJonathan Chew return (-1);
1449d5d7cf4eSJonathan Chew
1450d5d7cf4eSJonathan Chew return (cpu_node[cpuid].node);
1451d5d7cf4eSJonathan Chew }
1452d5d7cf4eSJonathan Chew
1453d5d7cf4eSJonathan Chew
1454d5d7cf4eSJonathan Chew /*
1455d5d7cf4eSJonathan Chew * Return node number for given proximity domain/system locality
1456d5d7cf4eSJonathan Chew */
1457d5d7cf4eSJonathan Chew static int
lgrp_plat_domain_to_node(node_domain_map_t * node_domain,int node_cnt,uint32_t domain)1458d5d7cf4eSJonathan Chew lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt,
1459d5d7cf4eSJonathan Chew uint32_t domain)
1460d5d7cf4eSJonathan Chew {
1461d5d7cf4eSJonathan Chew uint_t node;
1462d5d7cf4eSJonathan Chew uint_t start;
1463d5d7cf4eSJonathan Chew
1464d5d7cf4eSJonathan Chew /*
1465d5d7cf4eSJonathan Chew * Hash proximity domain ID into node to domain mapping table (array),
1466d5d7cf4eSJonathan Chew * search for entry with matching proximity domain ID, and return index
1467d5d7cf4eSJonathan Chew * of matching entry as node ID.
1468d5d7cf4eSJonathan Chew */
1469d5d7cf4eSJonathan Chew node = start = NODE_DOMAIN_HASH(domain, node_cnt);
1470d5d7cf4eSJonathan Chew do {
1471a3114836SGerry Liu if (node_domain[node].exists) {
1472a3114836SGerry Liu membar_consumer();
1473a3114836SGerry Liu if (node_domain[node].prox_domain == domain)
1474a3114836SGerry Liu return (node);
1475a3114836SGerry Liu }
1476d5d7cf4eSJonathan Chew node = (node + 1) % node_cnt;
1477d5d7cf4eSJonathan Chew } while (node != start);
1478d5d7cf4eSJonathan Chew return (-1);
1479d5d7cf4eSJonathan Chew }
1480d5d7cf4eSJonathan Chew
1481d5d7cf4eSJonathan Chew
1482d5d7cf4eSJonathan Chew /*
1483d5d7cf4eSJonathan Chew * Get NUMA configuration of machine
1484d5d7cf4eSJonathan Chew */
1485d5d7cf4eSJonathan Chew static void
lgrp_plat_get_numa_config(void)1486d5d7cf4eSJonathan Chew lgrp_plat_get_numa_config(void)
1487d5d7cf4eSJonathan Chew {
1488d5d7cf4eSJonathan Chew uint_t probe_op;
1489d5d7cf4eSJonathan Chew
1490d5d7cf4eSJonathan Chew /*
1491d5d7cf4eSJonathan Chew * Read boot property with CPU to APIC ID mapping table/array to
1492d5d7cf4eSJonathan Chew * determine number of CPUs
1493d5d7cf4eSJonathan Chew */
1494d5d7cf4eSJonathan Chew lgrp_plat_apic_ncpus = lgrp_plat_process_cpu_apicids(NULL);
1495d5d7cf4eSJonathan Chew
1496d5d7cf4eSJonathan Chew /*
1497d5d7cf4eSJonathan Chew * Determine which CPUs and memory are local to each other and number
1498d5d7cf4eSJonathan Chew * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT)
1499d5d7cf4eSJonathan Chew */
1500d5d7cf4eSJonathan Chew if (lgrp_plat_apic_ncpus > 0) {
1501d5d7cf4eSJonathan Chew int retval;
1502d5d7cf4eSJonathan Chew
1503a3114836SGerry Liu /* Reserve enough resources if CPU DR is enabled. */
1504a3114836SGerry Liu if (plat_dr_support_cpu() && max_ncpus > lgrp_plat_apic_ncpus)
1505a3114836SGerry Liu lgrp_plat_cpu_node_nentries = max_ncpus;
1506a3114836SGerry Liu else
1507a3114836SGerry Liu lgrp_plat_cpu_node_nentries = lgrp_plat_apic_ncpus;
1508a3114836SGerry Liu
1509d5d7cf4eSJonathan Chew /*
1510d5d7cf4eSJonathan Chew * Temporarily allocate boot memory to use for CPU to node
1511d5d7cf4eSJonathan Chew * mapping since kernel memory allocator isn't alive yet
1512d5d7cf4eSJonathan Chew */
1513d5d7cf4eSJonathan Chew lgrp_plat_cpu_node = (cpu_node_map_t *)BOP_ALLOC(bootops,
1514a3114836SGerry Liu NULL, lgrp_plat_cpu_node_nentries * sizeof (cpu_node_map_t),
1515d5d7cf4eSJonathan Chew sizeof (int));
1516d5d7cf4eSJonathan Chew
1517d5d7cf4eSJonathan Chew ASSERT(lgrp_plat_cpu_node != NULL);
1518d5d7cf4eSJonathan Chew if (lgrp_plat_cpu_node) {
1519d5d7cf4eSJonathan Chew bzero(lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries *
1520d5d7cf4eSJonathan Chew sizeof (cpu_node_map_t));
1521a3114836SGerry Liu } else {
1522a3114836SGerry Liu lgrp_plat_cpu_node_nentries = 0;
1523d5d7cf4eSJonathan Chew }
1524d5d7cf4eSJonathan Chew
1525d5d7cf4eSJonathan Chew /*
1526d5d7cf4eSJonathan Chew * Fill in CPU to node ID mapping table with APIC ID for each
1527d5d7cf4eSJonathan Chew * CPU
1528d5d7cf4eSJonathan Chew */
1529d5d7cf4eSJonathan Chew (void) lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node);
1530d5d7cf4eSJonathan Chew
1531a3114836SGerry Liu retval = lgrp_plat_process_srat(srat_ptr, msct_ptr,
1532d5d7cf4eSJonathan Chew &lgrp_plat_prox_domain_min,
1533d5d7cf4eSJonathan Chew lgrp_plat_node_domain, lgrp_plat_cpu_node,
1534a3114836SGerry Liu lgrp_plat_apic_ncpus, lgrp_plat_memnode_info);
1535d5d7cf4eSJonathan Chew if (retval <= 0) {
1536d5d7cf4eSJonathan Chew lgrp_plat_srat_error = retval;
1537d5d7cf4eSJonathan Chew lgrp_plat_node_cnt = 1;
1538d5d7cf4eSJonathan Chew } else {
1539d5d7cf4eSJonathan Chew lgrp_plat_srat_error = 0;
1540d5d7cf4eSJonathan Chew lgrp_plat_node_cnt = retval;
1541d5d7cf4eSJonathan Chew }
1542d5d7cf4eSJonathan Chew }
1543d5d7cf4eSJonathan Chew
1544d5d7cf4eSJonathan Chew /*
1545d5d7cf4eSJonathan Chew * Try to use PCI config space registers on Opteron if there's an error
1546d5d7cf4eSJonathan Chew * processing CPU to APIC ID mapping or SRAT
1547d5d7cf4eSJonathan Chew */
1548d5d7cf4eSJonathan Chew if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) &&
1549d5d7cf4eSJonathan Chew is_opteron())
1550d5d7cf4eSJonathan Chew opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv,
1551a3114836SGerry Liu lgrp_plat_memnode_info);
1552d5d7cf4eSJonathan Chew
1553d5d7cf4eSJonathan Chew /*
1554d5d7cf4eSJonathan Chew * Don't bother to setup system for multiple lgroups and only use one
1555d5d7cf4eSJonathan Chew * memory node when memory is interleaved between any nodes or there is
1556d5d7cf4eSJonathan Chew * only one NUMA node
1557d5d7cf4eSJonathan Chew */
1558d5d7cf4eSJonathan Chew if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) {
1559d5d7cf4eSJonathan Chew lgrp_plat_node_cnt = max_mem_nodes = 1;
1560d5d7cf4eSJonathan Chew (void) lgrp_topo_ht_limit_set(1);
1561d5d7cf4eSJonathan Chew return;
1562dae2fa37Sjjc }
15637c478bd9Sstevel@tonic-gate
15647c478bd9Sstevel@tonic-gate /*
1565d5d7cf4eSJonathan Chew * Leaf lgroups on x86/x64 architectures contain one physical
1566d5d7cf4eSJonathan Chew * processor chip. Tune lgrp_expand_proc_thresh and
1567d5d7cf4eSJonathan Chew * lgrp_expand_proc_diff so that lgrp_choose() will spread
1568d5d7cf4eSJonathan Chew * things out aggressively.
15697c478bd9Sstevel@tonic-gate */
1570d5d7cf4eSJonathan Chew lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2;
1571d5d7cf4eSJonathan Chew lgrp_expand_proc_diff = 0;
15727c478bd9Sstevel@tonic-gate
1573d5d7cf4eSJonathan Chew /*
1574d5d7cf4eSJonathan Chew * There should be one memnode (physical page free list(s)) for
1575a3114836SGerry Liu * each node if memory DR is disabled.
1576d5d7cf4eSJonathan Chew */
1577d5d7cf4eSJonathan Chew max_mem_nodes = lgrp_plat_node_cnt;
1578dae2fa37Sjjc
15797c478bd9Sstevel@tonic-gate /*
1580d5d7cf4eSJonathan Chew * Initialize min and max latency before reading SLIT or probing
15817c478bd9Sstevel@tonic-gate */
1582d5d7cf4eSJonathan Chew lgrp_plat_lat_stats.latency_min = -1;
1583d5d7cf4eSJonathan Chew lgrp_plat_lat_stats.latency_max = 0;
15847c478bd9Sstevel@tonic-gate
15857c478bd9Sstevel@tonic-gate /*
1586d5d7cf4eSJonathan Chew * Determine how far each NUMA node is from each other by
1587d5d7cf4eSJonathan Chew * reading ACPI System Locality Information Table (SLIT) if it
1588d5d7cf4eSJonathan Chew * exists
15897c478bd9Sstevel@tonic-gate */
1590d5d7cf4eSJonathan Chew lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr,
1591a3114836SGerry Liu lgrp_plat_node_domain, lgrp_plat_node_cnt, lgrp_plat_memnode_info,
1592d5d7cf4eSJonathan Chew &lgrp_plat_lat_stats);
1593a3114836SGerry Liu
1594a3114836SGerry Liu /*
1595a3114836SGerry Liu * Disable support of CPU/memory DR operations if multiple locality
1596a3114836SGerry Liu * domains exist in system and either of following is true.
1597a3114836SGerry Liu * 1) Failed to process SLIT table.
1598a3114836SGerry Liu * 2) Latency probing is enabled by user.
1599a3114836SGerry Liu */
1600a3114836SGerry Liu if (lgrp_plat_node_cnt > 1 &&
1601a3114836SGerry Liu (plat_dr_support_cpu() || plat_dr_support_memory())) {
1602a3114836SGerry Liu if (!lgrp_plat_slit_enable || lgrp_plat_slit_error != 0 ||
1603a3114836SGerry Liu !lgrp_plat_srat_enable || lgrp_plat_srat_error != 0 ||
1604a3114836SGerry Liu lgrp_plat_apic_ncpus <= 0) {
1605a3114836SGerry Liu cmn_err(CE_CONT,
1606a3114836SGerry Liu "?lgrp: failed to process ACPI SRAT/SLIT table, "
1607a3114836SGerry Liu "disable support of CPU/memory DR operations.");
1608a3114836SGerry Liu plat_dr_disable_cpu();
1609a3114836SGerry Liu plat_dr_disable_memory();
1610a3114836SGerry Liu } else if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) {
1611a3114836SGerry Liu cmn_err(CE_CONT,
1612a3114836SGerry Liu "?lgrp: latency probing enabled by user, "
1613a3114836SGerry Liu "disable support of CPU/memory DR operations.");
1614a3114836SGerry Liu plat_dr_disable_cpu();
1615a3114836SGerry Liu plat_dr_disable_memory();
1616a3114836SGerry Liu }
1617a3114836SGerry Liu }
1618a3114836SGerry Liu
1619a3114836SGerry Liu /* Done if succeeded to process SLIT table. */
1620d5d7cf4eSJonathan Chew if (lgrp_plat_slit_error == 0)
1621d5d7cf4eSJonathan Chew return;
1622dae2fa37Sjjc
1623d5d7cf4eSJonathan Chew /*
1624d5d7cf4eSJonathan Chew * Probe to determine latency between NUMA nodes when SLIT
1625d5d7cf4eSJonathan Chew * doesn't exist or make sense
1626d5d7cf4eSJonathan Chew */
1627d5d7cf4eSJonathan Chew lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE;
16287c478bd9Sstevel@tonic-gate
1629d5d7cf4eSJonathan Chew /*
1630d5d7cf4eSJonathan Chew * Specify whether to probe using vendor ID register or page copy
1631d5d7cf4eSJonathan Chew * if hasn't been specified already or is overspecified
1632d5d7cf4eSJonathan Chew */
1633d5d7cf4eSJonathan Chew probe_op = lgrp_plat_probe_flags &
1634d5d7cf4eSJonathan Chew (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
16357c478bd9Sstevel@tonic-gate
1636d5d7cf4eSJonathan Chew if (probe_op == 0 ||
1637d5d7cf4eSJonathan Chew probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) {
1638d5d7cf4eSJonathan Chew lgrp_plat_probe_flags &=
1639d5d7cf4eSJonathan Chew ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
1640d5d7cf4eSJonathan Chew if (is_opteron())
1641d5d7cf4eSJonathan Chew lgrp_plat_probe_flags |=
1642d5d7cf4eSJonathan Chew LGRP_PLAT_PROBE_VENDOR;
1643d5d7cf4eSJonathan Chew else
1644d5d7cf4eSJonathan Chew lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY;
1645d5d7cf4eSJonathan Chew }
16467c478bd9Sstevel@tonic-gate
16477c478bd9Sstevel@tonic-gate /*
1648d5d7cf4eSJonathan Chew * Probing errors can mess up the lgroup topology and
1649d5d7cf4eSJonathan Chew * force us fall back to a 2 level lgroup topology.
1650d5d7cf4eSJonathan Chew * Here we bound how tall the lgroup topology can grow
1651d5d7cf4eSJonathan Chew * in hopes of avoiding any anamolies in probing from
1652d5d7cf4eSJonathan Chew * messing up the lgroup topology by limiting the
1653d5d7cf4eSJonathan Chew * accuracy of the latency topology.
1654d5d7cf4eSJonathan Chew *
1655d5d7cf4eSJonathan Chew * Assume that nodes will at least be configured in a
1656d5d7cf4eSJonathan Chew * ring, so limit height of lgroup topology to be less
1657d5d7cf4eSJonathan Chew * than number of nodes on a system with 4 or more
1658d5d7cf4eSJonathan Chew * nodes
16597c478bd9Sstevel@tonic-gate */
1660d5d7cf4eSJonathan Chew if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() ==
1661d5d7cf4eSJonathan Chew lgrp_topo_ht_limit_default())
1662d5d7cf4eSJonathan Chew (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
16637c478bd9Sstevel@tonic-gate }
16647c478bd9Sstevel@tonic-gate
16657c478bd9Sstevel@tonic-gate
16667c478bd9Sstevel@tonic-gate /*
16672e2c009bSjjc * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to
16682e2c009bSjjc * be considered same
16697c478bd9Sstevel@tonic-gate */
16702e2c009bSjjc #define LGRP_LAT_TOLERANCE_SHIFT 4
16712e2c009bSjjc
16722e2c009bSjjc int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT;
16732e2c009bSjjc
16742e2c009bSjjc
16752e2c009bSjjc /*
16762e2c009bSjjc * Adjust latencies between nodes to be symmetric, normalize latencies between
16772e2c009bSjjc * any nodes that are within some tolerance to be same, and make local
16782e2c009bSjjc * latencies be same
16792e2c009bSjjc */
16802e2c009bSjjc static void
lgrp_plat_latency_adjust(memnode_phys_addr_map_t * memnode_info,lgrp_plat_latency_stats_t * lat_stats,lgrp_plat_probe_stats_t * probe_stats)1681a3114836SGerry Liu lgrp_plat_latency_adjust(memnode_phys_addr_map_t *memnode_info,
16822e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
16837c478bd9Sstevel@tonic-gate {
16842e2c009bSjjc int i;
16852e2c009bSjjc int j;
16862e2c009bSjjc int k;
16872e2c009bSjjc int l;
16882e2c009bSjjc u_longlong_t max;
16892e2c009bSjjc u_longlong_t min;
16902e2c009bSjjc u_longlong_t t;
16912e2c009bSjjc u_longlong_t t1;
16922e2c009bSjjc u_longlong_t t2;
16932e2c009bSjjc const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL;
16942e2c009bSjjc int lat_corrected[MAX_NODES][MAX_NODES];
16957c478bd9Sstevel@tonic-gate
1696584b574aSToomas Soome t = 0;
16977c478bd9Sstevel@tonic-gate /*
16982e2c009bSjjc * Nothing to do when this is an UMA machine or don't have args needed
16997c478bd9Sstevel@tonic-gate */
17002e2c009bSjjc if (max_mem_nodes == 1)
17012e2c009bSjjc return;
17022e2c009bSjjc
1703a3114836SGerry Liu ASSERT(memnode_info != NULL && lat_stats != NULL &&
17042e2c009bSjjc probe_stats != NULL);
17057c478bd9Sstevel@tonic-gate
17067c478bd9Sstevel@tonic-gate /*
17072e2c009bSjjc * Make sure that latencies are symmetric between any two nodes
17082e2c009bSjjc * (ie. latency(node0, node1) == latency(node1, node0))
17097c478bd9Sstevel@tonic-gate */
17102e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
1711a3114836SGerry Liu if (!memnode_info[i].exists)
17122e2c009bSjjc continue;
17137c478bd9Sstevel@tonic-gate
17142e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) {
1715a3114836SGerry Liu if (!memnode_info[j].exists)
17162e2c009bSjjc continue;
17177c478bd9Sstevel@tonic-gate
17182e2c009bSjjc t1 = lat_stats->latencies[i][j];
17192e2c009bSjjc t2 = lat_stats->latencies[j][i];
17202e2c009bSjjc
17212e2c009bSjjc if (t1 == 0 || t2 == 0 || t1 == t2)
17222e2c009bSjjc continue;
17232e2c009bSjjc
17242e2c009bSjjc /*
17252e2c009bSjjc * Latencies should be same
17262e2c009bSjjc * - Use minimum of two latencies which should be same
17272e2c009bSjjc * - Track suspect probe times not within tolerance of
17282e2c009bSjjc * min value
17292e2c009bSjjc * - Remember how much values are corrected by
17302e2c009bSjjc */
17312e2c009bSjjc if (t1 > t2) {
17322e2c009bSjjc t = t2;
17332e2c009bSjjc probe_stats->probe_errors[i][j] += t1 - t2;
17342e2c009bSjjc if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) {
17352e2c009bSjjc probe_stats->probe_suspect[i][j]++;
17362e2c009bSjjc probe_stats->probe_suspect[j][i]++;
17372e2c009bSjjc }
17382e2c009bSjjc } else if (t2 > t1) {
17392e2c009bSjjc t = t1;
17402e2c009bSjjc probe_stats->probe_errors[j][i] += t2 - t1;
17412e2c009bSjjc if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) {
17422e2c009bSjjc probe_stats->probe_suspect[i][j]++;
17432e2c009bSjjc probe_stats->probe_suspect[j][i]++;
17442e2c009bSjjc }
17452e2c009bSjjc }
17462e2c009bSjjc
17472e2c009bSjjc lat_stats->latencies[i][j] =
17482e2c009bSjjc lat_stats->latencies[j][i] = t;
17492e2c009bSjjc lgrp_config(cflag, t1, t);
17502e2c009bSjjc lgrp_config(cflag, t2, t);
17512e2c009bSjjc }
17527c478bd9Sstevel@tonic-gate }
17537c478bd9Sstevel@tonic-gate
17547c478bd9Sstevel@tonic-gate /*
17552e2c009bSjjc * Keep track of which latencies get corrected
17567c478bd9Sstevel@tonic-gate */
17572e2c009bSjjc for (i = 0; i < MAX_NODES; i++)
17582e2c009bSjjc for (j = 0; j < MAX_NODES; j++)
17592e2c009bSjjc lat_corrected[i][j] = 0;
17607c478bd9Sstevel@tonic-gate
17617c478bd9Sstevel@tonic-gate /*
17622e2c009bSjjc * For every two nodes, see whether there is another pair of nodes which
17632e2c009bSjjc * are about the same distance apart and make the latencies be the same
17642e2c009bSjjc * if they are close enough together
17657c478bd9Sstevel@tonic-gate */
17667c478bd9Sstevel@tonic-gate for (i = 0; i < lgrp_plat_node_cnt; i++) {
17672e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) {
1768a3114836SGerry Liu if (!memnode_info[j].exists)
17692e2c009bSjjc continue;
17702e2c009bSjjc /*
17712e2c009bSjjc * Pick one pair of nodes (i, j)
17722e2c009bSjjc * and get latency between them
17732e2c009bSjjc */
17742e2c009bSjjc t1 = lat_stats->latencies[i][j];
17752e2c009bSjjc
17762e2c009bSjjc /*
17772e2c009bSjjc * Skip this pair of nodes if there isn't a latency
17782e2c009bSjjc * for it yet
17792e2c009bSjjc */
17802e2c009bSjjc if (t1 == 0)
17812e2c009bSjjc continue;
17822e2c009bSjjc
17832e2c009bSjjc for (k = 0; k < lgrp_plat_node_cnt; k++) {
17842e2c009bSjjc for (l = 0; l < lgrp_plat_node_cnt; l++) {
1785a3114836SGerry Liu if (!memnode_info[l].exists)
17862e2c009bSjjc continue;
17872e2c009bSjjc /*
17882e2c009bSjjc * Pick another pair of nodes (k, l)
17892e2c009bSjjc * not same as (i, j) and get latency
17902e2c009bSjjc * between them
17912e2c009bSjjc */
17922e2c009bSjjc if (k == i && l == j)
17932e2c009bSjjc continue;
17942e2c009bSjjc
17952e2c009bSjjc t2 = lat_stats->latencies[k][l];
17962e2c009bSjjc
17972e2c009bSjjc /*
17982e2c009bSjjc * Skip this pair of nodes if there
17992e2c009bSjjc * isn't a latency for it yet
18002e2c009bSjjc */
18012e2c009bSjjc
18022e2c009bSjjc if (t2 == 0)
18032e2c009bSjjc continue;
18042e2c009bSjjc
18052e2c009bSjjc /*
18062e2c009bSjjc * Skip nodes (k, l) if they already
18072e2c009bSjjc * have same latency as (i, j) or
18082e2c009bSjjc * their latency isn't close enough to
18092e2c009bSjjc * be considered/made the same
18102e2c009bSjjc */
18112e2c009bSjjc if (t1 == t2 || (t1 > t2 && t1 - t2 >
18122e2c009bSjjc t1 >> lgrp_plat_probe_lt_shift) ||
18132e2c009bSjjc (t2 > t1 && t2 - t1 >
18142e2c009bSjjc t2 >> lgrp_plat_probe_lt_shift))
18152e2c009bSjjc continue;
18162e2c009bSjjc
18172e2c009bSjjc /*
18182e2c009bSjjc * Make latency(i, j) same as
18192e2c009bSjjc * latency(k, l), try to use latency
18202e2c009bSjjc * that has been adjusted already to get
18212e2c009bSjjc * more consistency (if possible), and
18222e2c009bSjjc * remember which latencies were
18232e2c009bSjjc * adjusted for next time
18242e2c009bSjjc */
18252e2c009bSjjc if (lat_corrected[i][j]) {
18262e2c009bSjjc t = t1;
18272e2c009bSjjc lgrp_config(cflag, t2, t);
18282e2c009bSjjc t2 = t;
18292e2c009bSjjc } else if (lat_corrected[k][l]) {
18302e2c009bSjjc t = t2;
18312e2c009bSjjc lgrp_config(cflag, t1, t);
18322e2c009bSjjc t1 = t;
18332e2c009bSjjc } else {
18342e2c009bSjjc if (t1 > t2)
18352e2c009bSjjc t = t2;
18362e2c009bSjjc else
18372e2c009bSjjc t = t1;
18382e2c009bSjjc lgrp_config(cflag, t1, t);
18392e2c009bSjjc lgrp_config(cflag, t2, t);
18402e2c009bSjjc t1 = t2 = t;
18412e2c009bSjjc }
18422e2c009bSjjc
18432e2c009bSjjc lat_stats->latencies[i][j] =
18442e2c009bSjjc lat_stats->latencies[k][l] = t;
18452e2c009bSjjc
18462e2c009bSjjc lat_corrected[i][j] =
18472e2c009bSjjc lat_corrected[k][l] = 1;
18482e2c009bSjjc }
18492e2c009bSjjc }
18502e2c009bSjjc }
18512e2c009bSjjc }
18522e2c009bSjjc
18532e2c009bSjjc /*
18542e2c009bSjjc * Local latencies should be same
18552e2c009bSjjc * - Find min and max local latencies
18562e2c009bSjjc * - Make all local latencies be minimum
18572e2c009bSjjc */
18582e2c009bSjjc min = -1;
18592e2c009bSjjc max = 0;
18602e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
1861a3114836SGerry Liu if (!memnode_info[i].exists)
18622e2c009bSjjc continue;
18632e2c009bSjjc t = lat_stats->latencies[i][i];
18642e2c009bSjjc if (t == 0)
18652e2c009bSjjc continue;
18662e2c009bSjjc if (min == -1 || t < min)
18672e2c009bSjjc min = t;
18682e2c009bSjjc if (t > max)
18692e2c009bSjjc max = t;
18702e2c009bSjjc }
18712e2c009bSjjc if (min != max) {
18722e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
18732e2c009bSjjc int local;
18742e2c009bSjjc
1875a3114836SGerry Liu if (!memnode_info[i].exists)
18762e2c009bSjjc continue;
18772e2c009bSjjc
18782e2c009bSjjc local = lat_stats->latencies[i][i];
18792e2c009bSjjc if (local == 0)
18802e2c009bSjjc continue;
18812e2c009bSjjc
18822e2c009bSjjc /*
18832e2c009bSjjc * Track suspect probe times that aren't within
18842e2c009bSjjc * tolerance of minimum local latency and how much
18852e2c009bSjjc * probe times are corrected by
18862e2c009bSjjc */
18872e2c009bSjjc if (local - min > min >> lgrp_plat_probe_lt_shift)
18882e2c009bSjjc probe_stats->probe_suspect[i][i]++;
18892e2c009bSjjc
18902e2c009bSjjc probe_stats->probe_errors[i][i] += local - min;
18912e2c009bSjjc
18922e2c009bSjjc /*
18932e2c009bSjjc * Make local latencies be minimum
18942e2c009bSjjc */
18952e2c009bSjjc lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min);
18962e2c009bSjjc lat_stats->latencies[i][i] = min;
18972e2c009bSjjc }
18982e2c009bSjjc }
18992e2c009bSjjc
19002e2c009bSjjc /*
19012e2c009bSjjc * Determine max probe time again since just adjusted latencies
19022e2c009bSjjc */
19032e2c009bSjjc lat_stats->latency_max = 0;
19042e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
19052e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) {
1906a3114836SGerry Liu if (!memnode_info[j].exists)
19072e2c009bSjjc continue;
19082e2c009bSjjc t = lat_stats->latencies[i][j];
19092e2c009bSjjc if (t > lat_stats->latency_max)
19102e2c009bSjjc lat_stats->latency_max = t;
19112e2c009bSjjc }
19122e2c009bSjjc }
19132e2c009bSjjc }
19142e2c009bSjjc
19152e2c009bSjjc
19162e2c009bSjjc /*
19172e2c009bSjjc * Verify following about latencies between nodes:
19182e2c009bSjjc *
19192e2c009bSjjc * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a))
19202e2c009bSjjc * - Local latencies same
19212e2c009bSjjc * - Local < remote
19222e2c009bSjjc * - Number of latencies seen is reasonable
19232e2c009bSjjc * - Number of occurrences of a given latency should be more than 1
19242e2c009bSjjc *
19252e2c009bSjjc * Returns:
19262e2c009bSjjc * 0 Success
19272e2c009bSjjc * -1 Not symmetric
19282e2c009bSjjc * -2 Local latencies not same
19292e2c009bSjjc * -3 Local >= remote
19302e2c009bSjjc */
19312e2c009bSjjc static int
lgrp_plat_latency_verify(memnode_phys_addr_map_t * memnode_info,lgrp_plat_latency_stats_t * lat_stats)1932a3114836SGerry Liu lgrp_plat_latency_verify(memnode_phys_addr_map_t *memnode_info,
19332e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats)
19342e2c009bSjjc {
19352e2c009bSjjc int i;
19362e2c009bSjjc int j;
19372e2c009bSjjc u_longlong_t t1;
19382e2c009bSjjc u_longlong_t t2;
19392e2c009bSjjc
1940a3114836SGerry Liu ASSERT(memnode_info != NULL && lat_stats != NULL);
19412e2c009bSjjc
19422e2c009bSjjc /*
19432e2c009bSjjc * Nothing to do when this is an UMA machine, lgroup topology is
19442e2c009bSjjc * limited to 2 levels, or there aren't any probe times yet
19452e2c009bSjjc */
19462e2c009bSjjc if (max_mem_nodes == 1 || lgrp_topo_levels < 2 ||
19472e2c009bSjjc lat_stats->latencies[0][0] == 0)
19482e2c009bSjjc return (0);
19492e2c009bSjjc
19502e2c009bSjjc /*
19512e2c009bSjjc * Make sure that latencies are symmetric between any two nodes
19522e2c009bSjjc * (ie. latency(node0, node1) == latency(node1, node0))
19532e2c009bSjjc */
19542e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
1955a3114836SGerry Liu if (!memnode_info[i].exists)
19562e2c009bSjjc continue;
19572e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) {
1958a3114836SGerry Liu if (!memnode_info[j].exists)
19592e2c009bSjjc continue;
19602e2c009bSjjc t1 = lat_stats->latencies[i][j];
19612e2c009bSjjc t2 = lat_stats->latencies[j][i];
19622e2c009bSjjc
19632e2c009bSjjc if (t1 == 0 || t2 == 0 || t1 == t2)
19642e2c009bSjjc continue;
19652e2c009bSjjc
19662e2c009bSjjc return (-1);
19672e2c009bSjjc }
19682e2c009bSjjc }
19692e2c009bSjjc
19702e2c009bSjjc /*
19712e2c009bSjjc * Local latencies should be same
19722e2c009bSjjc */
19732e2c009bSjjc t1 = lat_stats->latencies[0][0];
19742e2c009bSjjc for (i = 1; i < lgrp_plat_node_cnt; i++) {
1975a3114836SGerry Liu if (!memnode_info[i].exists)
19762e2c009bSjjc continue;
19772e2c009bSjjc
19782e2c009bSjjc t2 = lat_stats->latencies[i][i];
19792e2c009bSjjc if (t2 == 0)
19802e2c009bSjjc continue;
19812e2c009bSjjc
19822e2c009bSjjc if (t1 == 0) {
19832e2c009bSjjc t1 = t2;
19842e2c009bSjjc continue;
19852e2c009bSjjc }
19862e2c009bSjjc
19872e2c009bSjjc if (t1 != t2)
19882e2c009bSjjc return (-2);
19892e2c009bSjjc }
19902e2c009bSjjc
19912e2c009bSjjc /*
19922e2c009bSjjc * Local latencies should be less than remote
19932e2c009bSjjc */
19942e2c009bSjjc if (t1) {
19952e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
19962e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) {
1997a3114836SGerry Liu if (!memnode_info[j].exists)
19982e2c009bSjjc continue;
19992e2c009bSjjc t2 = lat_stats->latencies[i][j];
20002e2c009bSjjc if (i == j || t2 == 0)
20012e2c009bSjjc continue;
20022e2c009bSjjc
20032e2c009bSjjc if (t1 >= t2)
20042e2c009bSjjc return (-3);
20052e2c009bSjjc }
20062e2c009bSjjc }
20072e2c009bSjjc }
20082e2c009bSjjc
20092e2c009bSjjc return (0);
20102e2c009bSjjc }
20112e2c009bSjjc
20122e2c009bSjjc
2013d5d7cf4eSJonathan Chew /*
2014d5d7cf4eSJonathan Chew * Platform-specific initialization
2015d5d7cf4eSJonathan Chew */
2016d5d7cf4eSJonathan Chew static void
lgrp_plat_main_init(void)2017d5d7cf4eSJonathan Chew lgrp_plat_main_init(void)
2018d5d7cf4eSJonathan Chew {
2019d5d7cf4eSJonathan Chew int curnode;
2020d5d7cf4eSJonathan Chew int ht_limit;
2021d5d7cf4eSJonathan Chew int i;
2022d5d7cf4eSJonathan Chew
2023d5d7cf4eSJonathan Chew /*
2024d5d7cf4eSJonathan Chew * Print a notice that MPO is disabled when memory is interleaved
2025d5d7cf4eSJonathan Chew * across nodes....Would do this when it is discovered, but can't
2026d5d7cf4eSJonathan Chew * because it happens way too early during boot....
2027d5d7cf4eSJonathan Chew */
2028d5d7cf4eSJonathan Chew if (lgrp_plat_mem_intrlv)
2029d5d7cf4eSJonathan Chew cmn_err(CE_NOTE,
2030d5d7cf4eSJonathan Chew "MPO disabled because memory is interleaved\n");
2031d5d7cf4eSJonathan Chew
2032d5d7cf4eSJonathan Chew /*
2033d5d7cf4eSJonathan Chew * Don't bother to do any probing if it is disabled, there is only one
2034d5d7cf4eSJonathan Chew * node, or the height of the lgroup topology less than or equal to 2
2035d5d7cf4eSJonathan Chew */
2036d5d7cf4eSJonathan Chew ht_limit = lgrp_topo_ht_limit();
2037d5d7cf4eSJonathan Chew if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
2038d5d7cf4eSJonathan Chew max_mem_nodes == 1 || ht_limit <= 2) {
2039d5d7cf4eSJonathan Chew /*
2040d5d7cf4eSJonathan Chew * Setup lgroup latencies for 2 level lgroup topology
2041d5d7cf4eSJonathan Chew * (ie. local and remote only) if they haven't been set yet
2042d5d7cf4eSJonathan Chew */
2043d5d7cf4eSJonathan Chew if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 &&
2044d5d7cf4eSJonathan Chew lgrp_plat_lat_stats.latency_max == 0)
2045a3114836SGerry Liu lgrp_plat_2level_setup(&lgrp_plat_lat_stats);
2046d5d7cf4eSJonathan Chew return;
2047d5d7cf4eSJonathan Chew }
2048d5d7cf4eSJonathan Chew
2049d5d7cf4eSJonathan Chew if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
2050d5d7cf4eSJonathan Chew /*
2051d5d7cf4eSJonathan Chew * Should have been able to probe from CPU 0 when it was added
2052d5d7cf4eSJonathan Chew * to lgroup hierarchy, but may not have been able to then
2053d5d7cf4eSJonathan Chew * because it happens so early in boot that gethrtime() hasn't
2054d5d7cf4eSJonathan Chew * been initialized. (:-(
2055d5d7cf4eSJonathan Chew */
2056d5d7cf4eSJonathan Chew curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node,
2057d5d7cf4eSJonathan Chew lgrp_plat_cpu_node_nentries);
2058d5d7cf4eSJonathan Chew ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt);
2059d5d7cf4eSJonathan Chew if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0)
2060d5d7cf4eSJonathan Chew lgrp_plat_probe();
2061d5d7cf4eSJonathan Chew
2062d5d7cf4eSJonathan Chew return;
2063d5d7cf4eSJonathan Chew }
2064d5d7cf4eSJonathan Chew
2065d5d7cf4eSJonathan Chew /*
2066d5d7cf4eSJonathan Chew * When probing memory, use one page for every sample to determine
2067d5d7cf4eSJonathan Chew * lgroup topology and taking multiple samples
2068d5d7cf4eSJonathan Chew */
2069d5d7cf4eSJonathan Chew if (lgrp_plat_probe_mem_config.probe_memsize == 0)
2070d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE *
2071d5d7cf4eSJonathan Chew lgrp_plat_probe_nsamples;
2072d5d7cf4eSJonathan Chew
2073d5d7cf4eSJonathan Chew /*
2074d5d7cf4eSJonathan Chew * Map memory in each node needed for probing to determine latency
2075d5d7cf4eSJonathan Chew * topology
2076d5d7cf4eSJonathan Chew */
2077d5d7cf4eSJonathan Chew for (i = 0; i < lgrp_plat_node_cnt; i++) {
2078d5d7cf4eSJonathan Chew int mnode;
2079d5d7cf4eSJonathan Chew
2080d5d7cf4eSJonathan Chew /*
2081d5d7cf4eSJonathan Chew * Skip this node and leave its probe page NULL
2082d5d7cf4eSJonathan Chew * if it doesn't have any memory
2083d5d7cf4eSJonathan Chew */
2084a3114836SGerry Liu mnode = i;
2085d5d7cf4eSJonathan Chew if (!mem_node_config[mnode].exists) {
2086d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_va[i] = NULL;
2087d5d7cf4eSJonathan Chew continue;
2088d5d7cf4eSJonathan Chew }
2089d5d7cf4eSJonathan Chew
2090d5d7cf4eSJonathan Chew /*
2091d5d7cf4eSJonathan Chew * Allocate one kernel virtual page
2092d5d7cf4eSJonathan Chew */
2093d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena,
2094d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP);
2095d5d7cf4eSJonathan Chew if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) {
2096d5d7cf4eSJonathan Chew cmn_err(CE_WARN,
2097d5d7cf4eSJonathan Chew "lgrp_plat_main_init: couldn't allocate memory");
2098d5d7cf4eSJonathan Chew return;
2099d5d7cf4eSJonathan Chew }
2100d5d7cf4eSJonathan Chew
2101d5d7cf4eSJonathan Chew /*
2102d5d7cf4eSJonathan Chew * Get PFN for first page in each node
2103d5d7cf4eSJonathan Chew */
2104d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_pfn[i] =
2105d5d7cf4eSJonathan Chew mem_node_config[mnode].physbase;
2106d5d7cf4eSJonathan Chew
2107d5d7cf4eSJonathan Chew /*
2108d5d7cf4eSJonathan Chew * Map virtual page to first page in node
2109d5d7cf4eSJonathan Chew */
2110d5d7cf4eSJonathan Chew hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i],
2111d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_memsize,
2112d5d7cf4eSJonathan Chew lgrp_plat_probe_mem_config.probe_pfn[i],
2113d5d7cf4eSJonathan Chew PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE,
2114d5d7cf4eSJonathan Chew HAT_LOAD_NOCONSIST);
2115d5d7cf4eSJonathan Chew }
2116d5d7cf4eSJonathan Chew
2117d5d7cf4eSJonathan Chew /*
2118d5d7cf4eSJonathan Chew * Probe from current CPU
2119d5d7cf4eSJonathan Chew */
2120d5d7cf4eSJonathan Chew lgrp_plat_probe();
2121d5d7cf4eSJonathan Chew }
2122d5d7cf4eSJonathan Chew
2123d5d7cf4eSJonathan Chew
21242e2c009bSjjc /*
21252e2c009bSjjc * Return the number of free, allocatable, or installed
21262e2c009bSjjc * pages in an lgroup
21272e2c009bSjjc * This is a copy of the MAX_MEM_NODES == 1 version of the routine
21282e2c009bSjjc * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup
21292e2c009bSjjc */
21302e2c009bSjjc static pgcnt_t
lgrp_plat_mem_size_default(lgrp_handle_t lgrphand,lgrp_mem_query_t query)21312e2c009bSjjc lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query)
21322e2c009bSjjc {
2133a3114836SGerry Liu _NOTE(ARGUNUSED(lgrphand));
2134a3114836SGerry Liu
21352e2c009bSjjc struct memlist *mlist;
21362e2c009bSjjc pgcnt_t npgs = 0;
21372e2c009bSjjc extern struct memlist *phys_avail;
21382e2c009bSjjc extern struct memlist *phys_install;
21392e2c009bSjjc
21402e2c009bSjjc switch (query) {
21412e2c009bSjjc case LGRP_MEM_SIZE_FREE:
21422e2c009bSjjc return ((pgcnt_t)freemem);
21432e2c009bSjjc case LGRP_MEM_SIZE_AVAIL:
21442e2c009bSjjc memlist_read_lock();
214556f33205SJonathan Adams for (mlist = phys_avail; mlist; mlist = mlist->ml_next)
214656f33205SJonathan Adams npgs += btop(mlist->ml_size);
21472e2c009bSjjc memlist_read_unlock();
21482e2c009bSjjc return (npgs);
21492e2c009bSjjc case LGRP_MEM_SIZE_INSTALL:
21502e2c009bSjjc memlist_read_lock();
215156f33205SJonathan Adams for (mlist = phys_install; mlist; mlist = mlist->ml_next)
215256f33205SJonathan Adams npgs += btop(mlist->ml_size);
21532e2c009bSjjc memlist_read_unlock();
21542e2c009bSjjc return (npgs);
21552e2c009bSjjc default:
21562e2c009bSjjc return ((pgcnt_t)0);
21572e2c009bSjjc }
21582e2c009bSjjc }
21592e2c009bSjjc
21602e2c009bSjjc
21612e2c009bSjjc /*
21622e2c009bSjjc * Update node to proximity domain mappings for given domain and return node ID
21632e2c009bSjjc */
21642e2c009bSjjc static int
lgrp_plat_node_domain_update(node_domain_map_t * node_domain,int node_cnt,uint32_t domain)2165d821f0f0Sjjc lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt,
2166d821f0f0Sjjc uint32_t domain)
21672e2c009bSjjc {
21682e2c009bSjjc uint_t node;
21692e2c009bSjjc uint_t start;
21702e2c009bSjjc
21712e2c009bSjjc /*
21722e2c009bSjjc * Hash proximity domain ID into node to domain mapping table (array)
21732e2c009bSjjc * and add entry for it into first non-existent or matching entry found
21742e2c009bSjjc */
2175d821f0f0Sjjc node = start = NODE_DOMAIN_HASH(domain, node_cnt);
21762e2c009bSjjc do {
21777c478bd9Sstevel@tonic-gate /*
21782e2c009bSjjc * Entry doesn't exist yet, so create one for this proximity
21792e2c009bSjjc * domain and return node ID which is index into mapping table.
21807c478bd9Sstevel@tonic-gate */
21812e2c009bSjjc if (!node_domain[node].exists) {
21822e2c009bSjjc node_domain[node].prox_domain = domain;
2183a3114836SGerry Liu membar_producer();
2184a3114836SGerry Liu node_domain[node].exists = 1;
21852e2c009bSjjc return (node);
21862e2c009bSjjc }
21872e2c009bSjjc
21882e2c009bSjjc /*
21892e2c009bSjjc * Entry exists for this proximity domain already, so just
21902e2c009bSjjc * return node ID (index into table).
21912e2c009bSjjc */
21922e2c009bSjjc if (node_domain[node].prox_domain == domain)
21932e2c009bSjjc return (node);
2194d821f0f0Sjjc node = NODE_DOMAIN_HASH(node + 1, node_cnt);
21952e2c009bSjjc } while (node != start);
21962e2c009bSjjc
21972e2c009bSjjc /*
21982e2c009bSjjc * Ran out of supported number of entries which shouldn't happen....
21992e2c009bSjjc */
22002e2c009bSjjc ASSERT(node != start);
22012e2c009bSjjc return (-1);
22022e2c009bSjjc }
22032e2c009bSjjc
22042e2c009bSjjc /*
22052e2c009bSjjc * Update node memory information for given proximity domain with specified
22062e2c009bSjjc * starting and ending physical address range (and return positive numbers for
22072e2c009bSjjc * success and negative ones for errors)
22082e2c009bSjjc */
22092e2c009bSjjc static int
lgrp_plat_memnode_info_update(node_domain_map_t * node_domain,int node_cnt,memnode_phys_addr_map_t * memnode_info,int memnode_cnt,uint64_t start,uint64_t end,uint32_t domain,uint32_t device_id)2210a3114836SGerry Liu lgrp_plat_memnode_info_update(node_domain_map_t *node_domain, int node_cnt,
2211a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info, int memnode_cnt, uint64_t start,
2212a3114836SGerry Liu uint64_t end, uint32_t domain, uint32_t device_id)
22132e2c009bSjjc {
2214a3114836SGerry Liu int node, mnode;
22152e2c009bSjjc
22162e2c009bSjjc /*
22172e2c009bSjjc * Get node number for proximity domain
22182e2c009bSjjc */
2219d821f0f0Sjjc node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain);
22202e2c009bSjjc if (node == -1) {
2221d821f0f0Sjjc node = lgrp_plat_node_domain_update(node_domain, node_cnt,
2222d821f0f0Sjjc domain);
22232e2c009bSjjc if (node == -1)
22242e2c009bSjjc return (-1);
22252e2c009bSjjc }
22262e2c009bSjjc
2227a3114836SGerry Liu /*
2228a3114836SGerry Liu * This function is called during boot if device_id is
2229a3114836SGerry Liu * ACPI_MEMNODE_DEVID_BOOT, otherwise it's called at runtime for
2230a3114836SGerry Liu * memory DR operations.
2231a3114836SGerry Liu */
2232a3114836SGerry Liu if (device_id != ACPI_MEMNODE_DEVID_BOOT) {
2233a3114836SGerry Liu ASSERT(lgrp_plat_max_mem_node <= memnode_cnt);
2234a3114836SGerry Liu
2235a3114836SGerry Liu for (mnode = lgrp_plat_node_cnt;
2236a3114836SGerry Liu mnode < lgrp_plat_max_mem_node; mnode++) {
2237a3114836SGerry Liu if (memnode_info[mnode].exists &&
2238a3114836SGerry Liu memnode_info[mnode].prox_domain == domain &&
2239a3114836SGerry Liu memnode_info[mnode].device_id == device_id) {
2240a3114836SGerry Liu if (btop(start) < memnode_info[mnode].start)
2241a3114836SGerry Liu memnode_info[mnode].start = btop(start);
2242a3114836SGerry Liu if (btop(end) > memnode_info[mnode].end)
2243a3114836SGerry Liu memnode_info[mnode].end = btop(end);
2244a3114836SGerry Liu return (1);
2245a3114836SGerry Liu }
2246a3114836SGerry Liu }
2247a3114836SGerry Liu
2248a3114836SGerry Liu if (lgrp_plat_max_mem_node >= memnode_cnt) {
2249a3114836SGerry Liu return (-3);
2250a3114836SGerry Liu } else {
2251a3114836SGerry Liu lgrp_plat_max_mem_node++;
2252a3114836SGerry Liu memnode_info[mnode].start = btop(start);
2253a3114836SGerry Liu memnode_info[mnode].end = btop(end);
2254a3114836SGerry Liu memnode_info[mnode].prox_domain = domain;
2255a3114836SGerry Liu memnode_info[mnode].device_id = device_id;
2256a3114836SGerry Liu memnode_info[mnode].lgrphand = node;
2257a3114836SGerry Liu membar_producer();
2258a3114836SGerry Liu memnode_info[mnode].exists = 1;
2259a3114836SGerry Liu return (0);
2260a3114836SGerry Liu }
2261a3114836SGerry Liu }
2262a3114836SGerry Liu
22632e2c009bSjjc /*
22642e2c009bSjjc * Create entry in table for node if it doesn't exist
22652e2c009bSjjc */
2266a3114836SGerry Liu ASSERT(node < memnode_cnt);
2267a3114836SGerry Liu if (!memnode_info[node].exists) {
2268a3114836SGerry Liu memnode_info[node].start = btop(start);
2269a3114836SGerry Liu memnode_info[node].end = btop(end);
2270a3114836SGerry Liu memnode_info[node].prox_domain = domain;
2271a3114836SGerry Liu memnode_info[node].device_id = device_id;
2272a3114836SGerry Liu memnode_info[node].lgrphand = node;
2273a3114836SGerry Liu membar_producer();
2274a3114836SGerry Liu memnode_info[node].exists = 1;
22752e2c009bSjjc return (0);
22762e2c009bSjjc }
22772e2c009bSjjc
22782e2c009bSjjc /*
22792e2c009bSjjc * Entry already exists for this proximity domain
22802e2c009bSjjc *
22812e2c009bSjjc * There may be more than one SRAT memory entry for a domain, so we may
22822e2c009bSjjc * need to update existing start or end address for the node.
22832e2c009bSjjc */
2284a3114836SGerry Liu if (memnode_info[node].prox_domain == domain) {
2285a3114836SGerry Liu if (btop(start) < memnode_info[node].start)
2286a3114836SGerry Liu memnode_info[node].start = btop(start);
2287a3114836SGerry Liu if (btop(end) > memnode_info[node].end)
2288a3114836SGerry Liu memnode_info[node].end = btop(end);
22892e2c009bSjjc return (1);
22902e2c009bSjjc }
22912e2c009bSjjc return (-2);
22922e2c009bSjjc }
22932e2c009bSjjc
22942e2c009bSjjc
229581d9ccb6SJonathan Chew /*
2296a3114836SGerry Liu * Have to sort nodes by starting physical address because plat_mnode_xcheck()
2297a3114836SGerry Liu * assumes and expects memnodes to be sorted in ascending order by physical
2298a3114836SGerry Liu * address.
229981d9ccb6SJonathan Chew */
230081d9ccb6SJonathan Chew static void
lgrp_plat_node_sort(node_domain_map_t * node_domain,int node_cnt,cpu_node_map_t * cpu_node,int cpu_count,memnode_phys_addr_map_t * memnode_info)230181d9ccb6SJonathan Chew lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt,
2302a3114836SGerry Liu cpu_node_map_t *cpu_node, int cpu_count,
2303a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info)
230481d9ccb6SJonathan Chew {
230581d9ccb6SJonathan Chew boolean_t found;
230681d9ccb6SJonathan Chew int i;
230781d9ccb6SJonathan Chew int j;
230881d9ccb6SJonathan Chew int n;
230981d9ccb6SJonathan Chew boolean_t sorted;
231081d9ccb6SJonathan Chew boolean_t swapped;
231181d9ccb6SJonathan Chew
231281d9ccb6SJonathan Chew if (!lgrp_plat_node_sort_enable || node_cnt <= 1 ||
2313a3114836SGerry Liu node_domain == NULL || memnode_info == NULL)
231481d9ccb6SJonathan Chew return;
231581d9ccb6SJonathan Chew
231681d9ccb6SJonathan Chew /*
231781d9ccb6SJonathan Chew * Sorted already?
231881d9ccb6SJonathan Chew */
231981d9ccb6SJonathan Chew sorted = B_TRUE;
232081d9ccb6SJonathan Chew for (i = 0; i < node_cnt - 1; i++) {
232181d9ccb6SJonathan Chew /*
232281d9ccb6SJonathan Chew * Skip entries that don't exist
232381d9ccb6SJonathan Chew */
2324a3114836SGerry Liu if (!memnode_info[i].exists)
232581d9ccb6SJonathan Chew continue;
232681d9ccb6SJonathan Chew
232781d9ccb6SJonathan Chew /*
232881d9ccb6SJonathan Chew * Try to find next existing entry to compare against
232981d9ccb6SJonathan Chew */
233081d9ccb6SJonathan Chew found = B_FALSE;
233181d9ccb6SJonathan Chew for (j = i + 1; j < node_cnt; j++) {
2332a3114836SGerry Liu if (memnode_info[j].exists) {
233381d9ccb6SJonathan Chew found = B_TRUE;
233481d9ccb6SJonathan Chew break;
233581d9ccb6SJonathan Chew }
233681d9ccb6SJonathan Chew }
233781d9ccb6SJonathan Chew
233881d9ccb6SJonathan Chew /*
233981d9ccb6SJonathan Chew * Done if no more existing entries to compare against
234081d9ccb6SJonathan Chew */
234181d9ccb6SJonathan Chew if (found == B_FALSE)
234281d9ccb6SJonathan Chew break;
234381d9ccb6SJonathan Chew
234481d9ccb6SJonathan Chew /*
234581d9ccb6SJonathan Chew * Not sorted if starting address of current entry is bigger
234681d9ccb6SJonathan Chew * than starting address of next existing entry
234781d9ccb6SJonathan Chew */
2348a3114836SGerry Liu if (memnode_info[i].start > memnode_info[j].start) {
234981d9ccb6SJonathan Chew sorted = B_FALSE;
235081d9ccb6SJonathan Chew break;
235181d9ccb6SJonathan Chew }
235281d9ccb6SJonathan Chew }
235381d9ccb6SJonathan Chew
235481d9ccb6SJonathan Chew /*
235581d9ccb6SJonathan Chew * Don't need to sort if sorted already
235681d9ccb6SJonathan Chew */
235781d9ccb6SJonathan Chew if (sorted == B_TRUE)
235881d9ccb6SJonathan Chew return;
235981d9ccb6SJonathan Chew
236081d9ccb6SJonathan Chew /*
236181d9ccb6SJonathan Chew * Just use bubble sort since number of nodes is small
236281d9ccb6SJonathan Chew */
236381d9ccb6SJonathan Chew n = node_cnt;
236481d9ccb6SJonathan Chew do {
236581d9ccb6SJonathan Chew swapped = B_FALSE;
236681d9ccb6SJonathan Chew n--;
236781d9ccb6SJonathan Chew for (i = 0; i < n; i++) {
236881d9ccb6SJonathan Chew /*
236981d9ccb6SJonathan Chew * Skip entries that don't exist
237081d9ccb6SJonathan Chew */
2371a3114836SGerry Liu if (!memnode_info[i].exists)
237281d9ccb6SJonathan Chew continue;
237381d9ccb6SJonathan Chew
237481d9ccb6SJonathan Chew /*
237581d9ccb6SJonathan Chew * Try to find next existing entry to compare against
237681d9ccb6SJonathan Chew */
237781d9ccb6SJonathan Chew found = B_FALSE;
237881d9ccb6SJonathan Chew for (j = i + 1; j <= n; j++) {
2379a3114836SGerry Liu if (memnode_info[j].exists) {
238081d9ccb6SJonathan Chew found = B_TRUE;
238181d9ccb6SJonathan Chew break;
238281d9ccb6SJonathan Chew }
238381d9ccb6SJonathan Chew }
238481d9ccb6SJonathan Chew
238581d9ccb6SJonathan Chew /*
238681d9ccb6SJonathan Chew * Done if no more existing entries to compare against
238781d9ccb6SJonathan Chew */
238881d9ccb6SJonathan Chew if (found == B_FALSE)
238981d9ccb6SJonathan Chew break;
239081d9ccb6SJonathan Chew
2391a3114836SGerry Liu if (memnode_info[i].start > memnode_info[j].start) {
2392a3114836SGerry Liu memnode_phys_addr_map_t save_addr;
239381d9ccb6SJonathan Chew node_domain_map_t save_node;
239481d9ccb6SJonathan Chew
239581d9ccb6SJonathan Chew /*
239681d9ccb6SJonathan Chew * Swap node to proxmity domain ID assignments
239781d9ccb6SJonathan Chew */
239881d9ccb6SJonathan Chew bcopy(&node_domain[i], &save_node,
239981d9ccb6SJonathan Chew sizeof (node_domain_map_t));
240081d9ccb6SJonathan Chew bcopy(&node_domain[j], &node_domain[i],
240181d9ccb6SJonathan Chew sizeof (node_domain_map_t));
240281d9ccb6SJonathan Chew bcopy(&save_node, &node_domain[j],
240381d9ccb6SJonathan Chew sizeof (node_domain_map_t));
240481d9ccb6SJonathan Chew
240581d9ccb6SJonathan Chew /*
240681d9ccb6SJonathan Chew * Swap node to physical memory assignments
240781d9ccb6SJonathan Chew */
2408a3114836SGerry Liu bcopy(&memnode_info[i], &save_addr,
2409a3114836SGerry Liu sizeof (memnode_phys_addr_map_t));
2410a3114836SGerry Liu bcopy(&memnode_info[j], &memnode_info[i],
2411a3114836SGerry Liu sizeof (memnode_phys_addr_map_t));
2412a3114836SGerry Liu bcopy(&save_addr, &memnode_info[j],
2413a3114836SGerry Liu sizeof (memnode_phys_addr_map_t));
241481d9ccb6SJonathan Chew swapped = B_TRUE;
241581d9ccb6SJonathan Chew }
241681d9ccb6SJonathan Chew }
241781d9ccb6SJonathan Chew } while (swapped == B_TRUE);
241881d9ccb6SJonathan Chew
241981d9ccb6SJonathan Chew /*
242081d9ccb6SJonathan Chew * Check to make sure that CPUs assigned to correct node IDs now since
242181d9ccb6SJonathan Chew * node to proximity domain ID assignments may have been changed above
242281d9ccb6SJonathan Chew */
242381d9ccb6SJonathan Chew if (n == node_cnt - 1 || cpu_node == NULL || cpu_count < 1)
242481d9ccb6SJonathan Chew return;
242581d9ccb6SJonathan Chew for (i = 0; i < cpu_count; i++) {
242681d9ccb6SJonathan Chew int node;
242781d9ccb6SJonathan Chew
242881d9ccb6SJonathan Chew node = lgrp_plat_domain_to_node(node_domain, node_cnt,
242981d9ccb6SJonathan Chew cpu_node[i].prox_domain);
243081d9ccb6SJonathan Chew if (cpu_node[i].node != node)
243181d9ccb6SJonathan Chew cpu_node[i].node = node;
243281d9ccb6SJonathan Chew }
243381d9ccb6SJonathan Chew
243481d9ccb6SJonathan Chew }
243581d9ccb6SJonathan Chew
243681d9ccb6SJonathan Chew
24372e2c009bSjjc /*
24382e2c009bSjjc * Return time needed to probe from current CPU to memory in given node
24392e2c009bSjjc */
24402e2c009bSjjc static hrtime_t
lgrp_plat_probe_time(int to,cpu_node_map_t * cpu_node,int cpu_node_nentries,lgrp_plat_probe_mem_config_t * probe_mem_config,lgrp_plat_latency_stats_t * lat_stats,lgrp_plat_probe_stats_t * probe_stats)2441d5d7cf4eSJonathan Chew lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, int cpu_node_nentries,
24422e2c009bSjjc lgrp_plat_probe_mem_config_t *probe_mem_config,
24432e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
24442e2c009bSjjc {
24452e2c009bSjjc caddr_t buf;
24462e2c009bSjjc hrtime_t elapsed;
24472e2c009bSjjc hrtime_t end;
24482e2c009bSjjc int from;
24492e2c009bSjjc int i;
24502e2c009bSjjc int ipl;
24512e2c009bSjjc hrtime_t max;
24522e2c009bSjjc hrtime_t min;
24532e2c009bSjjc hrtime_t start;
24542e2c009bSjjc extern int use_sse_pagecopy;
24552e2c009bSjjc
24562e2c009bSjjc /*
24572e2c009bSjjc * Determine ID of node containing current CPU
24582e2c009bSjjc */
2459d5d7cf4eSJonathan Chew from = lgrp_plat_cpu_to_node(CPU, cpu_node, cpu_node_nentries);
24602e2c009bSjjc ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
24612e2c009bSjjc
24622e2c009bSjjc /*
24632e2c009bSjjc * Do common work for probing main memory
24642e2c009bSjjc */
24652e2c009bSjjc if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) {
24662e2c009bSjjc /*
24672e2c009bSjjc * Skip probing any nodes without memory and
24682e2c009bSjjc * set probe time to 0
24692e2c009bSjjc */
24702e2c009bSjjc if (probe_mem_config->probe_va[to] == NULL) {
24712e2c009bSjjc lat_stats->latencies[from][to] = 0;
24722e2c009bSjjc return (0);
24732e2c009bSjjc }
24742e2c009bSjjc
24752e2c009bSjjc /*
24762e2c009bSjjc * Invalidate caches once instead of once every sample
24772e2c009bSjjc * which should cut cost of probing by a lot
24782e2c009bSjjc */
24792e2c009bSjjc probe_stats->flush_cost = gethrtime();
24802e2c009bSjjc invalidate_cache();
24812e2c009bSjjc probe_stats->flush_cost = gethrtime() -
24822e2c009bSjjc probe_stats->flush_cost;
24832e2c009bSjjc probe_stats->probe_cost_total += probe_stats->flush_cost;
24842e2c009bSjjc }
24852e2c009bSjjc
24862e2c009bSjjc /*
24872e2c009bSjjc * Probe from current CPU to given memory using specified operation
24882e2c009bSjjc * and take specified number of samples
24892e2c009bSjjc */
24902e2c009bSjjc max = 0;
24912e2c009bSjjc min = -1;
24922e2c009bSjjc for (i = 0; i < lgrp_plat_probe_nsamples; i++) {
24932e2c009bSjjc probe_stats->probe_cost = gethrtime();
24942e2c009bSjjc
24952e2c009bSjjc /*
24962e2c009bSjjc * Can't measure probe time if gethrtime() isn't working yet
24972e2c009bSjjc */
24982e2c009bSjjc if (probe_stats->probe_cost == 0 && gethrtime() == 0)
24992e2c009bSjjc return (0);
25002e2c009bSjjc
25012e2c009bSjjc if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
25022e2c009bSjjc /*
25032e2c009bSjjc * Measure how long it takes to read vendor ID from
25042e2c009bSjjc * Northbridge
25052e2c009bSjjc */
25062e2c009bSjjc elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads);
25072e2c009bSjjc } else {
25082e2c009bSjjc /*
25092e2c009bSjjc * Measure how long it takes to copy page
25102e2c009bSjjc * on top of itself
25112e2c009bSjjc */
25122e2c009bSjjc buf = probe_mem_config->probe_va[to] + (i * PAGESIZE);
25132e2c009bSjjc
25142e2c009bSjjc kpreempt_disable();
25152e2c009bSjjc ipl = splhigh();
25162e2c009bSjjc start = gethrtime();
25172e2c009bSjjc if (use_sse_pagecopy)
25182e2c009bSjjc hwblkpagecopy(buf, buf);
25192e2c009bSjjc else
25202e2c009bSjjc bcopy(buf, buf, PAGESIZE);
25212e2c009bSjjc end = gethrtime();
25222e2c009bSjjc elapsed = end - start;
25232e2c009bSjjc splx(ipl);
25242e2c009bSjjc kpreempt_enable();
25252e2c009bSjjc }
25262e2c009bSjjc
25272e2c009bSjjc probe_stats->probe_cost = gethrtime() -
25282e2c009bSjjc probe_stats->probe_cost;
25292e2c009bSjjc probe_stats->probe_cost_total += probe_stats->probe_cost;
25302e2c009bSjjc
25312e2c009bSjjc if (min == -1 || elapsed < min)
25322e2c009bSjjc min = elapsed;
25332e2c009bSjjc if (elapsed > max)
25342e2c009bSjjc max = elapsed;
25352e2c009bSjjc }
25362e2c009bSjjc
25372e2c009bSjjc /*
25382e2c009bSjjc * Update minimum and maximum probe times between
25392e2c009bSjjc * these two nodes
25402e2c009bSjjc */
25412e2c009bSjjc if (min < probe_stats->probe_min[from][to] ||
25422e2c009bSjjc probe_stats->probe_min[from][to] == 0)
25432e2c009bSjjc probe_stats->probe_min[from][to] = min;
25442e2c009bSjjc
25452e2c009bSjjc if (max > probe_stats->probe_max[from][to])
25462e2c009bSjjc probe_stats->probe_max[from][to] = max;
25472e2c009bSjjc
25482e2c009bSjjc return (min);
25492e2c009bSjjc }
25502e2c009bSjjc
25512e2c009bSjjc
2552dae2fa37Sjjc /*
2553d821f0f0Sjjc * Read boot property with CPU to APIC ID array, fill in CPU to node ID
2554d5d7cf4eSJonathan Chew * mapping table with APIC ID for each CPU (if pointer to table isn't NULL),
2555d5d7cf4eSJonathan Chew * and return number of CPU APIC IDs.
2556dae2fa37Sjjc *
2557dae2fa37Sjjc * NOTE: This code assumes that CPU IDs are assigned in order that they appear
2558dae2fa37Sjjc * in in cpu_apicid_array boot property which is based on and follows
2559dae2fa37Sjjc * same ordering as processor list in ACPI MADT. If the code in
2560dae2fa37Sjjc * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns
2561dae2fa37Sjjc * CPU IDs ever changes, then this code will need to change too....
2562dae2fa37Sjjc */
2563dae2fa37Sjjc static int
lgrp_plat_process_cpu_apicids(cpu_node_map_t * cpu_node)2564d821f0f0Sjjc lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node)
2565dae2fa37Sjjc {
2566d821f0f0Sjjc int boot_prop_len;
2567dae2fa37Sjjc char *boot_prop_name = BP_CPU_APICID_ARRAY;
25687bc870e6SToomas Soome uint32_t *cpu_apicid_array;
2569dae2fa37Sjjc int i;
2570d821f0f0Sjjc int n;
2571dae2fa37Sjjc
2572dae2fa37Sjjc /*
2573dae2fa37Sjjc * Check length of property value
2574dae2fa37Sjjc */
2575dae2fa37Sjjc boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name);
25767bc870e6SToomas Soome if (boot_prop_len <= 0)
2577d5d7cf4eSJonathan Chew return (-1);
2578d821f0f0Sjjc
2579d821f0f0Sjjc /*
2580a3114836SGerry Liu * Calculate number of entries in array and return when the system is
2581a3114836SGerry Liu * not very interesting for NUMA. It's not interesting for NUMA if
2582a3114836SGerry Liu * system has only one CPU and doesn't support CPU hotplug.
2583d821f0f0Sjjc */
25847bc870e6SToomas Soome n = boot_prop_len / sizeof (*cpu_apicid_array);
2585a3114836SGerry Liu if (n == 1 && !plat_dr_support_cpu())
2586d5d7cf4eSJonathan Chew return (-2);
2587dae2fa37Sjjc
25887bc870e6SToomas Soome cpu_apicid_array = (uint32_t *)BOP_ALLOC(bootops, NULL, boot_prop_len,
25897bc870e6SToomas Soome sizeof (*cpu_apicid_array));
2590dae2fa37Sjjc /*
2591dae2fa37Sjjc * Get CPU to APIC ID property value
2592dae2fa37Sjjc */
25937bc870e6SToomas Soome if (cpu_apicid_array == NULL ||
25947bc870e6SToomas Soome BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0)
2595d5d7cf4eSJonathan Chew return (-3);
2596d5d7cf4eSJonathan Chew
2597d5d7cf4eSJonathan Chew /*
2598d5d7cf4eSJonathan Chew * Just return number of CPU APIC IDs if CPU to node mapping table is
2599d5d7cf4eSJonathan Chew * NULL
2600d5d7cf4eSJonathan Chew */
2601a3114836SGerry Liu if (cpu_node == NULL) {
2602a3114836SGerry Liu if (plat_dr_support_cpu() && n >= boot_ncpus) {
2603a3114836SGerry Liu return (boot_ncpus);
2604a3114836SGerry Liu } else {
2605a3114836SGerry Liu return (n);
2606a3114836SGerry Liu }
2607a3114836SGerry Liu }
2608dae2fa37Sjjc
2609dae2fa37Sjjc /*
2610dae2fa37Sjjc * Fill in CPU to node ID mapping table with APIC ID for each CPU
2611dae2fa37Sjjc */
2612d821f0f0Sjjc for (i = 0; i < n; i++) {
2613a3114836SGerry Liu /* Only add boot CPUs into the map if CPU DR is enabled. */
2614a3114836SGerry Liu if (plat_dr_support_cpu() && i >= boot_ncpus)
2615a3114836SGerry Liu break;
2616dae2fa37Sjjc cpu_node[i].exists = 1;
2617dae2fa37Sjjc cpu_node[i].apicid = cpu_apicid_array[i];
2618a3114836SGerry Liu cpu_node[i].prox_domain = UINT32_MAX;
2619a3114836SGerry Liu cpu_node[i].node = UINT_MAX;
2620dae2fa37Sjjc }
2621dae2fa37Sjjc
2622d821f0f0Sjjc /*
2623d821f0f0Sjjc * Return number of CPUs based on number of APIC IDs
2624d821f0f0Sjjc */
2625a3114836SGerry Liu return (i);
2626dae2fa37Sjjc }
2627dae2fa37Sjjc
2628dae2fa37Sjjc
26292e2c009bSjjc /*
26302e2c009bSjjc * Read ACPI System Locality Information Table (SLIT) to determine how far each
26312e2c009bSjjc * NUMA node is from each other
26322e2c009bSjjc */
26332e2c009bSjjc static int
lgrp_plat_process_slit(ACPI_TABLE_SLIT * tp,node_domain_map_t * node_domain,uint_t node_cnt,memnode_phys_addr_map_t * memnode_info,lgrp_plat_latency_stats_t * lat_stats)26347bc870e6SToomas Soome lgrp_plat_process_slit(ACPI_TABLE_SLIT *tp,
2635a3114836SGerry Liu node_domain_map_t *node_domain, uint_t node_cnt,
2636a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats)
26372e2c009bSjjc {
26382e2c009bSjjc int i;
26392e2c009bSjjc int j;
2640a3114836SGerry Liu int src;
2641a3114836SGerry Liu int dst;
26422e2c009bSjjc int localities;
26432e2c009bSjjc hrtime_t max;
26442e2c009bSjjc hrtime_t min;
26452e2c009bSjjc int retval;
26462e2c009bSjjc uint8_t *slit_entries;
26472e2c009bSjjc
26482e2c009bSjjc if (tp == NULL || !lgrp_plat_slit_enable)
26492e2c009bSjjc return (1);
26502e2c009bSjjc
26512e2c009bSjjc if (lat_stats == NULL)
26522e2c009bSjjc return (2);
26532e2c009bSjjc
26547bc870e6SToomas Soome localities = tp->LocalityCount;
26552e2c009bSjjc
26562e2c009bSjjc min = lat_stats->latency_min;
26572e2c009bSjjc max = lat_stats->latency_max;
26582e2c009bSjjc
26592e2c009bSjjc /*
26602e2c009bSjjc * Fill in latency matrix based on SLIT entries
26612e2c009bSjjc */
26627bc870e6SToomas Soome slit_entries = tp->Entry;
26632e2c009bSjjc for (i = 0; i < localities; i++) {
2664a3114836SGerry Liu src = lgrp_plat_domain_to_node(node_domain,
2665a3114836SGerry Liu node_cnt, i);
2666a3114836SGerry Liu if (src == -1)
2667a3114836SGerry Liu continue;
2668a3114836SGerry Liu
26692e2c009bSjjc for (j = 0; j < localities; j++) {
26702e2c009bSjjc uint8_t latency;
26712e2c009bSjjc
2672a3114836SGerry Liu dst = lgrp_plat_domain_to_node(node_domain,
2673a3114836SGerry Liu node_cnt, j);
2674a3114836SGerry Liu if (dst == -1)
2675a3114836SGerry Liu continue;
2676a3114836SGerry Liu
26772e2c009bSjjc latency = slit_entries[(i * localities) + j];
2678a3114836SGerry Liu lat_stats->latencies[src][dst] = latency;
26795b7cf7f0Sjjc if (latency < min || min == -1)
26802e2c009bSjjc min = latency;
26812e2c009bSjjc if (latency > max)
26822e2c009bSjjc max = latency;
26837c478bd9Sstevel@tonic-gate }
26842e2c009bSjjc }
26857c478bd9Sstevel@tonic-gate
26862e2c009bSjjc /*
26872e2c009bSjjc * Verify that latencies/distances given in SLIT look reasonable
26882e2c009bSjjc */
2689a3114836SGerry Liu retval = lgrp_plat_latency_verify(memnode_info, lat_stats);
26902e2c009bSjjc
26912e2c009bSjjc if (retval) {
26927c478bd9Sstevel@tonic-gate /*
26932e2c009bSjjc * Reinitialize (zero) latency table since SLIT doesn't look
26942e2c009bSjjc * right
26957c478bd9Sstevel@tonic-gate */
26962e2c009bSjjc for (i = 0; i < localities; i++) {
26972e2c009bSjjc for (j = 0; j < localities; j++)
26982e2c009bSjjc lat_stats->latencies[i][j] = 0;
26997c478bd9Sstevel@tonic-gate }
27002e2c009bSjjc } else {
27017c478bd9Sstevel@tonic-gate /*
27022e2c009bSjjc * Update min and max latencies seen since SLIT looks valid
27037c478bd9Sstevel@tonic-gate */
27042e2c009bSjjc lat_stats->latency_min = min;
27052e2c009bSjjc lat_stats->latency_max = max;
27067c478bd9Sstevel@tonic-gate }
27077c478bd9Sstevel@tonic-gate
27082e2c009bSjjc return (retval);
27097c478bd9Sstevel@tonic-gate }
27107c478bd9Sstevel@tonic-gate
27112e2c009bSjjc
2712a3114836SGerry Liu /*
2713a3114836SGerry Liu * Update lgrp latencies according to information returned by ACPI _SLI method.
2714a3114836SGerry Liu */
2715a3114836SGerry Liu static int
lgrp_plat_process_sli(uint32_t domain_id,uchar_t * sli_info,uint32_t sli_cnt,node_domain_map_t * node_domain,uint_t node_cnt,lgrp_plat_latency_stats_t * lat_stats)2716a3114836SGerry Liu lgrp_plat_process_sli(uint32_t domain_id, uchar_t *sli_info,
2717a3114836SGerry Liu uint32_t sli_cnt, node_domain_map_t *node_domain, uint_t node_cnt,
2718a3114836SGerry Liu lgrp_plat_latency_stats_t *lat_stats)
2719a3114836SGerry Liu {
2720a3114836SGerry Liu int i;
2721a3114836SGerry Liu int src, dst;
2722a3114836SGerry Liu uint8_t latency;
2723a3114836SGerry Liu hrtime_t max, min;
2724a3114836SGerry Liu
2725a3114836SGerry Liu if (lat_stats == NULL || sli_info == NULL ||
2726a3114836SGerry Liu sli_cnt == 0 || domain_id >= sli_cnt)
2727a3114836SGerry Liu return (-1);
2728a3114836SGerry Liu
2729a3114836SGerry Liu src = lgrp_plat_domain_to_node(node_domain, node_cnt, domain_id);
2730a3114836SGerry Liu if (src == -1) {
2731a3114836SGerry Liu src = lgrp_plat_node_domain_update(node_domain, node_cnt,
2732a3114836SGerry Liu domain_id);
2733a3114836SGerry Liu if (src == -1)
2734a3114836SGerry Liu return (-1);
2735a3114836SGerry Liu }
2736a3114836SGerry Liu
2737a3114836SGerry Liu /*
2738a3114836SGerry Liu * Don't update latency info if topology has been flattened to 2 levels.
2739a3114836SGerry Liu */
2740a3114836SGerry Liu if (lgrp_plat_topo_flatten != 0) {
2741a3114836SGerry Liu return (0);
2742a3114836SGerry Liu }
2743a3114836SGerry Liu
2744a3114836SGerry Liu /*
2745a3114836SGerry Liu * Latency information for proximity domain is ready.
2746a3114836SGerry Liu * TODO: support adjusting latency information at runtime.
2747a3114836SGerry Liu */
2748a3114836SGerry Liu if (lat_stats->latencies[src][src] != 0) {
2749a3114836SGerry Liu return (0);
2750a3114836SGerry Liu }
2751a3114836SGerry Liu
2752a3114836SGerry Liu /* Validate latency information. */
2753a3114836SGerry Liu for (i = 0; i < sli_cnt; i++) {
2754a3114836SGerry Liu if (i == domain_id) {
2755a3114836SGerry Liu if (sli_info[i] != ACPI_SLIT_SELF_LATENCY ||
2756a3114836SGerry Liu sli_info[sli_cnt + i] != ACPI_SLIT_SELF_LATENCY) {
2757a3114836SGerry Liu return (-1);
2758a3114836SGerry Liu }
2759a3114836SGerry Liu } else {
2760a3114836SGerry Liu if (sli_info[i] <= ACPI_SLIT_SELF_LATENCY ||
2761a3114836SGerry Liu sli_info[sli_cnt + i] <= ACPI_SLIT_SELF_LATENCY ||
2762a3114836SGerry Liu sli_info[i] != sli_info[sli_cnt + i]) {
2763a3114836SGerry Liu return (-1);
2764a3114836SGerry Liu }
2765a3114836SGerry Liu }
2766a3114836SGerry Liu }
2767a3114836SGerry Liu
2768a3114836SGerry Liu min = lat_stats->latency_min;
2769a3114836SGerry Liu max = lat_stats->latency_max;
2770a3114836SGerry Liu for (i = 0; i < sli_cnt; i++) {
2771a3114836SGerry Liu dst = lgrp_plat_domain_to_node(node_domain, node_cnt, i);
2772a3114836SGerry Liu if (dst == -1)
2773a3114836SGerry Liu continue;
2774a3114836SGerry Liu
2775a3114836SGerry Liu ASSERT(sli_info[i] == sli_info[sli_cnt + i]);
2776a3114836SGerry Liu
2777a3114836SGerry Liu /* Update row in latencies matrix. */
2778a3114836SGerry Liu latency = sli_info[i];
2779a3114836SGerry Liu lat_stats->latencies[src][dst] = latency;
2780a3114836SGerry Liu if (latency < min || min == -1)
2781a3114836SGerry Liu min = latency;
2782a3114836SGerry Liu if (latency > max)
2783a3114836SGerry Liu max = latency;
2784a3114836SGerry Liu
2785a3114836SGerry Liu /* Update column in latencies matrix. */
2786a3114836SGerry Liu latency = sli_info[sli_cnt + i];
2787a3114836SGerry Liu lat_stats->latencies[dst][src] = latency;
2788a3114836SGerry Liu if (latency < min || min == -1)
2789a3114836SGerry Liu min = latency;
2790a3114836SGerry Liu if (latency > max)
2791a3114836SGerry Liu max = latency;
2792a3114836SGerry Liu }
2793a3114836SGerry Liu lat_stats->latency_min = min;
2794a3114836SGerry Liu lat_stats->latency_max = max;
2795a3114836SGerry Liu
2796a3114836SGerry Liu return (0);
2797a3114836SGerry Liu }
2798a3114836SGerry Liu
2799a3114836SGerry Liu
28007c478bd9Sstevel@tonic-gate /*
28012e2c009bSjjc * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs
2802d821f0f0Sjjc * and memory are local to each other in the same NUMA node and return number
2803d821f0f0Sjjc * of nodes
28047c478bd9Sstevel@tonic-gate */
28052e2c009bSjjc static int
lgrp_plat_process_srat(ACPI_TABLE_SRAT * tp,ACPI_TABLE_MSCT * mp,uint32_t * prox_domain_min,node_domain_map_t * node_domain,cpu_node_map_t * cpu_node,int cpu_count,memnode_phys_addr_map_t * memnode_info)28067bc870e6SToomas Soome lgrp_plat_process_srat(ACPI_TABLE_SRAT *tp, ACPI_TABLE_MSCT *mp,
2807a3114836SGerry Liu uint32_t *prox_domain_min, node_domain_map_t *node_domain,
2808a3114836SGerry Liu cpu_node_map_t *cpu_node, int cpu_count,
2809a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info)
28107c478bd9Sstevel@tonic-gate {
28117bc870e6SToomas Soome ACPI_SUBTABLE_HEADER *item, *srat_end;
28122e2c009bSjjc int i;
2813d821f0f0Sjjc int node_cnt;
2814dae2fa37Sjjc int proc_entry_count;
2815a3114836SGerry Liu int rc;
28167c478bd9Sstevel@tonic-gate
2817d821f0f0Sjjc /*
2818d821f0f0Sjjc * Nothing to do when no SRAT or disabled
2819d821f0f0Sjjc */
28202e2c009bSjjc if (tp == NULL || !lgrp_plat_srat_enable)
2821d821f0f0Sjjc return (-1);
28222e2c009bSjjc
28232e2c009bSjjc /*
2824a3114836SGerry Liu * Try to get domain information from MSCT table.
2825a3114836SGerry Liu * ACPI4.0: OSPM will use information provided by the MSCT only
2826a3114836SGerry Liu * when the System Resource Affinity Table (SRAT) exists.
2827a3114836SGerry Liu */
2828a3114836SGerry Liu node_cnt = lgrp_plat_msct_domains(mp, prox_domain_min);
2829a3114836SGerry Liu if (node_cnt <= 0) {
2830a3114836SGerry Liu /*
2831a3114836SGerry Liu * Determine number of nodes by counting number of proximity
2832a3114836SGerry Liu * domains in SRAT.
2833a3114836SGerry Liu */
2834a3114836SGerry Liu node_cnt = lgrp_plat_srat_domains(tp, prox_domain_min);
2835a3114836SGerry Liu }
2836a3114836SGerry Liu /*
2837a3114836SGerry Liu * Return if number of nodes is 1 or less since don't need to read SRAT.
28382e2c009bSjjc */
2839d821f0f0Sjjc if (node_cnt == 1)
2840d821f0f0Sjjc return (1);
2841d821f0f0Sjjc else if (node_cnt <= 0)
2842d821f0f0Sjjc return (-2);
28432e2c009bSjjc
28442e2c009bSjjc /*
28452e2c009bSjjc * Walk through SRAT, examining each CPU and memory entry to determine
28462e2c009bSjjc * which CPUs and memory belong to which node.
28472e2c009bSjjc */
28487bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp));
28497bc870e6SToomas Soome srat_end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp);
2850dae2fa37Sjjc proc_entry_count = 0;
28515b7cf7f0Sjjc while (item < srat_end) {
28522e2c009bSjjc uint32_t apic_id;
28532e2c009bSjjc uint32_t domain;
28542e2c009bSjjc uint64_t end;
28552e2c009bSjjc uint64_t length;
28562e2c009bSjjc uint64_t start;
28572e2c009bSjjc
28587bc870e6SToomas Soome switch (item->Type) {
28597bc870e6SToomas Soome case ACPI_SRAT_TYPE_CPU_AFFINITY: { /* CPU entry */
28607bc870e6SToomas Soome ACPI_SRAT_CPU_AFFINITY *cpu =
28617bc870e6SToomas Soome (ACPI_SRAT_CPU_AFFINITY *) item;
28627bc870e6SToomas Soome
28637bc870e6SToomas Soome if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED) ||
28642e2c009bSjjc cpu_node == NULL)
28652e2c009bSjjc break;
28662e2c009bSjjc
28672e2c009bSjjc /*
28682e2c009bSjjc * Calculate domain (node) ID and fill in APIC ID to
28692e2c009bSjjc * domain/node mapping table
28702e2c009bSjjc */
28717bc870e6SToomas Soome domain = cpu->ProximityDomainLo;
28722e2c009bSjjc for (i = 0; i < 3; i++) {
28737bc870e6SToomas Soome domain += cpu->ProximityDomainHi[i] <<
28742e2c009bSjjc ((i + 1) * 8);
28752e2c009bSjjc }
28767bc870e6SToomas Soome apic_id = cpu->ApicId;
28772e2c009bSjjc
2878a3114836SGerry Liu rc = lgrp_plat_cpu_node_update(node_domain, node_cnt,
2879a3114836SGerry Liu cpu_node, cpu_count, apic_id, domain);
2880a3114836SGerry Liu if (rc < 0)
2881d821f0f0Sjjc return (-3);
2882a3114836SGerry Liu else if (rc == 0)
2883a3114836SGerry Liu proc_entry_count++;
28842e2c009bSjjc break;
28857bc870e6SToomas Soome }
28867bc870e6SToomas Soome case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { /* memory entry */
28877bc870e6SToomas Soome ACPI_SRAT_MEM_AFFINITY *mem =
28887bc870e6SToomas Soome (ACPI_SRAT_MEM_AFFINITY *)item;
28892e2c009bSjjc
28907bc870e6SToomas Soome if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED) ||
2891a3114836SGerry Liu memnode_info == NULL)
28922e2c009bSjjc break;
28932e2c009bSjjc
28942e2c009bSjjc /*
28952e2c009bSjjc * Get domain (node) ID and fill in domain/node
28962e2c009bSjjc * to memory mapping table
28972e2c009bSjjc */
28987bc870e6SToomas Soome domain = mem->ProximityDomain;
28997bc870e6SToomas Soome start = mem->BaseAddress;
29007bc870e6SToomas Soome length = mem->Length;
29012e2c009bSjjc end = start + length - 1;
29022e2c009bSjjc
2903a3114836SGerry Liu /*
2904a3114836SGerry Liu * According to ACPI 4.0, both ENABLE and HOTPLUG flags
2905a3114836SGerry Liu * may be set for memory address range entries in SRAT
2906a3114836SGerry Liu * table which are reserved for memory hot plug.
2907a3114836SGerry Liu * We intersect memory address ranges in SRAT table
2908a3114836SGerry Liu * with memory ranges in physinstalled to filter out
2909a3114836SGerry Liu * memory address ranges reserved for hot plug.
2910a3114836SGerry Liu */
29117bc870e6SToomas Soome if (mem->Flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
2912a3114836SGerry Liu uint64_t rstart = UINT64_MAX;
2913a3114836SGerry Liu uint64_t rend = 0;
2914a3114836SGerry Liu struct memlist *ml;
2915a3114836SGerry Liu extern struct bootops *bootops;
2916a3114836SGerry Liu
2917a3114836SGerry Liu memlist_read_lock();
2918a3114836SGerry Liu for (ml = bootops->boot_mem->physinstalled;
2919a3114836SGerry Liu ml; ml = ml->ml_next) {
2920a3114836SGerry Liu uint64_t tstart = ml->ml_address;
2921a3114836SGerry Liu uint64_t tend;
2922a3114836SGerry Liu
2923a3114836SGerry Liu tend = ml->ml_address + ml->ml_size;
2924a3114836SGerry Liu if (tstart > end || tend < start)
2925a3114836SGerry Liu continue;
2926a3114836SGerry Liu if (start > tstart)
2927a3114836SGerry Liu tstart = start;
2928a3114836SGerry Liu if (rstart > tstart)
2929a3114836SGerry Liu rstart = tstart;
2930a3114836SGerry Liu if (end < tend)
2931a3114836SGerry Liu tend = end;
2932a3114836SGerry Liu if (rend < tend)
2933a3114836SGerry Liu rend = tend;
2934a3114836SGerry Liu }
2935a3114836SGerry Liu memlist_read_unlock();
2936a3114836SGerry Liu start = rstart;
2937a3114836SGerry Liu end = rend;
2938a3114836SGerry Liu /* Skip this entry if no memory installed. */
2939a3114836SGerry Liu if (start > end)
2940a3114836SGerry Liu break;
2941a3114836SGerry Liu }
2942a3114836SGerry Liu
2943a3114836SGerry Liu if (lgrp_plat_memnode_info_update(node_domain,
2944a3114836SGerry Liu node_cnt, memnode_info, node_cnt,
2945a3114836SGerry Liu start, end, domain, ACPI_MEMNODE_DEVID_BOOT) < 0)
2946d821f0f0Sjjc return (-4);
29472e2c009bSjjc break;
29487bc870e6SToomas Soome }
29497bc870e6SToomas Soome case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { /* x2apic CPU */
29507bc870e6SToomas Soome ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu =
29517bc870e6SToomas Soome (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item;
2952a3114836SGerry Liu
29537bc870e6SToomas Soome if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED) ||
2954b6917abeSmishra cpu_node == NULL)
2955b6917abeSmishra break;
2956b6917abeSmishra
2957b6917abeSmishra /*
2958b6917abeSmishra * Calculate domain (node) ID and fill in APIC ID to
2959b6917abeSmishra * domain/node mapping table
2960b6917abeSmishra */
29617bc870e6SToomas Soome domain = x2cpu->ProximityDomain;
29627bc870e6SToomas Soome apic_id = x2cpu->ApicId;
2963b6917abeSmishra
2964a3114836SGerry Liu rc = lgrp_plat_cpu_node_update(node_domain, node_cnt,
2965a3114836SGerry Liu cpu_node, cpu_count, apic_id, domain);
2966a3114836SGerry Liu if (rc < 0)
2967b6917abeSmishra return (-3);
2968a3114836SGerry Liu else if (rc == 0)
2969a3114836SGerry Liu proc_entry_count++;
2970b6917abeSmishra break;
29717bc870e6SToomas Soome }
29722e2c009bSjjc default:
29732e2c009bSjjc break;
29742e2c009bSjjc }
29752e2c009bSjjc
29767bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length);
29772e2c009bSjjc }
2978dae2fa37Sjjc
2979dae2fa37Sjjc /*
2980dae2fa37Sjjc * Should have seen at least as many SRAT processor entries as CPUs
2981dae2fa37Sjjc */
2982d821f0f0Sjjc if (proc_entry_count < cpu_count)
2983d821f0f0Sjjc return (-5);
2984dae2fa37Sjjc
298581d9ccb6SJonathan Chew /*
298681d9ccb6SJonathan Chew * Need to sort nodes by starting physical address since VM system
298781d9ccb6SJonathan Chew * assumes and expects memnodes to be sorted in ascending order by
298881d9ccb6SJonathan Chew * physical address
298981d9ccb6SJonathan Chew */
299081d9ccb6SJonathan Chew lgrp_plat_node_sort(node_domain, node_cnt, cpu_node, cpu_count,
2991a3114836SGerry Liu memnode_info);
299281d9ccb6SJonathan Chew
2993d821f0f0Sjjc return (node_cnt);
29947c478bd9Sstevel@tonic-gate }
29957c478bd9Sstevel@tonic-gate
29962e2c009bSjjc
2997d5d7cf4eSJonathan Chew /*
2998d5d7cf4eSJonathan Chew * Allocate permanent memory for any temporary memory that we needed to
2999d5d7cf4eSJonathan Chew * allocate using BOP_ALLOC() before kmem_alloc() and VM system were
3000d5d7cf4eSJonathan Chew * initialized and copy everything from temporary to permanent memory since
3001d5d7cf4eSJonathan Chew * temporary boot memory will eventually be released during boot
3002d5d7cf4eSJonathan Chew */
3003d5d7cf4eSJonathan Chew static void
lgrp_plat_release_bootstrap(void)3004d5d7cf4eSJonathan Chew lgrp_plat_release_bootstrap(void)
3005d5d7cf4eSJonathan Chew {
3006d5d7cf4eSJonathan Chew void *buf;
3007d5d7cf4eSJonathan Chew size_t size;
3008d5d7cf4eSJonathan Chew
3009d5d7cf4eSJonathan Chew if (lgrp_plat_cpu_node_nentries > 0) {
3010d5d7cf4eSJonathan Chew size = lgrp_plat_cpu_node_nentries * sizeof (cpu_node_map_t);
3011d5d7cf4eSJonathan Chew buf = kmem_alloc(size, KM_SLEEP);
3012d5d7cf4eSJonathan Chew bcopy(lgrp_plat_cpu_node, buf, size);
3013d5d7cf4eSJonathan Chew lgrp_plat_cpu_node = buf;
3014d5d7cf4eSJonathan Chew }
3015d5d7cf4eSJonathan Chew }
3016d5d7cf4eSJonathan Chew
3017d5d7cf4eSJonathan Chew
30187c478bd9Sstevel@tonic-gate /*
30192e2c009bSjjc * Return number of proximity domains given in ACPI SRAT
30207c478bd9Sstevel@tonic-gate */
30212e2c009bSjjc static int
lgrp_plat_srat_domains(ACPI_TABLE_SRAT * tp,uint32_t * prox_domain_min)30227bc870e6SToomas Soome lgrp_plat_srat_domains(ACPI_TABLE_SRAT *tp, uint32_t *prox_domain_min)
30237c478bd9Sstevel@tonic-gate {
30242e2c009bSjjc int domain_cnt;
302581d9ccb6SJonathan Chew uint32_t domain_min;
30267bc870e6SToomas Soome ACPI_SUBTABLE_HEADER *item, *end;
30272e2c009bSjjc int i;
30282e2c009bSjjc node_domain_map_t node_domain[MAX_NODES];
30292e2c009bSjjc
30302e2c009bSjjc
30312e2c009bSjjc if (tp == NULL || !lgrp_plat_srat_enable)
30322e2c009bSjjc return (1);
30332e2c009bSjjc
303481d9ccb6SJonathan Chew /*
303581d9ccb6SJonathan Chew * Walk through SRAT to find minimum proximity domain ID
303681d9ccb6SJonathan Chew */
303781d9ccb6SJonathan Chew domain_min = UINT32_MAX;
30387bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp));
30397bc870e6SToomas Soome end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp);
304081d9ccb6SJonathan Chew while (item < end) {
304181d9ccb6SJonathan Chew uint32_t domain;
304281d9ccb6SJonathan Chew
30437bc870e6SToomas Soome switch (item->Type) {
30447bc870e6SToomas Soome case ACPI_SRAT_TYPE_CPU_AFFINITY: { /* CPU entry */
30457bc870e6SToomas Soome ACPI_SRAT_CPU_AFFINITY *cpu =
30467bc870e6SToomas Soome (ACPI_SRAT_CPU_AFFINITY *) item;
30477bc870e6SToomas Soome
30487bc870e6SToomas Soome if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED)) {
30497bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)
30507bc870e6SToomas Soome ((uintptr_t)item + item->Length);
305181d9ccb6SJonathan Chew continue;
305281d9ccb6SJonathan Chew }
30537bc870e6SToomas Soome domain = cpu->ProximityDomainLo;
305481d9ccb6SJonathan Chew for (i = 0; i < 3; i++) {
30557bc870e6SToomas Soome domain += cpu->ProximityDomainHi[i] <<
305681d9ccb6SJonathan Chew ((i + 1) * 8);
305781d9ccb6SJonathan Chew }
305881d9ccb6SJonathan Chew break;
30597bc870e6SToomas Soome }
30607bc870e6SToomas Soome case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { /* memory entry */
30617bc870e6SToomas Soome ACPI_SRAT_MEM_AFFINITY *mem =
30627bc870e6SToomas Soome (ACPI_SRAT_MEM_AFFINITY *)item;
306381d9ccb6SJonathan Chew
30647bc870e6SToomas Soome if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED)) {
30657bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)
30667bc870e6SToomas Soome ((uintptr_t)item + item->Length);
306781d9ccb6SJonathan Chew continue;
306881d9ccb6SJonathan Chew }
30697bc870e6SToomas Soome domain = mem->ProximityDomain;
307081d9ccb6SJonathan Chew break;
30717bc870e6SToomas Soome }
30727bc870e6SToomas Soome case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { /* x2apic CPU */
30737bc870e6SToomas Soome ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu =
30747bc870e6SToomas Soome (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item;
307581d9ccb6SJonathan Chew
30767bc870e6SToomas Soome if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED)) {
30777bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)
30787bc870e6SToomas Soome ((uintptr_t)item + item->Length);
307981d9ccb6SJonathan Chew continue;
308081d9ccb6SJonathan Chew }
30817bc870e6SToomas Soome domain = x2cpu->ProximityDomain;
308281d9ccb6SJonathan Chew break;
30837bc870e6SToomas Soome }
308481d9ccb6SJonathan Chew default:
30857bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item +
30867bc870e6SToomas Soome item->Length);
308781d9ccb6SJonathan Chew continue;
308881d9ccb6SJonathan Chew }
308981d9ccb6SJonathan Chew
309081d9ccb6SJonathan Chew /*
309181d9ccb6SJonathan Chew * Keep track of minimum proximity domain ID
309281d9ccb6SJonathan Chew */
309381d9ccb6SJonathan Chew if (domain < domain_min)
309481d9ccb6SJonathan Chew domain_min = domain;
309581d9ccb6SJonathan Chew
30967bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length);
309781d9ccb6SJonathan Chew }
309881d9ccb6SJonathan Chew if (lgrp_plat_domain_min_enable && prox_domain_min != NULL)
309981d9ccb6SJonathan Chew *prox_domain_min = domain_min;
310081d9ccb6SJonathan Chew
31012e2c009bSjjc /*
31022e2c009bSjjc * Walk through SRAT, examining each CPU and memory entry to determine
31032e2c009bSjjc * proximity domain ID for each.
31042e2c009bSjjc */
31052e2c009bSjjc domain_cnt = 0;
31067bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp));
31077bc870e6SToomas Soome end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp);
31082e2c009bSjjc bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t));
31092e2c009bSjjc while (item < end) {
31102e2c009bSjjc uint32_t domain;
31112e2c009bSjjc boolean_t overflow;
31122e2c009bSjjc uint_t start;
31132e2c009bSjjc
31147bc870e6SToomas Soome switch (item->Type) {
31157bc870e6SToomas Soome case ACPI_SRAT_TYPE_CPU_AFFINITY: { /* CPU entry */
31167bc870e6SToomas Soome ACPI_SRAT_CPU_AFFINITY *cpu =
31177bc870e6SToomas Soome (ACPI_SRAT_CPU_AFFINITY *) item;
31187bc870e6SToomas Soome
31197bc870e6SToomas Soome if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED)) {
31207bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)
31217bc870e6SToomas Soome ((uintptr_t)item + item->Length);
312281d9ccb6SJonathan Chew continue;
312381d9ccb6SJonathan Chew }
31247bc870e6SToomas Soome domain = cpu->ProximityDomainLo;
31252e2c009bSjjc for (i = 0; i < 3; i++) {
31267bc870e6SToomas Soome domain += cpu->ProximityDomainHi[i] <<
31272e2c009bSjjc ((i + 1) * 8);
31282e2c009bSjjc }
31292e2c009bSjjc break;
31307bc870e6SToomas Soome }
31317bc870e6SToomas Soome case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { /* memory entry */
31327bc870e6SToomas Soome ACPI_SRAT_MEM_AFFINITY *mem =
31337bc870e6SToomas Soome (ACPI_SRAT_MEM_AFFINITY *)item;
31342e2c009bSjjc
31357bc870e6SToomas Soome if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED)) {
31367bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)
31377bc870e6SToomas Soome ((uintptr_t)item + item->Length);
313881d9ccb6SJonathan Chew continue;
313981d9ccb6SJonathan Chew }
31407bc870e6SToomas Soome domain = mem->ProximityDomain;
31412e2c009bSjjc break;
31427bc870e6SToomas Soome }
31437bc870e6SToomas Soome case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { /* x2apic CPU */
31447bc870e6SToomas Soome ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu =
31457bc870e6SToomas Soome (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item;
31462e2c009bSjjc
31477bc870e6SToomas Soome if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED)) {
31487bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)
31497bc870e6SToomas Soome ((uintptr_t)item + item->Length);
315081d9ccb6SJonathan Chew continue;
315181d9ccb6SJonathan Chew }
31527bc870e6SToomas Soome domain = x2cpu->ProximityDomain;
3153b6917abeSmishra break;
31547bc870e6SToomas Soome }
31552e2c009bSjjc default:
31567bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item +
31577bc870e6SToomas Soome item->Length);
315881d9ccb6SJonathan Chew continue;
31592e2c009bSjjc }
31602e2c009bSjjc
31612e2c009bSjjc /*
31622e2c009bSjjc * Count and keep track of which proximity domain IDs seen
31632e2c009bSjjc */
31642e2c009bSjjc start = i = domain % MAX_NODES;
31652e2c009bSjjc overflow = B_TRUE;
31662e2c009bSjjc do {
31672e2c009bSjjc /*
31682e2c009bSjjc * Create entry for proximity domain and increment
31692e2c009bSjjc * count when no entry exists where proximity domain
31702e2c009bSjjc * hashed
31712e2c009bSjjc */
31722e2c009bSjjc if (!node_domain[i].exists) {
31732e2c009bSjjc node_domain[i].exists = 1;
31742e2c009bSjjc node_domain[i].prox_domain = domain;
31752e2c009bSjjc domain_cnt++;
31762e2c009bSjjc overflow = B_FALSE;
31772e2c009bSjjc break;
31782e2c009bSjjc }
31792e2c009bSjjc
31802e2c009bSjjc /*
31812e2c009bSjjc * Nothing to do when proximity domain seen already
31822e2c009bSjjc * and its entry exists
31832e2c009bSjjc */
31842e2c009bSjjc if (node_domain[i].prox_domain == domain) {
31852e2c009bSjjc overflow = B_FALSE;
31862e2c009bSjjc break;
31872e2c009bSjjc }
31882e2c009bSjjc
31892e2c009bSjjc /*
31902e2c009bSjjc * Entry exists where proximity domain hashed, but for
31912e2c009bSjjc * different proximity domain so keep search for empty
31922e2c009bSjjc * slot to put it or matching entry whichever comes
31932e2c009bSjjc * first.
31942e2c009bSjjc */
31952e2c009bSjjc i = (i + 1) % MAX_NODES;
31962e2c009bSjjc } while (i != start);
31972e2c009bSjjc
31982e2c009bSjjc /*
31992e2c009bSjjc * Didn't find empty or matching entry which means have more
32002e2c009bSjjc * proximity domains than supported nodes (:-(
32012e2c009bSjjc */
32022e2c009bSjjc ASSERT(overflow != B_TRUE);
32032e2c009bSjjc if (overflow == B_TRUE)
32042e2c009bSjjc return (-1);
32052e2c009bSjjc
32067bc870e6SToomas Soome item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length);
32072e2c009bSjjc }
32082e2c009bSjjc return (domain_cnt);
32097c478bd9Sstevel@tonic-gate }
32107c478bd9Sstevel@tonic-gate
32112e2c009bSjjc
3212a3114836SGerry Liu /*
3213a3114836SGerry Liu * Parse domain information in ACPI Maximum System Capability Table (MSCT).
3214a3114836SGerry Liu * MSCT table has been verified in function process_msct() in fakebop.c.
3215a3114836SGerry Liu */
3216a3114836SGerry Liu static int
lgrp_plat_msct_domains(ACPI_TABLE_MSCT * tp,uint32_t * prox_domain_min)32177bc870e6SToomas Soome lgrp_plat_msct_domains(ACPI_TABLE_MSCT *tp, uint32_t *prox_domain_min)
3218a3114836SGerry Liu {
3219a3114836SGerry Liu int last_seen = 0;
3220a3114836SGerry Liu uint32_t proxmin = UINT32_MAX;
32217bc870e6SToomas Soome ACPI_MSCT_PROXIMITY *item, *end;
3222a3114836SGerry Liu
3223a3114836SGerry Liu if (tp == NULL || lgrp_plat_msct_enable == 0)
3224a3114836SGerry Liu return (-1);
3225a3114836SGerry Liu
32267bc870e6SToomas Soome if (tp->MaxProximityDomains >= MAX_NODES) {
3227a3114836SGerry Liu cmn_err(CE_CONT,
3228a3114836SGerry Liu "?lgrp: too many proximity domains (%d), max %d supported, "
3229a3114836SGerry Liu "disable support of CPU/memory DR operations.",
32307bc870e6SToomas Soome tp->MaxProximityDomains + 1, MAX_NODES);
3231a3114836SGerry Liu plat_dr_disable_cpu();
3232a3114836SGerry Liu plat_dr_disable_memory();
3233a3114836SGerry Liu return (-1);
3234a3114836SGerry Liu }
3235a3114836SGerry Liu
3236a3114836SGerry Liu if (prox_domain_min != NULL) {
32377bc870e6SToomas Soome end = (void *)(tp->Header.Length + (uintptr_t)tp);
3238a3114836SGerry Liu for (item = (void *)((uintptr_t)tp +
32397bc870e6SToomas Soome tp->ProximityOffset); item < end;
32407bc870e6SToomas Soome item = (void *)(item->Length + (uintptr_t)item)) {
32417bc870e6SToomas Soome if (item->RangeStart < proxmin) {
32427bc870e6SToomas Soome proxmin = item->RangeStart;
3243a3114836SGerry Liu }
3244a3114836SGerry Liu
32457bc870e6SToomas Soome last_seen = item->RangeEnd - item->RangeStart + 1;
3246a3114836SGerry Liu /*
3247a3114836SGerry Liu * Break out if all proximity domains have been
3248a3114836SGerry Liu * processed. Some BIOSes may have unused items
3249a3114836SGerry Liu * at the end of MSCT table.
3250a3114836SGerry Liu */
32517bc870e6SToomas Soome if (last_seen > tp->MaxProximityDomains) {
3252a3114836SGerry Liu break;
3253a3114836SGerry Liu }
3254a3114836SGerry Liu }
3255a3114836SGerry Liu *prox_domain_min = proxmin;
3256a3114836SGerry Liu }
3257a3114836SGerry Liu
32587bc870e6SToomas Soome return (tp->MaxProximityDomains + 1);
3259a3114836SGerry Liu }
3260a3114836SGerry Liu
3261a3114836SGerry Liu
32627c478bd9Sstevel@tonic-gate /*
32632e2c009bSjjc * Set lgroup latencies for 2 level lgroup topology
32647c478bd9Sstevel@tonic-gate */
32652e2c009bSjjc static void
lgrp_plat_2level_setup(lgrp_plat_latency_stats_t * lat_stats)3266a3114836SGerry Liu lgrp_plat_2level_setup(lgrp_plat_latency_stats_t *lat_stats)
32677c478bd9Sstevel@tonic-gate {
3268a3114836SGerry Liu int i, j;
32692e2c009bSjjc
3270a3114836SGerry Liu ASSERT(lat_stats != NULL);
32717c478bd9Sstevel@tonic-gate
32722e2c009bSjjc if (lgrp_plat_node_cnt >= 4)
32732e2c009bSjjc cmn_err(CE_NOTE,
32742e2c009bSjjc "MPO only optimizing for local and remote\n");
32752e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) {
32762e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) {
32772e2c009bSjjc if (i == j)
32782e2c009bSjjc lat_stats->latencies[i][j] = 2;
32792e2c009bSjjc else
32802e2c009bSjjc lat_stats->latencies[i][j] = 3;
32812e2c009bSjjc }
32822e2c009bSjjc }
32832e2c009bSjjc lat_stats->latency_min = 2;
32842e2c009bSjjc lat_stats->latency_max = 3;
3285a3114836SGerry Liu /* TODO: check it. */
32862e2c009bSjjc lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0);
3287a3114836SGerry Liu lgrp_plat_topo_flatten = 1;
32887c478bd9Sstevel@tonic-gate }
32897c478bd9Sstevel@tonic-gate
32902e2c009bSjjc
32917c478bd9Sstevel@tonic-gate /*
32922e2c009bSjjc * The following Opteron specific constants, macros, types, and routines define
32932e2c009bSjjc * PCI configuration space registers and how to read them to determine the NUMA
32942e2c009bSjjc * configuration of *supported* Opteron processors. They provide the same
32952e2c009bSjjc * information that may be gotten from the ACPI System Resource Affinity Table
32962e2c009bSjjc * (SRAT) if it exists on the machine of interest.
32972e2c009bSjjc *
32982e2c009bSjjc * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family
32992e2c009bSjjc * of interest describes all of these registers and their contents. The main
33002e2c009bSjjc * registers used by this code to determine the NUMA configuration of the
33012e2c009bSjjc * machine are the node ID register for the number of NUMA nodes and the DRAM
33022e2c009bSjjc * address map registers for the physical address range of each node.
33032e2c009bSjjc *
33042e2c009bSjjc * NOTE: The format and how to determine the NUMA configuration using PCI
33052e2c009bSjjc * config space registers may change or may not be supported in future
33062e2c009bSjjc * Opteron processor families.
33077c478bd9Sstevel@tonic-gate */
33087c478bd9Sstevel@tonic-gate
33092e2c009bSjjc /*
33102e2c009bSjjc * How many bits to shift Opteron DRAM Address Map base and limit registers
33112e2c009bSjjc * to get actual value
33122e2c009bSjjc */
33132e2c009bSjjc #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */
33142e2c009bSjjc #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */
33152e2c009bSjjc
33162e2c009bSjjc #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */
33172e2c009bSjjc #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */
33182e2c009bSjjc
33192e2c009bSjjc #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */
33202e2c009bSjjc
33212e2c009bSjjc /*
33222e2c009bSjjc * Macros to derive addresses from Opteron DRAM Address Map registers
33232e2c009bSjjc */
33242e2c009bSjjc #define OPT_DRAMADDR_HI(reg) \
33252e2c009bSjjc (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \
33262e2c009bSjjc OPT_DRAMADDR_HI_LSHIFT_ADDR)
33272e2c009bSjjc
33282e2c009bSjjc #define OPT_DRAMADDR_LO(reg) \
33292e2c009bSjjc (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \
33302e2c009bSjjc OPT_DRAMADDR_LO_LSHIFT_ADDR)
33312e2c009bSjjc
33322e2c009bSjjc #define OPT_DRAMADDR(high, low) \
33332e2c009bSjjc (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low))
33342e2c009bSjjc
33352e2c009bSjjc /*
33362e2c009bSjjc * Bit masks defining what's in Opteron DRAM Address Map base register
33372e2c009bSjjc */
33382e2c009bSjjc #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */
33392e2c009bSjjc #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */
33402e2c009bSjjc #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */
33412e2c009bSjjc
33422e2c009bSjjc /*
33432e2c009bSjjc * Bit masks defining what's in Opteron DRAM Address Map limit register
33442e2c009bSjjc */
33452e2c009bSjjc #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */
33462e2c009bSjjc #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */
33472e2c009bSjjc
33482e2c009bSjjc
33492e2c009bSjjc /*
33502e2c009bSjjc * Opteron Node ID register in PCI configuration space contains
33512e2c009bSjjc * number of nodes in system, etc. for Opteron K8. The following
33522e2c009bSjjc * constants and macros define its contents, structure, and access.
33532e2c009bSjjc */
33542e2c009bSjjc
33552e2c009bSjjc /*
33562e2c009bSjjc * Bit masks defining what's in Opteron Node ID register
33572e2c009bSjjc */
33582e2c009bSjjc #define OPT_NODE_MASK_ID 0x7 /* node ID */
33592e2c009bSjjc #define OPT_NODE_MASK_CNT 0x70 /* node count */
33602e2c009bSjjc #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */
33612e2c009bSjjc #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */
33622e2c009bSjjc #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */
33632e2c009bSjjc
33642e2c009bSjjc /*
33652e2c009bSjjc * How many bits in Opteron Node ID register to shift right to get actual value
33662e2c009bSjjc */
33672e2c009bSjjc #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */
33682e2c009bSjjc
33692e2c009bSjjc /*
33702e2c009bSjjc * Macros to get values from Opteron Node ID register
33712e2c009bSjjc */
33722e2c009bSjjc #define OPT_NODE_CNT(reg) \
33732e2c009bSjjc ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT)
33742e2c009bSjjc
33752e2c009bSjjc /*
33762e2c009bSjjc * Macro to setup PCI Extended Configuration Space (ECS) address to give to
33772e2c009bSjjc * "in/out" instructions
33782e2c009bSjjc *
33792e2c009bSjjc * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any
33802e2c009bSjjc * other uses should just do MMIO to access PCI ECS.
33812e2c009bSjjc * Must enable special bit in Northbridge Configuration Register on
33822e2c009bSjjc * Greyhound for extended CF8 space access to be able to access PCI ECS
33832e2c009bSjjc * using "in/out" instructions and restore special bit after done
33842e2c009bSjjc * accessing PCI ECS.
33852e2c009bSjjc */
33862e2c009bSjjc #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \
33872e2c009bSjjc (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \
33882e2c009bSjjc (((function) & 0x7) << 8) | ((reg) & 0xfc) | \
33892e2c009bSjjc ((((reg) >> 8) & 0xf) << 24))
33902e2c009bSjjc
33912e2c009bSjjc /*
33922e2c009bSjjc * PCI configuration space registers accessed by specifying
33932e2c009bSjjc * a bus, device, function, and offset. The following constants
33942e2c009bSjjc * define the values needed to access Opteron K8 configuration
33952e2c009bSjjc * info to determine its node topology
33962e2c009bSjjc */
33972e2c009bSjjc
33982e2c009bSjjc #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */
33992e2c009bSjjc
34002e2c009bSjjc /*
34012e2c009bSjjc * Opteron PCI configuration space register function values
34022e2c009bSjjc */
34032e2c009bSjjc #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */
34042e2c009bSjjc #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */
34052e2c009bSjjc #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */
34062e2c009bSjjc #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */
34072e2c009bSjjc
34082e2c009bSjjc /*
34092e2c009bSjjc * PCI Configuration Space register offsets
34102e2c009bSjjc */
34112e2c009bSjjc #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */
34122e2c009bSjjc #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */
34132e2c009bSjjc #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */
34142e2c009bSjjc #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */
34152e2c009bSjjc
34162e2c009bSjjc /*
34172e2c009bSjjc * Opteron PCI Configuration Space device IDs for nodes
34182e2c009bSjjc */
34192e2c009bSjjc #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */
34202e2c009bSjjc
34217c478bd9Sstevel@tonic-gate
34222e2c009bSjjc /*
34232e2c009bSjjc * Opteron DRAM address map gives base and limit for physical memory in a node
34242e2c009bSjjc */
34252e2c009bSjjc typedef struct opt_dram_addr_map {
34262e2c009bSjjc uint32_t base_hi;
34272e2c009bSjjc uint32_t base_lo;
34282e2c009bSjjc uint32_t limit_hi;
34292e2c009bSjjc uint32_t limit_lo;
34302e2c009bSjjc } opt_dram_addr_map_t;
3431c39996a7Sstevel
3432c39996a7Sstevel
34332e2c009bSjjc /*
34342e2c009bSjjc * Supported AMD processor families
34352e2c009bSjjc */
34362e2c009bSjjc #define AMD_FAMILY_HAMMER 15
34372e2c009bSjjc #define AMD_FAMILY_GREYHOUND 16
34387c478bd9Sstevel@tonic-gate
34397c478bd9Sstevel@tonic-gate /*
34402e2c009bSjjc * Whether to have is_opteron() return 1 even when processor isn't supported
34417c478bd9Sstevel@tonic-gate */
34422e2c009bSjjc uint_t is_opteron_override = 0;
34437c478bd9Sstevel@tonic-gate
34447c478bd9Sstevel@tonic-gate /*
34452e2c009bSjjc * AMD processor family for current CPU
34467c478bd9Sstevel@tonic-gate */
34472e2c009bSjjc uint_t opt_family = 0;
34487c478bd9Sstevel@tonic-gate
34497c478bd9Sstevel@tonic-gate
34507c478bd9Sstevel@tonic-gate /*
34512e2c009bSjjc * Determine whether we're running on a supported AMD Opteron since reading
34522e2c009bSjjc * node count and DRAM address map registers may have different format or
34532e2c009bSjjc * may not be supported across processor families
34547c478bd9Sstevel@tonic-gate */
34552e2c009bSjjc static int
is_opteron(void)34562e2c009bSjjc is_opteron(void)
34577c478bd9Sstevel@tonic-gate {
34587c478bd9Sstevel@tonic-gate
34592e2c009bSjjc if (x86_vendor != X86_VENDOR_AMD)
34602e2c009bSjjc return (0);
34617c478bd9Sstevel@tonic-gate
34622e2c009bSjjc opt_family = cpuid_getfamily(CPU);
34632e2c009bSjjc if (opt_family == AMD_FAMILY_HAMMER ||
34642e2c009bSjjc opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override)
34652e2c009bSjjc return (1);
34662e2c009bSjjc else
34672e2c009bSjjc return (0);
34687c478bd9Sstevel@tonic-gate }
34697c478bd9Sstevel@tonic-gate
34702e2c009bSjjc
34717c478bd9Sstevel@tonic-gate /*
34722e2c009bSjjc * Determine NUMA configuration for Opteron from registers that live in PCI
34732e2c009bSjjc * configuration space
34747c478bd9Sstevel@tonic-gate */
34752e2c009bSjjc static void
opt_get_numa_config(uint_t * node_cnt,int * mem_intrlv,memnode_phys_addr_map_t * memnode_info)34762e2c009bSjjc opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
3477a3114836SGerry Liu memnode_phys_addr_map_t *memnode_info)
34787c478bd9Sstevel@tonic-gate {
34792e2c009bSjjc uint_t bus;
34802e2c009bSjjc uint_t dev;
34812e2c009bSjjc struct opt_dram_addr_map dram_map[MAX_NODES];
34822e2c009bSjjc uint_t node;
34832e2c009bSjjc uint_t node_info[MAX_NODES];
34842e2c009bSjjc uint_t off_hi;
34852e2c009bSjjc uint_t off_lo;
3486584b574aSToomas Soome uint64_t nb_cfg_reg;
34877c478bd9Sstevel@tonic-gate
34887c478bd9Sstevel@tonic-gate /*
34892e2c009bSjjc * Read configuration registers from PCI configuration space to
34902e2c009bSjjc * determine node information, which memory is in each node, etc.
34912e2c009bSjjc *
34922e2c009bSjjc * Write to PCI configuration space address register to specify
34932e2c009bSjjc * which configuration register to read and read/write PCI
34942e2c009bSjjc * configuration space data register to get/set contents
34957c478bd9Sstevel@tonic-gate */
34962e2c009bSjjc bus = OPT_PCS_BUS_CONFIG;
34972e2c009bSjjc dev = OPT_PCS_DEV_NODE0;
34982e2c009bSjjc off_hi = OPT_PCS_OFF_DRAMBASE_HI;
34992e2c009bSjjc off_lo = OPT_PCS_OFF_DRAMBASE_LO;
35007c478bd9Sstevel@tonic-gate
35012e2c009bSjjc /*
35022e2c009bSjjc * Read node ID register for node 0 to get node count
35032e2c009bSjjc */
35042e2c009bSjjc node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT,
35052e2c009bSjjc OPT_PCS_OFF_NODEID);
35062e2c009bSjjc *node_cnt = OPT_NODE_CNT(node_info[0]) + 1;
35077c478bd9Sstevel@tonic-gate
35087c478bd9Sstevel@tonic-gate /*
35092e2c009bSjjc * If number of nodes is more than maximum supported, then set node
35102e2c009bSjjc * count to 1 and treat system as UMA instead of NUMA.
35117c478bd9Sstevel@tonic-gate */
35122e2c009bSjjc if (*node_cnt > MAX_NODES) {
35132e2c009bSjjc *node_cnt = 1;
35142e2c009bSjjc return;
35152e2c009bSjjc }
35167c478bd9Sstevel@tonic-gate
35177c478bd9Sstevel@tonic-gate /*
35182e2c009bSjjc * For Greyhound, PCI Extended Configuration Space must be enabled to
35192e2c009bSjjc * read high DRAM address map base and limit registers
35207c478bd9Sstevel@tonic-gate */
3521584b574aSToomas Soome nb_cfg_reg = 0;
35222e2c009bSjjc if (opt_family == AMD_FAMILY_GREYHOUND) {
35232e2c009bSjjc nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG);
35242e2c009bSjjc if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
35252e2c009bSjjc wrmsr(MSR_AMD_NB_CFG,
35262e2c009bSjjc nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS);
35272e2c009bSjjc }
35282e2c009bSjjc
35292e2c009bSjjc for (node = 0; node < *node_cnt; node++) {
35302e2c009bSjjc uint32_t base_hi;
35312e2c009bSjjc uint32_t base_lo;
35322e2c009bSjjc uint32_t limit_hi;
35332e2c009bSjjc uint32_t limit_lo;
35342e2c009bSjjc
35352e2c009bSjjc /*
35362e2c009bSjjc * Read node ID register (except for node 0 which we just read)
35372e2c009bSjjc */
35382e2c009bSjjc if (node > 0) {
35392e2c009bSjjc node_info[node] = pci_getl_func(bus, dev,
35402e2c009bSjjc OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID);
35412e2c009bSjjc }
35422e2c009bSjjc
35432e2c009bSjjc /*
35442e2c009bSjjc * Read DRAM base and limit registers which specify
35452e2c009bSjjc * physical memory range of each node
35462e2c009bSjjc */
35472e2c009bSjjc if (opt_family != AMD_FAMILY_GREYHOUND)
35482e2c009bSjjc base_hi = 0;
35492e2c009bSjjc else {
35502e2c009bSjjc outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
35512e2c009bSjjc OPT_PCS_FUNC_ADDRMAP, off_hi));
35522e2c009bSjjc base_hi = dram_map[node].base_hi =
35532e2c009bSjjc inl(PCI_CONFDATA);
35542e2c009bSjjc }
35552e2c009bSjjc base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev,
35562e2c009bSjjc OPT_PCS_FUNC_ADDRMAP, off_lo);
35572e2c009bSjjc
35582e2c009bSjjc if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) &&
35592e2c009bSjjc mem_intrlv)
35602e2c009bSjjc *mem_intrlv = *mem_intrlv + 1;
35617c478bd9Sstevel@tonic-gate
35622e2c009bSjjc off_hi += 4; /* high limit register offset */
35632e2c009bSjjc if (opt_family != AMD_FAMILY_GREYHOUND)
35642e2c009bSjjc limit_hi = 0;
35652e2c009bSjjc else {
35662e2c009bSjjc outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
35672e2c009bSjjc OPT_PCS_FUNC_ADDRMAP, off_hi));
35682e2c009bSjjc limit_hi = dram_map[node].limit_hi =
35692e2c009bSjjc inl(PCI_CONFDATA);
35702e2c009bSjjc }
35712e2c009bSjjc
35722e2c009bSjjc off_lo += 4; /* low limit register offset */
35732e2c009bSjjc limit_lo = dram_map[node].limit_lo = pci_getl_func(bus,
35742e2c009bSjjc dev, OPT_PCS_FUNC_ADDRMAP, off_lo);
35752e2c009bSjjc
35762e2c009bSjjc /*
35772e2c009bSjjc * Increment device number to next node and register offsets
35782e2c009bSjjc * for DRAM base register of next node
35792e2c009bSjjc */
35802e2c009bSjjc off_hi += 4;
35812e2c009bSjjc off_lo += 4;
35822e2c009bSjjc dev++;
35832e2c009bSjjc
35842e2c009bSjjc /*
35852e2c009bSjjc * Both read and write enable bits must be enabled in DRAM
35862e2c009bSjjc * address map base register for physical memory to exist in
35872e2c009bSjjc * node
35882e2c009bSjjc */
35892e2c009bSjjc if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 ||
35902e2c009bSjjc (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) {
35912e2c009bSjjc /*
35922e2c009bSjjc * Mark node memory as non-existent and set start and
3593a3114836SGerry Liu * end addresses to be same in memnode_info[]
35942e2c009bSjjc */
3595a3114836SGerry Liu memnode_info[node].exists = 0;
3596a3114836SGerry Liu memnode_info[node].start = memnode_info[node].end =
35972e2c009bSjjc (pfn_t)-1;
35982e2c009bSjjc continue;
35992e2c009bSjjc }
36002e2c009bSjjc
36012e2c009bSjjc /*
36022e2c009bSjjc * Mark node memory as existing and remember physical address
36032e2c009bSjjc * range of each node for use later
36042e2c009bSjjc */
3605a3114836SGerry Liu memnode_info[node].exists = 1;
36062e2c009bSjjc
3607a3114836SGerry Liu memnode_info[node].start = btop(OPT_DRAMADDR(base_hi, base_lo));
36082e2c009bSjjc
3609a3114836SGerry Liu memnode_info[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) |
36102e2c009bSjjc OPT_DRAMADDR_LO_MASK_OFF);
36112e2c009bSjjc }
36122e2c009bSjjc
36132e2c009bSjjc /*
36142e2c009bSjjc * Restore PCI Extended Configuration Space enable bit
36152e2c009bSjjc */
36162e2c009bSjjc if (opt_family == AMD_FAMILY_GREYHOUND) {
36172e2c009bSjjc if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
36182e2c009bSjjc wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg);
36192e2c009bSjjc }
36207c478bd9Sstevel@tonic-gate }
36217c478bd9Sstevel@tonic-gate
36222e2c009bSjjc
36237c478bd9Sstevel@tonic-gate /*
36242e2c009bSjjc * Return average amount of time to read vendor ID register on Northbridge
36252e2c009bSjjc * N times on specified destination node from current CPU
36267c478bd9Sstevel@tonic-gate */
36272e2c009bSjjc static hrtime_t
opt_probe_vendor(int dest_node,int nreads)36282e2c009bSjjc opt_probe_vendor(int dest_node, int nreads)
36297c478bd9Sstevel@tonic-gate {
36302e2c009bSjjc int cnt;
36312e2c009bSjjc uint_t dev;
36322e2c009bSjjc /* LINTED: set but not used in function */
36338f33f62fSToomas Soome volatile uint_t dev_vendor __unused;
36342e2c009bSjjc hrtime_t elapsed;
36352e2c009bSjjc hrtime_t end;
36362e2c009bSjjc int ipl;
36372e2c009bSjjc hrtime_t start;
36382e2c009bSjjc
36392e2c009bSjjc dev = OPT_PCS_DEV_NODE0 + dest_node;
36402e2c009bSjjc kpreempt_disable();
36412e2c009bSjjc ipl = spl8();
36422e2c009bSjjc outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM,
36432e2c009bSjjc OPT_PCS_OFF_VENDOR));
36442e2c009bSjjc start = gethrtime();
36452e2c009bSjjc for (cnt = 0; cnt < nreads; cnt++)
36462e2c009bSjjc dev_vendor = inl(PCI_CONFDATA);
36472e2c009bSjjc end = gethrtime();
36482e2c009bSjjc elapsed = (end - start) / nreads;
36492e2c009bSjjc splx(ipl);
36502e2c009bSjjc kpreempt_enable();
36512e2c009bSjjc return (elapsed);
36527c478bd9Sstevel@tonic-gate }
3653