xref: /illumos-gate/usr/src/uts/intel/os/cpuid.c (revision 5cd084ed)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ee88d2b9Skchow  * Common Development and Distribution License (the "License").
6ee88d2b9Skchow  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
226e5580c9SFrank Van Der Linden  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23c2710388SDan Kimmel  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
2479ec9da8SYuri Pankov  * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
256eedf6a5SJosef 'Jeff' Sipek  * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
26e25cb0e7SJohn Levon  * Copyright 2020 Joyent, Inc.
27a65c38a3SRobert Mustacchi  * Copyright 2023 Oxide Computer Company
28*5cd084edSDan McDonald  * Copyright 2024 MNX Cloud, Inc.
297c478bd9Sstevel@tonic-gate  */
30cef70d2cSBill Holler /*
3141afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * Copyright (c) 2010, Intel Corporation.
32cef70d2cSBill Holler  * All rights reserved.
33cef70d2cSBill Holler  */
348031591dSSrihari Venkatesan /*
358031591dSSrihari Venkatesan  * Portions Copyright 2009 Advanced Micro Devices, Inc.
368031591dSSrihari Venkatesan  */
370ce813ffSRobert Mustacchi 
387c478bd9Sstevel@tonic-gate /*
390ce813ffSRobert Mustacchi  * CPU Identification logic
400ce813ffSRobert Mustacchi  *
410ce813ffSRobert Mustacchi  * The purpose of this file and its companion, cpuid_subr.c, is to help deal
420ce813ffSRobert Mustacchi  * with the identification of CPUs, their features, and their topologies. More
430ce813ffSRobert Mustacchi  * specifically, this file helps drive the following:
440ce813ffSRobert Mustacchi  *
450ce813ffSRobert Mustacchi  * 1. Enumeration of features of the processor which are used by the kernel to
460ce813ffSRobert Mustacchi  *    determine what features to enable or disable. These may be instruction set
470ce813ffSRobert Mustacchi  *    enhancements or features that we use.
480ce813ffSRobert Mustacchi  *
490ce813ffSRobert Mustacchi  * 2. Enumeration of instruction set architecture (ISA) additions that userland
500ce813ffSRobert Mustacchi  *    will be told about through the auxiliary vector.
510ce813ffSRobert Mustacchi  *
520ce813ffSRobert Mustacchi  * 3. Understanding the physical topology of the CPU such as the number of
530ce813ffSRobert Mustacchi  *    caches, how many cores it has, whether or not it supports symmetric
540ce813ffSRobert Mustacchi  *    multi-processing (SMT), etc.
550ce813ffSRobert Mustacchi  *
560ce813ffSRobert Mustacchi  * ------------------------
570ce813ffSRobert Mustacchi  * CPUID History and Basics
580ce813ffSRobert Mustacchi  * ------------------------
590ce813ffSRobert Mustacchi  *
600ce813ffSRobert Mustacchi  * The cpuid instruction was added by Intel roughly around the time that the
610ce813ffSRobert Mustacchi  * original Pentium was introduced. The purpose of cpuid was to tell in a
620ce813ffSRobert Mustacchi  * programmatic fashion information about the CPU that previously was guessed
630ce813ffSRobert Mustacchi  * at. For example, an important part of cpuid is that we can know what
640ce813ffSRobert Mustacchi  * extensions to the ISA exist. If you use an invalid opcode you would get a
650ce813ffSRobert Mustacchi  * #UD, so this method allows a program (whether a user program or the kernel)
660ce813ffSRobert Mustacchi  * to determine what exists without crashing or getting a SIGILL. Of course,
670ce813ffSRobert Mustacchi  * this was also during the era of the clones and the AMD Am5x86. The vendor
680ce813ffSRobert Mustacchi  * name shows up first in cpuid for a reason.
690ce813ffSRobert Mustacchi  *
700ce813ffSRobert Mustacchi  * cpuid information is broken down into ranges called a 'leaf'. Each leaf puts
710ce813ffSRobert Mustacchi  * unique values into the registers %eax, %ebx, %ecx, and %edx and each leaf has
720ce813ffSRobert Mustacchi  * its own meaning. The different leaves are broken down into different regions:
730ce813ffSRobert Mustacchi  *
740ce813ffSRobert Mustacchi  *	[ 0, 7fffffff ]			This region is called the 'basic'
750ce813ffSRobert Mustacchi  *					region. This region is generally defined
760ce813ffSRobert Mustacchi  *					by Intel, though some of the original
770ce813ffSRobert Mustacchi  *					portions have different meanings based
780ce813ffSRobert Mustacchi  *					on the manufacturer. These days, Intel
790ce813ffSRobert Mustacchi  *					adds most new features to this region.
800ce813ffSRobert Mustacchi  *					AMD adds non-Intel compatible
810ce813ffSRobert Mustacchi  *					information in the third, extended
820ce813ffSRobert Mustacchi  *					region. Intel uses this for everything
830ce813ffSRobert Mustacchi  *					including ISA extensions, CPU
840ce813ffSRobert Mustacchi  *					features, cache information, topology,
850ce813ffSRobert Mustacchi  *					and more.
860ce813ffSRobert Mustacchi  *
870ce813ffSRobert Mustacchi  *					There is a hole carved out of this
880ce813ffSRobert Mustacchi  *					region which is reserved for
890ce813ffSRobert Mustacchi  *					hypervisors.
900ce813ffSRobert Mustacchi  *
910ce813ffSRobert Mustacchi  *	[ 40000000, 4fffffff ]		This region, which is found in the
920ce813ffSRobert Mustacchi  *					middle of the previous region, is
930ce813ffSRobert Mustacchi  *					explicitly promised to never be used by
940ce813ffSRobert Mustacchi  *					CPUs. Instead, it is used by hypervisors
950ce813ffSRobert Mustacchi  *					to communicate information about
960ce813ffSRobert Mustacchi  *					themselves to the operating system. The
970ce813ffSRobert Mustacchi  *					values and details are unique for each
980ce813ffSRobert Mustacchi  *					hypervisor.
990ce813ffSRobert Mustacchi  *
1000ce813ffSRobert Mustacchi  *	[ 80000000, ffffffff ]		This region is called the 'extended'
1010ce813ffSRobert Mustacchi  *					region. Some of the low leaves mirror
1020ce813ffSRobert Mustacchi  *					parts of the basic leaves. This region
1030ce813ffSRobert Mustacchi  *					has generally been used by AMD for
1040ce813ffSRobert Mustacchi  *					various extensions. For example, AMD-
1050ce813ffSRobert Mustacchi  *					specific information about caches,
1060ce813ffSRobert Mustacchi  *					features, and topology are found in this
1070ce813ffSRobert Mustacchi  *					region.
1080ce813ffSRobert Mustacchi  *
1090ce813ffSRobert Mustacchi  * To specify a range, you place the desired leaf into %eax, zero %ebx, %ecx,
1100ce813ffSRobert Mustacchi  * and %edx, and then issue the cpuid instruction. At the first leaf in each of
1110ce813ffSRobert Mustacchi  * the ranges, one of the primary things returned is the maximum valid leaf in
1120ce813ffSRobert Mustacchi  * that range. This allows for discovery of what range of CPUID is valid.
1130ce813ffSRobert Mustacchi  *
1140ce813ffSRobert Mustacchi  * The CPUs have potentially surprising behavior when using an invalid leaf or
1150ce813ffSRobert Mustacchi  * unimplemented leaf. If the requested leaf is within the valid basic or
1160ce813ffSRobert Mustacchi  * extended range, but is unimplemented, then %eax, %ebx, %ecx, and %edx will be
1170ce813ffSRobert Mustacchi  * set to zero. However, if you specify a leaf that is outside of a valid range,
1180ce813ffSRobert Mustacchi  * then instead it will be filled with the last valid _basic_ leaf. For example,
1190ce813ffSRobert Mustacchi  * if the maximum basic value is on leaf 0x3, then issuing a cpuid for leaf 4 or
1200ce813ffSRobert Mustacchi  * an invalid extended leaf will return the information for leaf 3.
1210ce813ffSRobert Mustacchi  *
1220ce813ffSRobert Mustacchi  * Some leaves are broken down into sub-leaves. This means that the value
1230ce813ffSRobert Mustacchi  * depends on both the leaf asked for in %eax and a secondary register. For
1240ce813ffSRobert Mustacchi  * example, Intel uses the value in %ecx on leaf 7 to indicate a sub-leaf to get
1250ce813ffSRobert Mustacchi  * additional information. Or when getting topology information in leaf 0xb, the
1260ce813ffSRobert Mustacchi  * initial value in %ecx changes which level of the topology that you are
1270ce813ffSRobert Mustacchi  * getting information about.
1280ce813ffSRobert Mustacchi  *
1290ce813ffSRobert Mustacchi  * cpuid values are always kept to 32 bits regardless of whether or not the
1300ce813ffSRobert Mustacchi  * program is in 64-bit mode. When executing in 64-bit mode, the upper
1310ce813ffSRobert Mustacchi  * 32 bits of the register are always set to zero so that way the values are the
1320ce813ffSRobert Mustacchi  * same regardless of execution mode.
1330ce813ffSRobert Mustacchi  *
1340ce813ffSRobert Mustacchi  * ----------------------
1350ce813ffSRobert Mustacchi  * Identifying Processors
1360ce813ffSRobert Mustacchi  * ----------------------
1370ce813ffSRobert Mustacchi  *
1380ce813ffSRobert Mustacchi  * We can identify a processor in two steps. The first step looks at cpuid leaf
1390ce813ffSRobert Mustacchi  * 0. Leaf 0 contains the processor's vendor information. This is done by
1400ce813ffSRobert Mustacchi  * putting a 12 character string in %ebx, %ecx, and %edx. On AMD, it is
1410ce813ffSRobert Mustacchi  * 'AuthenticAMD' and on Intel it is 'GenuineIntel'.
1420ce813ffSRobert Mustacchi  *
1430ce813ffSRobert Mustacchi  * From there, a processor is identified by a combination of three different
1440ce813ffSRobert Mustacchi  * values:
1450ce813ffSRobert Mustacchi  *
1460ce813ffSRobert Mustacchi  *  1. Family
1470ce813ffSRobert Mustacchi  *  2. Model
1480ce813ffSRobert Mustacchi  *  3. Stepping
1490ce813ffSRobert Mustacchi  *
1500ce813ffSRobert Mustacchi  * Each vendor uses the family and model to uniquely identify a processor. The
1510ce813ffSRobert Mustacchi  * way that family and model are changed depends on the vendor. For example,
1520ce813ffSRobert Mustacchi  * Intel has been using family 0x6 for almost all of their processor since the
1530ce813ffSRobert Mustacchi  * Pentium Pro/Pentium II era, often called the P6. The model is used to
1540ce813ffSRobert Mustacchi  * identify the exact processor. Different models are often used for the client
1550ce813ffSRobert Mustacchi  * (consumer) and server parts. Even though each processor often has major
1560ce813ffSRobert Mustacchi  * architectural differences, they still are considered the same family by
1570ce813ffSRobert Mustacchi  * Intel.
1580ce813ffSRobert Mustacchi  *
1590ce813ffSRobert Mustacchi  * On the other hand, each major AMD architecture generally has its own family.
1600ce813ffSRobert Mustacchi  * For example, the K8 is family 0x10, Bulldozer 0x15, and Zen 0x17. Within it
16122e4c3acSKeith M Wesolowski  * the model number is used to help identify specific processors.  As AMD's
16222e4c3acSKeith M Wesolowski  * product lines have expanded, they have started putting a mixed bag of
16322e4c3acSKeith M Wesolowski  * processors into the same family, with each processor under a single
16422e4c3acSKeith M Wesolowski  * identifying banner (e.g., Milan, Cezanne) using a range of model numbers.  We
16522e4c3acSKeith M Wesolowski  * refer to each such collection as a processor family, distinct from cpuid
16622e4c3acSKeith M Wesolowski  * family.  Importantly, each processor family has a BIOS and Kernel Developer's
16722e4c3acSKeith M Wesolowski  * Guide (BKDG, older parts) or Processor Programming Reference (PPR) that
16822e4c3acSKeith M Wesolowski  * defines the processor family's non-architectural features.  In general, we'll
16922e4c3acSKeith M Wesolowski  * use "family" here to mean the family number reported by the cpuid instruction
17022e4c3acSKeith M Wesolowski  * and distinguish the processor family from it where appropriate.
1710ce813ffSRobert Mustacchi  *
1720ce813ffSRobert Mustacchi  * The stepping is used to refer to a revision of a specific microprocessor. The
1730ce813ffSRobert Mustacchi  * term comes from equipment used to produce masks that are used to create
1740ce813ffSRobert Mustacchi  * integrated circuits.
1750ce813ffSRobert Mustacchi  *
1760ce813ffSRobert Mustacchi  * The information is present in leaf 1, %eax. In technical documentation you
1770ce813ffSRobert Mustacchi  * will see the terms extended model and extended family. The original family,
1780ce813ffSRobert Mustacchi  * model, and stepping fields were each 4 bits wide. If the values in either
1790ce813ffSRobert Mustacchi  * are 0xf, then one is to consult the extended model and extended family, which
1800ce813ffSRobert Mustacchi  * take previously reserved bits and allow for a larger number of models and add
1810ce813ffSRobert Mustacchi  * 0xf to them.
1820ce813ffSRobert Mustacchi  *
1830ce813ffSRobert Mustacchi  * When we process this information, we store the full family, model, and
1840ce813ffSRobert Mustacchi  * stepping in the struct cpuid_info members cpi_family, cpi_model, and
1850ce813ffSRobert Mustacchi  * cpi_step, respectively. Whenever you are performing comparisons with the
1860ce813ffSRobert Mustacchi  * family, model, and stepping, you should use these members and not the raw
1870ce813ffSRobert Mustacchi  * values from cpuid. If you must use the raw values from cpuid directly, you
1880ce813ffSRobert Mustacchi  * must make sure that you add the extended model and family to the base model
1890ce813ffSRobert Mustacchi  * and family.
1900ce813ffSRobert Mustacchi  *
1910ce813ffSRobert Mustacchi  * In general, we do not use information about the family, model, and stepping
1920ce813ffSRobert Mustacchi  * to determine whether or not a feature is present; that is generally driven by
1930ce813ffSRobert Mustacchi  * specific leaves. However, when something we care about on the processor is
1940ce813ffSRobert Mustacchi  * not considered 'architectural' meaning that it is specific to a set of
1950ce813ffSRobert Mustacchi  * processors and not promised in the architecture model to be consistent from
1960ce813ffSRobert Mustacchi  * generation to generation, then we will fall back on this information. The
1970ce813ffSRobert Mustacchi  * most common cases where this comes up is when we have to workaround errata in
1980ce813ffSRobert Mustacchi  * the processor, are dealing with processor-specific features such as CPU
1990ce813ffSRobert Mustacchi  * performance counters, or we want to provide additional information for things
2000ce813ffSRobert Mustacchi  * such as fault management.
2010ce813ffSRobert Mustacchi  *
2020ce813ffSRobert Mustacchi  * While processors also do have a brand string, which is the name that people
2030ce813ffSRobert Mustacchi  * are familiar with when buying the processor, they are not meant for
2040ce813ffSRobert Mustacchi  * programmatic consumption. That is what the family, model, and stepping are
2050ce813ffSRobert Mustacchi  * for.
2060ce813ffSRobert Mustacchi  *
20722e4c3acSKeith M Wesolowski  * We use the x86_chiprev_t to encode a combination of vendor, processor family,
20822e4c3acSKeith M Wesolowski  * and stepping(s) that refer to a single or very closely related set of silicon
20922e4c3acSKeith M Wesolowski  * implementations; while there are sometimes more specific ways to learn of the
21022e4c3acSKeith M Wesolowski  * presence or absence of a particular erratum or workaround, one may generally
21122e4c3acSKeith M Wesolowski  * assume that all processors of the same chiprev have the same errata and we
21222e4c3acSKeith M Wesolowski  * have chosen to represent them this way precisely because that is how AMD
21322e4c3acSKeith M Wesolowski  * groups them in their revision guides (errata documentation).  The processor
21422e4c3acSKeith M Wesolowski  * family (x86_processor_family_t) may be extracted from the chiprev if that
21522e4c3acSKeith M Wesolowski  * level of detail is not needed.  Processor families are considered unordered
21622e4c3acSKeith M Wesolowski  * but revisions within a family may be compared for either an exact match or at
21722e4c3acSKeith M Wesolowski  * least as recent as a reference revision.  See the chiprev_xxx() functions
21822e4c3acSKeith M Wesolowski  * below.
21922e4c3acSKeith M Wesolowski  *
22022e4c3acSKeith M Wesolowski  * Similarly, each processor family implements a particular microarchitecture,
22122e4c3acSKeith M Wesolowski  * which itself may have multiple revisions.  In general, non-architectural
22222e4c3acSKeith M Wesolowski  * features are specific to a processor family, but some may exist across
22322e4c3acSKeith M Wesolowski  * families containing cores that implement the same microarchitectural revision
22422e4c3acSKeith M Wesolowski  * (and, such cores share common bugs, too).  We provide utility routines
22522e4c3acSKeith M Wesolowski  * analogous to those for extracting and comparing chiprevs for
22622e4c3acSKeith M Wesolowski  * microarchitectures as well; see the uarch_xxx() functions.
22722e4c3acSKeith M Wesolowski  *
22822e4c3acSKeith M Wesolowski  * Both chiprevs and uarchrevs are defined in x86_archext.h and both are at
22922e4c3acSKeith M Wesolowski  * present used and available only for AMD and AMD-like processors.
23022e4c3acSKeith M Wesolowski  *
2310ce813ffSRobert Mustacchi  * ------------
2320ce813ffSRobert Mustacchi  * CPUID Passes
2330ce813ffSRobert Mustacchi  * ------------
2340ce813ffSRobert Mustacchi  *
2350ce813ffSRobert Mustacchi  * As part of performing feature detection, we break this into several different
236ab5bb018SKeith M Wesolowski  * passes. There used to be a pass 0 that was done from assembly in locore.s to
237ab5bb018SKeith M Wesolowski  * support processors that have a missing or broken cpuid instruction (notably
238ab5bb018SKeith M Wesolowski  * certain Cyrix processors) but those were all 32-bit processors which are no
239ab5bb018SKeith M Wesolowski  * longer supported. Passes are no longer numbered explicitly to make it easier
240ab5bb018SKeith M Wesolowski  * to break them up or move them around as needed; however, they still have a
241ab5bb018SKeith M Wesolowski  * well-defined execution ordering enforced by the definition of cpuid_pass_t in
242ab5bb018SKeith M Wesolowski  * x86_archext.h. The external interface to execute a cpuid pass or determine
243ab5bb018SKeith M Wesolowski  * whether a pass has been completed consists of cpuid_execpass() and
244ab5bb018SKeith M Wesolowski  * cpuid_checkpass() respectively.  The passes now, in that execution order,
245ab5bb018SKeith M Wesolowski  * are as follows:
246ab5bb018SKeith M Wesolowski  *
247ab5bb018SKeith M Wesolowski  *	PRELUDE		This pass does not have any dependencies on system
248ab5bb018SKeith M Wesolowski  *			setup; in particular, unlike all subsequent passes it is
249ab5bb018SKeith M Wesolowski  *			guaranteed not to require PCI config space access.  It
250ab5bb018SKeith M Wesolowski  *			sets the flag indicating that the processor we are
251ab5bb018SKeith M Wesolowski  *			running on supports the cpuid instruction, which all
252ab5bb018SKeith M Wesolowski  *			64-bit processors do.  This would also be the place to
253ab5bb018SKeith M Wesolowski  *			add any other basic state that is required later on and
254ab5bb018SKeith M Wesolowski  *			can be learned without dependencies.
255ab5bb018SKeith M Wesolowski  *
256ab5bb018SKeith M Wesolowski  *	IDENT		Determine which vendor manufactured the CPU, the family,
257ab5bb018SKeith M Wesolowski  *			model, and stepping information, and compute basic
258ab5bb018SKeith M Wesolowski  *			identifying tags from those values.  This is done first
259ab5bb018SKeith M Wesolowski  *			so that machine-dependent code can control the features
260ab5bb018SKeith M Wesolowski  *			the cpuid instruction will report during subsequent
261ab5bb018SKeith M Wesolowski  *			passes if needed, and so that any intervening
262ab5bb018SKeith M Wesolowski  *			machine-dependent code that needs basic identity will
26322e4c3acSKeith M Wesolowski  *			have it available.  This includes synthesised
26422e4c3acSKeith M Wesolowski  *			identifiers such as chiprev and uarchrev as well as the
26522e4c3acSKeith M Wesolowski  *			values obtained directly from cpuid.  Prior to executing
26622e4c3acSKeith M Wesolowski  *			this pass, machine-depedent boot code is responsible for
26722e4c3acSKeith M Wesolowski  *			ensuring that the PCI configuration space access
26822e4c3acSKeith M Wesolowski  *			functions have been set up and, if necessary, that
26922e4c3acSKeith M Wesolowski  *			determine_platform() has been called.
270ab5bb018SKeith M Wesolowski  *
271ab5bb018SKeith M Wesolowski  *	BASIC		This is the primary pass and is responsible for doing a
2720ce813ffSRobert Mustacchi  *			large number of different things:
2730ce813ffSRobert Mustacchi  *
274ab5bb018SKeith M Wesolowski  *			1. Gathering a large number of feature flags to
2750ce813ffSRobert Mustacchi  *			determine which features the CPU support and which
2760ce813ffSRobert Mustacchi  *			indicate things that we need to do other work in the OS
2770ce813ffSRobert Mustacchi  *			to enable. Features detected this way are added to the
2780ce813ffSRobert Mustacchi  *			x86_featureset which can be queried to
2790ce813ffSRobert Mustacchi  *			determine what we should do. This includes processing
2800ce813ffSRobert Mustacchi  *			all of the basic and extended CPU features that we care
2810ce813ffSRobert Mustacchi  *			about.
2820ce813ffSRobert Mustacchi  *
283ab5bb018SKeith M Wesolowski  *			2. Determining the CPU's topology. This includes
2840ce813ffSRobert Mustacchi  *			information about how many cores and threads are present
2850ce813ffSRobert Mustacchi  *			in the package. It also is responsible for figuring out
2860ce813ffSRobert Mustacchi  *			which logical CPUs are potentially part of the same core
2870ce813ffSRobert Mustacchi  *			and what other resources they might share. For more
2880ce813ffSRobert Mustacchi  *			information see the 'Topology' section.
2890ce813ffSRobert Mustacchi  *
290ab5bb018SKeith M Wesolowski  *			3. Determining the set of CPU security-specific features
2910ce813ffSRobert Mustacchi  *			that we need to worry about and determine the
2920ce813ffSRobert Mustacchi  *			appropriate set of workarounds.
2930ce813ffSRobert Mustacchi  *
2940ce813ffSRobert Mustacchi  *			Pass 1 on the boot CPU occurs before KMDB is started.
2950ce813ffSRobert Mustacchi  *
296ab5bb018SKeith M Wesolowski  *	EXTENDED	The second pass is done after startup(). Here, we check
2970ce813ffSRobert Mustacchi  *			other miscellaneous features. Most of this is gathering
2980ce813ffSRobert Mustacchi  *			additional basic and extended features that we'll use in
2990ce813ffSRobert Mustacchi  *			later passes or for debugging support.
3000ce813ffSRobert Mustacchi  *
301ab5bb018SKeith M Wesolowski  *	DYNAMIC		The third pass occurs after the kernel memory allocator
3020ce813ffSRobert Mustacchi  *			has been fully initialized. This gathers information
3030ce813ffSRobert Mustacchi  *			where we might need dynamic memory available for our
3040ce813ffSRobert Mustacchi  *			uses. This includes several varying width leaves that
3050ce813ffSRobert Mustacchi  *			have cache information and the processor's brand string.
3060ce813ffSRobert Mustacchi  *
307ab5bb018SKeith M Wesolowski  *	RESOLVE		The fourth and final normal pass is performed after the
3080ce813ffSRobert Mustacchi  *			kernel has brought most everything online. This is
3090ce813ffSRobert Mustacchi  *			invoked from post_startup(). In this pass, we go through
3100ce813ffSRobert Mustacchi  *			the set of features that we have enabled and turn that
3110ce813ffSRobert Mustacchi  *			into the hardware auxiliary vector features that
3120ce813ffSRobert Mustacchi  *			userland receives. This is used by userland, primarily
3130ce813ffSRobert Mustacchi  *			by the run-time link-editor (RTLD), though userland
3140ce813ffSRobert Mustacchi  *			software could also refer to it directly.
3150ce813ffSRobert Mustacchi  *
316ab5bb018SKeith M Wesolowski  * The function that performs a pass is currently assumed to be infallible, and
317ab5bb018SKeith M Wesolowski  * all existing implementation are.  This simplifies callers by allowing
318ab5bb018SKeith M Wesolowski  * cpuid_execpass() to return void. Similarly, implementers do not need to check
319ab5bb018SKeith M Wesolowski  * for a NULL CPU argument; the current CPU's cpu_t is substituted if necessary.
320ab5bb018SKeith M Wesolowski  * Both of these assumptions can be relaxed if needed by future developments.
321ab5bb018SKeith M Wesolowski  * Tracking of completed states is handled by cpuid_execpass(). It is programmer
322ab5bb018SKeith M Wesolowski  * error to attempt to execute a pass before all previous passes have been
323ab5bb018SKeith M Wesolowski  * completed on the specified CPU, or to request cpuid information before the
324ab5bb018SKeith M Wesolowski  * pass that captures it has been executed.  These conditions can be tested
325ab5bb018SKeith M Wesolowski  * using cpuid_checkpass().
326ab5bb018SKeith M Wesolowski  *
327ab5bb018SKeith M Wesolowski  * The Microcode Pass
328ab5bb018SKeith M Wesolowski  *
329ab5bb018SKeith M Wesolowski  * After a microcode update, we do a selective rescan of the cpuid leaves to
330ab5bb018SKeith M Wesolowski  * determine what features have changed. Microcode updates can provide more
331ab5bb018SKeith M Wesolowski  * details about security related features to deal with issues like Spectre and
332ab5bb018SKeith M Wesolowski  * L1TF. On occasion, vendors have violated their contract and removed bits.
333ab5bb018SKeith M Wesolowski  * However, we don't try to detect that because that puts us in a situation that
334ab5bb018SKeith M Wesolowski  * we really can't deal with. As such, the only thing we rescan are security
335ab5bb018SKeith M Wesolowski  * related features today. See cpuid_pass_ucode().  This pass may be run in a
336ab5bb018SKeith M Wesolowski  * different sequence on APs and therefore is not part of the sequential order;
337ab5bb018SKeith M Wesolowski  * It is invoked directly instead of by cpuid_execpass() and its completion
338ab5bb018SKeith M Wesolowski  * status cannot be checked by cpuid_checkpass().  This could be integrated with
339ab5bb018SKeith M Wesolowski  * a more complex dependency mechanism if warranted by future developments.
340ab5bb018SKeith M Wesolowski  *
341ab5bb018SKeith M Wesolowski  * All of the passes are run on all CPUs. However, for the most part we only
342ab5bb018SKeith M Wesolowski  * care about what the boot CPU says about this information and use the other
343ab5bb018SKeith M Wesolowski  * CPUs as a rough guide to sanity check that we have the same feature set.
3440ce813ffSRobert Mustacchi  *
3450ce813ffSRobert Mustacchi  * We do not support running multiple logical CPUs with disjoint, let alone
3460ce813ffSRobert Mustacchi  * different, feature sets.
3470ce813ffSRobert Mustacchi  *
3480ce813ffSRobert Mustacchi  * ------------------
3490ce813ffSRobert Mustacchi  * Processor Topology
3500ce813ffSRobert Mustacchi  * ------------------
3510ce813ffSRobert Mustacchi  *
3520ce813ffSRobert Mustacchi  * One of the important things that we need to do is to understand the topology
3530ce813ffSRobert Mustacchi  * of the underlying processor. When we say topology in this case, we're trying
3540ce813ffSRobert Mustacchi  * to understand the relationship between the logical CPUs that the operating
3550ce813ffSRobert Mustacchi  * system sees and the underlying physical layout. Different logical CPUs may
3560ce813ffSRobert Mustacchi  * share different resources which can have important consequences for the
3570ce813ffSRobert Mustacchi  * performance of the system. For example, they may share caches, execution
3580ce813ffSRobert Mustacchi  * units, and more.
3590ce813ffSRobert Mustacchi  *
3600ce813ffSRobert Mustacchi  * The topology of the processor changes from generation to generation and
3610ce813ffSRobert Mustacchi  * vendor to vendor.  Along with that, different vendors use different
3620ce813ffSRobert Mustacchi  * terminology, and the operating system itself uses occasionally overlapping
3630ce813ffSRobert Mustacchi  * terminology. It's important to understand what this topology looks like so
3640ce813ffSRobert Mustacchi  * one can understand the different things that we try to calculate and
3650ce813ffSRobert Mustacchi  * determine.
3660ce813ffSRobert Mustacchi  *
3670ce813ffSRobert Mustacchi  * To get started, let's talk about a little bit of terminology that we've used
3680ce813ffSRobert Mustacchi  * so far, is used throughout this file, and is fairly generic across multiple
3690ce813ffSRobert Mustacchi  * vendors:
3700ce813ffSRobert Mustacchi  *
3710ce813ffSRobert Mustacchi  * CPU
3720ce813ffSRobert Mustacchi  *	A central processing unit (CPU) refers to a logical and/or virtual
3730ce813ffSRobert Mustacchi  *	entity that the operating system can execute instructions on. The
3740ce813ffSRobert Mustacchi  *	underlying resources for this CPU may be shared between multiple
3750ce813ffSRobert Mustacchi  *	entities; however, to the operating system it is a discrete unit.
3760ce813ffSRobert Mustacchi  *
3770ce813ffSRobert Mustacchi  * PROCESSOR and PACKAGE
3780ce813ffSRobert Mustacchi  *
3790ce813ffSRobert Mustacchi  *	Generally, when we use the term 'processor' on its own, we are referring
3800ce813ffSRobert Mustacchi  *	to the physical entity that one buys and plugs into a board. However,
3810ce813ffSRobert Mustacchi  *	because processor has been overloaded and one might see it used to mean
3820ce813ffSRobert Mustacchi  *	multiple different levels, we will instead use the term 'package' for
3830ce813ffSRobert Mustacchi  *	the rest of this file. The term package comes from the electrical
3840ce813ffSRobert Mustacchi  *	engineering side and refers to the physical entity that encloses the
3850ce813ffSRobert Mustacchi  *	electronics inside. Strictly speaking the package can contain more than
3860ce813ffSRobert Mustacchi  *	just the CPU, for example, on many processors it may also have what's
3870ce813ffSRobert Mustacchi  *	called an 'integrated graphical processing unit (GPU)'. Because the
3880ce813ffSRobert Mustacchi  *	package can encapsulate multiple units, it is the largest physical unit
3890ce813ffSRobert Mustacchi  *	that we refer to.
3900ce813ffSRobert Mustacchi  *
3910ce813ffSRobert Mustacchi  * SOCKET
3920ce813ffSRobert Mustacchi  *
3930ce813ffSRobert Mustacchi  *	A socket refers to unit on a system board (generally the motherboard)
3940ce813ffSRobert Mustacchi  *	that can receive a package. A single package, or processor, is plugged
3950ce813ffSRobert Mustacchi  *	into a single socket. A system may have multiple sockets. Often times,
3960ce813ffSRobert Mustacchi  *	the term socket is used interchangeably with package and refers to the
3970ce813ffSRobert Mustacchi  *	electrical component that has plugged in, and not the receptacle itself.
3980ce813ffSRobert Mustacchi  *
3990ce813ffSRobert Mustacchi  * CORE
4000ce813ffSRobert Mustacchi  *
4010ce813ffSRobert Mustacchi  *	A core refers to the physical instantiation of a CPU, generally, with a
4020ce813ffSRobert Mustacchi  *	full set of hardware resources available to it. A package may contain
4030ce813ffSRobert Mustacchi  *	multiple cores inside of it or it may just have a single one. A
4040ce813ffSRobert Mustacchi  *	processor with more than one core is often referred to as 'multi-core'.
4050ce813ffSRobert Mustacchi  *	In illumos, we will use the feature X86FSET_CMP to refer to a system
4060ce813ffSRobert Mustacchi  *	that has 'multi-core' processors.
4070ce813ffSRobert Mustacchi  *
4080ce813ffSRobert Mustacchi  *	A core may expose a single logical CPU to the operating system, or it
4090ce813ffSRobert Mustacchi  *	may expose multiple CPUs, which we call threads, defined below.
4100ce813ffSRobert Mustacchi  *
4110ce813ffSRobert Mustacchi  *	Some resources may still be shared by cores in the same package. For
4120ce813ffSRobert Mustacchi  *	example, many processors will share the level 3 cache between cores.
4130ce813ffSRobert Mustacchi  *	Some AMD generations share hardware resources between cores. For more
4140ce813ffSRobert Mustacchi  *	information on that see the section 'AMD Topology'.
4150ce813ffSRobert Mustacchi  *
4160ce813ffSRobert Mustacchi  * THREAD and STRAND
4170ce813ffSRobert Mustacchi  *
4180ce813ffSRobert Mustacchi  *	In this file, generally a thread refers to a hardware resources and not
4190ce813ffSRobert Mustacchi  *	the operating system's logical abstraction. A thread is always exposed
4200ce813ffSRobert Mustacchi  *	as an independent logical CPU to the operating system. A thread belongs
4210ce813ffSRobert Mustacchi  *	to a specific core. A core may have more than one thread. When that is
4220ce813ffSRobert Mustacchi  *	the case, the threads that are part of the same core are often referred
4230ce813ffSRobert Mustacchi  *	to as 'siblings'.
4240ce813ffSRobert Mustacchi  *
4250ce813ffSRobert Mustacchi  *	When multiple threads exist, this is generally referred to as
4260ce813ffSRobert Mustacchi  *	simultaneous multi-threading (SMT). When Intel introduced this in their
4270ce813ffSRobert Mustacchi  *	processors they called it hyper-threading (HT). When multiple threads
4280ce813ffSRobert Mustacchi  *	are active in a core, they split the resources of the core. For example,
4290ce813ffSRobert Mustacchi  *	two threads may share the same set of hardware execution units.
4300ce813ffSRobert Mustacchi  *
4310ce813ffSRobert Mustacchi  *	The operating system often uses the term 'strand' to refer to a thread.
4320ce813ffSRobert Mustacchi  *	This helps disambiguate it from the software concept.
4330ce813ffSRobert Mustacchi  *
4340ce813ffSRobert Mustacchi  * CHIP
4350ce813ffSRobert Mustacchi  *
4360ce813ffSRobert Mustacchi  *	Unfortunately, the term 'chip' is dramatically overloaded. At its most
4370ce813ffSRobert Mustacchi  *	base meaning, it is used to refer to a single integrated circuit, which
4380ce813ffSRobert Mustacchi  *	may or may not be the only thing in the package. In illumos, when you
4390ce813ffSRobert Mustacchi  *	see the term 'chip' it is almost always referring to the same thing as
4400ce813ffSRobert Mustacchi  *	the 'package'. However, many vendors may use chip to refer to one of
4410ce813ffSRobert Mustacchi  *	many integrated circuits that have been placed in the package. As an
4420ce813ffSRobert Mustacchi  *	example, see the subsequent definition.
4430ce813ffSRobert Mustacchi  *
4440ce813ffSRobert Mustacchi  *	To try and keep things consistent, we will only use chip when referring
4450ce813ffSRobert Mustacchi  *	to the entire integrated circuit package, with the exception of the
4460ce813ffSRobert Mustacchi  *	definition of multi-chip module (because it is in the name) and use the
4470ce813ffSRobert Mustacchi  *	term 'die' when we want the more general, potential sub-component
4480ce813ffSRobert Mustacchi  *	definition.
4490ce813ffSRobert Mustacchi  *
4500ce813ffSRobert Mustacchi  * DIE
4510ce813ffSRobert Mustacchi  *
4520ce813ffSRobert Mustacchi  *	A die refers to an integrated circuit. Inside of the package there may
4530ce813ffSRobert Mustacchi  *	be a single die or multiple dies. This is sometimes called a 'chip' in
4540ce813ffSRobert Mustacchi  *	vendor's parlance, but in this file, we use the term die to refer to a
4550ce813ffSRobert Mustacchi  *	subcomponent.
4560ce813ffSRobert Mustacchi  *
4570ce813ffSRobert Mustacchi  * MULTI-CHIP MODULE
4580ce813ffSRobert Mustacchi  *
4590ce813ffSRobert Mustacchi  *	A multi-chip module (MCM) refers to putting multiple distinct chips that
4600ce813ffSRobert Mustacchi  *	are connected together in the same package. When a multi-chip design is
4610ce813ffSRobert Mustacchi  *	used, generally each chip is manufactured independently and then joined
4620ce813ffSRobert Mustacchi  *	together in the package. For example, on AMD's Zen microarchitecture
4630ce813ffSRobert Mustacchi  *	(family 0x17), the package contains several dies (the second meaning of
4640ce813ffSRobert Mustacchi  *	chip from above) that are connected together.
4650ce813ffSRobert Mustacchi  *
4660ce813ffSRobert Mustacchi  * CACHE
4670ce813ffSRobert Mustacchi  *
4680ce813ffSRobert Mustacchi  *	A cache is a part of the processor that maintains copies of recently
4690ce813ffSRobert Mustacchi  *	accessed memory. Caches are split into levels and then into types.
4700ce813ffSRobert Mustacchi  *	Commonly there are one to three levels, called level one, two, and
4710ce813ffSRobert Mustacchi  *	three. The lower the level, the smaller it is, the closer it is to the
4720ce813ffSRobert Mustacchi  *	execution units of the CPU, and the faster it is to access. The layout
4730ce813ffSRobert Mustacchi  *	and design of the cache come in many different flavors, consult other
4740ce813ffSRobert Mustacchi  *	resources for a discussion of those.
4750ce813ffSRobert Mustacchi  *
4760ce813ffSRobert Mustacchi  *	Caches are generally split into two types, the instruction and data
4770ce813ffSRobert Mustacchi  *	cache. The caches contain what their names suggest, the instruction
4780ce813ffSRobert Mustacchi  *	cache has executable program text, while the data cache has all other
4790ce813ffSRobert Mustacchi  *	memory that the processor accesses. As of this writing, data is kept
4800ce813ffSRobert Mustacchi  *	coherent between all of the caches on x86, so if one modifies program
4810ce813ffSRobert Mustacchi  *	text before it is executed, that will be in the data cache, and the
4820ce813ffSRobert Mustacchi  *	instruction cache will be synchronized with that change when the
4830ce813ffSRobert Mustacchi  *	processor actually executes those instructions. This coherency also
4840ce813ffSRobert Mustacchi  *	covers the fact that data could show up in multiple caches.
4850ce813ffSRobert Mustacchi  *
4860ce813ffSRobert Mustacchi  *	Generally, the lowest level caches are specific to a core. However, the
4870ce813ffSRobert Mustacchi  *	last layer cache is shared between some number of cores. The number of
4880ce813ffSRobert Mustacchi  *	CPUs sharing this last level cache is important. This has implications
4890ce813ffSRobert Mustacchi  *	for the choices that the scheduler makes, as accessing memory that might
4900ce813ffSRobert Mustacchi  *	be in a remote cache after thread migration can be quite expensive.
4910ce813ffSRobert Mustacchi  *
4920ce813ffSRobert Mustacchi  *	Sometimes, the word cache is abbreviated with a '$', because in US
4930ce813ffSRobert Mustacchi  *	English the word cache is pronounced the same as cash. So L1D$ refers to
4940ce813ffSRobert Mustacchi  *	the L1 data cache, and L2$ would be the L2 cache. This will not be used
4950ce813ffSRobert Mustacchi  *	in the rest of this theory statement for clarity.
4960ce813ffSRobert Mustacchi  *
4970ce813ffSRobert Mustacchi  * MEMORY CONTROLLER
4980ce813ffSRobert Mustacchi  *
4990ce813ffSRobert Mustacchi  *	The memory controller is a component that provides access to DRAM. Each
5000ce813ffSRobert Mustacchi  *	memory controller can access a set number of DRAM channels. Each channel
5010ce813ffSRobert Mustacchi  *	can have a number of DIMMs (sticks of memory) associated with it. A
5020ce813ffSRobert Mustacchi  *	given package may have more than one memory controller. The association
5030ce813ffSRobert Mustacchi  *	of the memory controller to a group of cores is important as it is
5040ce813ffSRobert Mustacchi  *	cheaper to access memory on the controller that you are associated with.
5050ce813ffSRobert Mustacchi  *
5060ce813ffSRobert Mustacchi  * NUMA
5070ce813ffSRobert Mustacchi  *
5080ce813ffSRobert Mustacchi  *	NUMA or non-uniform memory access, describes a way that systems are
5090ce813ffSRobert Mustacchi  *	built. On x86, any processor core can address all of the memory in the
5100ce813ffSRobert Mustacchi  *	system. However, When using multiple sockets or possibly within a
5110ce813ffSRobert Mustacchi  *	multi-chip module, some of that memory is physically closer and some of
5120ce813ffSRobert Mustacchi  *	it is further. Memory that is further away is more expensive to access.
5130ce813ffSRobert Mustacchi  *	Consider the following image of multiple sockets with memory:
5140ce813ffSRobert Mustacchi  *
5150ce813ffSRobert Mustacchi  *	+--------+                                                +--------+
5160ce813ffSRobert Mustacchi  *	| DIMM A |         +----------+      +----------+         | DIMM D |
5170ce813ffSRobert Mustacchi  *	+--------+-+       |          |      |          |       +-+------+-+
5180ce813ffSRobert Mustacchi  *	  | DIMM B |=======| Socket 0 |======| Socket 1 |=======| DIMM E |
5190ce813ffSRobert Mustacchi  *	  +--------+-+     |          |      |          |     +-+------+-+
5200ce813ffSRobert Mustacchi  *	    | DIMM C |     +----------+      +----------+     | DIMM F |
5210ce813ffSRobert Mustacchi  *	    +--------+                                        +--------+
5220ce813ffSRobert Mustacchi  *
5230ce813ffSRobert Mustacchi  *	In this example, Socket 0 is closer to DIMMs A-C while Socket 1 is
5240ce813ffSRobert Mustacchi  *	closer to DIMMs D-F. This means that it is cheaper for socket 0 to
5250ce813ffSRobert Mustacchi  *	access DIMMs A-C and more expensive to access D-F as it has to go
5260ce813ffSRobert Mustacchi  *	through Socket 1 to get there. The inverse is true for Socket 1. DIMMs
5270ce813ffSRobert Mustacchi  *	D-F are cheaper than A-C. While the socket form is the most common, when
5280ce813ffSRobert Mustacchi  *	using multi-chip modules, this can also sometimes occur. For another
5290ce813ffSRobert Mustacchi  *	example of this that's more involved, see the AMD topology section.
5300ce813ffSRobert Mustacchi  *
5310ce813ffSRobert Mustacchi  *
5320ce813ffSRobert Mustacchi  * Intel Topology
5330ce813ffSRobert Mustacchi  * --------------
5340ce813ffSRobert Mustacchi  *
5350ce813ffSRobert Mustacchi  * Most Intel processors since Nehalem, (as of this writing the current gen
5360ce813ffSRobert Mustacchi  * is Skylake / Cannon Lake) follow a fairly similar pattern. The CPU portion of
5370ce813ffSRobert Mustacchi  * the package is a single monolithic die. MCMs currently aren't used. Most
5380ce813ffSRobert Mustacchi  * parts have three levels of caches, with the L3 cache being shared between
5390ce813ffSRobert Mustacchi  * all of the cores on the package. The L1/L2 cache is generally specific to
5400ce813ffSRobert Mustacchi  * an individual core. The following image shows at a simplified level what
5410ce813ffSRobert Mustacchi  * this looks like. The memory controller is commonly part of something called
5420ce813ffSRobert Mustacchi  * the 'Uncore', that used to be separate physical chips that were not a part of
5430ce813ffSRobert Mustacchi  * the package, but are now part of the same chip.
5440ce813ffSRobert Mustacchi  *
5450ce813ffSRobert Mustacchi  *  +-----------------------------------------------------------------------+
5460ce813ffSRobert Mustacchi  *  | Package                                                               |
5470ce813ffSRobert Mustacchi  *  |  +-------------------+  +-------------------+  +-------------------+  |
5480ce813ffSRobert Mustacchi  *  |  | Core              |  | Core              |  | Core              |  |
5490ce813ffSRobert Mustacchi  *  |  |  +--------+ +---+ |  |  +--------+ +---+ |  |  +--------+ +---+ |  |
5500ce813ffSRobert Mustacchi  *  |  |  | Thread | | L | |  |  | Thread | | L | |  |  | Thread | | L | |  |
5510ce813ffSRobert Mustacchi  *  |  |  +--------+ | 1 | |  |  +--------+ | 1 | |  |  +--------+ | 1 | |  |
5520ce813ffSRobert Mustacchi  *  |  |  +--------+ |   | |  |  +--------+ |   | |  |  +--------+ |   | |  |
5530ce813ffSRobert Mustacchi  *  |  |  | Thread | |   | |  |  | Thread | |   | |  |  | Thread | |   | |  |
5540ce813ffSRobert Mustacchi  *  |  |  +--------+ +---+ |  |  +--------+ +---+ |  |  +--------+ +---+ |  |
5550ce813ffSRobert Mustacchi  *  |  |  +--------------+ |  |  +--------------+ |  |  +--------------+ |  |
5560ce813ffSRobert Mustacchi  *  |  |  | L2 Cache     | |  |  | L2 Cache     | |  |  | L2 Cache     | |  |
5570ce813ffSRobert Mustacchi  *  |  |  +--------------+ |  |  +--------------+ |  |  +--------------+ |  |
5580ce813ffSRobert Mustacchi  *  |  +-------------------+  +-------------------+  +-------------------+  |
5590ce813ffSRobert Mustacchi  *  | +-------------------------------------------------------------------+ |
5600ce813ffSRobert Mustacchi  *  | |                         Shared L3 Cache                           | |
5610ce813ffSRobert Mustacchi  *  | +-------------------------------------------------------------------+ |
5620ce813ffSRobert Mustacchi  *  | +-------------------------------------------------------------------+ |
5630ce813ffSRobert Mustacchi  *  | |                        Memory Controller                          | |
5640ce813ffSRobert Mustacchi  *  | +-------------------------------------------------------------------+ |
5650ce813ffSRobert Mustacchi  *  +-----------------------------------------------------------------------+
5660ce813ffSRobert Mustacchi  *
5670ce813ffSRobert Mustacchi  * A side effect of this current architecture is that what we care about from a
5680ce813ffSRobert Mustacchi  * scheduling and topology perspective, is simplified. In general we care about
5690ce813ffSRobert Mustacchi  * understanding which logical CPUs are part of the same core and socket.
5700ce813ffSRobert Mustacchi  *
5710ce813ffSRobert Mustacchi  * To determine the relationship between threads and cores, Intel initially used
5720ce813ffSRobert Mustacchi  * the identifier in the advanced programmable interrupt controller (APIC). They
5730ce813ffSRobert Mustacchi  * also added cpuid leaf 4 to give additional information about the number of
5740ce813ffSRobert Mustacchi  * threads and CPUs in the processor. With the addition of x2apic (which
5750ce813ffSRobert Mustacchi  * increased the number of addressable logical CPUs from 8-bits to 32-bits), an
5760ce813ffSRobert Mustacchi  * additional cpuid topology leaf 0xB was added.
5770ce813ffSRobert Mustacchi  *
5780ce813ffSRobert Mustacchi  * AMD Topology
5790ce813ffSRobert Mustacchi  * ------------
5800ce813ffSRobert Mustacchi  *
5810ce813ffSRobert Mustacchi  * When discussing AMD topology, we want to break this into three distinct
5820ce813ffSRobert Mustacchi  * generations of topology. There's the basic topology that has been used in
5830ce813ffSRobert Mustacchi  * family 0xf+ (Opteron, Athlon64), there's the topology that was introduced
5840ce813ffSRobert Mustacchi  * with family 0x15 (Bulldozer), and there's the topology that was introduced
5850c21e245SRobert Mustacchi  * with family 0x17 (Zen), evolved more dramatically in Zen 2 (still family
5860c21e245SRobert Mustacchi  * 0x17), and tweaked slightly in Zen 3 (family 19h). AMD also has some
5870c21e245SRobert Mustacchi  * additional terminology that's worth talking about.
5880ce813ffSRobert Mustacchi  *
5890ce813ffSRobert Mustacchi  * Until the introduction of family 0x17 (Zen), AMD did not implement something
5900ce813ffSRobert Mustacchi  * that they considered SMT. Whether or not the AMD processors have SMT
5910ce813ffSRobert Mustacchi  * influences many things including scheduling and reliability, availability,
5920ce813ffSRobert Mustacchi  * and serviceability (RAS) features.
5930ce813ffSRobert Mustacchi  *
5940ce813ffSRobert Mustacchi  * NODE
5950ce813ffSRobert Mustacchi  *
5960ce813ffSRobert Mustacchi  *	AMD uses the term node to refer to a die that contains a number of cores
5970ce813ffSRobert Mustacchi  *	and I/O resources. Depending on the processor family and model, more
5980ce813ffSRobert Mustacchi  *	than one node can be present in the package. When there is more than one
5990ce813ffSRobert Mustacchi  *	node this indicates a multi-chip module. Usually each node has its own
6000ce813ffSRobert Mustacchi  *	access to memory and I/O devices. This is important and generally
6010ce813ffSRobert Mustacchi  *	different from the corresponding Intel Nehalem-Skylake+ processors. As a
6020ce813ffSRobert Mustacchi  *	result, we track this relationship in the operating system.
6030ce813ffSRobert Mustacchi  *
6040ce813ffSRobert Mustacchi  *	In processors with an L3 cache, the L3 cache is generally shared across
6050ce813ffSRobert Mustacchi  *	the entire node, though the way this is carved up varies from generation
6060ce813ffSRobert Mustacchi  *	to generation.
6070ce813ffSRobert Mustacchi  *
6080ce813ffSRobert Mustacchi  * BULLDOZER
6090ce813ffSRobert Mustacchi  *
6100ce813ffSRobert Mustacchi  *	Starting with the Bulldozer family (0x15) and continuing until the
6110ce813ffSRobert Mustacchi  *	introduction of the Zen microarchitecture, AMD introduced the idea of a
6120ce813ffSRobert Mustacchi  *	compute unit. In a compute unit, two traditional cores share a number of
6130ce813ffSRobert Mustacchi  *	hardware resources. Critically, they share the FPU, L1 instruction
6140ce813ffSRobert Mustacchi  *	cache, and the L2 cache. Several compute units were then combined inside
6150ce813ffSRobert Mustacchi  *	of a single node.  Because the integer execution units, L1 data cache,
6160ce813ffSRobert Mustacchi  *	and some other resources were not shared between the cores, AMD never
6170ce813ffSRobert Mustacchi  *	considered this to be SMT.
6180ce813ffSRobert Mustacchi  *
6190ce813ffSRobert Mustacchi  * ZEN
6200ce813ffSRobert Mustacchi  *
6210ce813ffSRobert Mustacchi  *	The Zen family (0x17) uses a multi-chip module (MCM) design, the module
6220ce813ffSRobert Mustacchi  *	is called Zeppelin. These modules are similar to the idea of nodes used
6230ce813ffSRobert Mustacchi  *	previously. Each of these nodes has two DRAM channels which all of the
6240ce813ffSRobert Mustacchi  *	cores in the node can access uniformly. These nodes are linked together
6250ce813ffSRobert Mustacchi  *	in the package, creating a NUMA environment.
6260ce813ffSRobert Mustacchi  *
6270ce813ffSRobert Mustacchi  *	The Zeppelin die itself contains two different 'core complexes'. Each
6280ce813ffSRobert Mustacchi  *	core complex consists of four cores which each have two threads, for a
6290ce813ffSRobert Mustacchi  *	total of 8 logical CPUs per complex. Unlike other generations,
6300ce813ffSRobert Mustacchi  *	where all the logical CPUs in a given node share the L3 cache, here each
6310ce813ffSRobert Mustacchi  *	core complex has its own shared L3 cache.
6320ce813ffSRobert Mustacchi  *
6330ce813ffSRobert Mustacchi  *	A further thing that we need to consider is that in some configurations,
6340ce813ffSRobert Mustacchi  *	particularly with the Threadripper line of processors, not every die
6350ce813ffSRobert Mustacchi  *	actually has its memory controllers wired up to actual memory channels.
6360ce813ffSRobert Mustacchi  *	This means that some cores have memory attached to them and others
6370ce813ffSRobert Mustacchi  *	don't.
6380ce813ffSRobert Mustacchi  *
6390ce813ffSRobert Mustacchi  *	To put Zen in perspective, consider the following images:
6400ce813ffSRobert Mustacchi  *
6410ce813ffSRobert Mustacchi  *      +--------------------------------------------------------+
6420ce813ffSRobert Mustacchi  *      | Core Complex                                           |
6430ce813ffSRobert Mustacchi  *      | +-------------------+    +-------------------+  +---+  |
6440ce813ffSRobert Mustacchi  *      | | Core       +----+ |    | Core       +----+ |  |   |  |
6450ce813ffSRobert Mustacchi  *      | | +--------+ | L2 | |    | +--------+ | L2 | |  |   |  |
6460ce813ffSRobert Mustacchi  *      | | | Thread | +----+ |    | | Thread | +----+ |  |   |  |
6470ce813ffSRobert Mustacchi  *      | | +--------+-+ +--+ |    | +--------+-+ +--+ |  | L |  |
6480ce813ffSRobert Mustacchi  *      | |   | Thread | |L1| |    |   | Thread | |L1| |  | 3 |  |
6490ce813ffSRobert Mustacchi  *      | |   +--------+ +--+ |    |   +--------+ +--+ |  |   |  |
6500ce813ffSRobert Mustacchi  *      | +-------------------+    +-------------------+  | C |  |
6510ce813ffSRobert Mustacchi  *      | +-------------------+    +-------------------+  | a |  |
6520ce813ffSRobert Mustacchi  *      | | Core       +----+ |    | Core       +----+ |  | c |  |
6530ce813ffSRobert Mustacchi  *      | | +--------+ | L2 | |    | +--------+ | L2 | |  | h |  |
6540ce813ffSRobert Mustacchi  *      | | | Thread | +----+ |    | | Thread | +----+ |  | e |  |
6550ce813ffSRobert Mustacchi  *      | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |   |  |
6560ce813ffSRobert Mustacchi  *      | |   | Thread | |L1| |    |   | Thread | |L1| |  |   |  |
6570ce813ffSRobert Mustacchi  *      | |   +--------+ +--+ |    |   +--------+ +--+ |  |   |  |
6580ce813ffSRobert Mustacchi  *      | +-------------------+    +-------------------+  +---+  |
6590ce813ffSRobert Mustacchi  *      |                                                        |
6600ce813ffSRobert Mustacchi  *	+--------------------------------------------------------+
6610ce813ffSRobert Mustacchi  *
6620ce813ffSRobert Mustacchi  *  This first image represents a single Zen core complex that consists of four
6630ce813ffSRobert Mustacchi  *  cores.
6640ce813ffSRobert Mustacchi  *
6650ce813ffSRobert Mustacchi  *
6660ce813ffSRobert Mustacchi  *	+--------------------------------------------------------+
6670ce813ffSRobert Mustacchi  *	| Zeppelin Die                                           |
6680ce813ffSRobert Mustacchi  *	|  +--------------------------------------------------+  |
6690ce813ffSRobert Mustacchi  *	|  |         I/O Units (PCIe, SATA, USB, etc.)        |  |
6700ce813ffSRobert Mustacchi  *	|  +--------------------------------------------------+  |
6710ce813ffSRobert Mustacchi  *      |                           HH                           |
6720ce813ffSRobert Mustacchi  *	|          +-----------+    HH    +-----------+          |
6730ce813ffSRobert Mustacchi  *	|          |           |    HH    |           |          |
6740ce813ffSRobert Mustacchi  *	|          |    Core   |==========|    Core   |          |
6750ce813ffSRobert Mustacchi  *	|          |  Complex  |==========|  Complex  |          |
6760ce813ffSRobert Mustacchi  *	|          |           |    HH    |           |          |
6770ce813ffSRobert Mustacchi  *	|          +-----------+    HH    +-----------+          |
6780ce813ffSRobert Mustacchi  *      |                           HH                           |
6790ce813ffSRobert Mustacchi  *	|  +--------------------------------------------------+  |
6800ce813ffSRobert Mustacchi  *	|  |                Memory Controller                 |  |
6810ce813ffSRobert Mustacchi  *	|  +--------------------------------------------------+  |
6820ce813ffSRobert Mustacchi  *      |                                                        |
6830ce813ffSRobert Mustacchi  *	+--------------------------------------------------------+
6840ce813ffSRobert Mustacchi  *
6850ce813ffSRobert Mustacchi  *  This image represents a single Zeppelin Die. Note how both cores are
6860ce813ffSRobert Mustacchi  *  connected to the same memory controller and I/O units. While each core
6870ce813ffSRobert Mustacchi  *  complex has its own L3 cache as seen in the first image, they both have
6880ce813ffSRobert Mustacchi  *  uniform access to memory.
6890ce813ffSRobert Mustacchi  *
6900ce813ffSRobert Mustacchi  *
6910ce813ffSRobert Mustacchi  *                      PP                     PP
6920ce813ffSRobert Mustacchi  *                      PP                     PP
6930ce813ffSRobert Mustacchi  *           +----------PP---------------------PP---------+
6940ce813ffSRobert Mustacchi  *           |          PP                     PP         |
6950ce813ffSRobert Mustacchi  *           |    +-----------+          +-----------+    |
6960ce813ffSRobert Mustacchi  *           |    |           |          |           |    |
6970ce813ffSRobert Mustacchi  *       MMMMMMMMM|  Zeppelin |==========|  Zeppelin |MMMMMMMMM
6980ce813ffSRobert Mustacchi  *       MMMMMMMMM|    Die    |==========|    Die    |MMMMMMMMM
6990ce813ffSRobert Mustacchi  *           |    |           |          |           |    |
7000ce813ffSRobert Mustacchi  *           |    +-----------+ooo    ...+-----------+    |
7010ce813ffSRobert Mustacchi  *           |          HH      ooo  ...       HH         |
7020ce813ffSRobert Mustacchi  *           |          HH        oo..         HH         |
7030ce813ffSRobert Mustacchi  *           |          HH        ..oo         HH         |
7040ce813ffSRobert Mustacchi  *           |          HH      ...  ooo       HH         |
7050ce813ffSRobert Mustacchi  *           |    +-----------+...    ooo+-----------+    |
7060ce813ffSRobert Mustacchi  *           |    |           |          |           |    |
7070ce813ffSRobert Mustacchi  *       MMMMMMMMM|  Zeppelin |==========|  Zeppelin |MMMMMMMMM
7080ce813ffSRobert Mustacchi  *       MMMMMMMMM|    Die    |==========|    Die    |MMMMMMMMM
7090ce813ffSRobert Mustacchi  *           |    |           |          |           |    |
7100ce813ffSRobert Mustacchi  *           |    +-----------+          +-----------+    |
7110ce813ffSRobert Mustacchi  *           |          PP                     PP         |
7120ce813ffSRobert Mustacchi  *           +----------PP---------------------PP---------+
7130ce813ffSRobert Mustacchi  *                      PP                     PP
7140ce813ffSRobert Mustacchi  *                      PP                     PP
7150ce813ffSRobert Mustacchi  *
7160ce813ffSRobert Mustacchi  *  This image represents a single Zen package. In this example, it has four
7170ce813ffSRobert Mustacchi  *  Zeppelin dies, though some configurations only have a single one. In this
7180ce813ffSRobert Mustacchi  *  example, each die is directly connected to the next. Also, each die is
7190ce813ffSRobert Mustacchi  *  represented as being connected to memory by the 'M' character and connected
7200ce813ffSRobert Mustacchi  *  to PCIe devices and other I/O, by the 'P' character. Because each Zeppelin
7210ce813ffSRobert Mustacchi  *  die is made up of two core complexes, we have multiple different NUMA
7220ce813ffSRobert Mustacchi  *  domains that we care about for these systems.
7230ce813ffSRobert Mustacchi  *
7240c21e245SRobert Mustacchi  * ZEN 2
7250c21e245SRobert Mustacchi  *
7260c21e245SRobert Mustacchi  *	Zen 2 changes things in a dramatic way from Zen 1. Whereas in Zen 1
7270c21e245SRobert Mustacchi  *	each Zeppelin Die had its own I/O die, that has been moved out of the
7280c21e245SRobert Mustacchi  *	core complex in Zen 2. The actual core complex looks pretty similar, but
7290c21e245SRobert Mustacchi  *	now the die actually looks much simpler:
7300c21e245SRobert Mustacchi  *
7310c21e245SRobert Mustacchi  *      +--------------------------------------------------------+
7320c21e245SRobert Mustacchi  *      | Zen 2 Core Complex Die    HH                           |
7330c21e245SRobert Mustacchi  *      |                           HH                           |
7340c21e245SRobert Mustacchi  *      |          +-----------+    HH    +-----------+          |
7350c21e245SRobert Mustacchi  *      |          |           |    HH    |           |          |
7360c21e245SRobert Mustacchi  *      |          |    Core   |==========|    Core   |          |
7370c21e245SRobert Mustacchi  *      |          |  Complex  |==========|  Complex  |          |
7380c21e245SRobert Mustacchi  *      |          |           |    HH    |           |          |
7390c21e245SRobert Mustacchi  *      |          +-----------+    HH    +-----------+          |
7400c21e245SRobert Mustacchi  *      |                           HH                           |
7410c21e245SRobert Mustacchi  *      |                           HH                           |
7420c21e245SRobert Mustacchi  *      +--------------------------------------------------------+
7430c21e245SRobert Mustacchi  *
7440c21e245SRobert Mustacchi  *	From here, when we add the central I/O die, this changes things a bit.
7450c21e245SRobert Mustacchi  *	Each die is connected to the I/O die, rather than trying to interconnect
7460c21e245SRobert Mustacchi  *	them directly. The following image takes the same Zen 1 image that we
7470c21e245SRobert Mustacchi  *	had earlier and shows what it looks like with the I/O die instead:
7480c21e245SRobert Mustacchi  *
7490c21e245SRobert Mustacchi  *                                 PP    PP
7500c21e245SRobert Mustacchi  *                                 PP    PP
7510c21e245SRobert Mustacchi  *           +---------------------PP----PP---------------------+
7520c21e245SRobert Mustacchi  *           |                     PP    PP                     |
7530c21e245SRobert Mustacchi  *           |  +-----------+      PP    PP      +-----------+  |
7540c21e245SRobert Mustacchi  *           |  |           |      PP    PP      |           |  |
7550c21e245SRobert Mustacchi  *           |  |   Zen 2   |    +-PP----PP-+    |   Zen 2   |  |
7560c21e245SRobert Mustacchi  *           |  |    Die   _|    | PP    PP |    |_   Die    |  |
7570c21e245SRobert Mustacchi  *           |  |         |o|oooo|          |oooo|o|         |  |
7580c21e245SRobert Mustacchi  *           |  +-----------+    |          |    +-----------+  |
7590c21e245SRobert Mustacchi  *           |                   |   I/O    |                   |
7600c21e245SRobert Mustacchi  *       MMMMMMMMMMMMMMMMMMMMMMMMMM  Die   MMMMMMMMMMMMMMMMMMMMMMMMMM
7610c21e245SRobert Mustacchi  *       MMMMMMMMMMMMMMMMMMMMMMMMMM        MMMMMMMMMMMMMMMMMMMMMMMMMM
7620c21e245SRobert Mustacchi  *           |                   |          |                   |
7630c21e245SRobert Mustacchi  *       MMMMMMMMMMMMMMMMMMMMMMMMMM        MMMMMMMMMMMMMMMMMMMMMMMMMM
7640c21e245SRobert Mustacchi  *       MMMMMMMMMMMMMMMMMMMMMMMMMM        MMMMMMMMMMMMMMMMMMMMMMMMMM
7650c21e245SRobert Mustacchi  *           |                   |          |                   |
7660c21e245SRobert Mustacchi  *           |  +-----------+    |          |    +-----------+  |
7670c21e245SRobert Mustacchi  *           |  |         |o|oooo| PP    PP |oooo|o|         |  |
7680c21e245SRobert Mustacchi  *           |  |   Zen 2  -|    +-PP----PP-+    |-  Zen 2   |  |
7690c21e245SRobert Mustacchi  *           |  |    Die    |      PP    PP      |    Die    |  |
7700c21e245SRobert Mustacchi  *           |  |           |      PP    PP      |           |  |
7710c21e245SRobert Mustacchi  *           |  +-----------+      PP    PP      +-----------+  |
7720c21e245SRobert Mustacchi  *           |                     PP    PP                     |
7730c21e245SRobert Mustacchi  *           +---------------------PP----PP---------------------+
7740c21e245SRobert Mustacchi  *                                 PP    PP
7750c21e245SRobert Mustacchi  *                                 PP    PP
7760c21e245SRobert Mustacchi  *
7770c21e245SRobert Mustacchi  *	The above has four core complex dies installed, though the Zen 2 EPYC
7780c21e245SRobert Mustacchi  *	and ThreadRipper parts allow for up to eight, while the Ryzen parts
7790c21e245SRobert Mustacchi  *	generally only have one to two. The more notable difference here is how
7800c21e245SRobert Mustacchi  *	everything communicates. Note that memory and PCIe come out of the
7810c21e245SRobert Mustacchi  *	central die. This changes the way that one die accesses a resource. It
7820c21e245SRobert Mustacchi  *	basically always has to go to the I/O die, where as in Zen 1 it may have
7830c21e245SRobert Mustacchi  *	satisfied it locally. In general, this ends up being a better strategy
7840c21e245SRobert Mustacchi  *	for most things, though it is possible to still treat everything in four
7850c21e245SRobert Mustacchi  *	distinct NUMA domains with each Zen 2 die slightly closer to some memory
7860c21e245SRobert Mustacchi  *	and PCIe than otherwise. This also impacts the 'amdzen' nexus driver as
7870c21e245SRobert Mustacchi  *	now there is only one 'node' present.
7880c21e245SRobert Mustacchi  *
7890c21e245SRobert Mustacchi  * ZEN 3
7900c21e245SRobert Mustacchi  *
7910c21e245SRobert Mustacchi  *	From an architectural perspective, Zen 3 is a much smaller change from
7920c21e245SRobert Mustacchi  *	Zen 2 than Zen 2 was from Zen 1, though it makes up for most of that in
7930c21e245SRobert Mustacchi  *	its microarchitectural changes. The biggest thing for us is how the die
7940c21e245SRobert Mustacchi  *	changes. In Zen 1 and Zen 2, each core complex still had its own L3
7950c21e245SRobert Mustacchi  *	cache. However, in Zen 3, the L3 is now shared between the entire core
7960c21e245SRobert Mustacchi  *	complex die and is no longer partitioned between each core complex. This
7970c21e245SRobert Mustacchi  *	means that all cores on the die can share the same L3 cache. Otherwise,
7980c21e245SRobert Mustacchi  *	the general layout of the overall package with various core complexes
7990c21e245SRobert Mustacchi  *	and an I/O die stays the same. Here's what the Core Complex Die looks
8000c21e245SRobert Mustacchi  *	like in a bit more detail:
8010c21e245SRobert Mustacchi  *
8020c21e245SRobert Mustacchi  *               +-------------------------------------------------+
8030c21e245SRobert Mustacchi  *               | Zen 3 Core Complex Die                          |
8040c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8050c21e245SRobert Mustacchi  *               | | Core       +----+ |    | Core       +----+ |  |
8060c21e245SRobert Mustacchi  *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
8070c21e245SRobert Mustacchi  *               | | | Thread | +----+ |    | | Thread | +----+ |  |
8080c21e245SRobert Mustacchi  *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
8090c21e245SRobert Mustacchi  *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
8100c21e245SRobert Mustacchi  *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
8110c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8120c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8130c21e245SRobert Mustacchi  *               | | Core       +----+ |    | Core       +----+ |  |
8140c21e245SRobert Mustacchi  *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
8150c21e245SRobert Mustacchi  *               | | | Thread | +----+ |    | | Thread | +----+ |  |
8160c21e245SRobert Mustacchi  *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
8170c21e245SRobert Mustacchi  *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
8180c21e245SRobert Mustacchi  *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
8190c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8200c21e245SRobert Mustacchi  *               |                                                 |
8210c21e245SRobert Mustacchi  *               | +--------------------------------------------+  |
8220c21e245SRobert Mustacchi  *               | |                 L3 Cache                   |  |
8230c21e245SRobert Mustacchi  *               | +--------------------------------------------+  |
8240c21e245SRobert Mustacchi  *               |                                                 |
8250c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8260c21e245SRobert Mustacchi  *               | | Core       +----+ |    | Core       +----+ |  |
8270c21e245SRobert Mustacchi  *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
8280c21e245SRobert Mustacchi  *               | | | Thread | +----+ |    | | Thread | +----+ |  |
8290c21e245SRobert Mustacchi  *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
8300c21e245SRobert Mustacchi  *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
8310c21e245SRobert Mustacchi  *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
8320c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8330c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8340c21e245SRobert Mustacchi  *               | | Core       +----+ |    | Core       +----+ |  |
8350c21e245SRobert Mustacchi  *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
8360c21e245SRobert Mustacchi  *               | | | Thread | +----+ |    | | Thread | +----+ |  |
8370c21e245SRobert Mustacchi  *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
8380c21e245SRobert Mustacchi  *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
8390c21e245SRobert Mustacchi  *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
8400c21e245SRobert Mustacchi  *               | +-------------------+    +-------------------+  |
8410c21e245SRobert Mustacchi  *               +-------------------------------------------------+
8420c21e245SRobert Mustacchi  *
8430c21e245SRobert Mustacchi  *	While it is not pictured, there are connections from the die to the
8440c21e245SRobert Mustacchi  *	broader data fabric and additional functional blocks to support that
8450c21e245SRobert Mustacchi  *	communication and coherency.
8460c21e245SRobert Mustacchi  *
8470ce813ffSRobert Mustacchi  * CPUID LEAVES
8480ce813ffSRobert Mustacchi  *
8490ce813ffSRobert Mustacchi  * There are a few different CPUID leaves that we can use to try and understand
8500ce813ffSRobert Mustacchi  * the actual state of the world. As part of the introduction of family 0xf, AMD
8510ce813ffSRobert Mustacchi  * added CPUID leaf 0x80000008. This leaf tells us the number of logical
8520ce813ffSRobert Mustacchi  * processors that are in the system. Because families before Zen didn't have
8530ce813ffSRobert Mustacchi  * SMT, this was always the number of cores that were in the system. However, it
8540ce813ffSRobert Mustacchi  * should always be thought of as the number of logical threads to be consistent
8550ce813ffSRobert Mustacchi  * between generations. In addition we also get the size of the APIC ID that is
8560ce813ffSRobert Mustacchi  * used to represent the number of logical processors. This is important for
8570ce813ffSRobert Mustacchi  * deriving topology information.
8580ce813ffSRobert Mustacchi  *
8590ce813ffSRobert Mustacchi  * In the Bulldozer family, AMD added leaf 0x8000001E. The information varies a
8600ce813ffSRobert Mustacchi  * bit between Bulldozer and later families, but it is quite useful in
8610ce813ffSRobert Mustacchi  * determining the topology information. Because this information has changed
8620ce813ffSRobert Mustacchi  * across family generations, it's worth calling out what these mean
8630ce813ffSRobert Mustacchi  * explicitly. The registers have the following meanings:
8640ce813ffSRobert Mustacchi  *
8650ce813ffSRobert Mustacchi  *	%eax	The APIC ID. The entire register is defined to have a 32-bit
8660ce813ffSRobert Mustacchi  *		APIC ID, even though on systems without x2apic support, it will
8670ce813ffSRobert Mustacchi  *		be limited to 8 bits.
8680ce813ffSRobert Mustacchi  *
8690ce813ffSRobert Mustacchi  *	%ebx	On Bulldozer-era systems this contains information about the
8700ce813ffSRobert Mustacchi  *		number of cores that are in a compute unit (cores that share
8710ce813ffSRobert Mustacchi  *		resources). It also contains a per-package compute unit ID that
8720ce813ffSRobert Mustacchi  *		identifies which compute unit the logical CPU is a part of.
8730ce813ffSRobert Mustacchi  *
8740ce813ffSRobert Mustacchi  *		On Zen-era systems this instead contains the number of threads
8750ce813ffSRobert Mustacchi  *		per core and the ID of the core that the logical CPU is a part
8760ce813ffSRobert Mustacchi  *		of. Note, this ID is unique only to the package, it is not
8770ce813ffSRobert Mustacchi  *		globally unique across the entire system.
8780ce813ffSRobert Mustacchi  *
8790ce813ffSRobert Mustacchi  *	%ecx	This contains the number of nodes that exist in the package. It
8800ce813ffSRobert Mustacchi  *		also contains an ID that identifies which node the logical CPU
8810ce813ffSRobert Mustacchi  *		is a part of.
8820ce813ffSRobert Mustacchi  *
8830ce813ffSRobert Mustacchi  * Finally, we also use cpuid leaf 0x8000001D to determine information about the
8840ce813ffSRobert Mustacchi  * cache layout to determine which logical CPUs are sharing which caches.
8850ce813ffSRobert Mustacchi  *
8860ce813ffSRobert Mustacchi  * illumos Topology
8870ce813ffSRobert Mustacchi  * ----------------
8880ce813ffSRobert Mustacchi  *
8890ce813ffSRobert Mustacchi  * Based on the above we synthesize the information into several different
8900ce813ffSRobert Mustacchi  * variables that we store in the 'struct cpuid_info'. We'll go into the details
8910ce813ffSRobert Mustacchi  * of what each member is supposed to represent and their uniqueness. In
8920ce813ffSRobert Mustacchi  * general, there are two levels of uniqueness that we care about. We care about
8930ce813ffSRobert Mustacchi  * an ID that is globally unique. That means that it will be unique across all
8940ce813ffSRobert Mustacchi  * entities in the system. For example, the default logical CPU ID is globally
8950ce813ffSRobert Mustacchi  * unique. On the other hand, there is some information that we only care about
8960ce813ffSRobert Mustacchi  * being unique within the context of a single package / socket. Here are the
8970ce813ffSRobert Mustacchi  * variables that we keep track of and their meaning.
8980ce813ffSRobert Mustacchi  *
8990ce813ffSRobert Mustacchi  * Several of the values that are asking for an identifier, with the exception
9000ce813ffSRobert Mustacchi  * of cpi_apicid, are allowed to be synthetic.
9010ce813ffSRobert Mustacchi  *
9020ce813ffSRobert Mustacchi  *
9030ce813ffSRobert Mustacchi  * cpi_apicid
9040ce813ffSRobert Mustacchi  *
9050ce813ffSRobert Mustacchi  *	This is the value of the CPU's APIC id. This should be the full 32-bit
9060ce813ffSRobert Mustacchi  *	ID if the CPU is using the x2apic. Otherwise, it should be the 8-bit
9070ce813ffSRobert Mustacchi  *	APIC ID. This value is globally unique between all logical CPUs across
9080ce813ffSRobert Mustacchi  *	all packages. This is usually required by the APIC.
9090ce813ffSRobert Mustacchi  *
9100ce813ffSRobert Mustacchi  * cpi_chipid
9110ce813ffSRobert Mustacchi  *
9120ce813ffSRobert Mustacchi  *	This value indicates the ID of the package that the logical CPU is a
9130ce813ffSRobert Mustacchi  *	part of. This value is allowed to be synthetic. It is usually derived by
9140ce813ffSRobert Mustacchi  *	taking the CPU's APIC ID and determining how many bits are used to
9150ce813ffSRobert Mustacchi  *	represent CPU cores in the package. All logical CPUs that are part of
9160ce813ffSRobert Mustacchi  *	the same package must have the same value.
9170ce813ffSRobert Mustacchi  *
9180ce813ffSRobert Mustacchi  * cpi_coreid
9190ce813ffSRobert Mustacchi  *
9200ce813ffSRobert Mustacchi  *	This represents the ID of a CPU core. Two logical CPUs should only have
9210ce813ffSRobert Mustacchi  *	the same cpi_coreid value if they are part of the same core. These
9220ce813ffSRobert Mustacchi  *	values may be synthetic. On systems that support SMT, this value is
9230ce813ffSRobert Mustacchi  *	usually derived from the APIC ID, otherwise it is often synthetic and
9240ce813ffSRobert Mustacchi  *	just set to the value of the cpu_id in the cpu_t.
9250ce813ffSRobert Mustacchi  *
9260ce813ffSRobert Mustacchi  * cpi_pkgcoreid
9270ce813ffSRobert Mustacchi  *
9280ce813ffSRobert Mustacchi  *	This is similar to the cpi_coreid in that logical CPUs that are part of
9290ce813ffSRobert Mustacchi  *	the same core should have the same ID. The main difference is that these
9300ce813ffSRobert Mustacchi  *	values are only required to be unique to a given socket.
9310ce813ffSRobert Mustacchi  *
9320ce813ffSRobert Mustacchi  * cpi_clogid
9330ce813ffSRobert Mustacchi  *
9340ce813ffSRobert Mustacchi  *	This represents the logical ID of a logical CPU. This value should be
9350ce813ffSRobert Mustacchi  *	unique within a given socket for each logical CPU. This is allowed to be
9360ce813ffSRobert Mustacchi  *	synthetic, though it is usually based off of the CPU's apic ID. The
9370ce813ffSRobert Mustacchi  *	broader system expects that logical CPUs that have are part of the same
9380ce813ffSRobert Mustacchi  *	core have contiguous numbers. For example, if there were two threads per
9390ce813ffSRobert Mustacchi  *	core, then the core IDs divided by two should be the same and the first
9400ce813ffSRobert Mustacchi  *	modulus two should be zero and the second one. For example, IDs 4 and 5
9410ce813ffSRobert Mustacchi  *	indicate two logical CPUs that are part of the same core. But IDs 5 and
9420ce813ffSRobert Mustacchi  *	6 represent two logical CPUs that are part of different cores.
9430ce813ffSRobert Mustacchi  *
9440ce813ffSRobert Mustacchi  *	While it is common for the cpi_coreid and the cpi_clogid to be derived
9450ce813ffSRobert Mustacchi  *	from the same source, strictly speaking, they don't have to be and the
9460ce813ffSRobert Mustacchi  *	two values should be considered logically independent. One should not
9470ce813ffSRobert Mustacchi  *	try to compare a logical CPU's cpi_coreid and cpi_clogid to determine
9480ce813ffSRobert Mustacchi  *	some kind of relationship. While this is tempting, we've seen cases on
9490ce813ffSRobert Mustacchi  *	AMD family 0xf where the system's cpu id is not related to its APIC ID.
9500ce813ffSRobert Mustacchi  *
9510ce813ffSRobert Mustacchi  * cpi_ncpu_per_chip
9520ce813ffSRobert Mustacchi  *
9530ce813ffSRobert Mustacchi  *	This value indicates the total number of logical CPUs that exist in the
9540ce813ffSRobert Mustacchi  *	physical package. Critically, this is not the number of logical CPUs
9550ce813ffSRobert Mustacchi  *	that exist for just the single core.
9560ce813ffSRobert Mustacchi  *
9570ce813ffSRobert Mustacchi  *	This value should be the same for all logical CPUs in the same package.
9580ce813ffSRobert Mustacchi  *
9590ce813ffSRobert Mustacchi  * cpi_ncore_per_chip
9600ce813ffSRobert Mustacchi  *
9610ce813ffSRobert Mustacchi  *	This value indicates the total number of physical CPU cores that exist
9620ce813ffSRobert Mustacchi  *	in the package. The system compares this value with cpi_ncpu_per_chip to
9630ce813ffSRobert Mustacchi  *	determine if simultaneous multi-threading (SMT) is enabled. When
9640ce813ffSRobert Mustacchi  *	cpi_ncpu_per_chip equals cpi_ncore_per_chip, then there is no SMT and
9650ce813ffSRobert Mustacchi  *	the X86FSET_HTT feature is not set. If this value is greater than one,
9660ce813ffSRobert Mustacchi  *	than we consider the processor to have the feature X86FSET_CMP, to
9670ce813ffSRobert Mustacchi  *	indicate that there is support for more than one core.
9680ce813ffSRobert Mustacchi  *
9690ce813ffSRobert Mustacchi  *	This value should be the same for all logical CPUs in the same package.
9700ce813ffSRobert Mustacchi  *
9710ce813ffSRobert Mustacchi  * cpi_procnodes_per_pkg
9720ce813ffSRobert Mustacchi  *
9730ce813ffSRobert Mustacchi  *	This value indicates the number of 'nodes' that exist in the package.
9740ce813ffSRobert Mustacchi  *	When processors are actually a multi-chip module, this represents the
9750ce813ffSRobert Mustacchi  *	number of such modules that exist in the package. Currently, on Intel
9760ce813ffSRobert Mustacchi  *	based systems this member is always set to 1.
9770ce813ffSRobert Mustacchi  *
9780ce813ffSRobert Mustacchi  *	This value should be the same for all logical CPUs in the same package.
9790ce813ffSRobert Mustacchi  *
9800ce813ffSRobert Mustacchi  * cpi_procnodeid
9810ce813ffSRobert Mustacchi  *
9820ce813ffSRobert Mustacchi  *	This value indicates the ID of the node that the logical CPU is a part
9830ce813ffSRobert Mustacchi  *	of. All logical CPUs that are in the same node must have the same value
9840ce813ffSRobert Mustacchi  *	here. This value must be unique across all of the packages in the
9850ce813ffSRobert Mustacchi  *	system.  On Intel based systems, this is currently set to the value in
9860ce813ffSRobert Mustacchi  *	cpi_chipid because there is only one node.
9870ce813ffSRobert Mustacchi  *
9880ce813ffSRobert Mustacchi  * cpi_cores_per_compunit
9890ce813ffSRobert Mustacchi  *
9900ce813ffSRobert Mustacchi  *	This value indicates the number of cores that are part of a compute
9910ce813ffSRobert Mustacchi  *	unit. See the AMD topology section for this. This member only has real
9920ce813ffSRobert Mustacchi  *	meaning currently for AMD Bulldozer family processors. For all other
9930ce813ffSRobert Mustacchi  *	processors, this should currently be set to 1.
9940ce813ffSRobert Mustacchi  *
9950ce813ffSRobert Mustacchi  * cpi_compunitid
9960ce813ffSRobert Mustacchi  *
9970ce813ffSRobert Mustacchi  *	This indicates the compute unit that the logical CPU belongs to. For
9980ce813ffSRobert Mustacchi  *	processors without AMD Bulldozer-style compute units this should be set
9990ce813ffSRobert Mustacchi  *	to the value of cpi_coreid.
10000ce813ffSRobert Mustacchi  *
10010ce813ffSRobert Mustacchi  * cpi_ncpu_shr_last_cache
10020ce813ffSRobert Mustacchi  *
10030ce813ffSRobert Mustacchi  *	This indicates the number of logical CPUs that are sharing the same last
10040ce813ffSRobert Mustacchi  *	level cache. This value should be the same for all CPUs that are sharing
10050ce813ffSRobert Mustacchi  *	that cache. The last cache refers to the cache that is closest to memory
10060ce813ffSRobert Mustacchi  *	and furthest away from the CPU.
10070ce813ffSRobert Mustacchi  *
10080ce813ffSRobert Mustacchi  * cpi_last_lvl_cacheid
10090ce813ffSRobert Mustacchi  *
10100ce813ffSRobert Mustacchi  *	This indicates the ID of the last cache that the logical CPU uses. This
10110ce813ffSRobert Mustacchi  *	cache is often shared between multiple logical CPUs and is the cache
10120ce813ffSRobert Mustacchi  *	that is closest to memory and furthest away from the CPU. This value
10130ce813ffSRobert Mustacchi  *	should be the same for a group of logical CPUs only if they actually
10140ce813ffSRobert Mustacchi  *	share the same last level cache. IDs should not overlap between
10150ce813ffSRobert Mustacchi  *	packages.
10160ce813ffSRobert Mustacchi  *
1017d6517bbdSRobert Mustacchi  * cpi_ncore_bits
1018d6517bbdSRobert Mustacchi  *
1019d6517bbdSRobert Mustacchi  *	This indicates the number of bits that are required to represent all of
1020d6517bbdSRobert Mustacchi  *	the cores in the system. As cores are derived based on their APIC IDs,
1021d6517bbdSRobert Mustacchi  *	we aren't guaranteed a run of APIC IDs starting from zero. It's OK for
1022d6517bbdSRobert Mustacchi  *	this value to be larger than the actual number of IDs that are present
1023d6517bbdSRobert Mustacchi  *	in the system. This is used to size tables by the CMI framework. It is
1024d6517bbdSRobert Mustacchi  *	only filled in for Intel and AMD CPUs.
1025d6517bbdSRobert Mustacchi  *
1026d6517bbdSRobert Mustacchi  * cpi_nthread_bits
1027d6517bbdSRobert Mustacchi  *
1028d6517bbdSRobert Mustacchi  *	This indicates the number of bits required to represent all of the IDs
1029d6517bbdSRobert Mustacchi  *	that cover the logical CPUs that exist on a given core. It's OK for this
1030d6517bbdSRobert Mustacchi  *	value to be larger than the actual number of IDs that are present in the
1031d6517bbdSRobert Mustacchi  *	system.  This is used to size tables by the CMI framework. It is
1032d6517bbdSRobert Mustacchi  *	only filled in for Intel and AMD CPUs.
1033d6517bbdSRobert Mustacchi  *
10340ce813ffSRobert Mustacchi  * -----------
10350ce813ffSRobert Mustacchi  * Hypervisors
10360ce813ffSRobert Mustacchi  * -----------
10370ce813ffSRobert Mustacchi  *
10380ce813ffSRobert Mustacchi  * If trying to manage the differences between vendors wasn't bad enough, it can
10390ce813ffSRobert Mustacchi  * get worse thanks to our friend hardware virtualization. Hypervisors are given
10400ce813ffSRobert Mustacchi  * the ability to interpose on all cpuid instructions and change them to suit
10410ce813ffSRobert Mustacchi  * their purposes. In general, this is necessary as the hypervisor wants to be
10420ce813ffSRobert Mustacchi  * able to present a more uniform set of features or not necessarily give the
10430ce813ffSRobert Mustacchi  * guest operating system kernel knowledge of all features so it can be
10440ce813ffSRobert Mustacchi  * more easily migrated between systems.
10450ce813ffSRobert Mustacchi  *
10460ce813ffSRobert Mustacchi  * When it comes to trying to determine topology information, this can be a
10470ce813ffSRobert Mustacchi  * double edged sword. When a hypervisor doesn't actually implement a cpuid
10480ce813ffSRobert Mustacchi  * leaf, it'll often return all zeros. Because of that, you'll often see various
10490ce813ffSRobert Mustacchi  * checks scattered about fields being non-zero before we assume we can use
10500ce813ffSRobert Mustacchi  * them.
10510ce813ffSRobert Mustacchi  *
10520ce813ffSRobert Mustacchi  * When it comes to topology information, the hypervisor is often incentivized
10530ce813ffSRobert Mustacchi  * to lie to you about topology. This is because it doesn't always actually
10540ce813ffSRobert Mustacchi  * guarantee that topology at all. The topology path we take in the system
10550ce813ffSRobert Mustacchi  * depends on how the CPU advertises itself. If it advertises itself as an Intel
10560ce813ffSRobert Mustacchi  * or AMD CPU, then we basically do our normal path. However, when they don't
10570ce813ffSRobert Mustacchi  * use an actual vendor, then that usually turns into multiple one-core CPUs
10580ce813ffSRobert Mustacchi  * that we enumerate that are often on different sockets. The actual behavior
10590ce813ffSRobert Mustacchi  * depends greatly on what the hypervisor actually exposes to us.
10600ce813ffSRobert Mustacchi  *
10610ce813ffSRobert Mustacchi  * --------------------
10620ce813ffSRobert Mustacchi  * Exposing Information
10630ce813ffSRobert Mustacchi  * --------------------
10640ce813ffSRobert Mustacchi  *
10650ce813ffSRobert Mustacchi  * We expose CPUID information in three different forms in the system.
10660ce813ffSRobert Mustacchi  *
10670ce813ffSRobert Mustacchi  * The first is through the x86_featureset variable. This is used in conjunction
10680ce813ffSRobert Mustacchi  * with the is_x86_feature() function. This is queried by x86-specific functions
10690ce813ffSRobert Mustacchi  * to determine which features are or aren't present in the system and to make
10700ce813ffSRobert Mustacchi  * decisions based upon them. For example, users of this include everything from
10710ce813ffSRobert Mustacchi  * parts of the system dedicated to reliability, availability, and
10720ce813ffSRobert Mustacchi  * serviceability (RAS), to making decisions about how to handle security
10730ce813ffSRobert Mustacchi  * mitigations, to various x86-specific drivers. General purpose or
10740ce813ffSRobert Mustacchi  * architecture independent drivers should never be calling this function.
10750ce813ffSRobert Mustacchi  *
10760ce813ffSRobert Mustacchi  * The second means is through the auxiliary vector. The auxiliary vector is a
10770ce813ffSRobert Mustacchi  * series of tagged data that the kernel passes down to a user program when it
10780ce813ffSRobert Mustacchi  * begins executing. This information is used to indicate to programs what
10790ce813ffSRobert Mustacchi  * instruction set extensions are present. For example, information about the
10800ce813ffSRobert Mustacchi  * CPU supporting the machine check architecture (MCA) wouldn't be passed down
10810ce813ffSRobert Mustacchi  * since user programs cannot make use of it. However, things like the AVX
10820ce813ffSRobert Mustacchi  * instruction sets are. Programs use this information to make run-time
10830ce813ffSRobert Mustacchi  * decisions about what features they should use. As an example, the run-time
10840ce813ffSRobert Mustacchi  * link-editor (rtld) can relocate different functions depending on the hardware
10850ce813ffSRobert Mustacchi  * support available.
10860ce813ffSRobert Mustacchi  *
10870ce813ffSRobert Mustacchi  * The final form is through a series of accessor functions that all have the
10880ce813ffSRobert Mustacchi  * form cpuid_get*. This is used by a number of different subsystems in the
10890ce813ffSRobert Mustacchi  * kernel to determine more detailed information about what we're running on,
10900ce813ffSRobert Mustacchi  * topology information, etc. Some of these subsystems include processor groups
10910ce813ffSRobert Mustacchi  * (uts/common/os/pg.c.), CPU Module Interface (uts/i86pc/os/cmi.c), ACPI,
10920ce813ffSRobert Mustacchi  * microcode, and performance monitoring. These functions all ASSERT that the
10930ce813ffSRobert Mustacchi  * CPU they're being called on has reached a certain cpuid pass. If the passes
10940ce813ffSRobert Mustacchi  * are rearranged, then this needs to be adjusted.
109565f20420SRobert Mustacchi  *
109665f20420SRobert Mustacchi  * -----------------------------------------------
109765f20420SRobert Mustacchi  * Speculative Execution CPU Side Channel Security
109865f20420SRobert Mustacchi  * -----------------------------------------------
109965f20420SRobert Mustacchi  *
110065f20420SRobert Mustacchi  * With the advent of the Spectre and Meltdown attacks which exploit speculative
110165f20420SRobert Mustacchi  * execution in the CPU to create side channels there have been a number of
110265f20420SRobert Mustacchi  * different attacks and corresponding issues that the operating system needs to
110365f20420SRobert Mustacchi  * mitigate against. The following list is some of the common, but not
110465f20420SRobert Mustacchi  * exhaustive, set of issues that we know about and have done some or need to do
110565f20420SRobert Mustacchi  * more work in the system to mitigate against:
110665f20420SRobert Mustacchi  *
110765f20420SRobert Mustacchi  *   - Spectre v1
110842cd1931SJohn Levon  *   - swapgs (Spectre v1 variant)
110965f20420SRobert Mustacchi  *   - Spectre v2
111065f20420SRobert Mustacchi  *   - Meltdown (Spectre v3)
111165f20420SRobert Mustacchi  *   - Rogue Register Read (Spectre v3a)
111265f20420SRobert Mustacchi  *   - Speculative Store Bypass (Spectre v4)
111365f20420SRobert Mustacchi  *   - ret2spec, SpectreRSB
111465f20420SRobert Mustacchi  *   - L1 Terminal Fault (L1TF)
111565f20420SRobert Mustacchi  *   - Microarchitectural Data Sampling (MDS)
1116*5cd084edSDan McDonald  *   - Register File Data Sampling (RFDS)
111765f20420SRobert Mustacchi  *
111865f20420SRobert Mustacchi  * Each of these requires different sets of mitigations and has different attack
111965f20420SRobert Mustacchi  * surfaces. For the most part, this discussion is about protecting the kernel
112065f20420SRobert Mustacchi  * from non-kernel executing environments such as user processes and hardware
112165f20420SRobert Mustacchi  * virtual machines. Unfortunately, there are a number of user vs. user
112265f20420SRobert Mustacchi  * scenarios that exist with these. The rest of this section will describe the
112365f20420SRobert Mustacchi  * overall approach that the system has taken to address these as well as their
112465f20420SRobert Mustacchi  * shortcomings. Unfortunately, not all of the above have been handled today.
112565f20420SRobert Mustacchi  *
112642cd1931SJohn Levon  * SPECTRE v2, ret2spec, SpectreRSB
112765f20420SRobert Mustacchi  *
112865f20420SRobert Mustacchi  * The second variant of the spectre attack focuses on performing branch target
112965f20420SRobert Mustacchi  * injection. This generally impacts indirect call instructions in the system.
1130651a12cbSRobert Mustacchi  * There are four different ways to mitigate this issue that are commonly
113165f20420SRobert Mustacchi  * described today:
113265f20420SRobert Mustacchi  *
113365f20420SRobert Mustacchi  *  1. Using Indirect Branch Restricted Speculation (IBRS).
113465f20420SRobert Mustacchi  *  2. Using Retpolines and RSB Stuffing
1135345881c5SDan McDonald  *  3. Using Enhanced Indirect Branch Restricted Speculation (eIBRS)
1136651a12cbSRobert Mustacchi  *  4. Using Automated Indirect Branch Restricted Speculation (AIBRS)
113765f20420SRobert Mustacchi  *
113865f20420SRobert Mustacchi  * IBRS uses a feature added to microcode to restrict speculation, among other
113965f20420SRobert Mustacchi  * things. This form of mitigation has not been used as it has been generally
114065f20420SRobert Mustacchi  * seen as too expensive and requires reactivation upon various transitions in
114165f20420SRobert Mustacchi  * the system.
114265f20420SRobert Mustacchi  *
114365f20420SRobert Mustacchi  * As a less impactful alternative to IBRS, retpolines were developed by
114465f20420SRobert Mustacchi  * Google. These basically require one to replace indirect calls with a specific
114565f20420SRobert Mustacchi  * trampoline that will cause speculation to fail and break the attack.
114665f20420SRobert Mustacchi  * Retpolines require compiler support. We always build with retpolines in the
114765f20420SRobert Mustacchi  * external thunk mode. This means that a traditional indirect call is replaced
114865f20420SRobert Mustacchi  * with a call to one of the __x86_indirect_thunk_<reg> functions. A side effect
114965f20420SRobert Mustacchi  * of this is that all indirect function calls are performed through a register.
115065f20420SRobert Mustacchi  *
115165f20420SRobert Mustacchi  * We have to use a common external location of the thunk and not inline it into
115265f20420SRobert Mustacchi  * the callsite so that way we can have a single place to patch these functions.
11539514ab44SRobert Mustacchi  * As it turns out, we currently have two different forms of retpolines that
115465f20420SRobert Mustacchi  * exist in the system:
115565f20420SRobert Mustacchi  *
115665f20420SRobert Mustacchi  *  1. A full retpoline
11579514ab44SRobert Mustacchi  *  2. A no-op version
115865f20420SRobert Mustacchi  *
11599514ab44SRobert Mustacchi  * The first one is used in the general case. Historically, there was an
11609514ab44SRobert Mustacchi  * AMD-specific optimized retopoline variant that was based around using a
11619514ab44SRobert Mustacchi  * serializing lfence instruction; however, in March 2022 it was announced that
11629514ab44SRobert Mustacchi  * this was actually still vulnerable to Spectre v2 and therefore we no longer
11639514ab44SRobert Mustacchi  * use it and it is no longer available in the system.
116465f20420SRobert Mustacchi  *
116565f20420SRobert Mustacchi  * The third form described above is the most curious. It turns out that the way
116665f20420SRobert Mustacchi  * that retpolines are implemented is that they rely on how speculation is
116765f20420SRobert Mustacchi  * performed on a 'ret' instruction. Intel has continued to optimize this
116865f20420SRobert Mustacchi  * process (which is partly why we need to have return stack buffer stuffing,
116965f20420SRobert Mustacchi  * but more on that in a bit) and in processors starting with Cascade Lake
117065f20420SRobert Mustacchi  * on the server side, it's dangerous to rely on retpolines. Instead, a new
1171345881c5SDan McDonald  * mechanism has been introduced called Enhanced IBRS (eIBRS).
117265f20420SRobert Mustacchi  *
1173345881c5SDan McDonald  * Unlike IBRS, eIBRS is designed to be enabled once at boot and left on each
117465f20420SRobert Mustacchi  * physical core. However, if this is the case, we don't want to use retpolines
1175345881c5SDan McDonald  * any more. Therefore if eIBRS is present, we end up turning each retpoline
117665f20420SRobert Mustacchi  * function (called a thunk) into a jmp instruction. This means that we're still
117765f20420SRobert Mustacchi  * paying the cost of an extra jump to the external thunk, but it gives us
117865f20420SRobert Mustacchi  * flexibility and the ability to have a single kernel image that works across a
117965f20420SRobert Mustacchi  * wide variety of systems and hardware features.
118065f20420SRobert Mustacchi  *
118165f20420SRobert Mustacchi  * Unfortunately, this alone is insufficient. First, Skylake systems have
118265f20420SRobert Mustacchi  * additional speculation for the Return Stack Buffer (RSB) which is used to
118365f20420SRobert Mustacchi  * return from call instructions which retpolines take advantage of. However,
118465f20420SRobert Mustacchi  * this problem is not just limited to Skylake and is actually more pernicious.
118565f20420SRobert Mustacchi  * The SpectreRSB paper introduces several more problems that can arise with
118665f20420SRobert Mustacchi  * dealing with this. The RSB can be poisoned just like the indirect branch
118765f20420SRobert Mustacchi  * predictor. This means that one needs to clear the RSB when transitioning
118865f20420SRobert Mustacchi  * between two different privilege domains. Some examples include:
118965f20420SRobert Mustacchi  *
119065f20420SRobert Mustacchi  *  - Switching between two different user processes
119165f20420SRobert Mustacchi  *  - Going between user land and the kernel
119265f20420SRobert Mustacchi  *  - Returning to the kernel from a hardware virtual machine
119365f20420SRobert Mustacchi  *
119465f20420SRobert Mustacchi  * Mitigating this involves combining a couple of different things. The first is
119565f20420SRobert Mustacchi  * SMEP (supervisor mode execution protection) which was introduced in Ivy
119665f20420SRobert Mustacchi  * Bridge. When an RSB entry refers to a user address and we're executing in the
119765f20420SRobert Mustacchi  * kernel, speculation through it will be stopped when SMEP is enabled. This
119865f20420SRobert Mustacchi  * protects against a number of the different cases that we would normally be
119965f20420SRobert Mustacchi  * worried about such as when we enter the kernel from user land.
120065f20420SRobert Mustacchi  *
120165f20420SRobert Mustacchi  * To prevent against additional manipulation of the RSB from other contexts
1202345881c5SDan McDonald  * such as a non-root VMX context attacking the kernel we first look to
1203345881c5SDan McDonald  * enhanced IBRS. When eIBRS is present and enabled, then there should be
1204345881c5SDan McDonald  * nothing else that we need to do to protect the kernel at this time.
120565f20420SRobert Mustacchi  *
1206345881c5SDan McDonald  * Unfortunately, eIBRS or not, we need to manually overwrite the contents of
1207345881c5SDan McDonald  * the return stack buffer. We do this through the x86_rsb_stuff() function.
1208345881c5SDan McDonald  * Currently this is employed on context switch and vmx_exit. The
1209345881c5SDan McDonald  * x86_rsb_stuff() function is disabled only when mitigations in general are.
121065f20420SRobert Mustacchi  *
121165f20420SRobert Mustacchi  * If SMEP is not present, then we would have to stuff the RSB every time we
121265f20420SRobert Mustacchi  * transitioned from user mode to the kernel, which isn't very practical right
121365f20420SRobert Mustacchi  * now.
121465f20420SRobert Mustacchi  *
121565f20420SRobert Mustacchi  * To fully protect user to user and vmx to vmx attacks from these classes of
121665f20420SRobert Mustacchi  * issues, we would also need to allow them to opt into performing an Indirect
121765f20420SRobert Mustacchi  * Branch Prediction Barrier (IBPB) on switch. This is not currently wired up.
121865f20420SRobert Mustacchi  *
1219651a12cbSRobert Mustacchi  * The fourth form of mitigation here is specific to AMD and is called Automated
1220651a12cbSRobert Mustacchi  * IBRS (AIBRS). This is similar in spirit to eIBRS; however rather than set the
1221651a12cbSRobert Mustacchi  * IBRS bit in MSR_IA32_SPEC_CTRL (0x48) we instead set a bit in the EFER
1222651a12cbSRobert Mustacchi  * (extended feature enable register) MSR. This bit basically says that IBRS
1223651a12cbSRobert Mustacchi  * acts as though it is always active when executing at CPL0 and when executing
1224651a12cbSRobert Mustacchi  * in the 'host' context when SEV-SNP is enabled.
1225651a12cbSRobert Mustacchi  *
1226651a12cbSRobert Mustacchi  * When this is active, AMD states that the RSB is cleared on VMEXIT and
1227651a12cbSRobert Mustacchi  * therefore it is unnecessary. While this handles RSB stuffing attacks from SVM
1228651a12cbSRobert Mustacchi  * to the kernel, we must still consider the remaining cases that exist, just
1229651a12cbSRobert Mustacchi  * like above. While traditionally AMD employed a 32 entry RSB allowing the
1230651a12cbSRobert Mustacchi  * traditional technique to work, this is not true on all CPUs. While a write to
1231651a12cbSRobert Mustacchi  * IBRS would clear the RSB if the processor supports more than 32 entries (but
1232651a12cbSRobert Mustacchi  * not otherwise), AMD states that as long as at leat a single 4 KiB unmapped
1233651a12cbSRobert Mustacchi  * guard page is present between user and kernel address spaces and SMEP is
1234651a12cbSRobert Mustacchi  * enabled, then there is no need to clear the RSB at all.
1235651a12cbSRobert Mustacchi  *
123665f20420SRobert Mustacchi  * By default, the system will enable RSB stuffing and the required variant of
123765f20420SRobert Mustacchi  * retpolines and store that information in the x86_spectrev2_mitigation value.
123865f20420SRobert Mustacchi  * This will be evaluated after a microcode update as well, though it is
123965f20420SRobert Mustacchi  * expected that microcode updates will not take away features. This may mean
124065f20420SRobert Mustacchi  * that a late loaded microcode may not end up in the optimal configuration
124165f20420SRobert Mustacchi  * (though this should be rare).
124265f20420SRobert Mustacchi  *
124365f20420SRobert Mustacchi  * Currently we do not build kmdb with retpolines or perform any additional side
124465f20420SRobert Mustacchi  * channel security mitigations for it. One complication with kmdb is that it
124565f20420SRobert Mustacchi  * requires its own retpoline thunks and it would need to adjust itself based on
124665f20420SRobert Mustacchi  * what the kernel does. The threat model of kmdb is more limited and therefore
124765f20420SRobert Mustacchi  * it may make more sense to investigate using prediction barriers as the whole
124865f20420SRobert Mustacchi  * system is only executing a single instruction at a time while in kmdb.
124965f20420SRobert Mustacchi  *
125042cd1931SJohn Levon  * SPECTRE v1, v4
125165f20420SRobert Mustacchi  *
125265f20420SRobert Mustacchi  * The v1 and v4 variants of spectre are not currently mitigated in the
125365f20420SRobert Mustacchi  * system and require other classes of changes to occur in the code.
125465f20420SRobert Mustacchi  *
125542cd1931SJohn Levon  * SPECTRE v1 (SWAPGS VARIANT)
125642cd1931SJohn Levon  *
125742cd1931SJohn Levon  * The class of Spectre v1 vulnerabilities aren't all about bounds checks, but
125842cd1931SJohn Levon  * can generally affect any branch-dependent code. The swapgs issue is one
125942cd1931SJohn Levon  * variant of this. If we are coming in from userspace, we can have code like
126042cd1931SJohn Levon  * this:
126142cd1931SJohn Levon  *
126242cd1931SJohn Levon  *	cmpw	$KCS_SEL, REGOFF_CS(%rsp)
126342cd1931SJohn Levon  *	je	1f
126442cd1931SJohn Levon  *	movq	$0, REGOFF_SAVFP(%rsp)
126542cd1931SJohn Levon  *	swapgs
126642cd1931SJohn Levon  *	1:
126742cd1931SJohn Levon  *	movq	%gs:CPU_THREAD, %rax
126842cd1931SJohn Levon  *
126942cd1931SJohn Levon  * If an attacker can cause a mis-speculation of the branch here, we could skip
127042cd1931SJohn Levon  * the needed swapgs, and use the /user/ %gsbase as the base of the %gs-based
127142cd1931SJohn Levon  * load. If subsequent code can act as the usual Spectre cache gadget, this
127242cd1931SJohn Levon  * would potentially allow KPTI bypass. To fix this, we need an lfence prior to
127342cd1931SJohn Levon  * any use of the %gs override.
127442cd1931SJohn Levon  *
127542cd1931SJohn Levon  * The other case is also an issue: if we're coming into a trap from kernel
127642cd1931SJohn Levon  * space, we could mis-speculate and swapgs the user %gsbase back in prior to
127742cd1931SJohn Levon  * using it. AMD systems are not vulnerable to this version, as a swapgs is
127842cd1931SJohn Levon  * serializing with respect to subsequent uses. But as AMD /does/ need the other
127942cd1931SJohn Levon  * case, and the fix is the same in both cases (an lfence at the branch target
128042cd1931SJohn Levon  * 1: in this example), we'll just do it unconditionally.
128142cd1931SJohn Levon  *
128242cd1931SJohn Levon  * Note that we don't enable user-space "wrgsbase" via CR4_FSGSBASE, making it
128342cd1931SJohn Levon  * harder for user-space to actually set a useful %gsbase value: although it's
128442cd1931SJohn Levon  * not clear, it might still be feasible via lwp_setprivate(), though, so we
128542cd1931SJohn Levon  * mitigate anyway.
128642cd1931SJohn Levon  *
128765f20420SRobert Mustacchi  * MELTDOWN
128865f20420SRobert Mustacchi  *
128965f20420SRobert Mustacchi  * Meltdown, or spectre v3, allowed a user process to read any data in their
129065f20420SRobert Mustacchi  * address space regardless of whether or not the page tables in question
129165f20420SRobert Mustacchi  * allowed the user to have the ability to read them. The solution to meltdown
129265f20420SRobert Mustacchi  * is kernel page table isolation. In this world, there are two page tables that
129365f20420SRobert Mustacchi  * are used for a process, one in user land and one in the kernel. To implement
129465f20420SRobert Mustacchi  * this we use per-CPU page tables and switch between the user and kernel
129565f20420SRobert Mustacchi  * variants when entering and exiting the kernel.  For more information about
129665f20420SRobert Mustacchi  * this process and how the trampolines work, please see the big theory
129765f20420SRobert Mustacchi  * statements and additional comments in:
129865f20420SRobert Mustacchi  *
129965f20420SRobert Mustacchi  *  - uts/i86pc/ml/kpti_trampolines.s
130065f20420SRobert Mustacchi  *  - uts/i86pc/vm/hat_i86.c
130165f20420SRobert Mustacchi  *
130265f20420SRobert Mustacchi  * While Meltdown only impacted Intel systems and there are also Intel systems
130365f20420SRobert Mustacchi  * that have Meltdown fixed (called Rogue Data Cache Load), we always have
130465f20420SRobert Mustacchi  * kernel page table isolation enabled. While this may at first seem weird, an
130565f20420SRobert Mustacchi  * important thing to remember is that you can't speculatively read an address
130665f20420SRobert Mustacchi  * if it's never in your page table at all. Having user processes without kernel
130765f20420SRobert Mustacchi  * pages present provides us with an important layer of defense in the kernel
130865f20420SRobert Mustacchi  * against any other side channel attacks that exist and have yet to be
130965f20420SRobert Mustacchi  * discovered. As such, kernel page table isolation (KPTI) is always enabled by
131065f20420SRobert Mustacchi  * default, no matter the x86 system.
131165f20420SRobert Mustacchi  *
131265f20420SRobert Mustacchi  * L1 TERMINAL FAULT
131365f20420SRobert Mustacchi  *
131465f20420SRobert Mustacchi  * L1 Terminal Fault (L1TF) takes advantage of an issue in how speculative
131565f20420SRobert Mustacchi  * execution uses page table entries. Effectively, it is two different problems.
131665f20420SRobert Mustacchi  * The first is that it ignores the not present bit in the page table entries
131765f20420SRobert Mustacchi  * when performing speculative execution. This means that something can
131865f20420SRobert Mustacchi  * speculatively read the listed physical address if it's present in the L1
131965f20420SRobert Mustacchi  * cache under certain conditions (see Intel's documentation for the full set of
132065f20420SRobert Mustacchi  * conditions). Secondly, this can be used to bypass hardware virtualization
132165f20420SRobert Mustacchi  * extended page tables (EPT) that are part of Intel's hardware virtual machine
132265f20420SRobert Mustacchi  * instructions.
132365f20420SRobert Mustacchi  *
132465f20420SRobert Mustacchi  * For the non-hardware virtualized case, this is relatively easy to deal with.
132565f20420SRobert Mustacchi  * We must make sure that all unmapped pages have an address of zero. This means
132665f20420SRobert Mustacchi  * that they could read the first 4k of physical memory; however, we never use
132765f20420SRobert Mustacchi  * that first page in the operating system and always skip putting it in our
132865f20420SRobert Mustacchi  * memory map, even if firmware tells us we can use it in our memory map. While
132965f20420SRobert Mustacchi  * other systems try to put extra metadata in the address and reserved bits,
133065f20420SRobert Mustacchi  * which led to this being problematic in those cases, we do not.
133165f20420SRobert Mustacchi  *
133265f20420SRobert Mustacchi  * For hardware virtual machines things are more complicated. Because they can
133365f20420SRobert Mustacchi  * construct their own page tables, it isn't hard for them to perform this
133465f20420SRobert Mustacchi  * attack against any physical address. The one wrinkle is that this physical
133565f20420SRobert Mustacchi  * address must be in the L1 data cache. Thus Intel added an MSR that we can use
133665f20420SRobert Mustacchi  * to flush the L1 data cache. We wrap this up in the function
133765f20420SRobert Mustacchi  * spec_uarch_flush(). This function is also used in the mitigation of
133865f20420SRobert Mustacchi  * microarchitectural data sampling (MDS) discussed later on. Kernel based
133965f20420SRobert Mustacchi  * hypervisors such as KVM or bhyve are responsible for performing this before
134065f20420SRobert Mustacchi  * entering the guest.
134165f20420SRobert Mustacchi  *
134265f20420SRobert Mustacchi  * Because this attack takes place in the L1 cache, there's another wrinkle
134365f20420SRobert Mustacchi  * here. The L1 cache is shared between all logical CPUs in a core in most Intel
134465f20420SRobert Mustacchi  * designs. This means that when a thread enters a hardware virtualized context
134565f20420SRobert Mustacchi  * and flushes the L1 data cache, the other thread on the processor may then go
134665f20420SRobert Mustacchi  * ahead and put new data in it that can be potentially attacked. While one
134765f20420SRobert Mustacchi  * solution is to disable SMT on the system, another option that is available is
134865f20420SRobert Mustacchi  * to use a feature for hardware virtualization called 'SMT exclusion'. This
134965f20420SRobert Mustacchi  * goes through and makes sure that if a HVM is being scheduled on one thread,
135065f20420SRobert Mustacchi  * then the thing on the other thread is from the same hardware virtual machine.
135165f20420SRobert Mustacchi  * If an interrupt comes in or the guest exits to the broader system, then the
135265f20420SRobert Mustacchi  * other SMT thread will be kicked out.
135365f20420SRobert Mustacchi  *
135465f20420SRobert Mustacchi  * L1TF can be fully mitigated by hardware. If the RDCL_NO feature is set in the
135565f20420SRobert Mustacchi  * architecture capabilities MSR (MSR_IA32_ARCH_CAPABILITIES), then we will not
135665f20420SRobert Mustacchi  * perform L1TF related mitigations.
135765f20420SRobert Mustacchi  *
135865f20420SRobert Mustacchi  * MICROARCHITECTURAL DATA SAMPLING
135965f20420SRobert Mustacchi  *
136065f20420SRobert Mustacchi  * Microarchitectural data sampling (MDS) is a combination of four discrete
136165f20420SRobert Mustacchi  * vulnerabilities that are similar issues affecting various parts of the CPU's
136265f20420SRobert Mustacchi  * microarchitectural implementation around load, store, and fill buffers.
136365f20420SRobert Mustacchi  * Specifically it is made up of the following subcomponents:
136465f20420SRobert Mustacchi  *
136565f20420SRobert Mustacchi  *  1. Microarchitectural Store Buffer Data Sampling (MSBDS)
136665f20420SRobert Mustacchi  *  2. Microarchitectural Fill Buffer Data Sampling (MFBDS)
136765f20420SRobert Mustacchi  *  3. Microarchitectural Load Port Data Sampling (MLPDS)
136865f20420SRobert Mustacchi  *  4. Microarchitectural Data Sampling Uncacheable Memory (MDSUM)
136965f20420SRobert Mustacchi  *
137065f20420SRobert Mustacchi  * To begin addressing these, Intel has introduced another feature in microcode
137165f20420SRobert Mustacchi  * called MD_CLEAR. This changes the verw instruction to operate in a different
137265f20420SRobert Mustacchi  * way. This allows us to execute the verw instruction in a particular way to
137365f20420SRobert Mustacchi  * flush the state of the affected parts. The L1TF L1D flush mechanism is also
137465f20420SRobert Mustacchi  * updated when this microcode is present to flush this state.
137565f20420SRobert Mustacchi  *
137665f20420SRobert Mustacchi  * Primarily we need to flush this state whenever we transition from the kernel
137765f20420SRobert Mustacchi  * to a less privileged context such as user mode or an HVM guest. MSBDS is a
137865f20420SRobert Mustacchi  * little bit different. Here the structures are statically sized when a logical
137965f20420SRobert Mustacchi  * CPU is in use and resized when it goes to sleep. Therefore, we also need to
138065f20420SRobert Mustacchi  * flush the microarchitectural state before the CPU goes idles by calling hlt,
138165f20420SRobert Mustacchi  * mwait, or another ACPI method. To perform these flushes, we call
138265f20420SRobert Mustacchi  * x86_md_clear() at all of these transition points.
138365f20420SRobert Mustacchi  *
138465f20420SRobert Mustacchi  * If hardware enumerates RDCL_NO, indicating that it is not vulnerable to L1TF,
138565f20420SRobert Mustacchi  * then we change the spec_uarch_flush() function to point to x86_md_clear(). If
138665f20420SRobert Mustacchi  * MDS_NO has been set, then this is fully mitigated and x86_md_clear() becomes
138765f20420SRobert Mustacchi  * a no-op.
138865f20420SRobert Mustacchi  *
138965f20420SRobert Mustacchi  * Unfortunately, with this issue hyperthreading rears its ugly head. In
139065f20420SRobert Mustacchi  * particular, everything we've discussed above is only valid for a single
139165f20420SRobert Mustacchi  * thread executing on a core. In the case where you have hyper-threading
139265f20420SRobert Mustacchi  * present, this attack can be performed between threads. The theoretical fix
139365f20420SRobert Mustacchi  * for this is to ensure that both threads are always in the same security
139465f20420SRobert Mustacchi  * domain. This means that they are executing in the same ring and mutually
139565f20420SRobert Mustacchi  * trust each other. Practically speaking, this would mean that a system call
139665f20420SRobert Mustacchi  * would have to issue an inter-processor interrupt (IPI) to the other thread.
139765f20420SRobert Mustacchi  * Rather than implement this, we recommend that one disables hyper-threading
139865f20420SRobert Mustacchi  * through the use of psradm -aS.
139965f20420SRobert Mustacchi  *
1400e25cb0e7SJohn Levon  * TSX ASYNCHRONOUS ABORT
1401e25cb0e7SJohn Levon  *
1402e25cb0e7SJohn Levon  * TSX Asynchronous Abort (TAA) is another side-channel vulnerability that
1403e25cb0e7SJohn Levon  * behaves like MDS, but leverages Intel's transactional instructions as another
1404e25cb0e7SJohn Levon  * vector. Effectively, when a transaction hits one of these cases (unmapped
1405e25cb0e7SJohn Levon  * page, various cache snoop activity, etc.) then the same data can be exposed
1406e25cb0e7SJohn Levon  * as in the case of MDS. This means that you can attack your twin.
1407e25cb0e7SJohn Levon  *
1408e25cb0e7SJohn Levon  * Intel has described that there are two different ways that we can mitigate
1409e25cb0e7SJohn Levon  * this problem on affected processors:
1410e25cb0e7SJohn Levon  *
1411e25cb0e7SJohn Levon  *   1) We can use the same techniques used to deal with MDS. Flushing the
1412e25cb0e7SJohn Levon  *      microarchitectural buffers and disabling hyperthreading will mitigate
1413e25cb0e7SJohn Levon  *      this in the same way.
1414e25cb0e7SJohn Levon  *
1415e25cb0e7SJohn Levon  *   2) Using microcode to disable TSX.
1416e25cb0e7SJohn Levon  *
1417e25cb0e7SJohn Levon  * Now, most processors that are subject to MDS (as in they don't have MDS_NO in
1418e25cb0e7SJohn Levon  * the IA32_ARCH_CAPABILITIES MSR) will not receive microcode to disable TSX.
1419e25cb0e7SJohn Levon  * That's OK as we're already doing all such mitigations. On the other hand,
1420e25cb0e7SJohn Levon  * processors with MDS_NO are all supposed to receive microcode updates that
1421e25cb0e7SJohn Levon  * enumerate support for disabling TSX. In general, we'd rather use this method
1422e25cb0e7SJohn Levon  * when available as it doesn't require disabling hyperthreading to be
1423e25cb0e7SJohn Levon  * effective. Currently we basically are relying on microcode for processors
1424e25cb0e7SJohn Levon  * that enumerate MDS_NO.
1425e25cb0e7SJohn Levon  *
1426*5cd084edSDan McDonald  * Another MDS-variant in a few select Intel Atom CPUs is Register File Data
1427*5cd084edSDan McDonald  * Sampling: RFDS. This allows an attacker to sample values that were in any
1428*5cd084edSDan McDonald  * of integer, floating point, or vector registers. This was discovered by
1429*5cd084edSDan McDonald  * Intel during internal validation work.  The existence of the RFDS_NO
1430*5cd084edSDan McDonald  * capability, or the LACK of a RFDS_CLEAR capability, means we do not have to
1431*5cd084edSDan McDonald  * act. Intel has said some CPU models immune to RFDS MAY NOT enumerate
1432*5cd084edSDan McDonald  * RFDS_NO. If RFDS_NO is not set, but RFDS_CLEAR is, we must set x86_md_clear,
1433*5cd084edSDan McDonald  * and make sure it's using VERW. Unlike MDS, RFDS can't be helped by the
1434*5cd084edSDan McDonald  * MSR that L1D uses.
1435*5cd084edSDan McDonald  *
1436e25cb0e7SJohn Levon  * The microcode features are enumerated as part of the IA32_ARCH_CAPABILITIES.
1437e25cb0e7SJohn Levon  * When bit 7 (IA32_ARCH_CAP_TSX_CTRL) is present, then we are given two
1438e25cb0e7SJohn Levon  * different powers. The first allows us to cause all transactions to
1439e25cb0e7SJohn Levon  * immediately abort. The second gives us a means of disabling TSX completely,
1440e25cb0e7SJohn Levon  * which includes removing it from cpuid. If we have support for this in
1441e25cb0e7SJohn Levon  * microcode during the first cpuid pass, then we'll disable TSX completely such
1442e25cb0e7SJohn Levon  * that user land never has a chance to observe the bit. However, if we are late
1443e25cb0e7SJohn Levon  * loading the microcode, then we must use the functionality to cause
1444e25cb0e7SJohn Levon  * transactions to automatically abort. This is necessary for user land's sake.
1445e25cb0e7SJohn Levon  * Once a program sees a cpuid bit, it must not be taken away.
1446e25cb0e7SJohn Levon  *
1447e25cb0e7SJohn Levon  * We track whether or not we should do this based on what cpuid pass we're in.
1448e25cb0e7SJohn Levon  * Whenever we hit cpuid_scan_security() on the boot CPU and we're still on pass
1449e25cb0e7SJohn Levon  * 1 of the cpuid logic, then we can completely turn off TSX. Notably this
1450ab5bb018SKeith M Wesolowski  * should happen twice. Once in the normal cpuid_pass_basic() code and then a
1451ab5bb018SKeith M Wesolowski  * second time after we do the initial microcode update.  As a result we need to
1452ab5bb018SKeith M Wesolowski  * be careful in cpuid_apply_tsx() to only use the MSR if we've loaded a
1453ab5bb018SKeith M Wesolowski  * suitable microcode on the current CPU (which happens prior to
1454ab5bb018SKeith M Wesolowski  * cpuid_pass_ucode()).
1455e25cb0e7SJohn Levon  *
1456e25cb0e7SJohn Levon  * If TAA has been fixed, then it will be enumerated in IA32_ARCH_CAPABILITIES
1457e25cb0e7SJohn Levon  * as TAA_NO. In such a case, we will still disable TSX: it's proven to be an
1458e25cb0e7SJohn Levon  * unfortunate feature in a number of ways, and taking the opportunity to
1459e25cb0e7SJohn Levon  * finally be able to turn it off is likely to be of benefit in the future.
1460e25cb0e7SJohn Levon  *
146165f20420SRobert Mustacchi  * SUMMARY
146265f20420SRobert Mustacchi  *
146365f20420SRobert Mustacchi  * The following table attempts to summarize the mitigations for various issues
146465f20420SRobert Mustacchi  * and what's done in various places:
146565f20420SRobert Mustacchi  *
146665f20420SRobert Mustacchi  *  - Spectre v1: Not currently mitigated
146742cd1931SJohn Levon  *  - swapgs: lfences after swapgs paths
1468651a12cbSRobert Mustacchi  *  - Spectre v2: Retpolines/RSB Stuffing or eIBRS/AIBRS if HW support
146965f20420SRobert Mustacchi  *  - Meltdown: Kernel Page Table Isolation
147065f20420SRobert Mustacchi  *  - Spectre v3a: Updated CPU microcode
147165f20420SRobert Mustacchi  *  - Spectre v4: Not currently mitigated
147265f20420SRobert Mustacchi  *  - SpectreRSB: SMEP and RSB Stuffing
147342cd1931SJohn Levon  *  - L1TF: spec_uarch_flush, SMT exclusion, requires microcode
1474e25cb0e7SJohn Levon  *  - MDS: x86_md_clear, requires microcode, disabling SMT
1475e25cb0e7SJohn Levon  *  - TAA: x86_md_clear and disabling SMT OR microcode and disabling TSX
1476*5cd084edSDan McDonald  *  - RFDS: microcode with x86_md_clear if RFDS_CLEAR set and RFDS_NO not.
147765f20420SRobert Mustacchi  *
147865f20420SRobert Mustacchi  * The following table indicates the x86 feature set bits that indicate that a
147965f20420SRobert Mustacchi  * given problem has been solved or a notable feature is present:
148065f20420SRobert Mustacchi  *
148165f20420SRobert Mustacchi  *  - RDCL_NO: Meltdown, L1TF, MSBDS subset of MDS
148265f20420SRobert Mustacchi  *  - MDS_NO: All forms of MDS
1483e25cb0e7SJohn Levon  *  - TAA_NO: TAA
1484*5cd084edSDan McDonald  *  - RFDS_NO: RFDS
14857c478bd9Sstevel@tonic-gate  */
14867c478bd9Sstevel@tonic-gate 
14877c478bd9Sstevel@tonic-gate #include <sys/types.h>
14887c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
14897c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h>
14907c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
14917c478bd9Sstevel@tonic-gate #include <sys/systm.h>
14927c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
14937c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
14947c478bd9Sstevel@tonic-gate #include <sys/sunndi.h>
14957c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
14967c478bd9Sstevel@tonic-gate #include <sys/processor.h>
14975b8a6efeSbholler #include <sys/sysmacros.h>
1498fb2f18f8Sesaxe #include <sys/pg.h>
14997c478bd9Sstevel@tonic-gate #include <sys/fp.h>
15007c478bd9Sstevel@tonic-gate #include <sys/controlregs.h>
15017c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
1502dfea898aSKuriakose Kuruvilla #include <sys/auxv_386.h>
15037c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
15048031591dSSrihari Venkatesan #include <sys/pci_cfgspace.h>
15052428aad8SPatrick Mooney #include <sys/comm_page.h>
150674ecdb51SJohn Levon #include <sys/mach_mmu.h>
150701add34aSRobert Mustacchi #include <sys/ucode.h>
15082428aad8SPatrick Mooney #include <sys/tsc.h>
150965f20420SRobert Mustacchi #include <sys/kobj.h>
151065f20420SRobert Mustacchi #include <sys/asm_misc.h>
1511dd23d762SRobert Mustacchi #include <sys/bitmap.h>
15127c478bd9Sstevel@tonic-gate 
1513e4b86885SCheng Sean Ye #ifdef __xpv
1514e4b86885SCheng Sean Ye #include <sys/hypervisor.h>
1515e774b42bSBill Holler #else
1516e774b42bSBill Holler #include <sys/ontrap.h>
1517e4b86885SCheng Sean Ye #endif
1518e4b86885SCheng Sean Ye 
15197c478bd9Sstevel@tonic-gate uint_t x86_vendor = X86_VENDOR_IntelClone;
15207c478bd9Sstevel@tonic-gate uint_t x86_type = X86_TYPE_OTHER;
152186c1f4dcSVikram Hegde uint_t x86_clflush_size = 0;
15227c478bd9Sstevel@tonic-gate 
152374ecdb51SJohn Levon #if defined(__xpv)
152474ecdb51SJohn Levon int x86_use_pcid = 0;
152574ecdb51SJohn Levon int x86_use_invpcid = 0;
152674ecdb51SJohn Levon #else
152774ecdb51SJohn Levon int x86_use_pcid = -1;
152874ecdb51SJohn Levon int x86_use_invpcid = -1;
152974ecdb51SJohn Levon #endif
153074ecdb51SJohn Levon 
153165f20420SRobert Mustacchi typedef enum {
153265f20420SRobert Mustacchi 	X86_SPECTREV2_RETPOLINE,
153365f20420SRobert Mustacchi 	X86_SPECTREV2_ENHANCED_IBRS,
1534651a12cbSRobert Mustacchi 	X86_SPECTREV2_AUTO_IBRS,
153565f20420SRobert Mustacchi 	X86_SPECTREV2_DISABLED
153665f20420SRobert Mustacchi } x86_spectrev2_mitigation_t;
153765f20420SRobert Mustacchi 
153865f20420SRobert Mustacchi uint_t x86_disable_spectrev2 = 0;
153965f20420SRobert Mustacchi static x86_spectrev2_mitigation_t x86_spectrev2_mitigation =
154065f20420SRobert Mustacchi     X86_SPECTREV2_RETPOLINE;
154165f20420SRobert Mustacchi 
1542e25cb0e7SJohn Levon /*
1543e25cb0e7SJohn Levon  * The mitigation status for TAA:
1544e25cb0e7SJohn Levon  * X86_TAA_NOTHING -- no mitigation available for TAA side-channels
1545e25cb0e7SJohn Levon  * X86_TAA_DISABLED -- mitigation disabled via x86_disable_taa
1546e25cb0e7SJohn Levon  * X86_TAA_MD_CLEAR -- MDS mitigation also suffices for TAA
1547e25cb0e7SJohn Levon  * X86_TAA_TSX_FORCE_ABORT -- transactions are forced to abort
1548e25cb0e7SJohn Levon  * X86_TAA_TSX_DISABLE -- force abort transactions and hide from CPUID
1549e25cb0e7SJohn Levon  * X86_TAA_HW_MITIGATED -- TSX potentially active but H/W not TAA-vulnerable
1550e25cb0e7SJohn Levon  */
1551e25cb0e7SJohn Levon typedef enum {
1552e25cb0e7SJohn Levon 	X86_TAA_NOTHING,
1553e25cb0e7SJohn Levon 	X86_TAA_DISABLED,
1554e25cb0e7SJohn Levon 	X86_TAA_MD_CLEAR,
1555e25cb0e7SJohn Levon 	X86_TAA_TSX_FORCE_ABORT,
1556e25cb0e7SJohn Levon 	X86_TAA_TSX_DISABLE,
1557e25cb0e7SJohn Levon 	X86_TAA_HW_MITIGATED
1558e25cb0e7SJohn Levon } x86_taa_mitigation_t;
1559e25cb0e7SJohn Levon 
1560e25cb0e7SJohn Levon uint_t x86_disable_taa = 0;
1561e25cb0e7SJohn Levon static x86_taa_mitigation_t x86_taa_mitigation = X86_TAA_NOTHING;
1562e25cb0e7SJohn Levon 
15637c478bd9Sstevel@tonic-gate uint_t pentiumpro_bug4046376;
15647c478bd9Sstevel@tonic-gate 
1565dfea898aSKuriakose Kuruvilla uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
15667417cfdeSKuriakose Kuruvilla 
1567dfea898aSKuriakose Kuruvilla static char *x86_feature_names[NUM_X86_FEATURES] = {
15687417cfdeSKuriakose Kuruvilla 	"lgpg",
15697417cfdeSKuriakose Kuruvilla 	"tsc",
15707417cfdeSKuriakose Kuruvilla 	"msr",
15717417cfdeSKuriakose Kuruvilla 	"mtrr",
15727417cfdeSKuriakose Kuruvilla 	"pge",
15737417cfdeSKuriakose Kuruvilla 	"de",
15747417cfdeSKuriakose Kuruvilla 	"cmov",
15757417cfdeSKuriakose Kuruvilla 	"mmx",
15767417cfdeSKuriakose Kuruvilla 	"mca",
15777417cfdeSKuriakose Kuruvilla 	"pae",
15787417cfdeSKuriakose Kuruvilla 	"cv8",
15797417cfdeSKuriakose Kuruvilla 	"pat",
15807417cfdeSKuriakose Kuruvilla 	"sep",
15817417cfdeSKuriakose Kuruvilla 	"sse",
15827417cfdeSKuriakose Kuruvilla 	"sse2",
15837417cfdeSKuriakose Kuruvilla 	"htt",
15847417cfdeSKuriakose Kuruvilla 	"asysc",
15857417cfdeSKuriakose Kuruvilla 	"nx",
15867417cfdeSKuriakose Kuruvilla 	"sse3",
15877417cfdeSKuriakose Kuruvilla 	"cx16",
15887417cfdeSKuriakose Kuruvilla 	"cmp",
15897417cfdeSKuriakose Kuruvilla 	"tscp",
15907417cfdeSKuriakose Kuruvilla 	"mwait",
15917417cfdeSKuriakose Kuruvilla 	"sse4a",
15927417cfdeSKuriakose Kuruvilla 	"cpuid",
15937417cfdeSKuriakose Kuruvilla 	"ssse3",
15947417cfdeSKuriakose Kuruvilla 	"sse4_1",
15957417cfdeSKuriakose Kuruvilla 	"sse4_2",
15967417cfdeSKuriakose Kuruvilla 	"1gpg",
15977417cfdeSKuriakose Kuruvilla 	"clfsh",
15987417cfdeSKuriakose Kuruvilla 	"64",
15997417cfdeSKuriakose Kuruvilla 	"aes",
16007af88ac7SKuriakose Kuruvilla 	"pclmulqdq",
16017af88ac7SKuriakose Kuruvilla 	"xsave",
1602faa20166SBryan Cantrill 	"avx",
1603faa20166SBryan Cantrill 	"vmx",
16047660e73fSHans Rosenfeld 	"svm",
1605ebb8ac07SRobert Mustacchi 	"topoext",
1606ebb8ac07SRobert Mustacchi 	"f16c",
16076eedf6a5SJosef 'Jeff' Sipek 	"rdrand",
16086eedf6a5SJosef 'Jeff' Sipek 	"x2apic",
1609245ac945SRobert Mustacchi 	"avx2",
1610245ac945SRobert Mustacchi 	"bmi1",
1611245ac945SRobert Mustacchi 	"bmi2",
1612799823bbSRobert Mustacchi 	"fma",
16133ce2fcdcSRobert Mustacchi 	"smep",
16148889c875SRobert Mustacchi 	"smap",
16158889c875SRobert Mustacchi 	"adx",
1616088d69f8SJerry Jelinek 	"rdseed",
1617088d69f8SJerry Jelinek 	"mpx",
1618088d69f8SJerry Jelinek 	"avx512f",
1619088d69f8SJerry Jelinek 	"avx512dq",
1620088d69f8SJerry Jelinek 	"avx512pf",
1621088d69f8SJerry Jelinek 	"avx512er",
1622088d69f8SJerry Jelinek 	"avx512cd",
1623088d69f8SJerry Jelinek 	"avx512bw",
1624088d69f8SJerry Jelinek 	"avx512vl",
1625088d69f8SJerry Jelinek 	"avx512fma",
1626088d69f8SJerry Jelinek 	"avx512vbmi",
1627088d69f8SJerry Jelinek 	"avx512_vpopcntdq",
1628088d69f8SJerry Jelinek 	"avx512_4vnniw",
1629088d69f8SJerry Jelinek 	"avx512_4fmaps",
1630088d69f8SJerry Jelinek 	"xsaveopt",
1631088d69f8SJerry Jelinek 	"xsavec",
1632088d69f8SJerry Jelinek 	"xsaves",
1633088d69f8SJerry Jelinek 	"sha",
1634088d69f8SJerry Jelinek 	"umip",
1635088d69f8SJerry Jelinek 	"pku",
1636088d69f8SJerry Jelinek 	"ospke",
163774ecdb51SJohn Levon 	"pcid",
163874ecdb51SJohn Levon 	"invpcid",
163901add34aSRobert Mustacchi 	"ibrs",
164001add34aSRobert Mustacchi 	"ibpb",
164101add34aSRobert Mustacchi 	"stibp",
164201add34aSRobert Mustacchi 	"ssbd",
164301add34aSRobert Mustacchi 	"ssbd_virt",
164401add34aSRobert Mustacchi 	"rdcl_no",
164501add34aSRobert Mustacchi 	"ibrs_all",
164601add34aSRobert Mustacchi 	"rsba",
164701add34aSRobert Mustacchi 	"ssb_no",
1648c7749d0fSJohn Levon 	"stibp_all",
1649c7749d0fSJohn Levon 	"flush_cmd",
1650cff040f3SRobert Mustacchi 	"l1d_vmentry_no",
1651cff040f3SRobert Mustacchi 	"fsgsbase",
1652cff040f3SRobert Mustacchi 	"clflushopt",
1653cff040f3SRobert Mustacchi 	"clwb",
1654cff040f3SRobert Mustacchi 	"monitorx",
1655cff040f3SRobert Mustacchi 	"clzero",
1656cff040f3SRobert Mustacchi 	"xop",
1657cff040f3SRobert Mustacchi 	"fma4",
1658e4f6ce70SRobert Mustacchi 	"tbm",
1659d0e58ef5SRobert Mustacchi 	"avx512_vnni",
1660a9cc46cfSRobert Mustacchi 	"amd_pcec",
16613d21c6bfSRobert Mustacchi 	"md_clear",
1662f2dbfd32SRobert Mustacchi 	"mds_no",
1663f2dbfd32SRobert Mustacchi 	"core_thermal",
1664e25cb0e7SJohn Levon 	"pkg_thermal",
1665e25cb0e7SJohn Levon 	"tsx_ctrl",
1666a47ab03eSRobert Mustacchi 	"taa_no",
16675edbd2feSRobert Mustacchi 	"ppin",
16685edbd2feSRobert Mustacchi 	"vaes",
1669beed421eSPatrick Mooney 	"vpclmulqdq",
167056726c7eSRobert Mustacchi 	"lfence_serializing",
167156726c7eSRobert Mustacchi 	"gfni",
167256726c7eSRobert Mustacchi 	"avx512_vp2intersect",
167356726c7eSRobert Mustacchi 	"avx512_bitalg",
167456726c7eSRobert Mustacchi 	"avx512_vbmi2",
1675651a12cbSRobert Mustacchi 	"avx512_bf16",
1676*5cd084edSDan McDonald 	"auto_ibrs",
1677*5cd084edSDan McDonald 	"rfds_no",
1678*5cd084edSDan McDonald 	"rfds_clear"
1679faa20166SBryan Cantrill };
16807417cfdeSKuriakose Kuruvilla 
16817417cfdeSKuriakose Kuruvilla boolean_t
is_x86_feature(void * featureset,uint_t feature)16827417cfdeSKuriakose Kuruvilla is_x86_feature(void *featureset, uint_t feature)
16837417cfdeSKuriakose Kuruvilla {
16847417cfdeSKuriakose Kuruvilla 	ASSERT(feature < NUM_X86_FEATURES);
16857417cfdeSKuriakose Kuruvilla 	return (BT_TEST((ulong_t *)featureset, feature));
16867417cfdeSKuriakose Kuruvilla }
16877417cfdeSKuriakose Kuruvilla 
16887417cfdeSKuriakose Kuruvilla void
add_x86_feature(void * featureset,uint_t feature)16897417cfdeSKuriakose Kuruvilla add_x86_feature(void *featureset, uint_t feature)
16907417cfdeSKuriakose Kuruvilla {
16917417cfdeSKuriakose Kuruvilla 	ASSERT(feature < NUM_X86_FEATURES);
16927417cfdeSKuriakose Kuruvilla 	BT_SET((ulong_t *)featureset, feature);
16937417cfdeSKuriakose Kuruvilla }
16947417cfdeSKuriakose Kuruvilla 
16957417cfdeSKuriakose Kuruvilla void
remove_x86_feature(void * featureset,uint_t feature)16967417cfdeSKuriakose Kuruvilla remove_x86_feature(void *featureset, uint_t feature)
16977417cfdeSKuriakose Kuruvilla {
16987417cfdeSKuriakose Kuruvilla 	ASSERT(feature < NUM_X86_FEATURES);
16997417cfdeSKuriakose Kuruvilla 	BT_CLEAR((ulong_t *)featureset, feature);
17007417cfdeSKuriakose Kuruvilla }
17017417cfdeSKuriakose Kuruvilla 
17027417cfdeSKuriakose Kuruvilla boolean_t
compare_x86_featureset(void * setA,void * setB)17037417cfdeSKuriakose Kuruvilla compare_x86_featureset(void *setA, void *setB)
17047417cfdeSKuriakose Kuruvilla {
17057417cfdeSKuriakose Kuruvilla 	/*
17067417cfdeSKuriakose Kuruvilla 	 * We assume that the unused bits of the bitmap are always zero.
17077417cfdeSKuriakose Kuruvilla 	 */
17087417cfdeSKuriakose Kuruvilla 	if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
17097417cfdeSKuriakose Kuruvilla 		return (B_TRUE);
17107417cfdeSKuriakose Kuruvilla 	} else {
17117417cfdeSKuriakose Kuruvilla 		return (B_FALSE);
17127417cfdeSKuriakose Kuruvilla 	}
17137417cfdeSKuriakose Kuruvilla }
17147417cfdeSKuriakose Kuruvilla 
17157417cfdeSKuriakose Kuruvilla void
print_x86_featureset(void * featureset)17167417cfdeSKuriakose Kuruvilla print_x86_featureset(void *featureset)
17177417cfdeSKuriakose Kuruvilla {
17187417cfdeSKuriakose Kuruvilla 	uint_t i;
17197417cfdeSKuriakose Kuruvilla 
17207417cfdeSKuriakose Kuruvilla 	for (i = 0; i < NUM_X86_FEATURES; i++) {
17217417cfdeSKuriakose Kuruvilla 		if (is_x86_feature(featureset, i)) {
17227417cfdeSKuriakose Kuruvilla 			cmn_err(CE_CONT, "?x86_feature: %s\n",
17237417cfdeSKuriakose Kuruvilla 			    x86_feature_names[i]);
17247417cfdeSKuriakose Kuruvilla 		}
17257417cfdeSKuriakose Kuruvilla 	}
17267417cfdeSKuriakose Kuruvilla }
17277417cfdeSKuriakose Kuruvilla 
1728088d69f8SJerry Jelinek /* Note: This is the maximum size for the CPU, not the size of the structure. */
17297af88ac7SKuriakose Kuruvilla static size_t xsave_state_size = 0;
17307af88ac7SKuriakose Kuruvilla uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
17317af88ac7SKuriakose Kuruvilla boolean_t xsave_force_disable = B_FALSE;
17323ce2fcdcSRobert Mustacchi extern int disable_smap;
17337af88ac7SKuriakose Kuruvilla 
17347997e108SSurya Prakki /*
173579ec9da8SYuri Pankov  * This is set to platform type we are running on.
17367997e108SSurya Prakki  */
1737349b53ddSStuart Maybee static int platform_type = -1;
1738349b53ddSStuart Maybee 
1739349b53ddSStuart Maybee #if !defined(__xpv)
1740349b53ddSStuart Maybee /*
1741349b53ddSStuart Maybee  * Variable to patch if hypervisor platform detection needs to be
1742349b53ddSStuart Maybee  * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
1743349b53ddSStuart Maybee  */
1744349b53ddSStuart Maybee int enable_platform_detection = 1;
1745349b53ddSStuart Maybee #endif
17467c478bd9Sstevel@tonic-gate 
1747f98fbcecSbholler /*
1748f98fbcecSbholler  * monitor/mwait info.
17495b8a6efeSbholler  *
17505b8a6efeSbholler  * size_actual and buf_actual are the real address and size allocated to get
17515b8a6efeSbholler  * proper mwait_buf alignement.  buf_actual and size_actual should be passed
17525b8a6efeSbholler  * to kmem_free().  Currently kmem_alloc() and mwait happen to both use
17535b8a6efeSbholler  * processor cache-line alignment, but this is not guarantied in the furture.
1754f98fbcecSbholler  */
1755f98fbcecSbholler struct mwait_info {
1756f98fbcecSbholler 	size_t		mon_min;	/* min size to avoid missed wakeups */
1757f98fbcecSbholler 	size_t		mon_max;	/* size to avoid false wakeups */
17585b8a6efeSbholler 	size_t		size_actual;	/* size actually allocated */
17595b8a6efeSbholler 	void		*buf_actual;	/* memory actually allocated */
1760f98fbcecSbholler 	uint32_t	support;	/* processor support of monitor/mwait */
1761f98fbcecSbholler };
1762f98fbcecSbholler 
17637af88ac7SKuriakose Kuruvilla /*
17647af88ac7SKuriakose Kuruvilla  * xsave/xrestor info.
17657af88ac7SKuriakose Kuruvilla  *
1766088d69f8SJerry Jelinek  * This structure contains HW feature bits and the size of the xsave save area.
1767088d69f8SJerry Jelinek  * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
1768088d69f8SJerry Jelinek  * (xsave_state) to describe the xsave layout. However, at runtime the
1769088d69f8SJerry Jelinek  * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
1770088d69f8SJerry Jelinek  * xsave_state structure simply represents the legacy layout of the beginning
1771088d69f8SJerry Jelinek  * of the xsave area.
17727af88ac7SKuriakose Kuruvilla  */
17737af88ac7SKuriakose Kuruvilla struct xsave_info {
17747af88ac7SKuriakose Kuruvilla 	uint32_t	xsav_hw_features_low;   /* Supported HW features */
17757af88ac7SKuriakose Kuruvilla 	uint32_t	xsav_hw_features_high;  /* Supported HW features */
17767af88ac7SKuriakose Kuruvilla 	size_t		xsav_max_size;  /* max size save area for HW features */
17777af88ac7SKuriakose Kuruvilla 	size_t		ymm_size;	/* AVX: size of ymm save area */
17787af88ac7SKuriakose Kuruvilla 	size_t		ymm_offset;	/* AVX: offset for ymm save area */
1779088d69f8SJerry Jelinek 	size_t		bndregs_size;	/* MPX: size of bndregs save area */
1780088d69f8SJerry Jelinek 	size_t		bndregs_offset;	/* MPX: offset for bndregs save area */
1781088d69f8SJerry Jelinek 	size_t		bndcsr_size;	/* MPX: size of bndcsr save area */
1782088d69f8SJerry Jelinek 	size_t		bndcsr_offset;	/* MPX: offset for bndcsr save area */
1783088d69f8SJerry Jelinek 	size_t		opmask_size;	/* AVX512: size of opmask save */
1784088d69f8SJerry Jelinek 	size_t		opmask_offset;	/* AVX512: offset for opmask save */
1785088d69f8SJerry Jelinek 	size_t		zmmlo_size;	/* AVX512: size of zmm 256 save */
1786088d69f8SJerry Jelinek 	size_t		zmmlo_offset;	/* AVX512: offset for zmm 256 save */
1787088d69f8SJerry Jelinek 	size_t		zmmhi_size;	/* AVX512: size of zmm hi reg save */
1788088d69f8SJerry Jelinek 	size_t		zmmhi_offset;	/* AVX512: offset for zmm hi reg save */
17897af88ac7SKuriakose Kuruvilla };
17907af88ac7SKuriakose Kuruvilla 
17917af88ac7SKuriakose Kuruvilla 
17927c478bd9Sstevel@tonic-gate /*
17937c478bd9Sstevel@tonic-gate  * These constants determine how many of the elements of the
17947c478bd9Sstevel@tonic-gate  * cpuid we cache in the cpuid_info data structure; the
17957c478bd9Sstevel@tonic-gate  * remaining elements are accessible via the cpuid instruction.
17967c478bd9Sstevel@tonic-gate  */
17977c478bd9Sstevel@tonic-gate 
1798245ac945SRobert Mustacchi #define	NMAX_CPI_STD	8		/* eax = 0 .. 7 */
1799651a12cbSRobert Mustacchi #define	NMAX_CPI_EXTD	0x22		/* eax = 0x80000000 .. 0x80000021 */
1800dd23d762SRobert Mustacchi #define	NMAX_CPI_TOPO	0x10		/* Sanity check on leaf 8X26, 1F */
18018031591dSSrihari Venkatesan 
18028031591dSSrihari Venkatesan /*
18030ce813ffSRobert Mustacchi  * See the big theory statement for a more detailed explanation of what some of
18040ce813ffSRobert Mustacchi  * these members mean.
18058031591dSSrihari Venkatesan  */
18067c478bd9Sstevel@tonic-gate struct cpuid_info {
18077c478bd9Sstevel@tonic-gate 	uint_t cpi_pass;		/* last pass completed */
18087c478bd9Sstevel@tonic-gate 	/*
18097c478bd9Sstevel@tonic-gate 	 * standard function information
18107c478bd9Sstevel@tonic-gate 	 */
18117c478bd9Sstevel@tonic-gate 	uint_t cpi_maxeax;		/* fn 0: %eax */
18127c478bd9Sstevel@tonic-gate 	char cpi_vendorstr[13];		/* fn 0: %ebx:%ecx:%edx */
18137c478bd9Sstevel@tonic-gate 	uint_t cpi_vendor;		/* enum of cpi_vendorstr */
18147c478bd9Sstevel@tonic-gate 
18157c478bd9Sstevel@tonic-gate 	uint_t cpi_family;		/* fn 1: extended family */
18167c478bd9Sstevel@tonic-gate 	uint_t cpi_model;		/* fn 1: extended model */
18177c478bd9Sstevel@tonic-gate 	uint_t cpi_step;		/* fn 1: stepping */
18188031591dSSrihari Venkatesan 	chipid_t cpi_chipid;		/* fn 1: %ebx:  Intel: chip # */
18198031591dSSrihari Venkatesan 					/*		AMD: package/socket # */
18207c478bd9Sstevel@tonic-gate 	uint_t cpi_brandid;		/* fn 1: %ebx: brand ID */
18217c478bd9Sstevel@tonic-gate 	int cpi_clogid;			/* fn 1: %ebx: thread # */
18228949bcd6Sandrei 	uint_t cpi_ncpu_per_chip;	/* fn 1: %ebx: logical cpu count */
18237c478bd9Sstevel@tonic-gate 	uint8_t cpi_cacheinfo[16];	/* fn 2: intel-style cache desc */
18247c478bd9Sstevel@tonic-gate 	uint_t cpi_ncache;		/* fn 2: number of elements */
1825d129bde2Sesaxe 	uint_t cpi_ncpu_shr_last_cache;	/* fn 4: %eax: ncpus sharing cache */
1826d129bde2Sesaxe 	id_t cpi_last_lvl_cacheid;	/* fn 4: %eax: derived cache id */
18270ce813ffSRobert Mustacchi 	uint_t cpi_cache_leaf_size;	/* Number of cache elements */
18280ce813ffSRobert Mustacchi 					/* Intel fn: 4, AMD fn: 8000001d */
1829dd23d762SRobert Mustacchi 	struct cpuid_regs **cpi_cache_leaves;	/* Actual leaves from above */
1830245ac945SRobert Mustacchi 	struct cpuid_regs cpi_std[NMAX_CPI_STD];	/* 0 .. 7 */
183156726c7eSRobert Mustacchi 	struct cpuid_regs cpi_sub7[1];	/* Leaf 7, sub-leaf 1 */
18327c478bd9Sstevel@tonic-gate 	/*
18337c478bd9Sstevel@tonic-gate 	 * extended function information
18347c478bd9Sstevel@tonic-gate 	 */
18357c478bd9Sstevel@tonic-gate 	uint_t cpi_xmaxeax;		/* fn 0x80000000: %eax */
18367c478bd9Sstevel@tonic-gate 	char cpi_brandstr[49];		/* fn 0x8000000[234] */
18377c478bd9Sstevel@tonic-gate 	uint8_t cpi_pabits;		/* fn 0x80000006: %eax */
18388031591dSSrihari Venkatesan 	uint8_t	cpi_vabits;		/* fn 0x80000006: %eax */
1839088d69f8SJerry Jelinek 	uint8_t cpi_fp_amd_save;	/* AMD: FP error pointer save rqd. */
18408031591dSSrihari Venkatesan 	struct	cpuid_regs cpi_extd[NMAX_CPI_EXTD];	/* 0x800000XX */
18418031591dSSrihari Venkatesan 
184210569901Sgavinm 	id_t cpi_coreid;		/* same coreid => strands share core */
184310569901Sgavinm 	int cpi_pkgcoreid;		/* core number within single package */
18448949bcd6Sandrei 	uint_t cpi_ncore_per_chip;	/* AMD: fn 0x80000008: %ecx[7-0] */
18458949bcd6Sandrei 					/* Intel: fn 4: %eax[31-26] */
1846d6517bbdSRobert Mustacchi 
1847d6517bbdSRobert Mustacchi 	/*
1848d6517bbdSRobert Mustacchi 	 * These values represent the number of bits that are required to store
1849d6517bbdSRobert Mustacchi 	 * information about the number of cores and threads.
1850d6517bbdSRobert Mustacchi 	 */
1851d6517bbdSRobert Mustacchi 	uint_t cpi_ncore_bits;
1852d6517bbdSRobert Mustacchi 	uint_t cpi_nthread_bits;
18537c478bd9Sstevel@tonic-gate 	/*
18547c478bd9Sstevel@tonic-gate 	 * supported feature information
18557c478bd9Sstevel@tonic-gate 	 */
1856245ac945SRobert Mustacchi 	uint32_t cpi_support[6];
18577c478bd9Sstevel@tonic-gate #define	STD_EDX_FEATURES	0
18587c478bd9Sstevel@tonic-gate #define	AMD_EDX_FEATURES	1
18597c478bd9Sstevel@tonic-gate #define	TM_EDX_FEATURES		2
18607c478bd9Sstevel@tonic-gate #define	STD_ECX_FEATURES	3
1861ae115bc7Smrj #define	AMD_ECX_FEATURES	4
1862245ac945SRobert Mustacchi #define	STD_EBX_FEATURES	5
18638a40a695Sgavinm 	/*
18648a40a695Sgavinm 	 * Synthesized information, where known.
18658a40a695Sgavinm 	 */
186622e4c3acSKeith M Wesolowski 	x86_chiprev_t cpi_chiprev;	/* See X86_CHIPREV_* in x86_archext.h */
18678a40a695Sgavinm 	const char *cpi_chiprevstr;	/* May be NULL if chiprev unknown */
18688a40a695Sgavinm 	uint32_t cpi_socket;		/* Chip package/socket type */
186922e4c3acSKeith M Wesolowski 	x86_uarchrev_t cpi_uarchrev;	/* Microarchitecture and revision */
1870f98fbcecSbholler 
1871f98fbcecSbholler 	struct mwait_info cpi_mwait;	/* fn 5: monitor/mwait info */
1872b6917abeSmishra 	uint32_t cpi_apicid;
18738031591dSSrihari Venkatesan 	uint_t cpi_procnodeid;		/* AMD: nodeID on HT, Intel: chipid */
18748031591dSSrihari Venkatesan 	uint_t cpi_procnodes_per_pkg;	/* AMD: # of nodes in the package */
18758031591dSSrihari Venkatesan 					/* Intel: 1 */
18767660e73fSHans Rosenfeld 	uint_t cpi_compunitid;		/* AMD: ComputeUnit ID, Intel: coreid */
18777660e73fSHans Rosenfeld 	uint_t cpi_cores_per_compunit;	/* AMD: # of cores in the ComputeUnit */
18787af88ac7SKuriakose Kuruvilla 
18797af88ac7SKuriakose Kuruvilla 	struct xsave_info cpi_xsave;	/* fn D: xsave/xrestor info */
1880dd23d762SRobert Mustacchi 
1881dd23d762SRobert Mustacchi 	/*
1882dd23d762SRobert Mustacchi 	 * AMD and Intel extended topology information. Leaf 8X26 (AMD) and
1883dd23d762SRobert Mustacchi 	 * eventually leaf 0x1F (Intel).
1884dd23d762SRobert Mustacchi 	 */
1885dd23d762SRobert Mustacchi 	uint_t cpi_topo_nleaves;
1886dd23d762SRobert Mustacchi 	struct cpuid_regs cpi_topo[NMAX_CPI_TOPO];
18877c478bd9Sstevel@tonic-gate };
18887c478bd9Sstevel@tonic-gate 
18897c478bd9Sstevel@tonic-gate 
18907c478bd9Sstevel@tonic-gate static struct cpuid_info cpuid_info0;
18917c478bd9Sstevel@tonic-gate 
18927c478bd9Sstevel@tonic-gate /*
18937c478bd9Sstevel@tonic-gate  * These bit fields are defined by the Intel Application Note AP-485
18947c478bd9Sstevel@tonic-gate  * "Intel Processor Identification and the CPUID Instruction"
18957c478bd9Sstevel@tonic-gate  */
18967c478bd9Sstevel@tonic-gate #define	CPI_FAMILY_XTD(cpi)	BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
18977c478bd9Sstevel@tonic-gate #define	CPI_MODEL_XTD(cpi)	BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
18987c478bd9Sstevel@tonic-gate #define	CPI_TYPE(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
18997c478bd9Sstevel@tonic-gate #define	CPI_FAMILY(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
19007c478bd9Sstevel@tonic-gate #define	CPI_STEP(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
19017c478bd9Sstevel@tonic-gate #define	CPI_MODEL(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 7, 4)
19027c478bd9Sstevel@tonic-gate 
19037c478bd9Sstevel@tonic-gate #define	CPI_FEATURES_EDX(cpi)		((cpi)->cpi_std[1].cp_edx)
19047c478bd9Sstevel@tonic-gate #define	CPI_FEATURES_ECX(cpi)		((cpi)->cpi_std[1].cp_ecx)
19057c478bd9Sstevel@tonic-gate #define	CPI_FEATURES_XTD_EDX(cpi)	((cpi)->cpi_extd[1].cp_edx)
19067c478bd9Sstevel@tonic-gate #define	CPI_FEATURES_XTD_ECX(cpi)	((cpi)->cpi_extd[1].cp_ecx)
1907245ac945SRobert Mustacchi #define	CPI_FEATURES_7_0_EBX(cpi)	((cpi)->cpi_std[7].cp_ebx)
1908088d69f8SJerry Jelinek #define	CPI_FEATURES_7_0_ECX(cpi)	((cpi)->cpi_std[7].cp_ecx)
1909088d69f8SJerry Jelinek #define	CPI_FEATURES_7_0_EDX(cpi)	((cpi)->cpi_std[7].cp_edx)
191056726c7eSRobert Mustacchi #define	CPI_FEATURES_7_1_EAX(cpi)	((cpi)->cpi_sub7[0].cp_eax)
19117c478bd9Sstevel@tonic-gate 
19127c478bd9Sstevel@tonic-gate #define	CPI_BRANDID(cpi)	BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
19137c478bd9Sstevel@tonic-gate #define	CPI_CHUNKS(cpi)		BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
19147c478bd9Sstevel@tonic-gate #define	CPI_CPU_COUNT(cpi)	BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
19157c478bd9Sstevel@tonic-gate #define	CPI_APIC_ID(cpi)	BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)
19167c478bd9Sstevel@tonic-gate 
19177c478bd9Sstevel@tonic-gate #define	CPI_MAXEAX_MAX		0x100		/* sanity control */
19187c478bd9Sstevel@tonic-gate #define	CPI_XMAXEAX_MAX		0x80000100
1919d129bde2Sesaxe #define	CPI_FN4_ECX_MAX		0x20		/* sanity: max fn 4 levels */
1920b6917abeSmishra #define	CPI_FNB_ECX_MAX		0x20		/* sanity: max fn B levels */
1921d129bde2Sesaxe 
1922d129bde2Sesaxe /*
1923d129bde2Sesaxe  * Function 4 (Deterministic Cache Parameters) macros
1924d129bde2Sesaxe  * Defined by Intel Application Note AP-485
1925d129bde2Sesaxe  */
1926d129bde2Sesaxe #define	CPI_NUM_CORES(regs)		BITX((regs)->cp_eax, 31, 26)
1927d129bde2Sesaxe #define	CPI_NTHR_SHR_CACHE(regs)	BITX((regs)->cp_eax, 25, 14)
1928d129bde2Sesaxe #define	CPI_FULL_ASSOC_CACHE(regs)	BITX((regs)->cp_eax, 9, 9)
1929d129bde2Sesaxe #define	CPI_SELF_INIT_CACHE(regs)	BITX((regs)->cp_eax, 8, 8)
1930d129bde2Sesaxe #define	CPI_CACHE_LVL(regs)		BITX((regs)->cp_eax, 7, 5)
1931d129bde2Sesaxe #define	CPI_CACHE_TYPE(regs)		BITX((regs)->cp_eax, 4, 0)
1932dd23d762SRobert Mustacchi #define	CPI_CACHE_TYPE_DONE	0
1933dd23d762SRobert Mustacchi #define	CPI_CACHE_TYPE_DATA	1
1934dd23d762SRobert Mustacchi #define	CPI_CACHE_TYPE_INSTR	2
1935dd23d762SRobert Mustacchi #define	CPI_CACHE_TYPE_UNIFIED	3
1936b6917abeSmishra #define	CPI_CPU_LEVEL_TYPE(regs)	BITX((regs)->cp_ecx, 15, 8)
1937d129bde2Sesaxe 
1938d129bde2Sesaxe #define	CPI_CACHE_WAYS(regs)		BITX((regs)->cp_ebx, 31, 22)
1939d129bde2Sesaxe #define	CPI_CACHE_PARTS(regs)		BITX((regs)->cp_ebx, 21, 12)
1940d129bde2Sesaxe #define	CPI_CACHE_COH_LN_SZ(regs)	BITX((regs)->cp_ebx, 11, 0)
1941d129bde2Sesaxe 
1942d129bde2Sesaxe #define	CPI_CACHE_SETS(regs)		BITX((regs)->cp_ecx, 31, 0)
1943d129bde2Sesaxe 
1944d129bde2Sesaxe #define	CPI_PREFCH_STRIDE(regs)		BITX((regs)->cp_edx, 9, 0)
1945d129bde2Sesaxe 
19467c478bd9Sstevel@tonic-gate 
19475ff02082Sdmick /*
19485ff02082Sdmick  * A couple of shorthand macros to identify "later" P6-family chips
19495ff02082Sdmick  * like the Pentium M and Core.  First, the "older" P6-based stuff
19505ff02082Sdmick  * (loosely defined as "pre-Pentium-4"):
19515ff02082Sdmick  * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
19525ff02082Sdmick  */
19535ff02082Sdmick #define	IS_LEGACY_P6(cpi) (			\
1954c2710388SDan Kimmel 	cpi->cpi_family == 6 &&			\
19555ff02082Sdmick 		(cpi->cpi_model == 1 ||		\
19565ff02082Sdmick 		cpi->cpi_model == 3 ||		\
19575ff02082Sdmick 		cpi->cpi_model == 5 ||		\
19585ff02082Sdmick 		cpi->cpi_model == 6 ||		\
19595ff02082Sdmick 		cpi->cpi_model == 7 ||		\
19605ff02082Sdmick 		cpi->cpi_model == 8 ||		\
19615ff02082Sdmick 		cpi->cpi_model == 0xA ||	\
19625ff02082Sdmick 		cpi->cpi_model == 0xB)		\
19635ff02082Sdmick )
19645ff02082Sdmick 
19655ff02082Sdmick /* A "new F6" is everything with family 6 that's not the above */
19665ff02082Sdmick #define	IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))
19675ff02082Sdmick 
1968bf91205bSksadhukh /* Extended family/model support */
1969bf91205bSksadhukh #define	IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
1970bf91205bSksadhukh 	cpi->cpi_family >= 0xf)
1971bf91205bSksadhukh 
1972f98fbcecSbholler /*
1973f98fbcecSbholler  * Info for monitor/mwait idle loop.
1974f98fbcecSbholler  *
1975f98fbcecSbholler  * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
1976f98fbcecSbholler  * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
1977f98fbcecSbholler  * 2006.
1978f98fbcecSbholler  * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
1979f98fbcecSbholler  * Documentation Updates" #33633, Rev 2.05, December 2006.
1980f98fbcecSbholler  */
1981f98fbcecSbholler #define	MWAIT_SUPPORT		(0x00000001)	/* mwait supported */
1982f98fbcecSbholler #define	MWAIT_EXTENSIONS	(0x00000002)	/* extenstion supported */
1983f98fbcecSbholler #define	MWAIT_ECX_INT_ENABLE	(0x00000004)	/* ecx 1 extension supported */
1984f98fbcecSbholler #define	MWAIT_SUPPORTED(cpi)	((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
1985f98fbcecSbholler #define	MWAIT_INT_ENABLE(cpi)	((cpi)->cpi_std[5].cp_ecx & 0x2)
1986f98fbcecSbholler #define	MWAIT_EXTENSION(cpi)	((cpi)->cpi_std[5].cp_ecx & 0x1)
1987f98fbcecSbholler #define	MWAIT_SIZE_MIN(cpi)	BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
1988f98fbcecSbholler #define	MWAIT_SIZE_MAX(cpi)	BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
1989f98fbcecSbholler /*
1990f98fbcecSbholler  * Number of sub-cstates for a given c-state.
1991f98fbcecSbholler  */
1992f98fbcecSbholler #define	MWAIT_NUM_SUBC_STATES(cpi, c_state)			\
1993f98fbcecSbholler 	BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)
1994f98fbcecSbholler 
19957af88ac7SKuriakose Kuruvilla /*
19967af88ac7SKuriakose Kuruvilla  * XSAVE leaf 0xD enumeration
19977af88ac7SKuriakose Kuruvilla  */
19987af88ac7SKuriakose Kuruvilla #define	CPUID_LEAFD_2_YMM_OFFSET	576
19997af88ac7SKuriakose Kuruvilla #define	CPUID_LEAFD_2_YMM_SIZE		256
20007af88ac7SKuriakose Kuruvilla 
20010ce813ffSRobert Mustacchi /*
20020ce813ffSRobert Mustacchi  * Common extended leaf names to cut down on typos.
20030ce813ffSRobert Mustacchi  */
20040ce813ffSRobert Mustacchi #define	CPUID_LEAF_EXT_0		0x80000000
20050ce813ffSRobert Mustacchi #define	CPUID_LEAF_EXT_8		0x80000008
20060ce813ffSRobert Mustacchi #define	CPUID_LEAF_EXT_1d		0x8000001d
20070ce813ffSRobert Mustacchi #define	CPUID_LEAF_EXT_1e		0x8000001e
2008651a12cbSRobert Mustacchi #define	CPUID_LEAF_EXT_21		0x80000021
2009dd23d762SRobert Mustacchi #define	CPUID_LEAF_EXT_26		0x80000026
20100ce813ffSRobert Mustacchi 
2011e4b86885SCheng Sean Ye /*
2012dd23d762SRobert Mustacchi  * Functions we consume from cpuid_subr.c;  don't publish these in a header
2013e4b86885SCheng Sean Ye  * file to try and keep people using the expected cpuid_* interfaces.
2014e4b86885SCheng Sean Ye  */
2015e4b86885SCheng Sean Ye extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
201689e921d5SKuriakose Kuruvilla extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
201722e4c3acSKeith M Wesolowski extern x86_chiprev_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
2018e4b86885SCheng Sean Ye extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
201922e4c3acSKeith M Wesolowski extern x86_uarchrev_t _cpuid_uarchrev(uint_t, uint_t, uint_t, uint_t);
2020e4b86885SCheng Sean Ye extern uint_t _cpuid_vendorstr_to_vendorcode(char *);
20218a40a695Sgavinm 
2022ae115bc7Smrj /*
2023ae115bc7Smrj  * Apply up various platform-dependent restrictions where the
2024ae115bc7Smrj  * underlying platform restrictions mean the CPU can be marked
2025ae115bc7Smrj  * as less capable than its cpuid instruction would imply.
2026ae115bc7Smrj  */
2027843e1988Sjohnlev #if defined(__xpv)
2028843e1988Sjohnlev static void
platform_cpuid_mangle(uint_t vendor,uint32_t eax,struct cpuid_regs * cp)2029843e1988Sjohnlev platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
2030843e1988Sjohnlev {
2031843e1988Sjohnlev 	switch (eax) {
2032e4b86885SCheng Sean Ye 	case 1: {
2033e4b86885SCheng Sean Ye 		uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
2034e4b86885SCheng Sean Ye 		    0 : CPUID_INTC_EDX_MCA;
2035843e1988Sjohnlev 		cp->cp_edx &=
2036e4b86885SCheng Sean Ye 		    ~(mcamask |
2037e4b86885SCheng Sean Ye 		    CPUID_INTC_EDX_PSE |
2038843e1988Sjohnlev 		    CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
2039843e1988Sjohnlev 		    CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
2040843e1988Sjohnlev 		    CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
2041843e1988Sjohnlev 		    CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
2042843e1988Sjohnlev 		    CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
2043843e1988Sjohnlev 		break;
2044e4b86885SCheng Sean Ye 	}
2045843e1988Sjohnlev 
2046843e1988Sjohnlev 	case 0x80000001:
2047843e1988Sjohnlev 		cp->cp_edx &=
2048843e1988Sjohnlev 		    ~(CPUID_AMD_EDX_PSE |
2049843e1988Sjohnlev 		    CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
2050843e1988Sjohnlev 		    CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
2051843e1988Sjohnlev 		    CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
2052843e1988Sjohnlev 		    CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
2053843e1988Sjohnlev 		    CPUID_AMD_EDX_TSCP);
2054843e1988Sjohnlev 		cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
2055843e1988Sjohnlev 		break;
2056843e1988Sjohnlev 	default:
2057843e1988Sjohnlev 		break;
2058843e1988Sjohnlev 	}
2059ae115bc7Smrj 
2060843e1988Sjohnlev 	switch (vendor) {
2061843e1988Sjohnlev 	case X86_VENDOR_Intel:
2062843e1988Sjohnlev 		switch (eax) {
2063843e1988Sjohnlev 		case 4:
2064843e1988Sjohnlev 			/*
2065843e1988Sjohnlev 			 * Zero out the (ncores-per-chip - 1) field
2066843e1988Sjohnlev 			 */
2067843e1988Sjohnlev 			cp->cp_eax &= 0x03fffffff;
2068843e1988Sjohnlev 			break;
2069843e1988Sjohnlev 		default:
2070843e1988Sjohnlev 			break;
2071843e1988Sjohnlev 		}
2072843e1988Sjohnlev 		break;
2073843e1988Sjohnlev 	case X86_VENDOR_AMD:
20749b0429a1SPu Wen 	case X86_VENDOR_HYGON:
2075843e1988Sjohnlev 		switch (eax) {
20762ef50f01SJoe Bonasera 
20772ef50f01SJoe Bonasera 		case 0x80000001:
20782ef50f01SJoe Bonasera 			cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
20792ef50f01SJoe Bonasera 			break;
20802ef50f01SJoe Bonasera 
20810ce813ffSRobert Mustacchi 		case CPUID_LEAF_EXT_8:
2082843e1988Sjohnlev 			/*
2083843e1988Sjohnlev 			 * Zero out the (ncores-per-chip - 1) field
2084843e1988Sjohnlev 			 */
2085843e1988Sjohnlev 			cp->cp_ecx &= 0xffffff00;
2086843e1988Sjohnlev 			break;
2087843e1988Sjohnlev 		default:
2088843e1988Sjohnlev 			break;
2089843e1988Sjohnlev 		}
2090843e1988Sjohnlev 		break;
2091843e1988Sjohnlev 	default:
2092843e1988Sjohnlev 		break;
2093843e1988Sjohnlev 	}
2094843e1988Sjohnlev }
2095843e1988Sjohnlev #else
2096ae115bc7Smrj #define	platform_cpuid_mangle(vendor, eax, cp)	/* nothing */
2097843e1988Sjohnlev #endif
2098ae115bc7Smrj 
20997c478bd9Sstevel@tonic-gate /*
21007c478bd9Sstevel@tonic-gate  *  Some undocumented ways of patching the results of the cpuid
21017c478bd9Sstevel@tonic-gate  *  instruction to permit running Solaris 10 on future cpus that
21027c478bd9Sstevel@tonic-gate  *  we don't currently support.  Could be set to non-zero values
21037c478bd9Sstevel@tonic-gate  *  via settings in eeprom.
21047c478bd9Sstevel@tonic-gate  */
21057c478bd9Sstevel@tonic-gate 
21067c478bd9Sstevel@tonic-gate uint32_t cpuid_feature_ecx_include;
21077c478bd9Sstevel@tonic-gate uint32_t cpuid_feature_ecx_exclude;
21087c478bd9Sstevel@tonic-gate uint32_t cpuid_feature_edx_include;
21097c478bd9Sstevel@tonic-gate uint32_t cpuid_feature_edx_exclude;
21107c478bd9Sstevel@tonic-gate 
2111a3114836SGerry Liu /*
2112a3114836SGerry Liu  * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
2113a3114836SGerry Liu  */
2114ae115bc7Smrj void
cpuid_alloc_space(cpu_t * cpu)2115ae115bc7Smrj cpuid_alloc_space(cpu_t *cpu)
2116ae115bc7Smrj {
2117ae115bc7Smrj 	/*
2118ae115bc7Smrj 	 * By convention, cpu0 is the boot cpu, which is set up
2119ae115bc7Smrj 	 * before memory allocation is available.  All other cpus get
2120ae115bc7Smrj 	 * their cpuid_info struct allocated here.
2121ae115bc7Smrj 	 */
2122ae115bc7Smrj 	ASSERT(cpu->cpu_id != 0);
2123a3114836SGerry Liu 	ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
2124ae115bc7Smrj 	cpu->cpu_m.mcpu_cpi =
2125ae115bc7Smrj 	    kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
2126ae115bc7Smrj }
2127ae115bc7Smrj 
2128ae115bc7Smrj void
cpuid_free_space(cpu_t * cpu)2129ae115bc7Smrj cpuid_free_space(cpu_t *cpu)
2130ae115bc7Smrj {
2131d129bde2Sesaxe 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2132d129bde2Sesaxe 	int i;
2133d129bde2Sesaxe 
2134a3114836SGerry Liu 	ASSERT(cpi != NULL);
2135a3114836SGerry Liu 	ASSERT(cpi != &cpuid_info0);
2136d129bde2Sesaxe 
2137d129bde2Sesaxe 	/*
21380ce813ffSRobert Mustacchi 	 * Free up any cache leaf related dynamic storage. The first entry was
21390ce813ffSRobert Mustacchi 	 * cached from the standard cpuid storage, so we should not free it.
2140d129bde2Sesaxe 	 */
21410ce813ffSRobert Mustacchi 	for (i = 1; i < cpi->cpi_cache_leaf_size; i++)
21420ce813ffSRobert Mustacchi 		kmem_free(cpi->cpi_cache_leaves[i], sizeof (struct cpuid_regs));
21430ce813ffSRobert Mustacchi 	if (cpi->cpi_cache_leaf_size > 0)
21440ce813ffSRobert Mustacchi 		kmem_free(cpi->cpi_cache_leaves,
21450ce813ffSRobert Mustacchi 		    cpi->cpi_cache_leaf_size * sizeof (struct cpuid_regs *));
2146d129bde2Sesaxe 
2147a3114836SGerry Liu 	kmem_free(cpi, sizeof (*cpi));
2148a3114836SGerry Liu 	cpu->cpu_m.mcpu_cpi = NULL;
2149ae115bc7Smrj }
2150ae115bc7Smrj 
2151551bc2a6Smrj #if !defined(__xpv)
2152cfe84b82SMatt Amdur /*
2153cfe84b82SMatt Amdur  * Determine the type of the underlying platform. This is used to customize
2154cfe84b82SMatt Amdur  * initialization of various subsystems (e.g. TSC). determine_platform() must
2155cfe84b82SMatt Amdur  * only ever be called once to prevent two processors from seeing different
2156ab5bb018SKeith M Wesolowski  * values of platform_type. Must be called before cpuid_pass_ident(), the
2157ab5bb018SKeith M Wesolowski  * earliest consumer to execute; the identification pass will call
2158ab5bb018SKeith M Wesolowski  * synth_amd_info() to compute the chiprev, which in turn calls get_hwenv().
2159cfe84b82SMatt Amdur  */
2160cfe84b82SMatt Amdur void
determine_platform(void)2161cfe84b82SMatt Amdur determine_platform(void)
2162551bc2a6Smrj {
2163551bc2a6Smrj 	struct cpuid_regs cp;
216479ec9da8SYuri Pankov 	uint32_t base;
216579ec9da8SYuri Pankov 	uint32_t regs[4];
216679ec9da8SYuri Pankov 	char *hvstr = (char *)regs;
2167551bc2a6Smrj 
2168cfe84b82SMatt Amdur 	ASSERT(platform_type == -1);
2169cfe84b82SMatt Amdur 
2170349b53ddSStuart Maybee 	platform_type = HW_NATIVE;
2171349b53ddSStuart Maybee 
2172349b53ddSStuart Maybee 	if (!enable_platform_detection)
2173349b53ddSStuart Maybee 		return;
2174349b53ddSStuart Maybee 
2175551bc2a6Smrj 	/*
217679ec9da8SYuri Pankov 	 * If Hypervisor CPUID bit is set, try to determine hypervisor
217779ec9da8SYuri Pankov 	 * vendor signature, and set platform type accordingly.
217879ec9da8SYuri Pankov 	 *
217979ec9da8SYuri Pankov 	 * References:
218079ec9da8SYuri Pankov 	 * http://lkml.org/lkml/2008/10/1/246
218179ec9da8SYuri Pankov 	 * http://kb.vmware.com/kb/1009458
218279ec9da8SYuri Pankov 	 */
218379ec9da8SYuri Pankov 	cp.cp_eax = 0x1;
218479ec9da8SYuri Pankov 	(void) __cpuid_insn(&cp);
218579ec9da8SYuri Pankov 	if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
218679ec9da8SYuri Pankov 		cp.cp_eax = 0x40000000;
218779ec9da8SYuri Pankov 		(void) __cpuid_insn(&cp);
218879ec9da8SYuri Pankov 		regs[0] = cp.cp_ebx;
218979ec9da8SYuri Pankov 		regs[1] = cp.cp_ecx;
219079ec9da8SYuri Pankov 		regs[2] = cp.cp_edx;
219179ec9da8SYuri Pankov 		regs[3] = 0;
219279ec9da8SYuri Pankov 		if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
219379ec9da8SYuri Pankov 			platform_type = HW_XEN_HVM;
219479ec9da8SYuri Pankov 			return;
219579ec9da8SYuri Pankov 		}
219679ec9da8SYuri Pankov 		if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
219779ec9da8SYuri Pankov 			platform_type = HW_VMWARE;
219879ec9da8SYuri Pankov 			return;
219979ec9da8SYuri Pankov 		}
220079ec9da8SYuri Pankov 		if (strcmp(hvstr, HVSIG_KVM) == 0) {
220179ec9da8SYuri Pankov 			platform_type = HW_KVM;
220279ec9da8SYuri Pankov 			return;
220379ec9da8SYuri Pankov 		}
2204fbd54cb5SHans Rosenfeld 		if (strcmp(hvstr, HVSIG_BHYVE) == 0) {
2205fbd54cb5SHans Rosenfeld 			platform_type = HW_BHYVE;
2206fbd54cb5SHans Rosenfeld 			return;
2207fbd54cb5SHans Rosenfeld 		}
22082faf06a0SToomas Soome 		if (strcmp(hvstr, HVSIG_MICROSOFT) == 0) {
220979ec9da8SYuri Pankov 			platform_type = HW_MICROSOFT;
22102faf06a0SToomas Soome 			return;
22112faf06a0SToomas Soome 		}
22122faf06a0SToomas Soome 		if (strcmp(hvstr, HVSIG_QEMU_TCG) == 0) {
22132faf06a0SToomas Soome 			platform_type = HW_QEMU_TCG;
22142faf06a0SToomas Soome 			return;
22152faf06a0SToomas Soome 		}
221679ec9da8SYuri Pankov 	} else {
221779ec9da8SYuri Pankov 		/*
221879ec9da8SYuri Pankov 		 * Check older VMware hardware versions. VMware hypervisor is
221979ec9da8SYuri Pankov 		 * detected by performing an IN operation to VMware hypervisor
222079ec9da8SYuri Pankov 		 * port and checking that value returned in %ebx is VMware
222179ec9da8SYuri Pankov 		 * hypervisor magic value.
222279ec9da8SYuri Pankov 		 *
222379ec9da8SYuri Pankov 		 * References: http://kb.vmware.com/kb/1009458
222479ec9da8SYuri Pankov 		 */
222579ec9da8SYuri Pankov 		vmware_port(VMWARE_HVCMD_GETVERSION, regs);
222679ec9da8SYuri Pankov 		if (regs[1] == VMWARE_HVMAGIC) {
222779ec9da8SYuri Pankov 			platform_type = HW_VMWARE;
222879ec9da8SYuri Pankov 			return;
222979ec9da8SYuri Pankov 		}
223079ec9da8SYuri Pankov 	}
223179ec9da8SYuri Pankov 
223279ec9da8SYuri Pankov 	/*
223379ec9da8SYuri Pankov 	 * Check Xen hypervisor. In a fully virtualized domain,
223479ec9da8SYuri Pankov 	 * Xen's pseudo-cpuid function returns a string representing the
223579ec9da8SYuri Pankov 	 * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
223679ec9da8SYuri Pankov 	 * supported cpuid function. We need at least a (base + 2) leaf value
223779ec9da8SYuri Pankov 	 * to do what we want to do. Try different base values, since the
223879ec9da8SYuri Pankov 	 * hypervisor might use a different one depending on whether Hyper-V
223979ec9da8SYuri Pankov 	 * emulation is switched on by default or not.
2240551bc2a6Smrj 	 */
22416e5580c9SFrank Van Der Linden 	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
22426e5580c9SFrank Van Der Linden 		cp.cp_eax = base;
22436e5580c9SFrank Van Der Linden 		(void) __cpuid_insn(&cp);
224479ec9da8SYuri Pankov 		regs[0] = cp.cp_ebx;
224579ec9da8SYuri Pankov 		regs[1] = cp.cp_ecx;
224679ec9da8SYuri Pankov 		regs[2] = cp.cp_edx;
224779ec9da8SYuri Pankov 		regs[3] = 0;
224879ec9da8SYuri Pankov 		if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
22496e5580c9SFrank Van Der Linden 		    cp.cp_eax >= (base + 2)) {
225079ec9da8SYuri Pankov 			platform_type &= ~HW_NATIVE;
225179ec9da8SYuri Pankov 			platform_type |= HW_XEN_HVM;
22526e5580c9SFrank Van Der Linden 			return;
22536e5580c9SFrank Van Der Linden 		}
2254b9bfdccdSStuart Maybee 	}
2255b9bfdccdSStuart Maybee }
2256b9bfdccdSStuart Maybee 
2257b9bfdccdSStuart Maybee int
get_hwenv(void)2258b9bfdccdSStuart Maybee get_hwenv(void)
2259b9bfdccdSStuart Maybee {
2260cfe84b82SMatt Amdur 	ASSERT(platform_type != -1);
2261b9bfdccdSStuart Maybee 	return (platform_type);
2262b9bfdccdSStuart Maybee }
22637997e108SSurya Prakki 
2264b9bfdccdSStuart Maybee int
is_controldom(void)2265b9bfdccdSStuart Maybee is_controldom(void)
2266b9bfdccdSStuart Maybee {
2267b9bfdccdSStuart Maybee 	return (0);
2268551bc2a6Smrj }
2269b9bfdccdSStuart Maybee 
2270b9bfdccdSStuart Maybee #else
2271b9bfdccdSStuart Maybee 
2272b9bfdccdSStuart Maybee int
get_hwenv(void)2273b9bfdccdSStuart Maybee get_hwenv(void)
2274b9bfdccdSStuart Maybee {
2275b9bfdccdSStuart Maybee 	return (HW_XEN_PV);
2276b9bfdccdSStuart Maybee }
2277b9bfdccdSStuart Maybee 
2278b9bfdccdSStuart Maybee int
is_controldom(void)2279b9bfdccdSStuart Maybee is_controldom(void)
2280b9bfdccdSStuart Maybee {
2281b9bfdccdSStuart Maybee 	return (DOMAIN_IS_INITDOMAIN(xen_info));
2282b9bfdccdSStuart Maybee }
2283b9bfdccdSStuart Maybee 
2284551bc2a6Smrj #endif	/* __xpv */
2285551bc2a6Smrj 
2286dd23d762SRobert Mustacchi /*
2287dd23d762SRobert Mustacchi  * Gather the extended topology information. This should be the same for both
2288dd23d762SRobert Mustacchi  * AMD leaf 8X26 and Intel leaf 0x1F (though the data interpretation varies).
2289dd23d762SRobert Mustacchi  */
2290dd23d762SRobert Mustacchi static void
cpuid_gather_ext_topo_leaf(struct cpuid_info * cpi,uint32_t leaf)2291dd23d762SRobert Mustacchi cpuid_gather_ext_topo_leaf(struct cpuid_info *cpi, uint32_t leaf)
2292dd23d762SRobert Mustacchi {
2293dd23d762SRobert Mustacchi 	uint_t i;
2294dd23d762SRobert Mustacchi 
2295dd23d762SRobert Mustacchi 	for (i = 0; i < ARRAY_SIZE(cpi->cpi_topo); i++) {
2296dd23d762SRobert Mustacchi 		struct cpuid_regs *regs = &cpi->cpi_topo[i];
2297dd23d762SRobert Mustacchi 
2298dd23d762SRobert Mustacchi 		bzero(regs, sizeof (struct cpuid_regs));
2299dd23d762SRobert Mustacchi 		regs->cp_eax = leaf;
2300dd23d762SRobert Mustacchi 		regs->cp_ecx = i;
2301dd23d762SRobert Mustacchi 
2302dd23d762SRobert Mustacchi 		(void) __cpuid_insn(regs);
2303dd23d762SRobert Mustacchi 		if (CPUID_AMD_8X26_ECX_TYPE(regs->cp_ecx) ==
2304dd23d762SRobert Mustacchi 		    CPUID_AMD_8X26_TYPE_DONE) {
2305dd23d762SRobert Mustacchi 			break;
2306dd23d762SRobert Mustacchi 		}
2307dd23d762SRobert Mustacchi 	}
2308dd23d762SRobert Mustacchi 
2309dd23d762SRobert Mustacchi 	cpi->cpi_topo_nleaves = i;
2310dd23d762SRobert Mustacchi }
2311dd23d762SRobert Mustacchi 
23120ce813ffSRobert Mustacchi /*
23130ce813ffSRobert Mustacchi  * Make sure that we have gathered all of the CPUID leaves that we might need to
23140ce813ffSRobert Mustacchi  * determine topology. We assume that the standard leaf 1 has already been done
23150ce813ffSRobert Mustacchi  * and that xmaxeax has already been calculated.
23160ce813ffSRobert Mustacchi  */
23170ce813ffSRobert Mustacchi static void
cpuid_gather_amd_topology_leaves(cpu_t * cpu)23180ce813ffSRobert Mustacchi cpuid_gather_amd_topology_leaves(cpu_t *cpu)
23190ce813ffSRobert Mustacchi {
23200ce813ffSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
23210ce813ffSRobert Mustacchi 
23220ce813ffSRobert Mustacchi 	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
23230ce813ffSRobert Mustacchi 		struct cpuid_regs *cp;
23240ce813ffSRobert Mustacchi 
23250ce813ffSRobert Mustacchi 		cp = &cpi->cpi_extd[8];
23260ce813ffSRobert Mustacchi 		cp->cp_eax = CPUID_LEAF_EXT_8;
23270ce813ffSRobert Mustacchi 		(void) __cpuid_insn(cp);
23280ce813ffSRobert Mustacchi 		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_8, cp);
23290ce813ffSRobert Mustacchi 	}
23300ce813ffSRobert Mustacchi 
23310ce813ffSRobert Mustacchi 	if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
23320ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
23330ce813ffSRobert Mustacchi 		struct cpuid_regs *cp;
23340ce813ffSRobert Mustacchi 
23350ce813ffSRobert Mustacchi 		cp = &cpi->cpi_extd[0x1e];
23360ce813ffSRobert Mustacchi 		cp->cp_eax = CPUID_LEAF_EXT_1e;
23370ce813ffSRobert Mustacchi 		(void) __cpuid_insn(cp);
23380ce813ffSRobert Mustacchi 	}
2339dd23d762SRobert Mustacchi 
2340dd23d762SRobert Mustacchi 	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_26) {
2341dd23d762SRobert Mustacchi 		cpuid_gather_ext_topo_leaf(cpi, CPUID_LEAF_EXT_26);
2342dd23d762SRobert Mustacchi 	}
23430ce813ffSRobert Mustacchi }
23440ce813ffSRobert Mustacchi 
23450ce813ffSRobert Mustacchi /*
23460ce813ffSRobert Mustacchi  * Get the APIC ID for this processor. If Leaf B is present and valid, we prefer
23470ce813ffSRobert Mustacchi  * it to everything else. If not, and we're on an AMD system where 8000001e is
23480ce813ffSRobert Mustacchi  * valid, then we use that. Othewrise, we fall back to the default value for the
23490ce813ffSRobert Mustacchi  * APIC ID in leaf 1.
23500ce813ffSRobert Mustacchi  */
23510ce813ffSRobert Mustacchi static uint32_t
cpuid_gather_apicid(struct cpuid_info * cpi)23520ce813ffSRobert Mustacchi cpuid_gather_apicid(struct cpuid_info *cpi)
23530ce813ffSRobert Mustacchi {
23540ce813ffSRobert Mustacchi 	/*
2355dd23d762SRobert Mustacchi 	 * Leaf B changes based on the arguments to it. Because we don't cache
23560ce813ffSRobert Mustacchi 	 * it, we need to gather it again.
23570ce813ffSRobert Mustacchi 	 */
23580ce813ffSRobert Mustacchi 	if (cpi->cpi_maxeax >= 0xB) {
23590ce813ffSRobert Mustacchi 		struct cpuid_regs regs;
23600ce813ffSRobert Mustacchi 		struct cpuid_regs *cp;
23610ce813ffSRobert Mustacchi 
23620ce813ffSRobert Mustacchi 		cp = &regs;
23630ce813ffSRobert Mustacchi 		cp->cp_eax = 0xB;
23640ce813ffSRobert Mustacchi 		cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
23650ce813ffSRobert Mustacchi 		(void) __cpuid_insn(cp);
23660ce813ffSRobert Mustacchi 
23670ce813ffSRobert Mustacchi 		if (cp->cp_ebx != 0) {
23680ce813ffSRobert Mustacchi 			return (cp->cp_edx);
23690ce813ffSRobert Mustacchi 		}
23700ce813ffSRobert Mustacchi 	}
23710ce813ffSRobert Mustacchi 
23729b0429a1SPu Wen 	if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
23739b0429a1SPu Wen 	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
23740ce813ffSRobert Mustacchi 	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
23750ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
23760ce813ffSRobert Mustacchi 		return (cpi->cpi_extd[0x1e].cp_eax);
23770ce813ffSRobert Mustacchi 	}
23780ce813ffSRobert Mustacchi 
23790ce813ffSRobert Mustacchi 	return (CPI_APIC_ID(cpi));
23800ce813ffSRobert Mustacchi }
23810ce813ffSRobert Mustacchi 
23820ce813ffSRobert Mustacchi /*
23830ce813ffSRobert Mustacchi  * For AMD processors, attempt to calculate the number of chips and cores that
23840ce813ffSRobert Mustacchi  * exist. The way that we do this varies based on the generation, because the
23850ce813ffSRobert Mustacchi  * generations themselves have changed dramatically.
23860ce813ffSRobert Mustacchi  *
23870ce813ffSRobert Mustacchi  * If cpuid leaf 0x80000008 exists, that generally tells us the number of cores.
23880ce813ffSRobert Mustacchi  * However, with the advent of family 17h (Zen) it actually tells us the number
23890ce813ffSRobert Mustacchi  * of threads, so we need to look at leaf 0x8000001e if available to determine
23900ce813ffSRobert Mustacchi  * its value. Otherwise, for all prior families, the number of enabled cores is
23910ce813ffSRobert Mustacchi  * the same as threads.
23920ce813ffSRobert Mustacchi  *
23930ce813ffSRobert Mustacchi  * If we do not have leaf 0x80000008, then we assume that this processor does
23940ce813ffSRobert Mustacchi  * not have anything. AMD's older CPUID specification says there's no reason to
23950ce813ffSRobert Mustacchi  * fall back to leaf 1.
23960ce813ffSRobert Mustacchi  *
23970ce813ffSRobert Mustacchi  * In some virtualization cases we will not have leaf 8000001e or it will be
23980ce813ffSRobert Mustacchi  * zero. When that happens we assume the number of threads is one.
23990ce813ffSRobert Mustacchi  */
24000ce813ffSRobert Mustacchi static void
cpuid_amd_ncores(struct cpuid_info * cpi,uint_t * ncpus,uint_t * ncores)24010ce813ffSRobert Mustacchi cpuid_amd_ncores(struct cpuid_info *cpi, uint_t *ncpus, uint_t *ncores)
24020ce813ffSRobert Mustacchi {
24030ce813ffSRobert Mustacchi 	uint_t nthreads, nthread_per_core;
24040ce813ffSRobert Mustacchi 
24050ce813ffSRobert Mustacchi 	nthreads = nthread_per_core = 1;
24060ce813ffSRobert Mustacchi 
24070ce813ffSRobert Mustacchi 	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
24080ce813ffSRobert Mustacchi 		nthreads = BITX(cpi->cpi_extd[8].cp_ecx, 7, 0) + 1;
24090ce813ffSRobert Mustacchi 	} else if ((cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_HTT) != 0) {
24100ce813ffSRobert Mustacchi 		nthreads = CPI_CPU_COUNT(cpi);
24110ce813ffSRobert Mustacchi 	}
24120ce813ffSRobert Mustacchi 
24130ce813ffSRobert Mustacchi 	/*
24140ce813ffSRobert Mustacchi 	 * For us to have threads, and know about it, we have to be at least at
24150ce813ffSRobert Mustacchi 	 * family 17h and have the cpuid bit that says we have extended
24160ce813ffSRobert Mustacchi 	 * topology.
24170ce813ffSRobert Mustacchi 	 */
24180ce813ffSRobert Mustacchi 	if (cpi->cpi_family >= 0x17 &&
24190ce813ffSRobert Mustacchi 	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
24200ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
24210ce813ffSRobert Mustacchi 		nthread_per_core = BITX(cpi->cpi_extd[0x1e].cp_ebx, 15, 8) + 1;
24220ce813ffSRobert Mustacchi 	}
24230ce813ffSRobert Mustacchi 
24240ce813ffSRobert Mustacchi 	*ncpus = nthreads;
24250ce813ffSRobert Mustacchi 	*ncores = nthreads / nthread_per_core;
24260ce813ffSRobert Mustacchi }
24270ce813ffSRobert Mustacchi 
2428d6517bbdSRobert Mustacchi /*
2429d6517bbdSRobert Mustacchi  * Seed the initial values for the cores and threads for an Intel based
2430d6517bbdSRobert Mustacchi  * processor. These values will be overwritten if we detect that the processor
2431d6517bbdSRobert Mustacchi  * supports CPUID leaf 0xb.
2432d6517bbdSRobert Mustacchi  */
24330ce813ffSRobert Mustacchi static void
cpuid_intel_ncores(struct cpuid_info * cpi,uint_t * ncpus,uint_t * ncores)24340ce813ffSRobert Mustacchi cpuid_intel_ncores(struct cpuid_info *cpi, uint_t *ncpus, uint_t *ncores)
24350ce813ffSRobert Mustacchi {
2436d6517bbdSRobert Mustacchi 	/*
2437d6517bbdSRobert Mustacchi 	 * Only seed the number of physical cores from the first level leaf 4
2438d6517bbdSRobert Mustacchi 	 * information. The number of threads there indicate how many share the
2439d6517bbdSRobert Mustacchi 	 * L1 cache, which may or may not have anything to do with the number of
2440d6517bbdSRobert Mustacchi 	 * logical CPUs per core.
2441d6517bbdSRobert Mustacchi 	 */
24420ce813ffSRobert Mustacchi 	if (cpi->cpi_maxeax >= 4) {
24430ce813ffSRobert Mustacchi 		*ncores = BITX(cpi->cpi_std[4].cp_eax, 31, 26) + 1;
2444d6517bbdSRobert Mustacchi 	} else {
24450ce813ffSRobert Mustacchi 		*ncores = 1;
2446d6517bbdSRobert Mustacchi 	}
2447d6517bbdSRobert Mustacchi 
2448d6517bbdSRobert Mustacchi 	if ((cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_HTT) != 0) {
24490ce813ffSRobert Mustacchi 		*ncpus = CPI_CPU_COUNT(cpi);
24500ce813ffSRobert Mustacchi 	} else {
2451d6517bbdSRobert Mustacchi 		*ncpus = *ncores;
24520ce813ffSRobert Mustacchi 	}
24530ce813ffSRobert Mustacchi }
24540ce813ffSRobert Mustacchi 
24550ce813ffSRobert Mustacchi static boolean_t
cpuid_leafB_getids(cpu_t * cpu)24560ce813ffSRobert Mustacchi cpuid_leafB_getids(cpu_t *cpu)
24570ce813ffSRobert Mustacchi {
24580ce813ffSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
24590ce813ffSRobert Mustacchi 	struct cpuid_regs regs;
24600ce813ffSRobert Mustacchi 	struct cpuid_regs *cp;
24610ce813ffSRobert Mustacchi 
24620ce813ffSRobert Mustacchi 	if (cpi->cpi_maxeax < 0xB)
24630ce813ffSRobert Mustacchi 		return (B_FALSE);
24640ce813ffSRobert Mustacchi 
24650ce813ffSRobert Mustacchi 	cp = &regs;
24660ce813ffSRobert Mustacchi 	cp->cp_eax = 0xB;
24670ce813ffSRobert Mustacchi 	cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
24680ce813ffSRobert Mustacchi 
24690ce813ffSRobert Mustacchi 	(void) __cpuid_insn(cp);
24700ce813ffSRobert Mustacchi 
24710ce813ffSRobert Mustacchi 	/*
24720ce813ffSRobert Mustacchi 	 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
24730ce813ffSRobert Mustacchi 	 * indicates that the extended topology enumeration leaf is
24740ce813ffSRobert Mustacchi 	 * available.
24750ce813ffSRobert Mustacchi 	 */
24760ce813ffSRobert Mustacchi 	if (cp->cp_ebx != 0) {
24770ce813ffSRobert Mustacchi 		uint32_t x2apic_id = 0;
24780ce813ffSRobert Mustacchi 		uint_t coreid_shift = 0;
24790ce813ffSRobert Mustacchi 		uint_t ncpu_per_core = 1;
24800ce813ffSRobert Mustacchi 		uint_t chipid_shift = 0;
24810ce813ffSRobert Mustacchi 		uint_t ncpu_per_chip = 1;
24820ce813ffSRobert Mustacchi 		uint_t i;
24830ce813ffSRobert Mustacchi 		uint_t level;
24840ce813ffSRobert Mustacchi 
24850ce813ffSRobert Mustacchi 		for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
24860ce813ffSRobert Mustacchi 			cp->cp_eax = 0xB;
24870ce813ffSRobert Mustacchi 			cp->cp_ecx = i;
24880ce813ffSRobert Mustacchi 
24890ce813ffSRobert Mustacchi 			(void) __cpuid_insn(cp);
24900ce813ffSRobert Mustacchi 			level = CPI_CPU_LEVEL_TYPE(cp);
24910ce813ffSRobert Mustacchi 
24920ce813ffSRobert Mustacchi 			if (level == 1) {
24930ce813ffSRobert Mustacchi 				x2apic_id = cp->cp_edx;
24940ce813ffSRobert Mustacchi 				coreid_shift = BITX(cp->cp_eax, 4, 0);
24950ce813ffSRobert Mustacchi 				ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
24960ce813ffSRobert Mustacchi 			} else if (level == 2) {
24970ce813ffSRobert Mustacchi 				x2apic_id = cp->cp_edx;
24980ce813ffSRobert Mustacchi 				chipid_shift = BITX(cp->cp_eax, 4, 0);
24990ce813ffSRobert Mustacchi 				ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
25000ce813ffSRobert Mustacchi 			}
25010ce813ffSRobert Mustacchi 		}
25020ce813ffSRobert Mustacchi 
25030ce813ffSRobert Mustacchi 		/*
25040ce813ffSRobert Mustacchi 		 * cpi_apicid is taken care of in cpuid_gather_apicid.
25050ce813ffSRobert Mustacchi 		 */
25060ce813ffSRobert Mustacchi 		cpi->cpi_ncpu_per_chip = ncpu_per_chip;
25070ce813ffSRobert Mustacchi 		cpi->cpi_ncore_per_chip = ncpu_per_chip /
25080ce813ffSRobert Mustacchi 		    ncpu_per_core;
25090ce813ffSRobert Mustacchi 		cpi->cpi_chipid = x2apic_id >> chipid_shift;
25100ce813ffSRobert Mustacchi 		cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
25110ce813ffSRobert Mustacchi 		cpi->cpi_coreid = x2apic_id >> coreid_shift;
25120ce813ffSRobert Mustacchi 		cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
25130ce813ffSRobert Mustacchi 		cpi->cpi_procnodeid = cpi->cpi_chipid;
25140ce813ffSRobert Mustacchi 		cpi->cpi_compunitid = cpi->cpi_coreid;
25150ce813ffSRobert Mustacchi 
2516d6517bbdSRobert Mustacchi 		if (coreid_shift > 0 && chipid_shift > coreid_shift) {
2517d6517bbdSRobert Mustacchi 			cpi->cpi_nthread_bits = coreid_shift;
2518d6517bbdSRobert Mustacchi 			cpi->cpi_ncore_bits = chipid_shift - coreid_shift;
2519d6517bbdSRobert Mustacchi 		}
2520d6517bbdSRobert Mustacchi 
25210ce813ffSRobert Mustacchi 		return (B_TRUE);
25220ce813ffSRobert Mustacchi 	} else {
25230ce813ffSRobert Mustacchi 		return (B_FALSE);
25240ce813ffSRobert Mustacchi 	}
25250ce813ffSRobert Mustacchi }
25260ce813ffSRobert Mustacchi 
25278031591dSSrihari Venkatesan static void
cpuid_intel_getids(cpu_t * cpu,void * feature)25287417cfdeSKuriakose Kuruvilla cpuid_intel_getids(cpu_t *cpu, void *feature)
25298031591dSSrihari Venkatesan {
25308031591dSSrihari Venkatesan 	uint_t i;
25318031591dSSrihari Venkatesan 	uint_t chipid_shift = 0;
25328031591dSSrihari Venkatesan 	uint_t coreid_shift = 0;
25338031591dSSrihari Venkatesan 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
25348031591dSSrihari Venkatesan 
25350ce813ffSRobert Mustacchi 	/*
25360ce813ffSRobert Mustacchi 	 * There are no compute units or processor nodes currently on Intel.
25370ce813ffSRobert Mustacchi 	 * Always set these to one.
25380ce813ffSRobert Mustacchi 	 */
25390ce813ffSRobert Mustacchi 	cpi->cpi_procnodes_per_pkg = 1;
25400ce813ffSRobert Mustacchi 	cpi->cpi_cores_per_compunit = 1;
25410ce813ffSRobert Mustacchi 
25420ce813ffSRobert Mustacchi 	/*
25430ce813ffSRobert Mustacchi 	 * If cpuid Leaf B is present, use that to try and get this information.
25440ce813ffSRobert Mustacchi 	 * It will be the most accurate for Intel CPUs.
25450ce813ffSRobert Mustacchi 	 */
25460ce813ffSRobert Mustacchi 	if (cpuid_leafB_getids(cpu))
25470ce813ffSRobert Mustacchi 		return;
25480ce813ffSRobert Mustacchi 
2549d6517bbdSRobert Mustacchi 	/*
2550d6517bbdSRobert Mustacchi 	 * In this case, we have the leaf 1 and leaf 4 values for ncpu_per_chip
2551d6517bbdSRobert Mustacchi 	 * and ncore_per_chip. These represent the largest power of two values
2552d6517bbdSRobert Mustacchi 	 * that we need to cover all of the IDs in the system. Therefore, we use
2553d6517bbdSRobert Mustacchi 	 * those values to seed the number of bits needed to cover information
2554d6517bbdSRobert Mustacchi 	 * in the case when leaf B is not available. These values will probably
2555d6517bbdSRobert Mustacchi 	 * be larger than required, but that's OK.
2556d6517bbdSRobert Mustacchi 	 */
2557d6517bbdSRobert Mustacchi 	cpi->cpi_nthread_bits = ddi_fls(cpi->cpi_ncpu_per_chip);
2558d6517bbdSRobert Mustacchi 	cpi->cpi_ncore_bits = ddi_fls(cpi->cpi_ncore_per_chip);
2559d6517bbdSRobert Mustacchi 
25608031591dSSrihari Venkatesan 	for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
25618031591dSSrihari Venkatesan 		chipid_shift++;
25628031591dSSrihari Venkatesan 
25638031591dSSrihari Venkatesan 	cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
25648031591dSSrihari Venkatesan 	cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);
25658031591dSSrihari Venkatesan 
25667417cfdeSKuriakose Kuruvilla 	if (is_x86_feature(feature, X86FSET_CMP)) {
25678031591dSSrihari Venkatesan 		/*
25688031591dSSrihari Venkatesan 		 * Multi-core (and possibly multi-threaded)
25698031591dSSrihari Venkatesan 		 * processors.
25708031591dSSrihari Venkatesan 		 */
2571584b574aSToomas Soome 		uint_t ncpu_per_core = 0;
2572584b574aSToomas Soome 
25738031591dSSrihari Venkatesan 		if (cpi->cpi_ncore_per_chip == 1)
25748031591dSSrihari Venkatesan 			ncpu_per_core = cpi->cpi_ncpu_per_chip;
25758031591dSSrihari Venkatesan 		else if (cpi->cpi_ncore_per_chip > 1)
25768031591dSSrihari Venkatesan 			ncpu_per_core = cpi->cpi_ncpu_per_chip /
25778031591dSSrihari Venkatesan 			    cpi->cpi_ncore_per_chip;
25788031591dSSrihari Venkatesan 		/*
25798031591dSSrihari Venkatesan 		 * 8bit APIC IDs on dual core Pentiums
25808031591dSSrihari Venkatesan 		 * look like this:
25818031591dSSrihari Venkatesan 		 *
25828031591dSSrihari Venkatesan 		 * +-----------------------+------+------+
25838031591dSSrihari Venkatesan 		 * | Physical Package ID   |  MC  |  HT  |
25848031591dSSrihari Venkatesan 		 * +-----------------------+------+------+
25858031591dSSrihari Venkatesan 		 * <------- chipid -------->
25868031591dSSrihari Venkatesan 		 * <------- coreid --------------->
25878031591dSSrihari Venkatesan 		 *			   <--- clogid -->
25888031591dSSrihari Venkatesan 		 *			   <------>
25898031591dSSrihari Venkatesan 		 *			   pkgcoreid
25908031591dSSrihari Venkatesan 		 *
25918031591dSSrihari Venkatesan 		 * Where the number of bits necessary to
25928031591dSSrihari Venkatesan 		 * represent MC and HT fields together equals
25938031591dSSrihari Venkatesan 		 * to the minimum number of bits necessary to
25948031591dSSrihari Venkatesan 		 * store the value of cpi->cpi_ncpu_per_chip.
25958031591dSSrihari Venkatesan 		 * Of those bits, the MC part uses the number
25968031591dSSrihari Venkatesan 		 * of bits necessary to store the value of
25978031591dSSrihari Venkatesan 		 * cpi->cpi_ncore_per_chip.
25988031591dSSrihari Venkatesan 		 */
25998031591dSSrihari Venkatesan 		for (i = 1; i < ncpu_per_core; i <<= 1)
26008031591dSSrihari Venkatesan 			coreid_shift++;
26018031591dSSrihari Venkatesan 		cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
26028031591dSSrihari Venkatesan 		cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
26037417cfdeSKuriakose Kuruvilla 	} else if (is_x86_feature(feature, X86FSET_HTT)) {
26048031591dSSrihari Venkatesan 		/*
26058031591dSSrihari Venkatesan 		 * Single-core multi-threaded processors.
26068031591dSSrihari Venkatesan 		 */
26078031591dSSrihari Venkatesan 		cpi->cpi_coreid = cpi->cpi_chipid;
26088031591dSSrihari Venkatesan 		cpi->cpi_pkgcoreid = 0;
26090ce813ffSRobert Mustacchi 	} else {
26100ce813ffSRobert Mustacchi 		/*
26110ce813ffSRobert Mustacchi 		 * Single-core single-thread processors.
26120ce813ffSRobert Mustacchi 		 */
26130ce813ffSRobert Mustacchi 		cpi->cpi_coreid = cpu->cpu_id;
26140ce813ffSRobert Mustacchi 		cpi->cpi_pkgcoreid = 0;
26158031591dSSrihari Venkatesan 	}
26168031591dSSrihari Venkatesan 	cpi->cpi_procnodeid = cpi->cpi_chipid;
26177660e73fSHans Rosenfeld 	cpi->cpi_compunitid = cpi->cpi_coreid;
26188031591dSSrihari Venkatesan }
26198031591dSSrihari Venkatesan 
26200ce813ffSRobert Mustacchi /*
26210ce813ffSRobert Mustacchi  * Historically, AMD has had CMP chips with only a single thread per core.
26220ce813ffSRobert Mustacchi  * However, starting in family 17h (Zen), this has changed and they now have
26230ce813ffSRobert Mustacchi  * multiple threads. Our internal core id needs to be a unique value.
26240ce813ffSRobert Mustacchi  *
26250ce813ffSRobert Mustacchi  * To determine the core id of an AMD system, if we're from a family before 17h,
26260ce813ffSRobert Mustacchi  * then we just use the cpu id, as that gives us a good value that will be
26270ce813ffSRobert Mustacchi  * unique for each core. If instead, we're on family 17h or later, then we need
26280ce813ffSRobert Mustacchi  * to do something more complicated. CPUID leaf 0x8000001e can tell us
26290ce813ffSRobert Mustacchi  * how many threads are in the system. Based on that, we'll shift the APIC ID.
26300ce813ffSRobert Mustacchi  * We can't use the normal core id in that leaf as it's only unique within the
26310ce813ffSRobert Mustacchi  * socket, which is perfect for cpi_pkgcoreid, but not us.
26320ce813ffSRobert Mustacchi  */
26330ce813ffSRobert Mustacchi static id_t
cpuid_amd_get_coreid(cpu_t * cpu)26340ce813ffSRobert Mustacchi cpuid_amd_get_coreid(cpu_t *cpu)
26350ce813ffSRobert Mustacchi {
26360ce813ffSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
26370ce813ffSRobert Mustacchi 
26380ce813ffSRobert Mustacchi 	if (cpi->cpi_family >= 0x17 &&
26390ce813ffSRobert Mustacchi 	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
26400ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
26410ce813ffSRobert Mustacchi 		uint_t nthreads = BITX(cpi->cpi_extd[0x1e].cp_ebx, 15, 8) + 1;
26420ce813ffSRobert Mustacchi 		if (nthreads > 1) {
26430ce813ffSRobert Mustacchi 			VERIFY3U(nthreads, ==, 2);
26440ce813ffSRobert Mustacchi 			return (cpi->cpi_apicid >> 1);
26450ce813ffSRobert Mustacchi 		}
26460ce813ffSRobert Mustacchi 	}
26470ce813ffSRobert Mustacchi 
26480ce813ffSRobert Mustacchi 	return (cpu->cpu_id);
26490ce813ffSRobert Mustacchi }
26500ce813ffSRobert Mustacchi 
26510ce813ffSRobert Mustacchi /*
26520ce813ffSRobert Mustacchi  * IDs on AMD is a more challenging task. This is notable because of the
26530ce813ffSRobert Mustacchi  * following two facts:
26540ce813ffSRobert Mustacchi  *
26550ce813ffSRobert Mustacchi  *  1. Before family 0x17 (Zen), there was no support for SMT and there was
26560ce813ffSRobert Mustacchi  *     also no way to get an actual unique core id from the system. As such, we
26570ce813ffSRobert Mustacchi  *     synthesize this case by using cpu->cpu_id.  This scheme does not,
26580ce813ffSRobert Mustacchi  *     however, guarantee that sibling cores of a chip will have sequential
26590ce813ffSRobert Mustacchi  *     coreids starting at a multiple of the number of cores per chip - that is
2660ab5bb018SKeith M Wesolowski  *     usually the case, but if the APIC IDs have been set up in a different
26610ce813ffSRobert Mustacchi  *     order then we need to perform a few more gymnastics for the pkgcoreid.
26620ce813ffSRobert Mustacchi  *
26630ce813ffSRobert Mustacchi  *  2. In families 0x15 and 16x (Bulldozer and co.) the cores came in groups
26640ce813ffSRobert Mustacchi  *     called compute units. These compute units share the L1I cache, L2 cache,
26650ce813ffSRobert Mustacchi  *     and the FPU. To deal with this, a new topology leaf was added in
26660ce813ffSRobert Mustacchi  *     0x8000001e. However, parts of this leaf have different meanings
26670ce813ffSRobert Mustacchi  *     once we get to family 0x17.
26680ce813ffSRobert Mustacchi  */
26690ce813ffSRobert Mustacchi 
26708031591dSSrihari Venkatesan static void
cpuid_amd_getids(cpu_t * cpu,uchar_t * features)26710ce813ffSRobert Mustacchi cpuid_amd_getids(cpu_t *cpu, uchar_t *features)
26728031591dSSrihari Venkatesan {
26731fbe4a4fSSrihari Venkatesan 	int i, first_half, coreidsz;
26748031591dSSrihari Venkatesan 	uint32_t nb_caps_reg;
26758031591dSSrihari Venkatesan 	uint_t node2_1;
26768031591dSSrihari Venkatesan 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
26777660e73fSHans Rosenfeld 	struct cpuid_regs *cp;
26788031591dSSrihari Venkatesan 
26798031591dSSrihari Venkatesan 	/*
26800ce813ffSRobert Mustacchi 	 * Calculate the core id (this comes from hardware in family 0x17 if it
26810ce813ffSRobert Mustacchi 	 * hasn't been stripped by virtualization). We always set the compute
26820ce813ffSRobert Mustacchi 	 * unit id to the same value. Also, initialize the default number of
26830ce813ffSRobert Mustacchi 	 * cores per compute unit and nodes per package. This will be
26840ce813ffSRobert Mustacchi 	 * overwritten when we know information about a particular family.
26858031591dSSrihari Venkatesan 	 */
26860ce813ffSRobert Mustacchi 	cpi->cpi_coreid = cpuid_amd_get_coreid(cpu);
26870ce813ffSRobert Mustacchi 	cpi->cpi_compunitid = cpi->cpi_coreid;
26880ce813ffSRobert Mustacchi 	cpi->cpi_cores_per_compunit = 1;
26890ce813ffSRobert Mustacchi 	cpi->cpi_procnodes_per_pkg = 1;
26908031591dSSrihari Venkatesan 
26910ce813ffSRobert Mustacchi 	/*
26920ce813ffSRobert Mustacchi 	 * To construct the logical ID, we need to determine how many APIC IDs
26930ce813ffSRobert Mustacchi 	 * are dedicated to the cores and threads. This is provided for us in
26940ce813ffSRobert Mustacchi 	 * 0x80000008. However, if it's not present (say due to virtualization),
26950ce813ffSRobert Mustacchi 	 * then we assume it's one. This should be present on all 64-bit AMD
26960ce813ffSRobert Mustacchi 	 * processors.  It was added in family 0xf (Hammer).
26970ce813ffSRobert Mustacchi 	 */
26980ce813ffSRobert Mustacchi 	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
26998031591dSSrihari Venkatesan 		coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);
27008031591dSSrihari Venkatesan 
27018031591dSSrihari Venkatesan 		/*
27020ce813ffSRobert Mustacchi 		 * In AMD parlance chip is really a node while illumos
27030ce813ffSRobert Mustacchi 		 * uses chip as equivalent to socket/package.
27048031591dSSrihari Venkatesan 		 */
27051fbe4a4fSSrihari Venkatesan 		if (coreidsz == 0) {
27068031591dSSrihari Venkatesan 			/* Use legacy method */
27071fbe4a4fSSrihari Venkatesan 			for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
27081fbe4a4fSSrihari Venkatesan 				coreidsz++;
27091fbe4a4fSSrihari Venkatesan 			if (coreidsz == 0)
27101fbe4a4fSSrihari Venkatesan 				coreidsz = 1;
27111fbe4a4fSSrihari Venkatesan 		}
27128031591dSSrihari Venkatesan 	} else {
27138031591dSSrihari Venkatesan 		/* Assume single-core part */
271472b70389SJakub Jermar 		coreidsz = 1;
27158031591dSSrihari Venkatesan 	}
27160ce813ffSRobert Mustacchi 	cpi->cpi_clogid = cpi->cpi_apicid & ((1 << coreidsz) - 1);
27178031591dSSrihari Venkatesan 
27180ce813ffSRobert Mustacchi 	/*
271915a5383bSRobert Mustacchi 	 * The package core ID varies depending on the family. While it may be
272015a5383bSRobert Mustacchi 	 * tempting to use the CPUID_LEAF_EXT_1e %ebx core id, unfortunately,
272115a5383bSRobert Mustacchi 	 * this value is the core id in the given node. For non-virtualized
272215a5383bSRobert Mustacchi 	 * family 17h, we need to take the logical core id and shift off the
272315a5383bSRobert Mustacchi 	 * threads like we do when getting the core id.  Otherwise, we can use
272415a5383bSRobert Mustacchi 	 * the clogid as is. When family 17h is virtualized, the clogid should
272515a5383bSRobert Mustacchi 	 * be sufficient as if we don't have valid data in the leaf, then we
272615a5383bSRobert Mustacchi 	 * won't think we have SMT, in which case the cpi_clogid should be
27270ce813ffSRobert Mustacchi 	 * sufficient.
27280ce813ffSRobert Mustacchi 	 */
27290ce813ffSRobert Mustacchi 	if (cpi->cpi_family >= 0x17 &&
27300ce813ffSRobert Mustacchi 	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
27310ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e &&
27320ce813ffSRobert Mustacchi 	    cpi->cpi_extd[0x1e].cp_ebx != 0) {
273315a5383bSRobert Mustacchi 		uint_t nthreads = BITX(cpi->cpi_extd[0x1e].cp_ebx, 15, 8) + 1;
273415a5383bSRobert Mustacchi 		if (nthreads > 1) {
273515a5383bSRobert Mustacchi 			VERIFY3U(nthreads, ==, 2);
273615a5383bSRobert Mustacchi 			cpi->cpi_pkgcoreid = cpi->cpi_clogid >> 1;
273715a5383bSRobert Mustacchi 		} else {
273815a5383bSRobert Mustacchi 			cpi->cpi_pkgcoreid = cpi->cpi_clogid;
273915a5383bSRobert Mustacchi 		}
27400ce813ffSRobert Mustacchi 	} else {
27410ce813ffSRobert Mustacchi 		cpi->cpi_pkgcoreid = cpi->cpi_clogid;
27420ce813ffSRobert Mustacchi 	}
27438031591dSSrihari Venkatesan 
27440ce813ffSRobert Mustacchi 	/*
27450ce813ffSRobert Mustacchi 	 * Obtain the node ID and compute unit IDs. If we're on family 0x15
27460ce813ffSRobert Mustacchi 	 * (bulldozer) or newer, then we can derive all of this from leaf
27470ce813ffSRobert Mustacchi 	 * CPUID_LEAF_EXT_1e. Otherwise, the method varies by family.
27480ce813ffSRobert Mustacchi 	 */
27497660e73fSHans Rosenfeld 	if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
27500ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
27517660e73fSHans Rosenfeld 		cp = &cpi->cpi_extd[0x1e];
27527660e73fSHans Rosenfeld 
27537660e73fSHans Rosenfeld 		cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
27547660e73fSHans Rosenfeld 		cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);
27550ce813ffSRobert Mustacchi 
27560ce813ffSRobert Mustacchi 		/*
27570ce813ffSRobert Mustacchi 		 * For Bulldozer-era CPUs, recalculate the compute unit
27580ce813ffSRobert Mustacchi 		 * information.
27590ce813ffSRobert Mustacchi 		 */
27600ce813ffSRobert Mustacchi 		if (cpi->cpi_family >= 0x15 && cpi->cpi_family < 0x17) {
27610ce813ffSRobert Mustacchi 			cpi->cpi_cores_per_compunit =
27620ce813ffSRobert Mustacchi 			    BITX(cp->cp_ebx, 15, 8) + 1;
27630ce813ffSRobert Mustacchi 			cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0) +
27640ce813ffSRobert Mustacchi 			    (cpi->cpi_ncore_per_chip /
27650ce813ffSRobert Mustacchi 			    cpi->cpi_cores_per_compunit) *
27660ce813ffSRobert Mustacchi 			    (cpi->cpi_procnodeid /
27670ce813ffSRobert Mustacchi 			    cpi->cpi_procnodes_per_pkg);
27680ce813ffSRobert Mustacchi 		}
27697660e73fSHans Rosenfeld 	} else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
27701fbe4a4fSSrihari Venkatesan 		cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
27718031591dSSrihari Venkatesan 	} else if (cpi->cpi_family == 0x10) {
27728031591dSSrihari Venkatesan 		/*
27738031591dSSrihari Venkatesan 		 * See if we are a multi-node processor.
27748031591dSSrihari Venkatesan 		 * All processors in the system have the same number of nodes
27758031591dSSrihari Venkatesan 		 */
27768031591dSSrihari Venkatesan 		nb_caps_reg =  pci_getl_func(0, 24, 3, 0xe8);
27778031591dSSrihari Venkatesan 		if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
27788031591dSSrihari Venkatesan 			/* Single-node */
27791fbe4a4fSSrihari Venkatesan 			cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
27801fbe4a4fSSrihari Venkatesan 			    coreidsz);
27818031591dSSrihari Venkatesan 		} else {
27828031591dSSrihari Venkatesan 
27838031591dSSrihari Venkatesan 			/*
27848031591dSSrihari Venkatesan 			 * Multi-node revision D (2 nodes per package
27858031591dSSrihari Venkatesan 			 * are supported)
27868031591dSSrihari Venkatesan 			 */
27878031591dSSrihari Venkatesan 			cpi->cpi_procnodes_per_pkg = 2;
27888031591dSSrihari Venkatesan 
27898031591dSSrihari Venkatesan 			first_half = (cpi->cpi_pkgcoreid <=
27908031591dSSrihari Venkatesan 			    (cpi->cpi_ncore_per_chip/2 - 1));
27918031591dSSrihari Venkatesan 
27928031591dSSrihari Venkatesan 			if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
27938031591dSSrihari Venkatesan 				/* We are BSP */
27948031591dSSrihari Venkatesan 				cpi->cpi_procnodeid = (first_half ? 0 : 1);
27958031591dSSrihari Venkatesan 			} else {
27968031591dSSrihari Venkatesan 
27978031591dSSrihari Venkatesan 				/* We are AP */
27988031591dSSrihari Venkatesan 				/* NodeId[2:1] bits to use for reading F3xe8 */
27998031591dSSrihari Venkatesan 				node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;
28008031591dSSrihari Venkatesan 
28018031591dSSrihari Venkatesan 				nb_caps_reg =
28028031591dSSrihari Venkatesan 				    pci_getl_func(0, 24 + node2_1, 3, 0xe8);
28038031591dSSrihari Venkatesan 
28048031591dSSrihari Venkatesan 				/*
28058031591dSSrihari Venkatesan 				 * Check IntNodeNum bit (31:30, but bit 31 is
28068031591dSSrihari Venkatesan 				 * always 0 on dual-node processors)
28078031591dSSrihari Venkatesan 				 */
28088031591dSSrihari Venkatesan 				if (BITX(nb_caps_reg, 30, 30) == 0)
28098031591dSSrihari Venkatesan 					cpi->cpi_procnodeid = node2_1 +
28108031591dSSrihari Venkatesan 					    !first_half;
28118031591dSSrihari Venkatesan 				else
28128031591dSSrihari Venkatesan 					cpi->cpi_procnodeid = node2_1 +
28138031591dSSrihari Venkatesan 					    first_half;
28148031591dSSrihari Venkatesan 			}
28158031591dSSrihari Venkatesan 		}
28168031591dSSrihari Venkatesan 	} else {
28178031591dSSrihari Venkatesan 		cpi->cpi_procnodeid = 0;
28188031591dSSrihari Venkatesan 	}
28197660e73fSHans Rosenfeld 
28207660e73fSHans Rosenfeld 	cpi->cpi_chipid =
28217660e73fSHans Rosenfeld 	    cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;
2822d6517bbdSRobert Mustacchi 
2823d6517bbdSRobert Mustacchi 	cpi->cpi_ncore_bits = coreidsz;
2824d6517bbdSRobert Mustacchi 	cpi->cpi_nthread_bits = ddi_fls(cpi->cpi_ncpu_per_chip /
2825d6517bbdSRobert Mustacchi 	    cpi->cpi_ncore_per_chip);
28268031591dSSrihari Venkatesan }
28278031591dSSrihari Venkatesan 
2828455e370cSJohn Levon static void
spec_uarch_flush_noop(void)2829a9cc46cfSRobert Mustacchi spec_uarch_flush_noop(void)
2830455e370cSJohn Levon {
2831455e370cSJohn Levon }
2832455e370cSJohn Levon 
2833a9cc46cfSRobert Mustacchi /*
2834a9cc46cfSRobert Mustacchi  * When microcode is present that mitigates MDS, this wrmsr will also flush the
2835a9cc46cfSRobert Mustacchi  * MDS-related micro-architectural state that would normally happen by calling
2836a9cc46cfSRobert Mustacchi  * x86_md_clear().
2837a9cc46cfSRobert Mustacchi  */
2838455e370cSJohn Levon static void
spec_uarch_flush_msr(void)2839a9cc46cfSRobert Mustacchi spec_uarch_flush_msr(void)
2840455e370cSJohn Levon {
2841455e370cSJohn Levon 	wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
2842455e370cSJohn Levon }
2843455e370cSJohn Levon 
2844a9cc46cfSRobert Mustacchi /*
2845a9cc46cfSRobert Mustacchi  * This function points to a function that will flush certain
2846a9cc46cfSRobert Mustacchi  * micro-architectural state on the processor. This flush is used to mitigate
2847*5cd084edSDan McDonald  * three different classes of Intel CPU vulnerabilities: L1TF, MDS, and RFDS.
2848*5cd084edSDan McDonald  * This function can point to one of three functions:
2849a9cc46cfSRobert Mustacchi  *
2850a9cc46cfSRobert Mustacchi  * - A noop which is done because we either are vulnerable, but do not have
2851a9cc46cfSRobert Mustacchi  *   microcode available to help deal with a fix, or because we aren't
2852a9cc46cfSRobert Mustacchi  *   vulnerable.
2853a9cc46cfSRobert Mustacchi  *
2854a9cc46cfSRobert Mustacchi  * - spec_uarch_flush_msr which will issue an L1D flush and if microcode to
2855a9cc46cfSRobert Mustacchi  *   mitigate MDS is present, also perform the equivalent of the MDS flush;
2856a9cc46cfSRobert Mustacchi  *   however, it only flushes the MDS related micro-architectural state on the
2857a9cc46cfSRobert Mustacchi  *   current hyperthread, it does not do anything for the twin.
2858a9cc46cfSRobert Mustacchi  *
2859a9cc46cfSRobert Mustacchi  * - x86_md_clear which will flush the MDS related state. This is done when we
2860a9cc46cfSRobert Mustacchi  *   have a processor that is vulnerable to MDS, but is not vulnerable to L1TF
2861*5cd084edSDan McDonald  *   (RDCL_NO is set); or if the CPU is vulnerable to RFDS and indicates VERW
2862*5cd084edSDan McDonald  *   can clear it (RFDS_CLEAR is set).
2863a9cc46cfSRobert Mustacchi  */
2864a9cc46cfSRobert Mustacchi void (*spec_uarch_flush)(void) = spec_uarch_flush_noop;
2865a9cc46cfSRobert Mustacchi 
2866a9cc46cfSRobert Mustacchi static void
cpuid_update_md_clear(cpu_t * cpu,uchar_t * featureset)2867a9cc46cfSRobert Mustacchi cpuid_update_md_clear(cpu_t *cpu, uchar_t *featureset)
2868a9cc46cfSRobert Mustacchi {
2869a9cc46cfSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2870a9cc46cfSRobert Mustacchi 
2871*5cd084edSDan McDonald 	/* Non-Intel doesn't concern us here. */
2872*5cd084edSDan McDonald 	if (cpi->cpi_vendor != X86_VENDOR_Intel)
2873*5cd084edSDan McDonald 		return;
2874*5cd084edSDan McDonald 
2875a9cc46cfSRobert Mustacchi 	/*
2876a9cc46cfSRobert Mustacchi 	 * While RDCL_NO indicates that one of the MDS vulnerabilities (MSBDS)
2877a9cc46cfSRobert Mustacchi 	 * has been fixed in hardware, it doesn't cover everything related to
2878a9cc46cfSRobert Mustacchi 	 * MDS. Therefore we can only rely on MDS_NO to determine that we don't
2879a9cc46cfSRobert Mustacchi 	 * need to mitigate this.
2880*5cd084edSDan McDonald 	 *
2881*5cd084edSDan McDonald 	 * We must ALSO check the case of RFDS_NO and if RFDS_CLEAR is set,
2882*5cd084edSDan McDonald 	 * because of the small cases of RFDS.
2883a9cc46cfSRobert Mustacchi 	 */
2884a9cc46cfSRobert Mustacchi 
2885*5cd084edSDan McDonald 	if ((!is_x86_feature(featureset, X86FSET_MDS_NO) &&
2886*5cd084edSDan McDonald 	    is_x86_feature(featureset, X86FSET_MD_CLEAR)) ||
2887*5cd084edSDan McDonald 	    (!is_x86_feature(featureset, X86FSET_RFDS_NO) &&
2888*5cd084edSDan McDonald 	    is_x86_feature(featureset, X86FSET_RFDS_CLEAR))) {
288965f20420SRobert Mustacchi 		const uint8_t nop = NOP_INSTR;
289065f20420SRobert Mustacchi 		uint8_t *md = (uint8_t *)x86_md_clear;
289165f20420SRobert Mustacchi 
289265f20420SRobert Mustacchi 		*md = nop;
2893a9cc46cfSRobert Mustacchi 	}
2894a9cc46cfSRobert Mustacchi 
2895a9cc46cfSRobert Mustacchi 	membar_producer();
2896a9cc46cfSRobert Mustacchi }
2897a9cc46cfSRobert Mustacchi 
2898a9cc46cfSRobert Mustacchi static void
cpuid_update_l1d_flush(cpu_t * cpu,uchar_t * featureset)2899a9cc46cfSRobert Mustacchi cpuid_update_l1d_flush(cpu_t *cpu, uchar_t *featureset)
2900a9cc46cfSRobert Mustacchi {
2901*5cd084edSDan McDonald 	boolean_t need_l1d, need_mds, need_rfds;
2902a9cc46cfSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
2903a9cc46cfSRobert Mustacchi 
2904a9cc46cfSRobert Mustacchi 	/*
2905*5cd084edSDan McDonald 	 * If we're not on Intel or we've mitigated all of RDCL, MDS, and RFDS
2906*5cd084edSDan McDonald 	 * in hardware, then there's nothing left for us to do for enabling
2907*5cd084edSDan McDonald 	 * the flush. We can also go ahead and say that SMT exclusion is
2908c3377ee9SJohn Levon 	 * unnecessary.
2909a9cc46cfSRobert Mustacchi 	 */
2910a9cc46cfSRobert Mustacchi 	if (cpi->cpi_vendor != X86_VENDOR_Intel ||
2911a9cc46cfSRobert Mustacchi 	    (is_x86_feature(featureset, X86FSET_RDCL_NO) &&
2912*5cd084edSDan McDonald 	    is_x86_feature(featureset, X86FSET_MDS_NO) &&
2913*5cd084edSDan McDonald 	    is_x86_feature(featureset, X86FSET_RFDS_NO))) {
2914c3377ee9SJohn Levon 		extern int smt_exclusion;
2915c3377ee9SJohn Levon 		smt_exclusion = 0;
2916a9cc46cfSRobert Mustacchi 		spec_uarch_flush = spec_uarch_flush_noop;
2917a9cc46cfSRobert Mustacchi 		membar_producer();
2918a9cc46cfSRobert Mustacchi 		return;
2919a9cc46cfSRobert Mustacchi 	}
2920a9cc46cfSRobert Mustacchi 
2921a9cc46cfSRobert Mustacchi 	/*
2922a9cc46cfSRobert Mustacchi 	 * The locations where we need to perform an L1D flush are required both
2923a9cc46cfSRobert Mustacchi 	 * for mitigating L1TF and MDS. When verw support is present in
2924a9cc46cfSRobert Mustacchi 	 * microcode, then the L1D flush will take care of doing that as well.
2925a9cc46cfSRobert Mustacchi 	 * However, if we have a system where RDCL_NO is present, but we don't
2926a9cc46cfSRobert Mustacchi 	 * have MDS_NO, then we need to do a verw (x86_md_clear) and not a full
2927a9cc46cfSRobert Mustacchi 	 * L1D flush.
2928a9cc46cfSRobert Mustacchi 	 */
2929a9cc46cfSRobert Mustacchi 	if (!is_x86_feature(featureset, X86FSET_RDCL_NO) &&
2930a9cc46cfSRobert Mustacchi 	    is_x86_feature(featureset, X86FSET_FLUSH_CMD) &&
2931a9cc46cfSRobert Mustacchi 	    !is_x86_feature(featureset, X86FSET_L1D_VM_NO)) {
2932a9cc46cfSRobert Mustacchi 		need_l1d = B_TRUE;
2933a9cc46cfSRobert Mustacchi 	} else {
2934a9cc46cfSRobert Mustacchi 		need_l1d = B_FALSE;
2935a9cc46cfSRobert Mustacchi 	}
2936a9cc46cfSRobert Mustacchi 
2937a9cc46cfSRobert Mustacchi 	if (!is_x86_feature(featureset, X86FSET_MDS_NO) &&
2938a9cc46cfSRobert Mustacchi 	    is_x86_feature(featureset, X86FSET_MD_CLEAR)) {
2939a9cc46cfSRobert Mustacchi 		need_mds = B_TRUE;
2940a9cc46cfSRobert Mustacchi 	} else {
2941a9cc46cfSRobert Mustacchi 		need_mds = B_FALSE;
2942a9cc46cfSRobert Mustacchi 	}
2943a9cc46cfSRobert Mustacchi 
2944*5cd084edSDan McDonald 	if (!is_x86_feature(featureset, X86FSET_RFDS_NO) &&
2945*5cd084edSDan McDonald 	    is_x86_feature(featureset, X86FSET_RFDS_CLEAR)) {
2946*5cd084edSDan McDonald 		need_rfds = B_TRUE;
2947*5cd084edSDan McDonald 	} else {
2948*5cd084edSDan McDonald 		need_rfds = B_FALSE;
2949*5cd084edSDan McDonald 	}
2950*5cd084edSDan McDonald 
2951a9cc46cfSRobert Mustacchi 	if (need_l1d) {
2952*5cd084edSDan McDonald 		/*
2953*5cd084edSDan McDonald 		 * As of Feb, 2024, no CPU needs L1D *and* RFDS mitigation
2954*5cd084edSDan McDonald 		 * together. If the following VERIFY trips, we need to add
2955*5cd084edSDan McDonald 		 * further fixes here.
2956*5cd084edSDan McDonald 		 */
2957*5cd084edSDan McDonald 		VERIFY(!need_rfds);
2958a9cc46cfSRobert Mustacchi 		spec_uarch_flush = spec_uarch_flush_msr;
2959*5cd084edSDan McDonald 	} else if (need_mds || need_rfds) {
2960a9cc46cfSRobert Mustacchi 		spec_uarch_flush = x86_md_clear;
2961a9cc46cfSRobert Mustacchi 	} else {
2962a9cc46cfSRobert Mustacchi 		/*
2963a9cc46cfSRobert Mustacchi 		 * We have no hardware mitigations available to us.
2964a9cc46cfSRobert Mustacchi 		 */
2965a9cc46cfSRobert Mustacchi 		spec_uarch_flush = spec_uarch_flush_noop;
2966a9cc46cfSRobert Mustacchi 	}
2967a9cc46cfSRobert Mustacchi 	membar_producer();
2968a9cc46cfSRobert Mustacchi }
2969455e370cSJohn Levon 
297065f20420SRobert Mustacchi /*
297165f20420SRobert Mustacchi  * We default to enabling RSB mitigations.
2972345881c5SDan McDonald  *
2973345881c5SDan McDonald  * NOTE: We used to skip RSB mitigations with eIBRS, but developments around
2974345881c5SDan McDonald  * post-barrier RSB guessing suggests we should enable RSB mitigations always
2975345881c5SDan McDonald  * unless specifically instructed not to.
2976651a12cbSRobert Mustacchi  *
2977651a12cbSRobert Mustacchi  * AMD indicates that when Automatic IBRS is enabled we do not need to implement
2978651a12cbSRobert Mustacchi  * return stack buffer clearing for VMEXIT as it takes care of it. The manual
2979651a12cbSRobert Mustacchi  * also states that as long as SMEP and we maintain at least one page between
2980651a12cbSRobert Mustacchi  * the kernel and user space (we have much more of a red zone), then we do not
2981651a12cbSRobert Mustacchi  * need to clear the RSB. We constrain this to only when Automatic IRBS is
2982651a12cbSRobert Mustacchi  * present.
298365f20420SRobert Mustacchi  */
298465f20420SRobert Mustacchi static void
cpuid_patch_rsb(x86_spectrev2_mitigation_t mit)298565f20420SRobert Mustacchi cpuid_patch_rsb(x86_spectrev2_mitigation_t mit)
298665f20420SRobert Mustacchi {
298765f20420SRobert Mustacchi 	const uint8_t ret = RET_INSTR;
298865f20420SRobert Mustacchi 	uint8_t *stuff = (uint8_t *)x86_rsb_stuff;
298965f20420SRobert Mustacchi 
299065f20420SRobert Mustacchi 	switch (mit) {
2991651a12cbSRobert Mustacchi 	case X86_SPECTREV2_AUTO_IBRS:
299265f20420SRobert Mustacchi 	case X86_SPECTREV2_DISABLED:
299365f20420SRobert Mustacchi 		*stuff = ret;
299465f20420SRobert Mustacchi 		break;
299565f20420SRobert Mustacchi 	default:
299665f20420SRobert Mustacchi 		break;
299765f20420SRobert Mustacchi 	}
299865f20420SRobert Mustacchi }
299965f20420SRobert Mustacchi 
300065f20420SRobert Mustacchi static void
cpuid_patch_retpolines(x86_spectrev2_mitigation_t mit)300165f20420SRobert Mustacchi cpuid_patch_retpolines(x86_spectrev2_mitigation_t mit)
300265f20420SRobert Mustacchi {
300365f20420SRobert Mustacchi 	const char *thunks[] = { "_rax", "_rbx", "_rcx", "_rdx", "_rdi",
300465f20420SRobert Mustacchi 	    "_rsi", "_rbp", "_r8", "_r9", "_r10", "_r11", "_r12", "_r13",
300565f20420SRobert Mustacchi 	    "_r14", "_r15" };
300665f20420SRobert Mustacchi 	const uint_t nthunks = ARRAY_SIZE(thunks);
300765f20420SRobert Mustacchi 	const char *type;
300865f20420SRobert Mustacchi 	uint_t i;
300965f20420SRobert Mustacchi 
301065f20420SRobert Mustacchi 	if (mit == x86_spectrev2_mitigation)
301165f20420SRobert Mustacchi 		return;
301265f20420SRobert Mustacchi 
301365f20420SRobert Mustacchi 	switch (mit) {
301465f20420SRobert Mustacchi 	case X86_SPECTREV2_RETPOLINE:
301565f20420SRobert Mustacchi 		type = "gen";
301665f20420SRobert Mustacchi 		break;
3017651a12cbSRobert Mustacchi 	case X86_SPECTREV2_AUTO_IBRS:
301865f20420SRobert Mustacchi 	case X86_SPECTREV2_ENHANCED_IBRS:
301965f20420SRobert Mustacchi 	case X86_SPECTREV2_DISABLED:
302065f20420SRobert Mustacchi 		type = "jmp";
302165f20420SRobert Mustacchi 		break;
302265f20420SRobert Mustacchi 	default:
3023651a12cbSRobert Mustacchi 		panic("asked to update retpoline state with unknown state!");
302465f20420SRobert Mustacchi 	}
302565f20420SRobert Mustacchi 
302665f20420SRobert Mustacchi 	for (i = 0; i < nthunks; i++) {
302765f20420SRobert Mustacchi 		uintptr_t source, dest;
302865f20420SRobert Mustacchi 		int ssize, dsize;
302965f20420SRobert Mustacchi 		char sourcebuf[64], destbuf[64];
303065f20420SRobert Mustacchi 
303165f20420SRobert Mustacchi 		(void) snprintf(destbuf, sizeof (destbuf),
303265f20420SRobert Mustacchi 		    "__x86_indirect_thunk%s", thunks[i]);
303365f20420SRobert Mustacchi 		(void) snprintf(sourcebuf, sizeof (sourcebuf),
303465f20420SRobert Mustacchi 		    "__x86_indirect_thunk_%s%s", type, thunks[i]);
303565f20420SRobert Mustacchi 
303665f20420SRobert Mustacchi 		source = kobj_getelfsym(sourcebuf, NULL, &ssize);
303765f20420SRobert Mustacchi 		dest = kobj_getelfsym(destbuf, NULL, &dsize);
303865f20420SRobert Mustacchi 		VERIFY3U(source, !=, 0);
303965f20420SRobert Mustacchi 		VERIFY3U(dest, !=, 0);
304065f20420SRobert Mustacchi 		VERIFY3S(dsize, >=, ssize);
304165f20420SRobert Mustacchi 		bcopy((void *)source, (void *)dest, ssize);
304265f20420SRobert Mustacchi 	}
304365f20420SRobert Mustacchi }
304465f20420SRobert Mustacchi 
304565f20420SRobert Mustacchi static void
cpuid_enable_enhanced_ibrs(void)304665f20420SRobert Mustacchi cpuid_enable_enhanced_ibrs(void)
304765f20420SRobert Mustacchi {
304865f20420SRobert Mustacchi 	uint64_t val;
304965f20420SRobert Mustacchi 
305065f20420SRobert Mustacchi 	val = rdmsr(MSR_IA32_SPEC_CTRL);
305165f20420SRobert Mustacchi 	val |= IA32_SPEC_CTRL_IBRS;
305265f20420SRobert Mustacchi 	wrmsr(MSR_IA32_SPEC_CTRL, val);
305365f20420SRobert Mustacchi }
305465f20420SRobert Mustacchi 
3055651a12cbSRobert Mustacchi static void
cpuid_enable_auto_ibrs(void)3056651a12cbSRobert Mustacchi cpuid_enable_auto_ibrs(void)
3057651a12cbSRobert Mustacchi {
3058651a12cbSRobert Mustacchi 	uint64_t val;
3059651a12cbSRobert Mustacchi 
3060651a12cbSRobert Mustacchi 	val = rdmsr(MSR_AMD_EFER);
3061651a12cbSRobert Mustacchi 	val |= AMD_EFER_AIBRSE;
3062651a12cbSRobert Mustacchi 	wrmsr(MSR_AMD_EFER, val);
3063651a12cbSRobert Mustacchi }
3064651a12cbSRobert Mustacchi 
3065e25cb0e7SJohn Levon /*
3066e25cb0e7SJohn Levon  * Determine how we should mitigate TAA or if we need to. Regardless of TAA, if
3067e25cb0e7SJohn Levon  * we can disable TSX, we do so.
3068e25cb0e7SJohn Levon  *
3069e25cb0e7SJohn Levon  * This determination is done only on the boot CPU, potentially after loading
3070e25cb0e7SJohn Levon  * updated microcode.
3071e25cb0e7SJohn Levon  */
3072e25cb0e7SJohn Levon static void
cpuid_update_tsx(cpu_t * cpu,uchar_t * featureset)3073e25cb0e7SJohn Levon cpuid_update_tsx(cpu_t *cpu, uchar_t *featureset)
3074e25cb0e7SJohn Levon {
3075e25cb0e7SJohn Levon 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3076e25cb0e7SJohn Levon 
3077e25cb0e7SJohn Levon 	VERIFY(cpu->cpu_id == 0);
3078e25cb0e7SJohn Levon 
3079e25cb0e7SJohn Levon 	if (cpi->cpi_vendor != X86_VENDOR_Intel) {
3080e25cb0e7SJohn Levon 		x86_taa_mitigation = X86_TAA_HW_MITIGATED;
3081e25cb0e7SJohn Levon 		return;
3082e25cb0e7SJohn Levon 	}
3083e25cb0e7SJohn Levon 
3084e25cb0e7SJohn Levon 	if (x86_disable_taa) {
3085e25cb0e7SJohn Levon 		x86_taa_mitigation = X86_TAA_DISABLED;
3086e25cb0e7SJohn Levon 		return;
3087e25cb0e7SJohn Levon 	}
3088e25cb0e7SJohn Levon 
3089e25cb0e7SJohn Levon 	/*
3090e25cb0e7SJohn Levon 	 * If we do not have the ability to disable TSX, then our only
3091e25cb0e7SJohn Levon 	 * mitigation options are in hardware (TAA_NO), or by using our existing
3092e25cb0e7SJohn Levon 	 * MDS mitigation as described above.  The latter relies upon us having
3093e25cb0e7SJohn Levon 	 * configured MDS mitigations correctly! This includes disabling SMT if
3094e25cb0e7SJohn Levon 	 * we want to cross-CPU-thread protection.
3095e25cb0e7SJohn Levon 	 */
3096e25cb0e7SJohn Levon 	if (!is_x86_feature(featureset, X86FSET_TSX_CTRL)) {
3097e25cb0e7SJohn Levon 		/*
3098e25cb0e7SJohn Levon 		 * It's not clear whether any parts will enumerate TAA_NO
3099e25cb0e7SJohn Levon 		 * *without* TSX_CTRL, but let's mark it as such if we see this.
3100e25cb0e7SJohn Levon 		 */
3101e25cb0e7SJohn Levon 		if (is_x86_feature(featureset, X86FSET_TAA_NO)) {
3102e25cb0e7SJohn Levon 			x86_taa_mitigation = X86_TAA_HW_MITIGATED;
3103e25cb0e7SJohn Levon 			return;
3104e25cb0e7SJohn Levon 		}
3105e25cb0e7SJohn Levon 
3106e25cb0e7SJohn Levon 		if (is_x86_feature(featureset, X86FSET_MD_CLEAR) &&
3107e25cb0e7SJohn Levon 		    !is_x86_feature(featureset, X86FSET_MDS_NO)) {
3108e25cb0e7SJohn Levon 			x86_taa_mitigation = X86_TAA_MD_CLEAR;
3109e25cb0e7SJohn Levon 		} else {
3110e25cb0e7SJohn Levon 			x86_taa_mitigation = X86_TAA_NOTHING;
3111e25cb0e7SJohn Levon 		}
3112e25cb0e7SJohn Levon 		return;
3113e25cb0e7SJohn Levon 	}
3114e25cb0e7SJohn Levon 
3115e25cb0e7SJohn Levon 	/*
3116e25cb0e7SJohn Levon 	 * We have TSX_CTRL, but we can only fully disable TSX if we're early
3117e25cb0e7SJohn Levon 	 * enough in boot.
3118e25cb0e7SJohn Levon 	 *
3119e25cb0e7SJohn Levon 	 * Otherwise, we'll fall back to causing transactions to abort as our
3120e25cb0e7SJohn Levon 	 * mitigation. TSX-using code will always take the fallback path.
3121e25cb0e7SJohn Levon 	 */
3122e25cb0e7SJohn Levon 	if (cpi->cpi_pass < 4) {
3123e25cb0e7SJohn Levon 		x86_taa_mitigation = X86_TAA_TSX_DISABLE;
3124e25cb0e7SJohn Levon 	} else {
3125e25cb0e7SJohn Levon 		x86_taa_mitigation = X86_TAA_TSX_FORCE_ABORT;
3126e25cb0e7SJohn Levon 	}
3127e25cb0e7SJohn Levon }
3128e25cb0e7SJohn Levon 
3129e25cb0e7SJohn Levon /*
3130e25cb0e7SJohn Levon  * As mentioned, we should only touch the MSR when we've got a suitable
3131e25cb0e7SJohn Levon  * microcode loaded on this CPU.
3132e25cb0e7SJohn Levon  */
3133e25cb0e7SJohn Levon static void
cpuid_apply_tsx(x86_taa_mitigation_t taa,uchar_t * featureset)3134e25cb0e7SJohn Levon cpuid_apply_tsx(x86_taa_mitigation_t taa, uchar_t *featureset)
3135e25cb0e7SJohn Levon {
3136e25cb0e7SJohn Levon 	uint64_t val;
3137e25cb0e7SJohn Levon 
3138e25cb0e7SJohn Levon 	switch (taa) {
3139e25cb0e7SJohn Levon 	case X86_TAA_TSX_DISABLE:
3140e25cb0e7SJohn Levon 		if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
3141e25cb0e7SJohn Levon 			return;
3142e25cb0e7SJohn Levon 		val = rdmsr(MSR_IA32_TSX_CTRL);
3143e25cb0e7SJohn Levon 		val |= IA32_TSX_CTRL_CPUID_CLEAR | IA32_TSX_CTRL_RTM_DISABLE;
3144e25cb0e7SJohn Levon 		wrmsr(MSR_IA32_TSX_CTRL, val);
3145e25cb0e7SJohn Levon 		break;
3146e25cb0e7SJohn Levon 	case X86_TAA_TSX_FORCE_ABORT:
3147e25cb0e7SJohn Levon 		if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
3148e25cb0e7SJohn Levon 			return;
3149e25cb0e7SJohn Levon 		val = rdmsr(MSR_IA32_TSX_CTRL);
3150e25cb0e7SJohn Levon 		val |= IA32_TSX_CTRL_RTM_DISABLE;
3151e25cb0e7SJohn Levon 		wrmsr(MSR_IA32_TSX_CTRL, val);
3152e25cb0e7SJohn Levon 		break;
3153e25cb0e7SJohn Levon 	case X86_TAA_HW_MITIGATED:
3154e25cb0e7SJohn Levon 	case X86_TAA_MD_CLEAR:
3155e25cb0e7SJohn Levon 	case X86_TAA_DISABLED:
3156e25cb0e7SJohn Levon 	case X86_TAA_NOTHING:
3157e25cb0e7SJohn Levon 		break;
3158e25cb0e7SJohn Levon 	}
3159e25cb0e7SJohn Levon }
3160e25cb0e7SJohn Levon 
316101add34aSRobert Mustacchi static void
cpuid_scan_security(cpu_t * cpu,uchar_t * featureset)316201add34aSRobert Mustacchi cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
316301add34aSRobert Mustacchi {
316401add34aSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
316565f20420SRobert Mustacchi 	x86_spectrev2_mitigation_t v2mit;
316601add34aSRobert Mustacchi 
31679b0429a1SPu Wen 	if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
31689b0429a1SPu Wen 	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
31690ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
317001add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBPB)
317101add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_IBPB);
317201add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS)
317301add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_IBRS);
317401add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP)
317501add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_STIBP);
317601add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP_ALL)
317701add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_STIBP_ALL);
317801add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSBD)
317901add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_SSBD);
318001add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_VIRT_SSBD)
318101add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_SSBD_VIRT);
318201add34aSRobert Mustacchi 		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSB_NO)
318301add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_SSB_NO);
3184651a12cbSRobert Mustacchi 
318565f20420SRobert Mustacchi 		/*
3186651a12cbSRobert Mustacchi 		 * Rather than Enhanced IBRS, AMD has a different feature that
3187651a12cbSRobert Mustacchi 		 * is a bit in EFER that can be enabled and will basically do
3188651a12cbSRobert Mustacchi 		 * the right thing while executing in the kernel.
318965f20420SRobert Mustacchi 		 */
3190651a12cbSRobert Mustacchi 		if (cpi->cpi_vendor == X86_VENDOR_AMD &&
3191651a12cbSRobert Mustacchi 		    (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PREFER_IBRS) &&
3192651a12cbSRobert Mustacchi 		    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_21 &&
3193651a12cbSRobert Mustacchi 		    (cpi->cpi_extd[0x21].cp_eax & CPUID_AMD_8X21_EAX_AIBRS)) {
3194651a12cbSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_AUTO_IBRS);
319565f20420SRobert Mustacchi 		}
319665f20420SRobert Mustacchi 
319701add34aSRobert Mustacchi 	} else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
319801add34aSRobert Mustacchi 	    cpi->cpi_maxeax >= 7) {
319901add34aSRobert Mustacchi 		struct cpuid_regs *ecp;
320001add34aSRobert Mustacchi 		ecp = &cpi->cpi_std[7];
320101add34aSRobert Mustacchi 
3202a9cc46cfSRobert Mustacchi 		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_MD_CLEAR) {
3203a9cc46cfSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_MD_CLEAR);
3204a9cc46cfSRobert Mustacchi 		}
3205a9cc46cfSRobert Mustacchi 
320601add34aSRobert Mustacchi 		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SPEC_CTRL) {
320701add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_IBRS);
320801add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_IBPB);
320901add34aSRobert Mustacchi 		}
321001add34aSRobert Mustacchi 
321101add34aSRobert Mustacchi 		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_STIBP) {
321201add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_STIBP);
321301add34aSRobert Mustacchi 		}
321401add34aSRobert Mustacchi 
321501add34aSRobert Mustacchi 		/*
321601add34aSRobert Mustacchi 		 * Don't read the arch caps MSR on xpv where we lack the
321701add34aSRobert Mustacchi 		 * on_trap().
321801add34aSRobert Mustacchi 		 */
321901add34aSRobert Mustacchi #ifndef __xpv
322001add34aSRobert Mustacchi 		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_ARCH_CAPS) {
322101add34aSRobert Mustacchi 			on_trap_data_t otd;
322201add34aSRobert Mustacchi 
322301add34aSRobert Mustacchi 			/*
322401add34aSRobert Mustacchi 			 * Be paranoid and assume we'll get a #GP.
322501add34aSRobert Mustacchi 			 */
322601add34aSRobert Mustacchi 			if (!on_trap(&otd, OT_DATA_ACCESS)) {
322701add34aSRobert Mustacchi 				uint64_t reg;
322801add34aSRobert Mustacchi 
322901add34aSRobert Mustacchi 				reg = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
323001add34aSRobert Mustacchi 				if (reg & IA32_ARCH_CAP_RDCL_NO) {
323101add34aSRobert Mustacchi 					add_x86_feature(featureset,
323201add34aSRobert Mustacchi 					    X86FSET_RDCL_NO);
323301add34aSRobert Mustacchi 				}
323401add34aSRobert Mustacchi 				if (reg & IA32_ARCH_CAP_IBRS_ALL) {
323501add34aSRobert Mustacchi 					add_x86_feature(featureset,
323601add34aSRobert Mustacchi 					    X86FSET_IBRS_ALL);
323701add34aSRobert Mustacchi 				}
323801add34aSRobert Mustacchi 				if (reg & IA32_ARCH_CAP_RSBA) {
323901add34aSRobert Mustacchi 					add_x86_feature(featureset,
324001add34aSRobert Mustacchi 					    X86FSET_RSBA);
324101add34aSRobert Mustacchi 				}
3242c7749d0fSJohn Levon 				if (reg & IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) {
3243c7749d0fSJohn Levon 					add_x86_feature(featureset,
3244c7749d0fSJohn Levon 					    X86FSET_L1D_VM_NO);
3245c7749d0fSJohn Levon 				}
324601add34aSRobert Mustacchi 				if (reg & IA32_ARCH_CAP_SSB_NO) {
324701add34aSRobert Mustacchi 					add_x86_feature(featureset,
324801add34aSRobert Mustacchi 					    X86FSET_SSB_NO);
324901add34aSRobert Mustacchi 				}
3250a9cc46cfSRobert Mustacchi 				if (reg & IA32_ARCH_CAP_MDS_NO) {
3251a9cc46cfSRobert Mustacchi 					add_x86_feature(featureset,
3252a9cc46cfSRobert Mustacchi 					    X86FSET_MDS_NO);
3253a9cc46cfSRobert Mustacchi 				}
3254e25cb0e7SJohn Levon 				if (reg & IA32_ARCH_CAP_TSX_CTRL) {
3255e25cb0e7SJohn Levon 					add_x86_feature(featureset,
3256e25cb0e7SJohn Levon 					    X86FSET_TSX_CTRL);
3257e25cb0e7SJohn Levon 				}
3258e25cb0e7SJohn Levon 				if (reg & IA32_ARCH_CAP_TAA_NO) {
3259e25cb0e7SJohn Levon 					add_x86_feature(featureset,
3260e25cb0e7SJohn Levon 					    X86FSET_TAA_NO);
3261e25cb0e7SJohn Levon 				}
3262*5cd084edSDan McDonald 				if (reg & IA32_ARCH_CAP_RFDS_NO) {
3263*5cd084edSDan McDonald 					add_x86_feature(featureset,
3264*5cd084edSDan McDonald 					    X86FSET_RFDS_NO);
3265*5cd084edSDan McDonald 				}
3266*5cd084edSDan McDonald 				if (reg & IA32_ARCH_CAP_RFDS_CLEAR) {
3267*5cd084edSDan McDonald 					add_x86_feature(featureset,
3268*5cd084edSDan McDonald 					    X86FSET_RFDS_CLEAR);
3269*5cd084edSDan McDonald 				}
327001add34aSRobert Mustacchi 			}
327101add34aSRobert Mustacchi 			no_trap();
327201add34aSRobert Mustacchi 		}
327301add34aSRobert Mustacchi #endif	/* !__xpv */
327401add34aSRobert Mustacchi 
327501add34aSRobert Mustacchi 		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
327601add34aSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_SSBD);
3277c7749d0fSJohn Levon 
3278c7749d0fSJohn Levon 		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
3279c7749d0fSJohn Levon 			add_x86_feature(featureset, X86FSET_FLUSH_CMD);
328001add34aSRobert Mustacchi 	}
3281455e370cSJohn Levon 
3282e25cb0e7SJohn Levon 	/*
3283e25cb0e7SJohn Levon 	 * Take care of certain mitigations on the non-boot CPU. The boot CPU
3284e25cb0e7SJohn Levon 	 * will have already run this function and determined what we need to
3285e25cb0e7SJohn Levon 	 * do. This gives us a hook for per-HW thread mitigations such as
3286e25cb0e7SJohn Levon 	 * enhanced IBRS, or disabling TSX.
3287e25cb0e7SJohn Levon 	 */
328865f20420SRobert Mustacchi 	if (cpu->cpu_id != 0) {
3289651a12cbSRobert Mustacchi 		switch (x86_spectrev2_mitigation) {
3290651a12cbSRobert Mustacchi 		case X86_SPECTREV2_ENHANCED_IBRS:
329165f20420SRobert Mustacchi 			cpuid_enable_enhanced_ibrs();
3292651a12cbSRobert Mustacchi 			break;
3293651a12cbSRobert Mustacchi 		case X86_SPECTREV2_AUTO_IBRS:
3294651a12cbSRobert Mustacchi 			cpuid_enable_auto_ibrs();
3295651a12cbSRobert Mustacchi 			break;
3296651a12cbSRobert Mustacchi 		default:
3297651a12cbSRobert Mustacchi 			break;
329865f20420SRobert Mustacchi 		}
3299e25cb0e7SJohn Levon 
3300e25cb0e7SJohn Levon 		cpuid_apply_tsx(x86_taa_mitigation, featureset);
3301455e370cSJohn Levon 		return;
330265f20420SRobert Mustacchi 	}
330365f20420SRobert Mustacchi 
330465f20420SRobert Mustacchi 	/*
330565f20420SRobert Mustacchi 	 * Go through and initialize various security mechanisms that we should
3306e25cb0e7SJohn Levon 	 * only do on a single CPU. This includes Spectre V2, L1TF, MDS, and
3307e25cb0e7SJohn Levon 	 * TAA.
330865f20420SRobert Mustacchi 	 */
330965f20420SRobert Mustacchi 
331065f20420SRobert Mustacchi 	/*
331165f20420SRobert Mustacchi 	 * By default we've come in with retpolines enabled. Check whether we
3312651a12cbSRobert Mustacchi 	 * should disable them or enable enhanced or automatic IBRS. RSB
3313651a12cbSRobert Mustacchi 	 * stuffing is enabled by default. Note, we do not allow the use of AMD
3314651a12cbSRobert Mustacchi 	 * optimized retpolines as it was disclosed by AMD in March 2022 that
3315651a12cbSRobert Mustacchi 	 * they were still vulnerable. Prior to that point, we used them.
331665f20420SRobert Mustacchi 	 */
331765f20420SRobert Mustacchi 	if (x86_disable_spectrev2 != 0) {
331865f20420SRobert Mustacchi 		v2mit = X86_SPECTREV2_DISABLED;
3319651a12cbSRobert Mustacchi 	} else if (is_x86_feature(featureset, X86FSET_AUTO_IBRS)) {
3320651a12cbSRobert Mustacchi 		cpuid_enable_auto_ibrs();
3321651a12cbSRobert Mustacchi 		v2mit = X86_SPECTREV2_AUTO_IBRS;
332265f20420SRobert Mustacchi 	} else if (is_x86_feature(featureset, X86FSET_IBRS_ALL)) {
332365f20420SRobert Mustacchi 		cpuid_enable_enhanced_ibrs();
332465f20420SRobert Mustacchi 		v2mit = X86_SPECTREV2_ENHANCED_IBRS;
332565f20420SRobert Mustacchi 	} else {
332665f20420SRobert Mustacchi 		v2mit = X86_SPECTREV2_RETPOLINE;
332765f20420SRobert Mustacchi 	}
332865f20420SRobert Mustacchi 
332965f20420SRobert Mustacchi 	cpuid_patch_retpolines(v2mit);
333065f20420SRobert Mustacchi 	cpuid_patch_rsb(v2mit);
333165f20420SRobert Mustacchi 	x86_spectrev2_mitigation = v2mit;
333265f20420SRobert Mustacchi 	membar_producer();
3333455e370cSJohn Levon 
3334455e370cSJohn Levon 	/*
3335a9cc46cfSRobert Mustacchi 	 * We need to determine what changes are required for mitigating L1TF
3336c3377ee9SJohn Levon 	 * and MDS. If the CPU suffers from either of them, then SMT exclusion
3337c3377ee9SJohn Levon 	 * is required.
3338455e370cSJohn Levon 	 *
3339a9cc46cfSRobert Mustacchi 	 * If any of these are present, then we need to flush u-arch state at
3340a9cc46cfSRobert Mustacchi 	 * various points. For MDS, we need to do so whenever we change to a
3341a9cc46cfSRobert Mustacchi 	 * lesser privilege level or we are halting the CPU. For L1TF we need to
3342a9cc46cfSRobert Mustacchi 	 * flush the L1D cache at VM entry. When we have microcode that handles
3343a9cc46cfSRobert Mustacchi 	 * MDS, the L1D flush also clears the other u-arch state that the
3344c3377ee9SJohn Levon 	 * md_clear does.
3345455e370cSJohn Levon 	 */
3346455e370cSJohn Levon 
3347455e370cSJohn Levon 	/*
3348a9cc46cfSRobert Mustacchi 	 * Update whether or not we need to be taking explicit action against
3349*5cd084edSDan McDonald 	 * MDS or RFDS.
3350455e370cSJohn Levon 	 */
3351a9cc46cfSRobert Mustacchi 	cpuid_update_md_clear(cpu, featureset);
3352455e370cSJohn Levon 
3353a9cc46cfSRobert Mustacchi 	/*
3354c3377ee9SJohn Levon 	 * Determine whether SMT exclusion is required and whether or not we
3355c3377ee9SJohn Levon 	 * need to perform an l1d flush.
3356a9cc46cfSRobert Mustacchi 	 */
3357a9cc46cfSRobert Mustacchi 	cpuid_update_l1d_flush(cpu, featureset);
3358e25cb0e7SJohn Levon 
3359e25cb0e7SJohn Levon 	/*
3360e25cb0e7SJohn Levon 	 * Determine what our mitigation strategy should be for TAA and then
3361e25cb0e7SJohn Levon 	 * also apply TAA mitigations.
3362e25cb0e7SJohn Levon 	 */
3363e25cb0e7SJohn Levon 	cpuid_update_tsx(cpu, featureset);
3364e25cb0e7SJohn Levon 	cpuid_apply_tsx(x86_taa_mitigation, featureset);
336501add34aSRobert Mustacchi }
336601add34aSRobert Mustacchi 
33677af88ac7SKuriakose Kuruvilla /*
33687af88ac7SKuriakose Kuruvilla  * Setup XFeature_Enabled_Mask register. Required by xsave feature.
33697af88ac7SKuriakose Kuruvilla  */
33707af88ac7SKuriakose Kuruvilla void
setup_xfem(void)33717af88ac7SKuriakose Kuruvilla setup_xfem(void)
33727af88ac7SKuriakose Kuruvilla {
33737af88ac7SKuriakose Kuruvilla 	uint64_t flags = XFEATURE_LEGACY_FP;
33747af88ac7SKuriakose Kuruvilla 
33757af88ac7SKuriakose Kuruvilla 	ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
33767af88ac7SKuriakose Kuruvilla 
33777af88ac7SKuriakose Kuruvilla 	if (is_x86_feature(x86_featureset, X86FSET_SSE))
33787af88ac7SKuriakose Kuruvilla 		flags |= XFEATURE_SSE;
33797af88ac7SKuriakose Kuruvilla 
33807af88ac7SKuriakose Kuruvilla 	if (is_x86_feature(x86_featureset, X86FSET_AVX))
33817af88ac7SKuriakose Kuruvilla 		flags |= XFEATURE_AVX;
33827af88ac7SKuriakose Kuruvilla 
3383088d69f8SJerry Jelinek 	if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
3384088d69f8SJerry Jelinek 		flags |= XFEATURE_AVX512;
3385088d69f8SJerry Jelinek 
33867af88ac7SKuriakose Kuruvilla 	set_xcr(XFEATURE_ENABLED_MASK, flags);
33877af88ac7SKuriakose Kuruvilla 
33887af88ac7SKuriakose Kuruvilla 	xsave_bv_all = flags;
33897af88ac7SKuriakose Kuruvilla }
33907af88ac7SKuriakose Kuruvilla 
33910ce813ffSRobert Mustacchi static void
cpuid_basic_topology(cpu_t * cpu,uchar_t * featureset)3392ab5bb018SKeith M Wesolowski cpuid_basic_topology(cpu_t *cpu, uchar_t *featureset)
33930ce813ffSRobert Mustacchi {
33940ce813ffSRobert Mustacchi 	struct cpuid_info *cpi;
33950ce813ffSRobert Mustacchi 
33960ce813ffSRobert Mustacchi 	cpi = cpu->cpu_m.mcpu_cpi;
33970ce813ffSRobert Mustacchi 
33989b0429a1SPu Wen 	if (cpi->cpi_vendor == X86_VENDOR_AMD ||
33999b0429a1SPu Wen 	    cpi->cpi_vendor == X86_VENDOR_HYGON) {
34000ce813ffSRobert Mustacchi 		cpuid_gather_amd_topology_leaves(cpu);
34010ce813ffSRobert Mustacchi 	}
34020ce813ffSRobert Mustacchi 
34030ce813ffSRobert Mustacchi 	cpi->cpi_apicid = cpuid_gather_apicid(cpi);
34040ce813ffSRobert Mustacchi 
34050ce813ffSRobert Mustacchi 	/*
34060ce813ffSRobert Mustacchi 	 * Before we can calculate the IDs that we should assign to this
34070ce813ffSRobert Mustacchi 	 * processor, we need to understand how many cores and threads it has.
34080ce813ffSRobert Mustacchi 	 */
34090ce813ffSRobert Mustacchi 	switch (cpi->cpi_vendor) {
34100ce813ffSRobert Mustacchi 	case X86_VENDOR_Intel:
34110ce813ffSRobert Mustacchi 		cpuid_intel_ncores(cpi, &cpi->cpi_ncpu_per_chip,
34120ce813ffSRobert Mustacchi 		    &cpi->cpi_ncore_per_chip);
34130ce813ffSRobert Mustacchi 		break;
34140ce813ffSRobert Mustacchi 	case X86_VENDOR_AMD:
34159b0429a1SPu Wen 	case X86_VENDOR_HYGON:
34160ce813ffSRobert Mustacchi 		cpuid_amd_ncores(cpi, &cpi->cpi_ncpu_per_chip,
34170ce813ffSRobert Mustacchi 		    &cpi->cpi_ncore_per_chip);
34180ce813ffSRobert Mustacchi 		break;
34190ce813ffSRobert Mustacchi 	default:
34200ce813ffSRobert Mustacchi 		/*
34210ce813ffSRobert Mustacchi 		 * If we have some other x86 compatible chip, it's not clear how
34220ce813ffSRobert Mustacchi 		 * they would behave. The most common case is virtualization
34230ce813ffSRobert Mustacchi 		 * today, though there are also 64-bit VIA chips. Assume that
34240ce813ffSRobert Mustacchi 		 * all we can get is the basic Leaf 1 HTT information.
34250ce813ffSRobert Mustacchi 		 */
34260ce813ffSRobert Mustacchi 		if ((cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_HTT) != 0) {
34270ce813ffSRobert Mustacchi 			cpi->cpi_ncore_per_chip = 1;
34280ce813ffSRobert Mustacchi 			cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
34290ce813ffSRobert Mustacchi 		}
34300ce813ffSRobert Mustacchi 		break;
34310ce813ffSRobert Mustacchi 	}
34320ce813ffSRobert Mustacchi 
34330ce813ffSRobert Mustacchi 	/*
34340ce813ffSRobert Mustacchi 	 * Based on the calculated number of threads and cores, potentially
34350ce813ffSRobert Mustacchi 	 * assign the HTT and CMT features.
34360ce813ffSRobert Mustacchi 	 */
34370ce813ffSRobert Mustacchi 	if (cpi->cpi_ncore_per_chip > 1) {
34380ce813ffSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_CMP);
34390ce813ffSRobert Mustacchi 	}
34400ce813ffSRobert Mustacchi 
34410ce813ffSRobert Mustacchi 	if (cpi->cpi_ncpu_per_chip > 1 &&
34420ce813ffSRobert Mustacchi 	    cpi->cpi_ncpu_per_chip != cpi->cpi_ncore_per_chip) {
34430ce813ffSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_HTT);
34440ce813ffSRobert Mustacchi 	}
34450ce813ffSRobert Mustacchi 
34460ce813ffSRobert Mustacchi 	/*
34470ce813ffSRobert Mustacchi 	 * Now that has been set up, we need to go through and calculate all of
34480ce813ffSRobert Mustacchi 	 * the rest of the parameters that exist. If we think the CPU doesn't
34490ce813ffSRobert Mustacchi 	 * have either SMT (HTT) or CMP, then we basically go through and fake
34500ce813ffSRobert Mustacchi 	 * up information in some way. The most likely case for this is
34510ce813ffSRobert Mustacchi 	 * virtualization where we have a lot of partial topology information.
34520ce813ffSRobert Mustacchi 	 */
34530ce813ffSRobert Mustacchi 	if (!is_x86_feature(featureset, X86FSET_HTT) &&
34540ce813ffSRobert Mustacchi 	    !is_x86_feature(featureset, X86FSET_CMP)) {
34550ce813ffSRobert Mustacchi 		/*
34560ce813ffSRobert Mustacchi 		 * This is a single core, single-threaded processor.
34570ce813ffSRobert Mustacchi 		 */
34580ce813ffSRobert Mustacchi 		cpi->cpi_procnodes_per_pkg = 1;
34590ce813ffSRobert Mustacchi 		cpi->cpi_cores_per_compunit = 1;
34600ce813ffSRobert Mustacchi 		cpi->cpi_compunitid = 0;
34610ce813ffSRobert Mustacchi 		cpi->cpi_chipid = -1;
34620ce813ffSRobert Mustacchi 		cpi->cpi_clogid = 0;
34630ce813ffSRobert Mustacchi 		cpi->cpi_coreid = cpu->cpu_id;
34640ce813ffSRobert Mustacchi 		cpi->cpi_pkgcoreid = 0;
34659b0429a1SPu Wen 		if (cpi->cpi_vendor == X86_VENDOR_AMD ||
34669b0429a1SPu Wen 		    cpi->cpi_vendor == X86_VENDOR_HYGON) {
34670ce813ffSRobert Mustacchi 			cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
34680ce813ffSRobert Mustacchi 		} else {
34690ce813ffSRobert Mustacchi 			cpi->cpi_procnodeid = cpi->cpi_chipid;
34700ce813ffSRobert Mustacchi 		}
34710ce813ffSRobert Mustacchi 	} else {
34720ce813ffSRobert Mustacchi 		switch (cpi->cpi_vendor) {
34730ce813ffSRobert Mustacchi 		case X86_VENDOR_Intel:
34740ce813ffSRobert Mustacchi 			cpuid_intel_getids(cpu, featureset);
34750ce813ffSRobert Mustacchi 			break;
34760ce813ffSRobert Mustacchi 		case X86_VENDOR_AMD:
34779b0429a1SPu Wen 		case X86_VENDOR_HYGON:
34780ce813ffSRobert Mustacchi 			cpuid_amd_getids(cpu, featureset);
34790ce813ffSRobert Mustacchi 			break;
34800ce813ffSRobert Mustacchi 		default:
34810ce813ffSRobert Mustacchi 			/*
34820ce813ffSRobert Mustacchi 			 * In this case, it's hard to say what we should do.
34830ce813ffSRobert Mustacchi 			 * We're going to model them to the OS as single core
34840ce813ffSRobert Mustacchi 			 * threads. We don't have a good identifier for them, so
34850ce813ffSRobert Mustacchi 			 * we're just going to use the cpu id all on a single
34860ce813ffSRobert Mustacchi 			 * chip.
34870ce813ffSRobert Mustacchi 			 *
34880ce813ffSRobert Mustacchi 			 * This case has historically been different from the
34890ce813ffSRobert Mustacchi 			 * case above where we don't have HTT or CMP. While they
34900ce813ffSRobert Mustacchi 			 * could be combined, we've opted to keep it separate to
34910ce813ffSRobert Mustacchi 			 * minimize the risk of topology changes in weird cases.
34920ce813ffSRobert Mustacchi 			 */
34930ce813ffSRobert Mustacchi 			cpi->cpi_procnodes_per_pkg = 1;
34940ce813ffSRobert Mustacchi 			cpi->cpi_cores_per_compunit = 1;
34950ce813ffSRobert Mustacchi 			cpi->cpi_chipid = 0;
34960ce813ffSRobert Mustacchi 			cpi->cpi_coreid = cpu->cpu_id;
34970ce813ffSRobert Mustacchi 			cpi->cpi_clogid = cpu->cpu_id;
34980ce813ffSRobert Mustacchi 			cpi->cpi_pkgcoreid = cpu->cpu_id;
34990ce813ffSRobert Mustacchi 			cpi->cpi_procnodeid = cpi->cpi_chipid;
35000ce813ffSRobert Mustacchi 			cpi->cpi_compunitid = cpi->cpi_coreid;
35010ce813ffSRobert Mustacchi 			break;
35020ce813ffSRobert Mustacchi 		}
35030ce813ffSRobert Mustacchi 	}
35040ce813ffSRobert Mustacchi }
35050ce813ffSRobert Mustacchi 
3506f2dbfd32SRobert Mustacchi /*
3507f2dbfd32SRobert Mustacchi  * Gather relevant CPU features from leaf 6 which covers thermal information. We
3508f2dbfd32SRobert Mustacchi  * always gather leaf 6 if it's supported; however, we only look for features on
3509f2dbfd32SRobert Mustacchi  * Intel systems as AMD does not currently define any of the features we look
3510f2dbfd32SRobert Mustacchi  * for below.
3511f2dbfd32SRobert Mustacchi  */
3512f2dbfd32SRobert Mustacchi static void
cpuid_basic_thermal(cpu_t * cpu,uchar_t * featureset)3513ab5bb018SKeith M Wesolowski cpuid_basic_thermal(cpu_t *cpu, uchar_t *featureset)
3514f2dbfd32SRobert Mustacchi {
3515f2dbfd32SRobert Mustacchi 	struct cpuid_regs *cp;
3516f2dbfd32SRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3517f2dbfd32SRobert Mustacchi 
3518f2dbfd32SRobert Mustacchi 	if (cpi->cpi_maxeax < 6) {
3519f2dbfd32SRobert Mustacchi 		return;
3520f2dbfd32SRobert Mustacchi 	}
3521f2dbfd32SRobert Mustacchi 
3522f2dbfd32SRobert Mustacchi 	cp = &cpi->cpi_std[6];
3523f2dbfd32SRobert Mustacchi 	cp->cp_eax = 6;
3524f2dbfd32SRobert Mustacchi 	cp->cp_ebx = cp->cp_ecx = cp->cp_edx = 0;
3525f2dbfd32SRobert Mustacchi 	(void) __cpuid_insn(cp);
3526f2dbfd32SRobert Mustacchi 	platform_cpuid_mangle(cpi->cpi_vendor, 6, cp);
3527f2dbfd32SRobert Mustacchi 
3528f2dbfd32SRobert Mustacchi 	if (cpi->cpi_vendor != X86_VENDOR_Intel) {
3529f2dbfd32SRobert Mustacchi 		return;
3530f2dbfd32SRobert Mustacchi 	}
3531f2dbfd32SRobert Mustacchi 
3532f2dbfd32SRobert Mustacchi 	if ((cp->cp_eax & CPUID_INTC_EAX_DTS) != 0) {
3533f2dbfd32SRobert Mustacchi 		add_x86_feature(featureset, X86FSET_CORE_THERMAL);
3534f2dbfd32SRobert Mustacchi 	}
3535f2dbfd32SRobert Mustacchi 
3536f2dbfd32SRobert Mustacchi 	if ((cp->cp_eax & CPUID_INTC_EAX_PTM) != 0) {
3537f2dbfd32SRobert Mustacchi 		add_x86_feature(featureset, X86FSET_PKG_THERMAL);
3538f2dbfd32SRobert Mustacchi 	}
3539f2dbfd32SRobert Mustacchi }
3540f2dbfd32SRobert Mustacchi 
354156726c7eSRobert Mustacchi /*
354256726c7eSRobert Mustacchi  * This is used when we discover that we have AVX support in cpuid. This
354356726c7eSRobert Mustacchi  * proceeds to scan for the rest of the AVX derived features.
354456726c7eSRobert Mustacchi  */
354556726c7eSRobert Mustacchi static void
cpuid_basic_avx(cpu_t * cpu,uchar_t * featureset)354656726c7eSRobert Mustacchi cpuid_basic_avx(cpu_t *cpu, uchar_t *featureset)
354756726c7eSRobert Mustacchi {
354856726c7eSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
354956726c7eSRobert Mustacchi 
355056726c7eSRobert Mustacchi 	/*
355156726c7eSRobert Mustacchi 	 * If we don't have AVX, don't bother with most of this.
355256726c7eSRobert Mustacchi 	 */
355356726c7eSRobert Mustacchi 	if ((cpi->cpi_std[1].cp_ecx & CPUID_INTC_ECX_AVX) == 0)
355456726c7eSRobert Mustacchi 		return;
355556726c7eSRobert Mustacchi 
355656726c7eSRobert Mustacchi 	add_x86_feature(featureset, X86FSET_AVX);
355756726c7eSRobert Mustacchi 
355856726c7eSRobert Mustacchi 	/*
355956726c7eSRobert Mustacchi 	 * Intel says we can't check these without also
356056726c7eSRobert Mustacchi 	 * checking AVX.
356156726c7eSRobert Mustacchi 	 */
356256726c7eSRobert Mustacchi 	if (cpi->cpi_std[1].cp_ecx & CPUID_INTC_ECX_F16C)
356356726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_F16C);
356456726c7eSRobert Mustacchi 
356556726c7eSRobert Mustacchi 	if (cpi->cpi_std[1].cp_ecx & CPUID_INTC_ECX_FMA)
356656726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_FMA);
356756726c7eSRobert Mustacchi 
356856726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_BMI1)
356956726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_BMI1);
357056726c7eSRobert Mustacchi 
357156726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_BMI2)
357256726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_BMI2);
357356726c7eSRobert Mustacchi 
357456726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX2)
357556726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX2);
357656726c7eSRobert Mustacchi 
357756726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_VAES)
357856726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_VAES);
357956726c7eSRobert Mustacchi 
358056726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_VPCLMULQDQ)
358156726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_VPCLMULQDQ);
358256726c7eSRobert Mustacchi 
358356726c7eSRobert Mustacchi 	/*
358456726c7eSRobert Mustacchi 	 * The rest of the AVX features require AVX512. Do not check them unless
358556726c7eSRobert Mustacchi 	 * it is present.
358656726c7eSRobert Mustacchi 	 */
358756726c7eSRobert Mustacchi 	if ((cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512F) == 0)
358856726c7eSRobert Mustacchi 		return;
358956726c7eSRobert Mustacchi 	add_x86_feature(featureset, X86FSET_AVX512F);
359056726c7eSRobert Mustacchi 
359156726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
359256726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512DQ);
359356726c7eSRobert Mustacchi 
359456726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
359556726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512FMA);
359656726c7eSRobert Mustacchi 
359756726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512PF)
359856726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512PF);
359956726c7eSRobert Mustacchi 
360056726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512ER)
360156726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512ER);
360256726c7eSRobert Mustacchi 
360356726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512CD)
360456726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512CD);
360556726c7eSRobert Mustacchi 
360656726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512BW)
360756726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512BW);
360856726c7eSRobert Mustacchi 
360956726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512VL)
361056726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512VL);
361156726c7eSRobert Mustacchi 
361256726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VBMI)
361356726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512VBMI);
361456726c7eSRobert Mustacchi 
361556726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VBMI2)
361656726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512_VBMI2);
361756726c7eSRobert Mustacchi 
361856726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VNNI)
361956726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512VNNI);
362056726c7eSRobert Mustacchi 
362156726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512BITALG)
362256726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512_BITALG);
362356726c7eSRobert Mustacchi 
362456726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
362556726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512VPOPCDQ);
362656726c7eSRobert Mustacchi 
362756726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_edx & CPUID_INTC_EDX_7_0_AVX5124NNIW)
362856726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512NNIW);
362956726c7eSRobert Mustacchi 
363056726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_edx & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
363156726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512FMAPS);
363256726c7eSRobert Mustacchi 
363356726c7eSRobert Mustacchi 	/*
363456726c7eSRobert Mustacchi 	 * More features here are in Leaf 7, subleaf 1. Don't bother checking if
363556726c7eSRobert Mustacchi 	 * we don't need to.
363656726c7eSRobert Mustacchi 	 */
363756726c7eSRobert Mustacchi 	if (cpi->cpi_std[7].cp_eax < 1)
363856726c7eSRobert Mustacchi 		return;
363956726c7eSRobert Mustacchi 
364056726c7eSRobert Mustacchi 	if (cpi->cpi_sub7[0].cp_eax & CPUID_INTC_EAX_7_1_AVX512_BF16)
364156726c7eSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_AVX512_BF16);
364256726c7eSRobert Mustacchi }
364356726c7eSRobert Mustacchi 
3644a47ab03eSRobert Mustacchi /*
3645a47ab03eSRobert Mustacchi  * PPIN is the protected processor inventory number. On AMD this is an actual
3646a47ab03eSRobert Mustacchi  * feature bit. However, on Intel systems we need to read the platform
3647a47ab03eSRobert Mustacchi  * information MSR if we're on a specific model.
3648a47ab03eSRobert Mustacchi  */
3649a47ab03eSRobert Mustacchi #if !defined(__xpv)
3650a47ab03eSRobert Mustacchi static void
cpuid_basic_ppin(cpu_t * cpu,uchar_t * featureset)3651ab5bb018SKeith M Wesolowski cpuid_basic_ppin(cpu_t *cpu, uchar_t *featureset)
3652a47ab03eSRobert Mustacchi {
3653a47ab03eSRobert Mustacchi 	on_trap_data_t otd;
3654a47ab03eSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
3655a47ab03eSRobert Mustacchi 
3656a47ab03eSRobert Mustacchi 	switch (cpi->cpi_vendor) {
3657a47ab03eSRobert Mustacchi 	case X86_VENDOR_AMD:
3658a47ab03eSRobert Mustacchi 		/*
3659a47ab03eSRobert Mustacchi 		 * This leaf will have already been gathered in the topology
3660a47ab03eSRobert Mustacchi 		 * functions.
3661a47ab03eSRobert Mustacchi 		 */
3662a47ab03eSRobert Mustacchi 		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
3663a47ab03eSRobert Mustacchi 			if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PPIN) {
3664a47ab03eSRobert Mustacchi 				add_x86_feature(featureset, X86FSET_PPIN);
3665a47ab03eSRobert Mustacchi 			}
3666a47ab03eSRobert Mustacchi 		}
3667a47ab03eSRobert Mustacchi 		break;
3668a47ab03eSRobert Mustacchi 	case X86_VENDOR_Intel:
3669a47ab03eSRobert Mustacchi 		if (cpi->cpi_family != 6)
3670a47ab03eSRobert Mustacchi 			break;
3671a47ab03eSRobert Mustacchi 		switch (cpi->cpi_model) {
3672a47ab03eSRobert Mustacchi 		case INTC_MODEL_IVYBRIDGE_XEON:
3673a47ab03eSRobert Mustacchi 		case INTC_MODEL_HASWELL_XEON:
3674a47ab03eSRobert Mustacchi 		case INTC_MODEL_BROADWELL_XEON:
3675a47ab03eSRobert Mustacchi 		case INTC_MODEL_BROADWELL_XEON_D:
3676a47ab03eSRobert Mustacchi 		case INTC_MODEL_SKYLAKE_XEON:
367715276eb5SRobert Mustacchi 		case INTC_MODEL_ICELAKE_XEON:
3678a47ab03eSRobert Mustacchi 			if (!on_trap(&otd, OT_DATA_ACCESS)) {
3679a47ab03eSRobert Mustacchi 				uint64_t value;
3680a47ab03eSRobert Mustacchi 
3681a47ab03eSRobert Mustacchi 				value = rdmsr(MSR_PLATFORM_INFO);
3682a47ab03eSRobert Mustacchi 				if ((value & MSR_PLATFORM_INFO_PPIN) != 0) {
3683a47ab03eSRobert Mustacchi 					add_x86_feature(featureset,
3684a47ab03eSRobert Mustacchi 					    X86FSET_PPIN);
3685a47ab03eSRobert Mustacchi 				}
3686a47ab03eSRobert Mustacchi 			}
3687a47ab03eSRobert Mustacchi 			no_trap();
3688a47ab03eSRobert Mustacchi 			break;
3689a47ab03eSRobert Mustacchi 		default:
3690a47ab03eSRobert Mustacchi 			break;
3691a47ab03eSRobert Mustacchi 		}
3692a47ab03eSRobert Mustacchi 		break;
3693a47ab03eSRobert Mustacchi 	default:
3694a47ab03eSRobert Mustacchi 		break;
3695a47ab03eSRobert Mustacchi 	}
3696a47ab03eSRobert Mustacchi }
3697a47ab03eSRobert Mustacchi #endif	/* ! __xpv */
3698a47ab03eSRobert Mustacchi 
3699ab5bb018SKeith M Wesolowski static void
cpuid_pass_prelude(cpu_t * cpu,void * arg)3700ab5bb018SKeith M Wesolowski cpuid_pass_prelude(cpu_t *cpu, void *arg)
3701ab5bb018SKeith M Wesolowski {
3702ab5bb018SKeith M Wesolowski 	uchar_t *featureset = (uchar_t *)arg;
3703ab5bb018SKeith M Wesolowski 
3704ab5bb018SKeith M Wesolowski 	/*
3705ab5bb018SKeith M Wesolowski 	 * We don't run on any processor that doesn't have cpuid, and could not
3706ab5bb018SKeith M Wesolowski 	 * possibly have arrived here.
3707ab5bb018SKeith M Wesolowski 	 */
3708ab5bb018SKeith M Wesolowski 	add_x86_feature(featureset, X86FSET_CPUID);
3709ab5bb018SKeith M Wesolowski }
3710ab5bb018SKeith M Wesolowski 
3711ab5bb018SKeith M Wesolowski static void
cpuid_pass_ident(cpu_t * cpu,void * arg __unused)3712ab5bb018SKeith M Wesolowski cpuid_pass_ident(cpu_t *cpu, void *arg __unused)
37137c478bd9Sstevel@tonic-gate {
37147c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi;
37158949bcd6Sandrei 	struct cpuid_regs *cp;
3716ae115bc7Smrj 
37177c478bd9Sstevel@tonic-gate 	/*
3718ab5bb018SKeith M Wesolowski 	 * We require that virtual/native detection be complete and that PCI
3719ab5bb018SKeith M Wesolowski 	 * config space access has been set up; at present there is no reliable
3720ab5bb018SKeith M Wesolowski 	 * way to determine the latter.
37217c478bd9Sstevel@tonic-gate 	 */
372222e4c3acSKeith M Wesolowski #if !defined(__xpv)
3723ab5bb018SKeith M Wesolowski 	ASSERT3S(platform_type, !=, -1);
372422e4c3acSKeith M Wesolowski #endif	/* !__xpv */
37257417cfdeSKuriakose Kuruvilla 
3726ae115bc7Smrj 	cpi = cpu->cpu_m.mcpu_cpi;
3727ae115bc7Smrj 	ASSERT(cpi != NULL);
3728ab5bb018SKeith M Wesolowski 
37297c478bd9Sstevel@tonic-gate 	cp = &cpi->cpi_std[0];
37308949bcd6Sandrei 	cp->cp_eax = 0;
37318949bcd6Sandrei 	cpi->cpi_maxeax = __cpuid_insn(cp);
37327c478bd9Sstevel@tonic-gate 	{
37337c478bd9Sstevel@tonic-gate 		uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
37347c478bd9Sstevel@tonic-gate 		*iptr++ = cp->cp_ebx;
37357c478bd9Sstevel@tonic-gate 		*iptr++ = cp->cp_edx;
37367c478bd9Sstevel@tonic-gate 		*iptr++ = cp->cp_ecx;
37377c478bd9Sstevel@tonic-gate 		*(char *)&cpi->cpi_vendorstr[12] = '\0';
37387c478bd9Sstevel@tonic-gate 	}
37397c478bd9Sstevel@tonic-gate 
3740e4b86885SCheng Sean Ye 	cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
37417c478bd9Sstevel@tonic-gate 	x86_vendor = cpi->cpi_vendor; /* for compatibility */
37427c478bd9Sstevel@tonic-gate 
37437c478bd9Sstevel@tonic-gate 	/*
37447c478bd9Sstevel@tonic-gate 	 * Limit the range in case of weird hardware
37457c478bd9Sstevel@tonic-gate 	 */
37467c478bd9Sstevel@tonic-gate 	if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
37477c478bd9Sstevel@tonic-gate 		cpi->cpi_maxeax = CPI_MAXEAX_MAX;
37487c478bd9Sstevel@tonic-gate 	if (cpi->cpi_maxeax < 1)
3749ab5bb018SKeith M Wesolowski 		return;
37507c478bd9Sstevel@tonic-gate 
37517c478bd9Sstevel@tonic-gate 	cp = &cpi->cpi_std[1];
37528949bcd6Sandrei 	cp->cp_eax = 1;
37538949bcd6Sandrei 	(void) __cpuid_insn(cp);
37547c478bd9Sstevel@tonic-gate 
37557c478bd9Sstevel@tonic-gate 	/*
37567c478bd9Sstevel@tonic-gate 	 * Extract identifying constants for easy access.
37577c478bd9Sstevel@tonic-gate 	 */
37587c478bd9Sstevel@tonic-gate 	cpi->cpi_model = CPI_MODEL(cpi);
37597c478bd9Sstevel@tonic-gate 	cpi->cpi_family = CPI_FAMILY(cpi);
37607c478bd9Sstevel@tonic-gate 
37615ff02082Sdmick 	if (cpi->cpi_family == 0xf)
37627c478bd9Sstevel@tonic-gate 		cpi->cpi_family += CPI_FAMILY_XTD(cpi);
37635ff02082Sdmick 
376468c91426Sdmick 	/*
3765875b116eSkchow 	 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
376668c91426Sdmick 	 * Intel, and presumably everyone else, uses model == 0xf, as
376768c91426Sdmick 	 * one would expect (max value means possible overflow).  Sigh.
376868c91426Sdmick 	 */
376968c91426Sdmick 
377068c91426Sdmick 	switch (cpi->cpi_vendor) {
3771bf91205bSksadhukh 	case X86_VENDOR_Intel:
3772bf91205bSksadhukh 		if (IS_EXTENDED_MODEL_INTEL(cpi))
3773bf91205bSksadhukh 			cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
3774447af253Sksadhukh 		break;
377568c91426Sdmick 	case X86_VENDOR_AMD:
3776875b116eSkchow 		if (CPI_FAMILY(cpi) == 0xf)
377768c91426Sdmick 			cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
377868c91426Sdmick 		break;
37799b0429a1SPu Wen 	case X86_VENDOR_HYGON:
37809b0429a1SPu Wen 		cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
37819b0429a1SPu Wen 		break;
378268c91426Sdmick 	default:
378368c91426Sdmick 		if (cpi->cpi_model == 0xf)
378468c91426Sdmick 			cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
378568c91426Sdmick 		break;
378668c91426Sdmick 	}
37877c478bd9Sstevel@tonic-gate 
37887c478bd9Sstevel@tonic-gate 	cpi->cpi_step = CPI_STEP(cpi);
37897c478bd9Sstevel@tonic-gate 	cpi->cpi_brandid = CPI_BRANDID(cpi);
37907c478bd9Sstevel@tonic-gate 
3791ab5bb018SKeith M Wesolowski 	/*
3792ab5bb018SKeith M Wesolowski 	 * Synthesize chip "revision" and socket type
3793ab5bb018SKeith M Wesolowski 	 */
3794ab5bb018SKeith M Wesolowski 	cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
3795ab5bb018SKeith M Wesolowski 	    cpi->cpi_model, cpi->cpi_step);
3796ab5bb018SKeith M Wesolowski 	cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
3797ab5bb018SKeith M Wesolowski 	    cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
3798ab5bb018SKeith M Wesolowski 	cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
3799ab5bb018SKeith M Wesolowski 	    cpi->cpi_model, cpi->cpi_step);
380022e4c3acSKeith M Wesolowski 	cpi->cpi_uarchrev = _cpuid_uarchrev(cpi->cpi_vendor, cpi->cpi_family,
380122e4c3acSKeith M Wesolowski 	    cpi->cpi_model, cpi->cpi_step);
3802ab5bb018SKeith M Wesolowski }
3803ab5bb018SKeith M Wesolowski 
3804ab5bb018SKeith M Wesolowski static void
cpuid_pass_basic(cpu_t * cpu,void * arg)3805ab5bb018SKeith M Wesolowski cpuid_pass_basic(cpu_t *cpu, void *arg)
3806ab5bb018SKeith M Wesolowski {
3807ab5bb018SKeith M Wesolowski 	uchar_t *featureset = (uchar_t *)arg;
3808ab5bb018SKeith M Wesolowski 	uint32_t mask_ecx, mask_edx;
3809ab5bb018SKeith M Wesolowski 	struct cpuid_info *cpi;
3810ab5bb018SKeith M Wesolowski 	struct cpuid_regs *cp;
3811ab5bb018SKeith M Wesolowski 	int xcpuid;
3812ab5bb018SKeith M Wesolowski #if !defined(__xpv)
3813ab5bb018SKeith M Wesolowski 	extern int idle_cpu_prefer_mwait;
3814ab5bb018SKeith M Wesolowski #endif
3815ab5bb018SKeith M Wesolowski 
3816ab5bb018SKeith M Wesolowski 	cpi = cpu->cpu_m.mcpu_cpi;
3817ab5bb018SKeith M Wesolowski 	ASSERT(cpi != NULL);
3818ab5bb018SKeith M Wesolowski 
3819ab5bb018SKeith M Wesolowski 	if (cpi->cpi_maxeax < 1)
3820ab5bb018SKeith M Wesolowski 		return;
3821ab5bb018SKeith M Wesolowski 
3822ab5bb018SKeith M Wesolowski 	/*
3823ab5bb018SKeith M Wesolowski 	 * This was filled during the identification pass.
3824ab5bb018SKeith M Wesolowski 	 */
3825ab5bb018SKeith M Wesolowski 	cp = &cpi->cpi_std[1];
3826ab5bb018SKeith M Wesolowski 
38277c478bd9Sstevel@tonic-gate 	/*
38287c478bd9Sstevel@tonic-gate 	 * *default* assumptions:
38297c478bd9Sstevel@tonic-gate 	 * - believe %edx feature word
38307c478bd9Sstevel@tonic-gate 	 * - ignore %ecx feature word
38317c478bd9Sstevel@tonic-gate 	 * - 32-bit virtual and physical addressing
38327c478bd9Sstevel@tonic-gate 	 */
38337c478bd9Sstevel@tonic-gate 	mask_edx = 0xffffffff;
38347c478bd9Sstevel@tonic-gate 	mask_ecx = 0;
38357c478bd9Sstevel@tonic-gate 
38367c478bd9Sstevel@tonic-gate 	cpi->cpi_pabits = cpi->cpi_vabits = 32;
38377c478bd9Sstevel@tonic-gate 
38387c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
38397c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
38407c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5)
38417c478bd9Sstevel@tonic-gate 			x86_type = X86_TYPE_P5;
38425ff02082Sdmick 		else if (IS_LEGACY_P6(cpi)) {
38437c478bd9Sstevel@tonic-gate 			x86_type = X86_TYPE_P6;
38447c478bd9Sstevel@tonic-gate 			pentiumpro_bug4046376 = 1;
38457c478bd9Sstevel@tonic-gate 			/*
38467c478bd9Sstevel@tonic-gate 			 * Clear the SEP bit when it was set erroneously
38477c478bd9Sstevel@tonic-gate 			 */
38487c478bd9Sstevel@tonic-gate 			if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
38497c478bd9Sstevel@tonic-gate 				cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
38505ff02082Sdmick 		} else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
38517c478bd9Sstevel@tonic-gate 			x86_type = X86_TYPE_P4;
38527c478bd9Sstevel@tonic-gate 			/*
38537c478bd9Sstevel@tonic-gate 			 * We don't currently depend on any of the %ecx
38547c478bd9Sstevel@tonic-gate 			 * features until Prescott, so we'll only check
38557c478bd9Sstevel@tonic-gate 			 * this from P4 onwards.  We might want to revisit
38567c478bd9Sstevel@tonic-gate 			 * that idea later.
38577c478bd9Sstevel@tonic-gate 			 */
38587c478bd9Sstevel@tonic-gate 			mask_ecx = 0xffffffff;
38597c478bd9Sstevel@tonic-gate 		} else if (cpi->cpi_family > 0xf)
38607c478bd9Sstevel@tonic-gate 			mask_ecx = 0xffffffff;
38617c622d23Sbholler 		/*
38627c622d23Sbholler 		 * We don't support MONITOR/MWAIT if leaf 5 is not available
38637c622d23Sbholler 		 * to obtain the monitor linesize.
38647c622d23Sbholler 		 */
38657c622d23Sbholler 		if (cpi->cpi_maxeax < 5)
38667c622d23Sbholler 			mask_ecx &= ~CPUID_INTC_ECX_MON;
38677c478bd9Sstevel@tonic-gate 		break;
38687c478bd9Sstevel@tonic-gate 	case X86_VENDOR_IntelClone:
38697c478bd9Sstevel@tonic-gate 	default:
38707c478bd9Sstevel@tonic-gate 		break;
38717c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
38727c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_108)
38737c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
38747c478bd9Sstevel@tonic-gate 			cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
38757c478bd9Sstevel@tonic-gate 			cpi->cpi_model = 0xc;
38767c478bd9Sstevel@tonic-gate 		} else
38777c478bd9Sstevel@tonic-gate #endif
38787c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5) {
38797c478bd9Sstevel@tonic-gate 			/*
38807c478bd9Sstevel@tonic-gate 			 * AMD K5 and K6
38817c478bd9Sstevel@tonic-gate 			 *
38827c478bd9Sstevel@tonic-gate 			 * These CPUs have an incomplete implementation
38837c478bd9Sstevel@tonic-gate 			 * of MCA/MCE which we mask away.
38847c478bd9Sstevel@tonic-gate 			 */
38858949bcd6Sandrei 			mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);
38868949bcd6Sandrei 
38878949bcd6Sandrei 			/*
38888949bcd6Sandrei 			 * Model 0 uses the wrong (APIC) bit
38898949bcd6Sandrei 			 * to indicate PGE.  Fix it here.
38908949bcd6Sandrei 			 */
38917c478bd9Sstevel@tonic-gate 			if (cpi->cpi_model == 0) {
38927c478bd9Sstevel@tonic-gate 				if (cp->cp_edx & 0x200) {
38937c478bd9Sstevel@tonic-gate 					cp->cp_edx &= ~0x200;
38947c478bd9Sstevel@tonic-gate 					cp->cp_edx |= CPUID_INTC_EDX_PGE;
38957c478bd9Sstevel@tonic-gate 				}
38968949bcd6Sandrei 			}
38978949bcd6Sandrei 
38988949bcd6Sandrei 			/*
38998949bcd6Sandrei 			 * Early models had problems w/ MMX; disable.
39008949bcd6Sandrei 			 */
39018949bcd6Sandrei 			if (cpi->cpi_model < 6)
39028949bcd6Sandrei 				mask_edx &= ~CPUID_INTC_EDX_MMX;
39037c478bd9Sstevel@tonic-gate 		}
39048949bcd6Sandrei 
39058949bcd6Sandrei 		/*
39068949bcd6Sandrei 		 * For newer families, SSE3 and CX16, at least, are valid;
39078949bcd6Sandrei 		 * enable all
39088949bcd6Sandrei 		 */
39098949bcd6Sandrei 		if (cpi->cpi_family >= 0xf)
39108949bcd6Sandrei 			mask_ecx = 0xffffffff;
39117c622d23Sbholler 		/*
39127c622d23Sbholler 		 * We don't support MONITOR/MWAIT if leaf 5 is not available
39137c622d23Sbholler 		 * to obtain the monitor linesize.
39147c622d23Sbholler 		 */
39157c622d23Sbholler 		if (cpi->cpi_maxeax < 5)
39167c622d23Sbholler 			mask_ecx &= ~CPUID_INTC_ECX_MON;
39175b8a6efeSbholler 
3918843e1988Sjohnlev #if !defined(__xpv)
39195b8a6efeSbholler 		/*
3920007d65bcSRobert Mustacchi 		 * AMD has not historically used MWAIT in the CPU's idle loop.
3921007d65bcSRobert Mustacchi 		 * Pre-family-10h Opterons do not have the MWAIT instruction. We
3922007d65bcSRobert Mustacchi 		 * know for certain that in at least family 17h, per AMD, mwait
3923007d65bcSRobert Mustacchi 		 * is preferred. Families in-between are less certain.
39245b8a6efeSbholler 		 */
3925007d65bcSRobert Mustacchi 		if (cpi->cpi_family < 0x17) {
3926007d65bcSRobert Mustacchi 			idle_cpu_prefer_mwait = 0;
3927007d65bcSRobert Mustacchi 		}
3928843e1988Sjohnlev #endif
39295b8a6efeSbholler 
39309b0429a1SPu Wen 		break;
39319b0429a1SPu Wen 	case X86_VENDOR_HYGON:
39329b0429a1SPu Wen 		/* Enable all for Hygon Dhyana CPU */
39339b0429a1SPu Wen 		mask_ecx = 0xffffffff;
39347c478bd9Sstevel@tonic-gate 		break;
39357c478bd9Sstevel@tonic-gate 	case X86_VENDOR_TM:
39367c478bd9Sstevel@tonic-gate 		/*
39377c478bd9Sstevel@tonic-gate 		 * workaround the NT workaround in CMS 4.1
39387c478bd9Sstevel@tonic-gate 		 */
39397c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
39407c478bd9Sstevel@tonic-gate 		    (cpi->cpi_step == 2 || cpi->cpi_step == 3))
39417c478bd9Sstevel@tonic-gate 			cp->cp_edx |= CPUID_INTC_EDX_CX8;
39427c478bd9Sstevel@tonic-gate 		break;
39437c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Centaur:
39447c478bd9Sstevel@tonic-gate 		/*
39457c478bd9Sstevel@tonic-gate 		 * workaround the NT workarounds again
39467c478bd9Sstevel@tonic-gate 		 */
39477c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 6)
39487c478bd9Sstevel@tonic-gate 			cp->cp_edx |= CPUID_INTC_EDX_CX8;
39497c478bd9Sstevel@tonic-gate 		break;
39507c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Cyrix:
39517c478bd9Sstevel@tonic-gate 		/*
39527c478bd9Sstevel@tonic-gate 		 * We rely heavily on the probing in locore
39537c478bd9Sstevel@tonic-gate 		 * to actually figure out what parts, if any,
39547c478bd9Sstevel@tonic-gate 		 * of the Cyrix cpuid instruction to believe.
39557c478bd9Sstevel@tonic-gate 		 */
39567c478bd9Sstevel@tonic-gate 		switch (x86_type) {
39577c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_486:
39587c478bd9Sstevel@tonic-gate 			mask_edx = 0;
39597c478bd9Sstevel@tonic-gate 			break;
39607c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_6x86:
39617c478bd9Sstevel@tonic-gate 			mask_edx = 0;
39627c478bd9Sstevel@tonic-gate 			break;
39637c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_6x86L:
39647c478bd9Sstevel@tonic-gate 			mask_edx =
39657c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_DE |
39667c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CX8;
39677c478bd9Sstevel@tonic-gate 			break;
39687c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_6x86MX:
39697c478bd9Sstevel@tonic-gate 			mask_edx =
39707c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_DE |
39717c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_MSR |
39727c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CX8 |
39737c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_PGE |
39747c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CMOV |
39757c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_MMX;
39767c478bd9Sstevel@tonic-gate 			break;
39777c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_GXm:
39787c478bd9Sstevel@tonic-gate 			mask_edx =
39797c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_MSR |
39807c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CX8 |
39817c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CMOV |
39827c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_MMX;
39837c478bd9Sstevel@tonic-gate 			break;
39847c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_MediaGX:
39857c478bd9Sstevel@tonic-gate 			break;
39867c478bd9Sstevel@tonic-gate 		case X86_TYPE_CYRIX_MII:
39877c478bd9Sstevel@tonic-gate 		case X86_TYPE_VIA_CYRIX_III:
39887c478bd9Sstevel@tonic-gate 			mask_edx =
39897c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_DE |
39907c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_TSC |
39917c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_MSR |
39927c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CX8 |
39937c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_PGE |
39947c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_CMOV |
39957c478bd9Sstevel@tonic-gate 			    CPUID_INTC_EDX_MMX;
39967c478bd9Sstevel@tonic-gate 			break;
39977c478bd9Sstevel@tonic-gate 		default:
39987c478bd9Sstevel@tonic-gate 			break;
39997c478bd9Sstevel@tonic-gate 		}
40007c478bd9Sstevel@tonic-gate 		break;
40017c478bd9Sstevel@tonic-gate 	}
40027c478bd9Sstevel@tonic-gate 
4003843e1988Sjohnlev #if defined(__xpv)
4004843e1988Sjohnlev 	/*
4005843e1988Sjohnlev 	 * Do not support MONITOR/MWAIT under a hypervisor
4006843e1988Sjohnlev 	 */
4007843e1988Sjohnlev 	mask_ecx &= ~CPUID_INTC_ECX_MON;
40087af88ac7SKuriakose Kuruvilla 	/*
40097af88ac7SKuriakose Kuruvilla 	 * Do not support XSAVE under a hypervisor for now
40107af88ac7SKuriakose Kuruvilla 	 */
40117af88ac7SKuriakose Kuruvilla 	xsave_force_disable = B_TRUE;
40127af88ac7SKuriakose Kuruvilla 
4013843e1988Sjohnlev #endif	/* __xpv */
4014843e1988Sjohnlev 
40157af88ac7SKuriakose Kuruvilla 	if (xsave_force_disable) {
40167af88ac7SKuriakose Kuruvilla 		mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
40177af88ac7SKuriakose Kuruvilla 		mask_ecx &= ~CPUID_INTC_ECX_AVX;
4018ebb8ac07SRobert Mustacchi 		mask_ecx &= ~CPUID_INTC_ECX_F16C;
4019245ac945SRobert Mustacchi 		mask_ecx &= ~CPUID_INTC_ECX_FMA;
40207af88ac7SKuriakose Kuruvilla 	}
40217af88ac7SKuriakose Kuruvilla 
40227c478bd9Sstevel@tonic-gate 	/*
40237c478bd9Sstevel@tonic-gate 	 * Now we've figured out the masks that determine
40247c478bd9Sstevel@tonic-gate 	 * which bits we choose to believe, apply the masks
40257c478bd9Sstevel@tonic-gate 	 * to the feature words, then map the kernel's view
40267c478bd9Sstevel@tonic-gate 	 * of these feature words into its feature word.
40277c478bd9Sstevel@tonic-gate 	 */
40287c478bd9Sstevel@tonic-gate 	cp->cp_edx &= mask_edx;
40297c478bd9Sstevel@tonic-gate 	cp->cp_ecx &= mask_ecx;
40307c478bd9Sstevel@tonic-gate 
40317c478bd9Sstevel@tonic-gate 	/*
4032ae115bc7Smrj 	 * apply any platform restrictions (we don't call this
4033ae115bc7Smrj 	 * immediately after __cpuid_insn here, because we need the
4034ae115bc7Smrj 	 * workarounds applied above first)
40357c478bd9Sstevel@tonic-gate 	 */
4036ae115bc7Smrj 	platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);
40377c478bd9Sstevel@tonic-gate 
4038245ac945SRobert Mustacchi 	/*
403974ac317bSRobert Mustacchi 	 * In addition to ecx and edx, Intel and AMD are storing a bunch of
404056726c7eSRobert Mustacchi 	 * instruction set extensions in leaf 7's ebx, ecx, and edx. Note, leaf
404156726c7eSRobert Mustacchi 	 * 7 has sub-leaves determined by ecx.
4042245ac945SRobert Mustacchi 	 */
404374ac317bSRobert Mustacchi 	if (cpi->cpi_maxeax >= 7) {
4044245ac945SRobert Mustacchi 		struct cpuid_regs *ecp;
4045245ac945SRobert Mustacchi 		ecp = &cpi->cpi_std[7];
4046245ac945SRobert Mustacchi 		ecp->cp_eax = 7;
4047245ac945SRobert Mustacchi 		ecp->cp_ecx = 0;
4048245ac945SRobert Mustacchi 		(void) __cpuid_insn(ecp);
404901add34aSRobert Mustacchi 
4050245ac945SRobert Mustacchi 		/*
4051088d69f8SJerry Jelinek 		 * If XSAVE has been disabled, just ignore all of the
405256726c7eSRobert Mustacchi 		 * extended-save-area dependent flags here. By removing most of
405356726c7eSRobert Mustacchi 		 * the leaf 7, sub-leaf 0 flags, that will ensure tha we don't
405456726c7eSRobert Mustacchi 		 * end up looking at additional xsave dependent leaves right
405556726c7eSRobert Mustacchi 		 * now.
4056245ac945SRobert Mustacchi 		 */
4057245ac945SRobert Mustacchi 		if (xsave_force_disable) {
4058245ac945SRobert Mustacchi 			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
4059245ac945SRobert Mustacchi 			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
4060245ac945SRobert Mustacchi 			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
4061088d69f8SJerry Jelinek 			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
4062088d69f8SJerry Jelinek 			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
4063088d69f8SJerry Jelinek 			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
4064088d69f8SJerry Jelinek 			ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
40655edbd2feSRobert Mustacchi 			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_VAES;
40665edbd2feSRobert Mustacchi 			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_VPCLMULQDQ;
406756726c7eSRobert Mustacchi 			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_GFNI;
4068245ac945SRobert Mustacchi 		}
4069799823bbSRobert Mustacchi 
4070799823bbSRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
4071799823bbSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_SMEP);
40723ce2fcdcSRobert Mustacchi 
40733ce2fcdcSRobert Mustacchi 		/*
40743ce2fcdcSRobert Mustacchi 		 * We check disable_smap here in addition to in startup_smap()
40753ce2fcdcSRobert Mustacchi 		 * to ensure CPUs that aren't the boot CPU don't accidentally
40763ce2fcdcSRobert Mustacchi 		 * include it in the feature set and thus generate a mismatched
407774ac317bSRobert Mustacchi 		 * x86 feature set across CPUs.
40783ce2fcdcSRobert Mustacchi 		 */
40793ce2fcdcSRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
40803ce2fcdcSRobert Mustacchi 		    disable_smap == 0)
40813ce2fcdcSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_SMAP);
4082088d69f8SJerry Jelinek 
40838889c875SRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED)
40848889c875SRobert Mustacchi 			add_x86_feature(featureset, X86FSET_RDSEED);
40858889c875SRobert Mustacchi 
40868889c875SRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
40878889c875SRobert Mustacchi 			add_x86_feature(featureset, X86FSET_ADX);
408874ac317bSRobert Mustacchi 
4089cff040f3SRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_FSGSBASE)
4090cff040f3SRobert Mustacchi 			add_x86_feature(featureset, X86FSET_FSGSBASE);
4091cff040f3SRobert Mustacchi 
4092cff040f3SRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_CLFLUSHOPT)
4093cff040f3SRobert Mustacchi 			add_x86_feature(featureset, X86FSET_CLFLUSHOPT);
4094cff040f3SRobert Mustacchi 
4095c4b98cefSRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_INVPCID)
4096c4b98cefSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_INVPCID);
409774ac317bSRobert Mustacchi 
40985edbd2feSRobert Mustacchi 		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
40995edbd2feSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_UMIP);
41005edbd2feSRobert Mustacchi 		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_PKU)
41015edbd2feSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_PKU);
41025edbd2feSRobert Mustacchi 		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
41035edbd2feSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_OSPKE);
410456726c7eSRobert Mustacchi 		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_GFNI)
410556726c7eSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_GFNI);
410656726c7eSRobert Mustacchi 
410756726c7eSRobert Mustacchi 		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_CLWB)
410856726c7eSRobert Mustacchi 			add_x86_feature(featureset, X86FSET_CLWB);
41095edbd2feSRobert Mustacchi 
4110c4b98cefSRobert Mustacchi 		if (cpi->cpi_vendor == X86_VENDOR_Intel) {
411174ac317bSRobert Mustacchi 			if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
411274ac317bSRobert Mustacchi 				add_x86_feature(featureset, X86FSET_MPX);
411356726c7eSRobert Mustacchi 		}
4114cff040f3SRobert Mustacchi 
411556726c7eSRobert Mustacchi 		/*
411656726c7eSRobert Mustacchi 		 * If we have subleaf 1 available, grab and store that. This is
411756726c7eSRobert Mustacchi 		 * used for more AVX and related features.
411856726c7eSRobert Mustacchi 		 */
411956726c7eSRobert Mustacchi 		if (ecp->cp_eax >= 1) {
412056726c7eSRobert Mustacchi 			struct cpuid_regs *c71;
412156726c7eSRobert Mustacchi 			c71 = &cpi->cpi_sub7[0];
412256726c7eSRobert Mustacchi 			c71->cp_eax = 7;
412356726c7eSRobert Mustacchi 			c71->cp_ecx = 1;
412456726c7eSRobert Mustacchi 			(void) __cpuid_insn(c71);
412574ac317bSRobert Mustacchi 		}
4126245ac945SRobert Mustacchi 	}
4127245ac945SRobert Mustacchi 
4128ae115bc7Smrj 	/*
4129ae115bc7Smrj 	 * fold in overrides from the "eeprom" mechanism
4130ae115bc7Smrj 	 */
41317c478bd9Sstevel@tonic-gate 	cp->cp_edx |= cpuid_feature_edx_include;
41327c478bd9Sstevel@tonic-gate 	cp->cp_edx &= ~cpuid_feature_edx_exclude;
41337c478bd9Sstevel@tonic-gate 
41347c478bd9Sstevel@tonic-gate 	cp->cp_ecx |= cpuid_feature_ecx_include;
41357c478bd9Sstevel@tonic-gate 	cp->cp_ecx &= ~cpuid_feature_ecx_exclude;
41367c478bd9Sstevel@tonic-gate 
41377417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
41387417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_LARGEPAGE);
41397417cfdeSKuriakose Kuruvilla 	}
41407417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
41417417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_TSC);
41427417cfdeSKuriakose Kuruvilla 	}
41437417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
41447417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_MSR);
41457417cfdeSKuriakose Kuruvilla 	}
41467417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
41477417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_MTRR);
41487417cfdeSKuriakose Kuruvilla 	}
41497417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
41507417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_PGE);
41517417cfdeSKuriakose Kuruvilla 	}
41527417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
41537417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_CMOV);
41547417cfdeSKuriakose Kuruvilla 	}
41557417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
41567417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_MMX);
41577417cfdeSKuriakose Kuruvilla 	}
41587c478bd9Sstevel@tonic-gate 	if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
41597417cfdeSKuriakose Kuruvilla 	    (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
41607417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_MCA);
41617417cfdeSKuriakose Kuruvilla 	}
41627417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
41637417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_PAE);
41647417cfdeSKuriakose Kuruvilla 	}
41657417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
41667417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_CX8);
41677417cfdeSKuriakose Kuruvilla 	}
41687417cfdeSKuriakose Kuruvilla 	if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
41697417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_CX16);
41707417cfdeSKuriakose Kuruvilla 	}
41717417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
41727417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_PAT);
41737417cfdeSKuriakose Kuruvilla 	}
41747417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
41757417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_SEP);
41767417cfdeSKuriakose Kuruvilla 	}
41777c478bd9Sstevel@tonic-gate 	if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
41787c478bd9Sstevel@tonic-gate 		/*
41797c478bd9Sstevel@tonic-gate 		 * In our implementation, fxsave/fxrstor
41807c478bd9Sstevel@tonic-gate 		 * are prerequisites before we'll even
41817c478bd9Sstevel@tonic-gate 		 * try and do SSE things.
41827c478bd9Sstevel@tonic-gate 		 */
41837417cfdeSKuriakose Kuruvilla 		if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
41847417cfdeSKuriakose Kuruvilla 			add_x86_feature(featureset, X86FSET_SSE);
41857417cfdeSKuriakose Kuruvilla 		}
41867417cfdeSKuriakose Kuruvilla 		if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
41877417cfdeSKuriakose Kuruvilla 			add_x86_feature(featureset, X86FSET_SSE2);
41887417cfdeSKuriakose Kuruvilla 		}
41897417cfdeSKuriakose Kuruvilla 		if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
41907417cfdeSKuriakose Kuruvilla 			add_x86_feature(featureset, X86FSET_SSE3);
41917417cfdeSKuriakose Kuruvilla 		}
419263408480SHans Rosenfeld 		if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
419363408480SHans Rosenfeld 			add_x86_feature(featureset, X86FSET_SSSE3);
419463408480SHans Rosenfeld 		}
419563408480SHans Rosenfeld 		if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
419663408480SHans Rosenfeld 			add_x86_feature(featureset, X86FSET_SSE4_1);
419763408480SHans Rosenfeld 		}
419863408480SHans Rosenfeld 		if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
419963408480SHans Rosenfeld 			add_x86_feature(featureset, X86FSET_SSE4_2);
420063408480SHans Rosenfeld 		}
420163408480SHans Rosenfeld 		if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
420263408480SHans Rosenfeld 			add_x86_feature(featureset, X86FSET_AES);
420363408480SHans Rosenfeld 		}
420463408480SHans Rosenfeld 		if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
420563408480SHans Rosenfeld 			add_x86_feature(featureset, X86FSET_PCLMULQDQ);
420663408480SHans Rosenfeld 		}
42077af88ac7SKuriakose Kuruvilla 
4208088d69f8SJerry Jelinek 		if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
4209088d69f8SJerry Jelinek 			add_x86_feature(featureset, X86FSET_SHA);
4210088d69f8SJerry Jelinek 
421163408480SHans Rosenfeld 		if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
421263408480SHans Rosenfeld 			add_x86_feature(featureset, X86FSET_XSAVE);
4213ebb8ac07SRobert Mustacchi 
4214088d69f8SJerry Jelinek 			/* We only test AVX & AVX512 when there is XSAVE */
421556726c7eSRobert Mustacchi 			cpuid_basic_avx(cpu, featureset);
4216d0f8ff6eSkk 		}
42177c478bd9Sstevel@tonic-gate 	}
421874ecdb51SJohn Levon 
4219c4b98cefSRobert Mustacchi 	if (cp->cp_ecx & CPUID_INTC_ECX_PCID) {
4220c4b98cefSRobert Mustacchi 		add_x86_feature(featureset, X86FSET_PCID);
422174ecdb51SJohn Levon 	}
422274ecdb51SJohn Levon 
42236eedf6a5SJosef 'Jeff' Sipek 	if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
42246eedf6a5SJosef 'Jeff' Sipek 		add_x86_feature(featureset, X86FSET_X2APIC);
42256eedf6a5SJosef 'Jeff' Sipek 	}
42267417cfdeSKuriakose Kuruvilla 	if (cp->cp_edx & CPUID_INTC_EDX_DE) {
42277417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_DE);
42287417cfdeSKuriakose Kuruvilla 	}
42291d1a3942SBill Holler #if !defined(__xpv)
4230f98fbcecSbholler 	if (cp->cp_ecx & CPUID_INTC_ECX_MON) {
42311d1a3942SBill Holler 
42321d1a3942SBill Holler 		/*
42331d1a3942SBill Holler 		 * We require the CLFLUSH instruction for erratum workaround
42341d1a3942SBill Holler 		 * to use MONITOR/MWAIT.
42351d1a3942SBill Holler 		 */
42361d1a3942SBill Holler 		if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
42371d1a3942SBill Holler 			cpi->cpi_mwait.support |= MWAIT_SUPPORT;
42387417cfdeSKuriakose Kuruvilla 			add_x86_feature(featureset, X86FSET_MWAIT);
42391d1a3942SBill Holler 		} else {
42401d1a3942SBill Holler 			extern int idle_cpu_assert_cflush_monitor;
42411d1a3942SBill Holler 
42421d1a3942SBill Holler 			/*
42431d1a3942SBill Holler 			 * All processors we are aware of which have
42441d1a3942SBill Holler 			 * MONITOR/MWAIT also have CLFLUSH.
42451d1a3942SBill Holler 			 */
42461d1a3942SBill Holler 			if (idle_cpu_assert_cflush_monitor) {
42471d1a3942SBill Holler 				ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
42481d1a3942SBill Holler 				    (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
42491d1a3942SBill Holler 			}
42501d1a3942SBill Holler 		}
4251f98fbcecSbholler 	}
42521d1a3942SBill Holler #endif	/* __xpv */
42537c478bd9Sstevel@tonic-gate 
4254faa20166SBryan Cantrill 	if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
4255faa20166SBryan Cantrill 		add_x86_feature(featureset, X86FSET_VMX);
4256faa20166SBryan Cantrill 	}
4257faa20166SBryan Cantrill 
4258ebb8ac07SRobert Mustacchi 	if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
4259ebb8ac07SRobert Mustacchi 		add_x86_feature(featureset, X86FSET_RDRAND);
4260ebb8ac07SRobert Mustacchi 
426186c1f4dcSVikram Hegde 	/*
4262faa20166SBryan Cantrill 	 * Only need it first time, rest of the cpus would follow suit.
426386c1f4dcSVikram Hegde 	 * we only capture this for the bootcpu.
426486c1f4dcSVikram Hegde 	 */
426586c1f4dcSVikram Hegde 	if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
42667417cfdeSKuriakose Kuruvilla 		add_x86_feature(featureset, X86FSET_CLFSH);
426786c1f4dcSVikram Hegde 		x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
426886c1f4dcSVikram Hegde 	}
42697417cfdeSKuriakose Kuruvilla 	if (is_x86_feature(featureset, X86FSET_PAE))
42707c478bd9Sstevel@tonic-gate 		cpi->cpi_pabits = 36;
42717c478bd9Sstevel@tonic-gate 
427274ac317bSRobert Mustacchi 	if (cpi->cpi_maxeax >= 0xD && !xsave_force_disable) {
4273088d69f8SJerry Jelinek 		struct cpuid_regs r, *ecp;
4274088d69f8SJerry Jelinek 
4275088d69f8SJerry Jelinek 		ecp = &r;
4276088d69f8SJerry Jelinek 		ecp->cp_eax = 0xD;
4277088d69f8SJerry Jelinek 		ecp->cp_ecx = 1;
4278088d69f8SJerry Jelinek 		ecp->cp_edx = ecp->cp_ebx = 0;
4279088d69f8SJerry Jelinek 		(void) __cpuid_insn(ecp);
4280088d69f8SJerry Jelinek 
4281088d69f8SJerry Jelinek 		if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
4282088d69f8SJerry Jelinek 			add_x86_feature(featureset, X86FSET_XSAVEOPT);
4283088d69f8SJerry Jelinek 		if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
4284088d69f8SJerry Jelinek 			add_x86_feature(featureset, X86FSET_XSAVEC);
4285088d69f8SJerry Jelinek 		if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
4286088d69f8SJerry Jelinek 			add_x86_feature(featureset, X86FSET_XSAVES);
4287a65c38a3SRobert Mustacchi 
4288a65c38a3SRobert Mustacchi 		/*
4289a65c38a3SRobert Mustacchi 		 * Zen 2 family processors suffer from erratum 1386 that causes
4290a65c38a3SRobert Mustacchi 		 * xsaves to not function correctly in some circumstances. There
4291a65c38a3SRobert Mustacchi 		 * are no supervisor states in Zen 2 and earlier. Practically
4292a65c38a3SRobert Mustacchi 		 * speaking this has no impact for us as we currently do not
4293a65c38a3SRobert Mustacchi 		 * leverage compressed xsave formats. To safeguard against
4294a65c38a3SRobert Mustacchi 		 * issues in the future where we may opt to using it, we remove
4295a65c38a3SRobert Mustacchi 		 * it from the feature set now. While Matisse has a microcode
4296a65c38a3SRobert Mustacchi 		 * update available with a fix, not all Zen 2 CPUs do so it's
4297a65c38a3SRobert Mustacchi 		 * simpler for the moment to unconditionally remove it.
4298a65c38a3SRobert Mustacchi 		 */
4299a65c38a3SRobert Mustacchi 		if (cpi->cpi_vendor == X86_VENDOR_AMD &&
4300a65c38a3SRobert Mustacchi 		    uarchrev_uarch(cpi->cpi_uarchrev) <= X86_UARCH_AMD_ZEN2) {
4301a65c38a3SRobert Mustacchi 			remove_x86_feature(featureset, X86FSET_XSAVES);
4302a65c38a3SRobert Mustacchi 		}
4303088d69f8SJerry Jelinek 	}
4304088d69f8SJerry Jelinek 
43057c478bd9Sstevel@tonic-gate 	/*
43067c478bd9Sstevel@tonic-gate 	 * Work on the "extended" feature information, doing
4307ab5bb018SKeith M Wesolowski 	 * some basic initialization to be used in the extended pass.
43087c478bd9Sstevel@tonic-gate 	 */
43097c478bd9Sstevel@tonic-gate 	xcpuid = 0;
43107c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
43117c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
4312c2710388SDan Kimmel 		/*
4313c2710388SDan Kimmel 		 * On KVM we know we will have proper support for extended
4314c2710388SDan Kimmel 		 * cpuid.
4315c2710388SDan Kimmel 		 */
4316c2710388SDan Kimmel 		if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
4317c2710388SDan Kimmel 		    (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
4318c2710388SDan Kimmel 		    (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
43197c478bd9Sstevel@tonic-gate 			xcpuid++;
43207c478bd9Sstevel@tonic-gate 		break;
43217c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
43227c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family > 5 ||
43237c478bd9Sstevel@tonic-gate 		    (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
43247c478bd9Sstevel@tonic-gate 			xcpuid++;
43257c478bd9Sstevel@tonic-gate 		break;
43267c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Cyrix:
43277c478bd9Sstevel@tonic-gate 		/*
43287c478bd9Sstevel@tonic-gate 		 * Only these Cyrix CPUs are -known- to support
43297c478bd9Sstevel@tonic-gate 		 * extended cpuid operations.
43307c478bd9Sstevel@tonic-gate 		 */
43317c478bd9Sstevel@tonic-gate 		if (x86_type == X86_TYPE_VIA_CYRIX_III ||
43327c478bd9Sstevel@tonic-gate 		    x86_type == X86_TYPE_CYRIX_GXm)
43337c478bd9Sstevel@tonic-gate 			xcpuid++;
43347c478bd9Sstevel@tonic-gate 		break;
43359b0429a1SPu Wen 	case X86_VENDOR_HYGON:
43367c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Centaur:
43377c478bd9Sstevel@tonic-gate 	case X86_VENDOR_TM:
43387c478bd9Sstevel@tonic-gate 	default:
43397c478bd9Sstevel@tonic-gate 		xcpuid++;
43407c478bd9Sstevel@tonic-gate 		break;
43417c478bd9Sstevel@tonic-gate 	}
43427c478bd9Sstevel@tonic-gate 
43437c478bd9Sstevel@tonic-gate 	if (xcpuid) {
43447c478bd9Sstevel@tonic-gate 		cp = &cpi->cpi_extd[0];
43450ce813ffSRobert Mustacchi 		cp->cp_eax = CPUID_LEAF_EXT_0;
43468949bcd6Sandrei 		cpi->cpi_xmaxeax = __cpuid_insn(cp);
43477c478bd9Sstevel@tonic-gate 	}
43487c478bd9Sstevel@tonic-gate 
43490ce813ffSRobert Mustacchi 	if (cpi->cpi_xmaxeax & CPUID_LEAF_EXT_0) {
43507c478bd9Sstevel@tonic-gate 
43517c478bd9Sstevel@tonic-gate 		if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
43527c478bd9Sstevel@tonic-gate 			cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;
43537c478bd9Sstevel@tonic-gate 
43547c478bd9Sstevel@tonic-gate 		switch (cpi->cpi_vendor) {
43557c478bd9Sstevel@tonic-gate 		case X86_VENDOR_Intel:
43567c478bd9Sstevel@tonic-gate 		case X86_VENDOR_AMD:
43579b0429a1SPu Wen 		case X86_VENDOR_HYGON:
43587c478bd9Sstevel@tonic-gate 			if (cpi->cpi_xmaxeax < 0x80000001)
43597c478bd9Sstevel@tonic-gate 				break;
43607c478bd9Sstevel@tonic-gate 			cp = &cpi->cpi_extd[1];
43618949bcd6Sandrei 			cp->cp_eax = 0x80000001;
43628949bcd6Sandrei 			(void) __cpuid_insn(cp);
4363ae115bc7Smrj 
43647c478bd9Sstevel@tonic-gate 			if (cpi->cpi_vendor == X86_VENDOR_AMD &&
43657c478bd9Sstevel@tonic-gate 			    cpi->cpi_family == 5 &&
43667c478bd9Sstevel@tonic-gate 			    cpi->cpi_model == 6 &&
43677c478bd9Sstevel@tonic-gate 			    cpi->cpi_step == 6) {
43687c478bd9Sstevel@tonic-gate 				/*
43697c478bd9Sstevel@tonic-gate 				 * K6 model 6 uses bit 10 to indicate SYSC
43707c478bd9Sstevel@tonic-gate 				 * Later models use bit 11. Fix it here.
43717c478bd9Sstevel@tonic-gate 				 */
43727c478bd9Sstevel@tonic-gate 				if (cp->cp_edx & 0x400) {
43737c478bd9Sstevel@tonic-gate 					cp->cp_edx &= ~0x400;
43747c478bd9Sstevel@tonic-gate 					cp->cp_edx |= CPUID_AMD_EDX_SYSC;
43757c478bd9Sstevel@tonic-gate 				}
43767c478bd9Sstevel@tonic-gate 			}
43777c478bd9Sstevel@tonic-gate 
4378ae115bc7Smrj 			platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);
4379ae115bc7Smrj 
43807c478bd9Sstevel@tonic-gate 			/*
43817c478bd9Sstevel@tonic-gate 			 * Compute the additions to the kernel's feature word.
43827c478bd9Sstevel@tonic-gate 			 */
43837417cfdeSKuriakose Kuruvilla 			if (cp->cp_edx & CPUID_AMD_EDX_NX) {
43847417cfdeSKuriakose Kuruvilla 				add_x86_feature(featureset, X86FSET_NX);
43857417cfdeSKuriakose Kuruvilla 			}
43867c478bd9Sstevel@tonic-gate 
438719397407SSherry Moore 			/*
438819397407SSherry Moore 			 * Regardless whether or not we boot 64-bit,
438919397407SSherry Moore 			 * we should have a way to identify whether
439019397407SSherry Moore 			 * the CPU is capable of running 64-bit.
439119397407SSherry Moore 			 */
43927417cfdeSKuriakose Kuruvilla 			if (cp->cp_edx & CPUID_AMD_EDX_LM) {
43937417cfdeSKuriakose Kuruvilla 				add_x86_feature(featureset, X86FSET_64);
43947417cfdeSKuriakose Kuruvilla 			}
439519397407SSherry Moore 
439602bc52beSkchow 			/* 1 GB large page - enable only for 64 bit kernel */
43977417cfdeSKuriakose Kuruvilla 			if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
43987417cfdeSKuriakose Kuruvilla 				add_x86_feature(featureset, X86FSET_1GPG);
43997417cfdeSKuriakose Kuruvilla 			}
440002bc52beSkchow 
44019b0429a1SPu Wen 			if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
44029b0429a1SPu Wen 			    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
4403f8801251Skk 			    (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
44047417cfdeSKuriakose Kuruvilla 			    (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
44057417cfdeSKuriakose Kuruvilla 				add_x86_feature(featureset, X86FSET_SSE4A);
44067417cfdeSKuriakose Kuruvilla 			}
4407f8801251Skk 
44087c478bd9Sstevel@tonic-gate 			/*
44097c478bd9Sstevel@tonic-gate 			 * It's really tricky to support syscall/sysret in
44107c478bd9Sstevel@tonic-gate 			 * the i386 kernel; we rely on sysenter/sysexit
44117c478bd9Sstevel@tonic-gate 			 * instead.  In the amd64 kernel, things are -way-
44127c478bd9Sstevel@tonic-gate 			 * better.
44137c478bd9Sstevel@tonic-gate 			 */
44147417cfdeSKuriakose Kuruvilla 			if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
44157417cfdeSKuriakose Kuruvilla 				add_x86_feature(featureset, X86FSET_ASYSC);
44167417cfdeSKuriakose Kuruvilla 			}
44177c478bd9Sstevel@tonic-gate 
44187c478bd9Sstevel@tonic-gate 			/*
44197c478bd9Sstevel@tonic-gate 			 * While we're thinking about system calls, note
44207c478bd9Sstevel@tonic-gate 			 * that AMD processors don't support sysenter
44217c478bd9Sstevel@tonic-gate 			 * in long mode at all, so don't try to program them.
44227c478bd9Sstevel@tonic-gate 			 */
44239b0429a1SPu Wen 			if (x86_vendor == X86_VENDOR_AMD ||
44249b0429a1SPu Wen 			    x86_vendor == X86_VENDOR_HYGON) {
44257417cfdeSKuriakose Kuruvilla 				remove_x86_feature(featureset, X86FSET_SEP);
44267417cfdeSKuriakose Kuruvilla 			}
4427cff040f3SRobert Mustacchi 
44287417cfdeSKuriakose Kuruvilla 			if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
44297417cfdeSKuriakose Kuruvilla 				add_x86_feature(featureset, X86FSET_TSCP);
44307417cfdeSKuriakose Kuruvilla 			}
4431faa20166SBryan Cantrill 
4432faa20166SBryan Cantrill 			if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
4433faa20166SBryan Cantrill 				add_x86_feature(featureset, X86FSET_SVM);
4434faa20166SBryan Cantrill 			}
44357660e73fSHans Rosenfeld 
44367660e73fSHans Rosenfeld 			if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
44377660e73fSHans Rosenfeld 				add_x86_feature(featureset, X86FSET_TOPOEXT);
44387660e73fSHans Rosenfeld 			}
4439cff040f3SRobert Mustacchi 
4440d0e58ef5SRobert Mustacchi 			if (cp->cp_ecx & CPUID_AMD_ECX_PCEC) {
4441d0e58ef5SRobert Mustacchi 				add_x86_feature(featureset, X86FSET_AMD_PCEC);
4442d0e58ef5SRobert Mustacchi 			}
4443d0e58ef5SRobert Mustacchi 
4444cff040f3SRobert Mustacchi 			if (cp->cp_ecx & CPUID_AMD_ECX_XOP) {
4445cff040f3SRobert Mustacchi 				add_x86_feature(featureset, X86FSET_XOP);
4446cff040f3SRobert Mustacchi 			}
4447cff040f3SRobert Mustacchi 
4448cff040f3SRobert Mustacchi 			if (cp->cp_ecx & CPUID_AMD_ECX_FMA4) {
4449cff040f3SRobert Mustacchi 				add_x86_feature(featureset, X86FSET_FMA4);
4450cff040f3SRobert Mustacchi 			}
4451cff040f3SRobert Mustacchi 
4452cff040f3SRobert Mustacchi 			if (cp->cp_ecx & CPUID_AMD_ECX_TBM) {
4453cff040f3SRobert Mustacchi 				add_x86_feature(featureset, X86FSET_TBM);
4454cff040f3SRobert Mustacchi 			}
4455cff040f3SRobert Mustacchi 
4456cff040f3SRobert Mustacchi 			if (cp->cp_ecx & CPUID_AMD_ECX_MONITORX) {
4457cff040f3SRobert Mustacchi 				add_x86_feature(featureset, X86FSET_MONITORX);
4458cff040f3SRobert Mustacchi 			}
44597c478bd9Sstevel@tonic-gate 			break;
44607c478bd9Sstevel@tonic-gate 		default:
44617c478bd9Sstevel@tonic-gate 			break;
44627c478bd9Sstevel@tonic-gate 		}
44637c478bd9Sstevel@tonic-gate 
44648949bcd6Sandrei 		/*
44658949bcd6Sandrei 		 * Get CPUID data about processor cores and hyperthreads.
44668949bcd6Sandrei 		 */
44677c478bd9Sstevel@tonic-gate 		switch (cpi->cpi_vendor) {
44687c478bd9Sstevel@tonic-gate 		case X86_VENDOR_Intel:
44698949bcd6Sandrei 			if (cpi->cpi_maxeax >= 4) {
44708949bcd6Sandrei 				cp = &cpi->cpi_std[4];
44718949bcd6Sandrei 				cp->cp_eax = 4;
44728949bcd6Sandrei 				cp->cp_ecx = 0;
44738949bcd6Sandrei 				(void) __cpuid_insn(cp);
4474ae115bc7Smrj 				platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
44758949bcd6Sandrei 			}
44768949bcd6Sandrei 			/*FALLTHROUGH*/
44777c478bd9Sstevel@tonic-gate 		case X86_VENDOR_AMD:
44789b0429a1SPu Wen 		case X86_VENDOR_HYGON:
44790ce813ffSRobert Mustacchi 			if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_8)
44807c478bd9Sstevel@tonic-gate 				break;
44817c478bd9Sstevel@tonic-gate 			cp = &cpi->cpi_extd[8];
44820ce813ffSRobert Mustacchi 			cp->cp_eax = CPUID_LEAF_EXT_8;
44838949bcd6Sandrei 			(void) __cpuid_insn(cp);
44840ce813ffSRobert Mustacchi 			platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_8,
44850ce813ffSRobert Mustacchi 			    cp);
4486ae115bc7Smrj 
4487cff040f3SRobert Mustacchi 			/*
4488cff040f3SRobert Mustacchi 			 * AMD uses ebx for some extended functions.
4489cff040f3SRobert Mustacchi 			 */
44909b0429a1SPu Wen 			if (cpi->cpi_vendor == X86_VENDOR_AMD ||
44919b0429a1SPu Wen 			    cpi->cpi_vendor == X86_VENDOR_HYGON) {
4492cff040f3SRobert Mustacchi 				/*
4493cff040f3SRobert Mustacchi 				 * While we're here, check for the AMD "Error
4494cff040f3SRobert Mustacchi 				 * Pointer Zero/Restore" feature. This can be
4495cff040f3SRobert Mustacchi 				 * used to setup the FP save handlers
4496cff040f3SRobert Mustacchi 				 * appropriately.
4497cff040f3SRobert Mustacchi 				 */
4498cff040f3SRobert Mustacchi 				if (cp->cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
4499cff040f3SRobert Mustacchi 					cpi->cpi_fp_amd_save = 0;
4500cff040f3SRobert Mustacchi 				} else {
4501cff040f3SRobert Mustacchi 					cpi->cpi_fp_amd_save = 1;
4502cff040f3SRobert Mustacchi 				}
4503cff040f3SRobert Mustacchi 
4504cff040f3SRobert Mustacchi 				if (cp->cp_ebx & CPUID_AMD_EBX_CLZERO) {
4505cff040f3SRobert Mustacchi 					add_x86_feature(featureset,
4506cff040f3SRobert Mustacchi 					    X86FSET_CLZERO);
4507cff040f3SRobert Mustacchi 				}
4508cff040f3SRobert Mustacchi 			}
4509cff040f3SRobert Mustacchi 
45107c478bd9Sstevel@tonic-gate 			/*
45117c478bd9Sstevel@tonic-gate 			 * Virtual and physical address limits from
45127c478bd9Sstevel@tonic-gate 			 * cpuid override previously guessed values.
45137c478bd9Sstevel@tonic-gate 			 */
45147c478bd9Sstevel@tonic-gate 			cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
45157c478bd9Sstevel@tonic-gate 			cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
45168949bcd6Sandrei 			break;
45178949bcd6Sandrei 		default:
45188949bcd6Sandrei 			break;
45198949bcd6Sandrei 		}
45207c478bd9Sstevel@tonic-gate 
45210e751525SEric Saxe 		/*
45220e751525SEric Saxe 		 * Get CPUID data about TSC Invariance in Deep C-State.
45230e751525SEric Saxe 		 */
45240e751525SEric Saxe 		switch (cpi->cpi_vendor) {
45250e751525SEric Saxe 		case X86_VENDOR_Intel:
4526bf9b145bSRobert Mustacchi 		case X86_VENDOR_AMD:
45279b0429a1SPu Wen 		case X86_VENDOR_HYGON:
45280e751525SEric Saxe 			if (cpi->cpi_maxeax >= 7) {
45290e751525SEric Saxe 				cp = &cpi->cpi_extd[7];
45300e751525SEric Saxe 				cp->cp_eax = 0x80000007;
45310e751525SEric Saxe 				cp->cp_ecx = 0;
45320e751525SEric Saxe 				(void) __cpuid_insn(cp);
45330e751525SEric Saxe 			}
45340e751525SEric Saxe 			break;
45350e751525SEric Saxe 		default:
45360e751525SEric Saxe 			break;
45370e751525SEric Saxe 		}
45387c478bd9Sstevel@tonic-gate 	}
45397c478bd9Sstevel@tonic-gate 
4540a47ab03eSRobert Mustacchi 	/*
4541ab5bb018SKeith M Wesolowski 	 * cpuid_basic_ppin assumes that cpuid_basic_topology has already been
4542a47ab03eSRobert Mustacchi 	 * run and thus gathered some of its dependent leaves.
4543a47ab03eSRobert Mustacchi 	 */
4544ab5bb018SKeith M Wesolowski 	cpuid_basic_topology(cpu, featureset);
4545ab5bb018SKeith M Wesolowski 	cpuid_basic_thermal(cpu, featureset);
4546a47ab03eSRobert Mustacchi #if !defined(__xpv)
4547ab5bb018SKeith M Wesolowski 	cpuid_basic_ppin(cpu, featureset);
4548a47ab03eSRobert Mustacchi #endif
45497c478bd9Sstevel@tonic-gate 
45509b0429a1SPu Wen 	if (cpi->cpi_vendor == X86_VENDOR_AMD ||
45519b0429a1SPu Wen 	    cpi->cpi_vendor == X86_VENDOR_HYGON) {
45520ce813ffSRobert Mustacchi 		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8 &&
4553088d69f8SJerry Jelinek 		    cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
4554088d69f8SJerry Jelinek 			/* Special handling for AMD FP not necessary. */
4555088d69f8SJerry Jelinek 			cpi->cpi_fp_amd_save = 0;
4556088d69f8SJerry Jelinek 		} else {
4557088d69f8SJerry Jelinek 			cpi->cpi_fp_amd_save = 1;
4558088d69f8SJerry Jelinek 		}
4559088d69f8SJerry Jelinek 	}
4560088d69f8SJerry Jelinek 
4561beed421eSPatrick Mooney 	/*
4562beed421eSPatrick Mooney 	 * Check (and potentially set) if lfence is serializing.
4563beed421eSPatrick Mooney 	 * This is useful for accurate rdtsc measurements and AMD retpolines.
4564beed421eSPatrick Mooney 	 */
4565beed421eSPatrick Mooney 	if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
4566beed421eSPatrick Mooney 	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
4567beed421eSPatrick Mooney 	    is_x86_feature(featureset, X86FSET_SSE2)) {
4568beed421eSPatrick Mooney 		/*
4569beed421eSPatrick Mooney 		 * The AMD white paper Software Techniques For Managing
4570beed421eSPatrick Mooney 		 * Speculation on AMD Processors details circumstances for when
4571beed421eSPatrick Mooney 		 * lfence instructions are serializing.
4572beed421eSPatrick Mooney 		 *
4573beed421eSPatrick Mooney 		 * On family 0xf and 0x11, it is inherently so.  On family 0x10
4574beed421eSPatrick Mooney 		 * and later (excluding 0x11), a bit in the DE_CFG MSR
4575beed421eSPatrick Mooney 		 * determines the lfence behavior.  Per that whitepaper, AMD has
4576beed421eSPatrick Mooney 		 * committed to supporting that MSR on all later CPUs.
4577beed421eSPatrick Mooney 		 */
4578beed421eSPatrick Mooney 		if (cpi->cpi_family == 0xf || cpi->cpi_family == 0x11) {
4579beed421eSPatrick Mooney 			add_x86_feature(featureset, X86FSET_LFENCE_SER);
4580beed421eSPatrick Mooney 		} else if (cpi->cpi_family >= 0x10) {
45816ce41887SToomas Soome #if !defined(__xpv)
45823df2e8b2SRobert Mustacchi 			uint64_t val;
4583beed421eSPatrick Mooney 
4584beed421eSPatrick Mooney 			/*
4585beed421eSPatrick Mooney 			 * Be careful when attempting to enable the bit, and
4586beed421eSPatrick Mooney 			 * verify that it was actually set in case we are
4587beed421eSPatrick Mooney 			 * running in a hypervisor which is less than faithful
4588beed421eSPatrick Mooney 			 * about its emulation of this feature.
4589beed421eSPatrick Mooney 			 */
4590beed421eSPatrick Mooney 			on_trap_data_t otd;
4591beed421eSPatrick Mooney 			if (!on_trap(&otd, OT_DATA_ACCESS)) {
4592beed421eSPatrick Mooney 				val = rdmsr(MSR_AMD_DE_CFG);
4593beed421eSPatrick Mooney 				val |= AMD_DE_CFG_LFENCE_DISPATCH;
4594beed421eSPatrick Mooney 				wrmsr(MSR_AMD_DE_CFG, val);
4595beed421eSPatrick Mooney 				val = rdmsr(MSR_AMD_DE_CFG);
45963df2e8b2SRobert Mustacchi 			} else {
45973df2e8b2SRobert Mustacchi 				val = 0;
4598beed421eSPatrick Mooney 			}
4599beed421eSPatrick Mooney 			no_trap();
4600beed421eSPatrick Mooney 
4601beed421eSPatrick Mooney 			if ((val & AMD_DE_CFG_LFENCE_DISPATCH) != 0) {
4602beed421eSPatrick Mooney 				add_x86_feature(featureset, X86FSET_LFENCE_SER);
4603beed421eSPatrick Mooney 			}
46046ce41887SToomas Soome #endif
4605beed421eSPatrick Mooney 		}
4606beed421eSPatrick Mooney 	} else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
4607beed421eSPatrick Mooney 	    is_x86_feature(featureset, X86FSET_SSE2)) {
4608beed421eSPatrick Mooney 		/*
4609beed421eSPatrick Mooney 		 * Documentation and other OSes indicate that lfence is always
4610beed421eSPatrick Mooney 		 * serializing on Intel CPUs.
4611beed421eSPatrick Mooney 		 */
4612beed421eSPatrick Mooney 		add_x86_feature(featureset, X86FSET_LFENCE_SER);
4613beed421eSPatrick Mooney 	}
4614beed421eSPatrick Mooney 
4615beed421eSPatrick Mooney 
461601add34aSRobert Mustacchi 	/*
4617651a12cbSRobert Mustacchi 	 * Check the processor leaves that are used for security features. Grab
4618651a12cbSRobert Mustacchi 	 * any additional processor-specific leaves that we may not have yet.
461901add34aSRobert Mustacchi 	 */
4620651a12cbSRobert Mustacchi 	switch (cpi->cpi_vendor) {
4621651a12cbSRobert Mustacchi 	case X86_VENDOR_AMD:
4622651a12cbSRobert Mustacchi 	case X86_VENDOR_HYGON:
4623651a12cbSRobert Mustacchi 		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_21) {
4624651a12cbSRobert Mustacchi 			cp = &cpi->cpi_extd[7];
4625651a12cbSRobert Mustacchi 			cp->cp_eax = CPUID_LEAF_EXT_21;
4626651a12cbSRobert Mustacchi 			cp->cp_ecx = 0;
4627651a12cbSRobert Mustacchi 			(void) __cpuid_insn(cp);
4628651a12cbSRobert Mustacchi 		}
4629651a12cbSRobert Mustacchi 		break;
4630651a12cbSRobert Mustacchi 	default:
4631651a12cbSRobert Mustacchi 		break;
4632651a12cbSRobert Mustacchi 	}
4633651a12cbSRobert Mustacchi 
463401add34aSRobert Mustacchi 	cpuid_scan_security(cpu, featureset);
46357c478bd9Sstevel@tonic-gate }
46367c478bd9Sstevel@tonic-gate 
46377c478bd9Sstevel@tonic-gate /*
46387c478bd9Sstevel@tonic-gate  * Make copies of the cpuid table entries we depend on, in
46397c478bd9Sstevel@tonic-gate  * part for ease of parsing now, in part so that we have only
46407c478bd9Sstevel@tonic-gate  * one place to correct any of it, in part for ease of
46417c478bd9Sstevel@tonic-gate  * later export to userland, and in part so we can look at
46427c478bd9Sstevel@tonic-gate  * this stuff in a crash dump.
46437c478bd9Sstevel@tonic-gate  */
46447c478bd9Sstevel@tonic-gate 
4645ab5bb018SKeith M Wesolowski static void
cpuid_pass_extended(cpu_t * cpu,void * _arg __unused)4646ab5bb018SKeith M Wesolowski cpuid_pass_extended(cpu_t *cpu, void *_arg __unused)
46477c478bd9Sstevel@tonic-gate {
46487c478bd9Sstevel@tonic-gate 	uint_t n, nmax;
46497c478bd9Sstevel@tonic-gate 	int i;
46508949bcd6Sandrei 	struct cpuid_regs *cp;
46517c478bd9Sstevel@tonic-gate 	uint8_t *dp;
46527c478bd9Sstevel@tonic-gate 	uint32_t *iptr;
46537c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
46547c478bd9Sstevel@tonic-gate 
46557c478bd9Sstevel@tonic-gate 	if (cpi->cpi_maxeax < 1)
4656ab5bb018SKeith M Wesolowski 		return;
46577c478bd9Sstevel@tonic-gate 
46587c478bd9Sstevel@tonic-gate 	if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
46597c478bd9Sstevel@tonic-gate 		nmax = NMAX_CPI_STD;
46607c478bd9Sstevel@tonic-gate 	/*
4661ab5bb018SKeith M Wesolowski 	 * (We already handled n == 0 and n == 1 in the basic pass)
46627c478bd9Sstevel@tonic-gate 	 */
46637c478bd9Sstevel@tonic-gate 	for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
466401add34aSRobert Mustacchi 		/*
4665ab5bb018SKeith M Wesolowski 		 * leaves 6 and 7 were handled in the basic pass
466601add34aSRobert Mustacchi 		 */
4667f2dbfd32SRobert Mustacchi 		if (n == 6 || n == 7)
466801add34aSRobert Mustacchi 			continue;
466901add34aSRobert Mustacchi 
46701fbb7a7cSSpencer Berger 		cp->cp_eax = n;
46711fbb7a7cSSpencer Berger 
4672d129bde2Sesaxe 		/*
4673d129bde2Sesaxe 		 * CPUID function 4 expects %ecx to be initialized
4674d129bde2Sesaxe 		 * with an index which indicates which cache to return
4675d129bde2Sesaxe 		 * information about. The OS is expected to call function 4
4676d129bde2Sesaxe 		 * with %ecx set to 0, 1, 2, ... until it returns with
4677d129bde2Sesaxe 		 * EAX[4:0] set to 0, which indicates there are no more
4678d129bde2Sesaxe 		 * caches.
4679d129bde2Sesaxe 		 *
4680d129bde2Sesaxe 		 * Here, populate cpi_std[4] with the information returned by
4681ab5bb018SKeith M Wesolowski 		 * function 4 when %ecx == 0, and do the rest in a later pass
4682d129bde2Sesaxe 		 * when dynamic memory allocation becomes available.
4683d129bde2Sesaxe 		 *
4684d129bde2Sesaxe 		 * Note: we need to explicitly initialize %ecx here, since
4685d129bde2Sesaxe 		 * function 4 may have been previously invoked.
4686d129bde2Sesaxe 		 */
468701add34aSRobert Mustacchi 		if (n == 4)
4688d129bde2Sesaxe 			cp->cp_ecx = 0;
4689d129bde2Sesaxe 
46908949bcd6Sandrei 		(void) __cpuid_insn(cp);
4691ae115bc7Smrj 		platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
46927c478bd9Sstevel@tonic-gate 		switch (n) {
46937c478bd9Sstevel@tonic-gate 		case 2:
46947c478bd9Sstevel@tonic-gate 			/*
46957c478bd9Sstevel@tonic-gate 			 * "the lower 8 bits of the %eax register
46967c478bd9Sstevel@tonic-gate 			 * contain a value that identifies the number
46977c478bd9Sstevel@tonic-gate 			 * of times the cpuid [instruction] has to be
46987c478bd9Sstevel@tonic-gate 			 * executed to obtain a complete image of the
46997c478bd9Sstevel@tonic-gate 			 * processor's caching systems."
47007c478bd9Sstevel@tonic-gate 			 *
47017c478bd9Sstevel@tonic-gate 			 * How *do* they make this stuff up?
47027c478bd9Sstevel@tonic-gate 			 */
47037c478bd9Sstevel@tonic-gate 			cpi->cpi_ncache = sizeof (*cp) *
47047c478bd9Sstevel@tonic-gate 			    BITX(cp->cp_eax, 7, 0);
47057c478bd9Sstevel@tonic-gate 			if (cpi->cpi_ncache == 0)
47067c478bd9Sstevel@tonic-gate 				break;
47077c478bd9Sstevel@tonic-gate 			cpi->cpi_ncache--;	/* skip count byte */
47087c478bd9Sstevel@tonic-gate 
47097c478bd9Sstevel@tonic-gate 			/*
47107c478bd9Sstevel@tonic-gate 			 * Well, for now, rather than attempt to implement
47117c478bd9Sstevel@tonic-gate 			 * this slightly dubious algorithm, we just look
47127c478bd9Sstevel@tonic-gate 			 * at the first 15 ..
47137c478bd9Sstevel@tonic-gate 			 */
47147c478bd9Sstevel@tonic-gate 			if (cpi->cpi_ncache > (sizeof (*cp) - 1))
47157c478bd9Sstevel@tonic-gate 				cpi->cpi_ncache = sizeof (*cp) - 1;
47167c478bd9Sstevel@tonic-gate 
47177c478bd9Sstevel@tonic-gate 			dp = cpi->cpi_cacheinfo;
47187c478bd9Sstevel@tonic-gate 			if (BITX(cp->cp_eax, 31, 31) == 0) {
47197c478bd9Sstevel@tonic-gate 				uint8_t *p = (void *)&cp->cp_eax;
472063d3f7dfSkk 				for (i = 1; i < 4; i++)
47217c478bd9Sstevel@tonic-gate 					if (p[i] != 0)
47227c478bd9Sstevel@tonic-gate 						*dp++ = p[i];
47237c478bd9Sstevel@tonic-gate 			}
47247c478bd9Sstevel@tonic-gate 			if (BITX(cp->cp_ebx, 31, 31) == 0) {
47257c478bd9Sstevel@tonic-gate 				uint8_t *p = (void *)&cp->cp_ebx;
47267c478bd9Sstevel@tonic-gate 				for (i = 0; i < 4; i++)
47277c478bd9Sstevel@tonic-gate 					if (p[i] != 0)
47287c478bd9Sstevel@tonic-gate 						*dp++ = p[i];
47297c478bd9Sstevel@tonic-gate 			}
47307c478bd9Sstevel@tonic-gate 			if (BITX(cp->cp_ecx, 31, 31) == 0) {
47317c478bd9Sstevel@tonic-gate 				uint8_t *p = (void *)&cp->cp_ecx;
47327c478bd9Sstevel@tonic-gate 				for (i = 0; i < 4; i++)
47337c478bd9Sstevel@tonic-gate 					if (p[i] != 0)
47347c478bd9Sstevel@tonic-gate 						*dp++ = p[i];
47357c478bd9Sstevel@tonic-gate 			}
47367c478bd9Sstevel@tonic-gate 			if (BITX(cp->cp_edx, 31, 31) == 0) {
47377c478bd9Sstevel@tonic-gate 				uint8_t *p = (void *)&cp->cp_edx;
47387c478bd9Sstevel@tonic-gate 				for (i = 0; i < 4; i++)
47397c478bd9Sstevel@tonic-gate 					if (p[i] != 0)
47407c478bd9Sstevel@tonic-gate 						*dp++ = p[i];
47417c478bd9Sstevel@tonic-gate 			}
47427c478bd9Sstevel@tonic-gate 			break;
4743f98fbcecSbholler 
47447c478bd9Sstevel@tonic-gate 		case 3:	/* Processor serial number, if PSN supported */
4745f98fbcecSbholler 			break;
4746f98fbcecSbholler 
47477c478bd9Sstevel@tonic-gate 		case 4:	/* Deterministic cache parameters */
4748f98fbcecSbholler 			break;
4749f98fbcecSbholler 
47507c478bd9Sstevel@tonic-gate 		case 5:	/* Monitor/Mwait parameters */
47515b8a6efeSbholler 		{
47525b8a6efeSbholler 			size_t mwait_size;
4753f98fbcecSbholler 
4754f98fbcecSbholler 			/*
4755ab5bb018SKeith M Wesolowski 			 * check cpi_mwait.support which was set in
4756ab5bb018SKeith M Wesolowski 			 * cpuid_pass_basic()
4757f98fbcecSbholler 			 */
4758f98fbcecSbholler 			if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
4759f98fbcecSbholler 				break;
4760f98fbcecSbholler 
47615b8a6efeSbholler 			/*
47625b8a6efeSbholler 			 * Protect ourself from insane mwait line size.
47635b8a6efeSbholler 			 * Workaround for incomplete hardware emulator(s).
47645b8a6efeSbholler 			 */
47655b8a6efeSbholler 			mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
47665b8a6efeSbholler 			if (mwait_size < sizeof (uint32_t) ||
47675b8a6efeSbholler 			    !ISP2(mwait_size)) {
47685b8a6efeSbholler #if DEBUG
47695b8a6efeSbholler 				cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
47705d8efbbcSSaurabh Misra 				    "size %ld", cpu->cpu_id, (long)mwait_size);
47715b8a6efeSbholler #endif
47725b8a6efeSbholler 				break;
47735b8a6efeSbholler 			}
47745b8a6efeSbholler 
4775f98fbcecSbholler 			cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
47765b8a6efeSbholler 			cpi->cpi_mwait.mon_max = mwait_size;
4777f98fbcecSbholler 			if (MWAIT_EXTENSION(cpi)) {
4778f98fbcecSbholler 				cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
4779f98fbcecSbholler 				if (MWAIT_INT_ENABLE(cpi))
4780f98fbcecSbholler 					cpi->cpi_mwait.support |=
4781f98fbcecSbholler 					    MWAIT_ECX_INT_ENABLE;
4782f98fbcecSbholler 			}
4783f98fbcecSbholler 			break;
47845b8a6efeSbholler 		}
47857c478bd9Sstevel@tonic-gate 		default:
47867c478bd9Sstevel@tonic-gate 			break;
47877c478bd9Sstevel@tonic-gate 		}
47887c478bd9Sstevel@tonic-gate 	}
47897c478bd9Sstevel@tonic-gate 
47907af88ac7SKuriakose Kuruvilla 	/*
47917af88ac7SKuriakose Kuruvilla 	 * XSAVE enumeration
47927af88ac7SKuriakose Kuruvilla 	 */
479363408480SHans Rosenfeld 	if (cpi->cpi_maxeax >= 0xD) {
47947af88ac7SKuriakose Kuruvilla 		struct cpuid_regs regs;
47957af88ac7SKuriakose Kuruvilla 		boolean_t cpuid_d_valid = B_TRUE;
47967af88ac7SKuriakose Kuruvilla 
47977af88ac7SKuriakose Kuruvilla 		cp = &regs;
47987af88ac7SKuriakose Kuruvilla 		cp->cp_eax = 0xD;
47997af88ac7SKuriakose Kuruvilla 		cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
48007af88ac7SKuriakose Kuruvilla 
48017af88ac7SKuriakose Kuruvilla 		(void) __cpuid_insn(cp);
48027af88ac7SKuriakose Kuruvilla 
48037af88ac7SKuriakose Kuruvilla 		/*
48047af88ac7SKuriakose Kuruvilla 		 * Sanity checks for debug
48057af88ac7SKuriakose Kuruvilla 		 */
48067af88ac7SKuriakose Kuruvilla 		if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
48077af88ac7SKuriakose Kuruvilla 		    (cp->cp_eax & XFEATURE_SSE) == 0) {
48087af88ac7SKuriakose Kuruvilla 			cpuid_d_valid = B_FALSE;
48097af88ac7SKuriakose Kuruvilla 		}
48107af88ac7SKuriakose Kuruvilla 
48117af88ac7SKuriakose Kuruvilla 		cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
48127af88ac7SKuriakose Kuruvilla 		cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
48137af88ac7SKuriakose Kuruvilla 		cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;
48147af88ac7SKuriakose Kuruvilla 
48157af88ac7SKuriakose Kuruvilla 		/*
48167af88ac7SKuriakose Kuruvilla 		 * If the hw supports AVX, get the size and offset in the save
48177af88ac7SKuriakose Kuruvilla 		 * area for the ymm state.
48187af88ac7SKuriakose Kuruvilla 		 */
48197af88ac7SKuriakose Kuruvilla 		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
48207af88ac7SKuriakose Kuruvilla 			cp->cp_eax = 0xD;
48217af88ac7SKuriakose Kuruvilla 			cp->cp_ecx = 2;
48227af88ac7SKuriakose Kuruvilla 			cp->cp_edx = cp->cp_ebx = 0;
48237af88ac7SKuriakose Kuruvilla 
48247af88ac7SKuriakose Kuruvilla 			(void) __cpuid_insn(cp);
48257af88ac7SKuriakose Kuruvilla 
48267af88ac7SKuriakose Kuruvilla 			if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
48277af88ac7SKuriakose Kuruvilla 			    cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
48287af88ac7SKuriakose Kuruvilla 				cpuid_d_valid = B_FALSE;
48297af88ac7SKuriakose Kuruvilla 			}
48307af88ac7SKuriakose Kuruvilla 
48317af88ac7SKuriakose Kuruvilla 			cpi->cpi_xsave.ymm_size = cp->cp_eax;
48327af88ac7SKuriakose Kuruvilla 			cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
48337af88ac7SKuriakose Kuruvilla 		}
48347af88ac7SKuriakose Kuruvilla 
4835088d69f8SJerry Jelinek 		/*
4836088d69f8SJerry Jelinek 		 * If the hw supports MPX, get the size and offset in the
4837088d69f8SJerry Jelinek 		 * save area for BNDREGS and BNDCSR.
4838088d69f8SJerry Jelinek 		 */
4839088d69f8SJerry Jelinek 		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
4840088d69f8SJerry Jelinek 			cp->cp_eax = 0xD;
4841088d69f8SJerry Jelinek 			cp->cp_ecx = 3;
4842088d69f8SJerry Jelinek 			cp->cp_edx = cp->cp_ebx = 0;
4843088d69f8SJerry Jelinek 
4844088d69f8SJerry Jelinek 			(void) __cpuid_insn(cp);
4845088d69f8SJerry Jelinek 
4846088d69f8SJerry Jelinek 			cpi->cpi_xsave.bndregs_size = cp->cp_eax;
4847088d69f8SJerry Jelinek 			cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;
4848088d69f8SJerry Jelinek 
4849088d69f8SJerry Jelinek 			cp->cp_eax = 0xD;
4850088d69f8SJerry Jelinek 			cp->cp_ecx = 4;
4851088d69f8SJerry Jelinek 			cp->cp_edx = cp->cp_ebx = 0;
4852088d69f8SJerry Jelinek 
4853088d69f8SJerry Jelinek 			(void) __cpuid_insn(cp);
4854088d69f8SJerry Jelinek 
4855088d69f8SJerry Jelinek 			cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
4856088d69f8SJerry Jelinek 			cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
4857088d69f8SJerry Jelinek 		}
4858088d69f8SJerry Jelinek 
4859088d69f8SJerry Jelinek 		/*
4860088d69f8SJerry Jelinek 		 * If the hw supports AVX512, get the size and offset in the
4861088d69f8SJerry Jelinek 		 * save area for the opmask registers and zmm state.
4862088d69f8SJerry Jelinek 		 */
4863088d69f8SJerry Jelinek 		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
4864088d69f8SJerry Jelinek 			cp->cp_eax = 0xD;
4865088d69f8SJerry Jelinek 			cp->cp_ecx = 5;
4866088d69f8SJerry Jelinek 			cp->cp_edx = cp->cp_ebx = 0;
4867088d69f8SJerry Jelinek 
4868088d69f8SJerry Jelinek 			(void) __cpuid_insn(cp);
4869088d69f8SJerry Jelinek 
4870088d69f8SJerry Jelinek 			cpi->cpi_xsave.opmask_size = cp->cp_eax;
4871088d69f8SJerry Jelinek 			cpi->cpi_xsave.opmask_offset = cp->cp_ebx;
4872088d69f8SJerry Jelinek 
4873088d69f8SJerry Jelinek 			cp->cp_eax = 0xD;
4874088d69f8SJerry Jelinek 			cp->cp_ecx = 6;
4875088d69f8SJerry Jelinek 			cp->cp_edx = cp->cp_ebx = 0;
4876088d69f8SJerry Jelinek 
4877088d69f8SJerry Jelinek 			(void) __cpuid_insn(cp);
4878088d69f8SJerry Jelinek 
4879088d69f8SJerry Jelinek 			cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
4880088d69f8SJerry Jelinek 			cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;
4881088d69f8SJerry Jelinek 
4882088d69f8SJerry Jelinek 			cp->cp_eax = 0xD;
4883088d69f8SJerry Jelinek 			cp->cp_ecx = 7;
4884088d69f8SJerry Jelinek 			cp->cp_edx = cp->cp_ebx = 0;
4885088d69f8SJerry Jelinek 
4886088d69f8SJerry Jelinek 			(void) __cpuid_insn(cp);
4887088d69f8SJerry Jelinek 
4888088d69f8SJerry Jelinek 			cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
4889088d69f8SJerry Jelinek 			cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
4890088d69f8SJerry Jelinek 		}
4891088d69f8SJerry Jelinek 
48927af88ac7SKuriakose Kuruvilla 		if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
48937af88ac7SKuriakose Kuruvilla 			xsave_state_size = 0;
48947af88ac7SKuriakose Kuruvilla 		} else if (cpuid_d_valid) {
48957af88ac7SKuriakose Kuruvilla 			xsave_state_size = cpi->cpi_xsave.xsav_max_size;
48967af88ac7SKuriakose Kuruvilla 		} else {
48977af88ac7SKuriakose Kuruvilla 			/* Broken CPUID 0xD, probably in HVM */
48987af88ac7SKuriakose Kuruvilla 			cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
48997af88ac7SKuriakose Kuruvilla 			    "value: hw_low = %d, hw_high = %d, xsave_size = %d"
49007af88ac7SKuriakose Kuruvilla 			    ", ymm_size = %d, ymm_offset = %d\n",
49017af88ac7SKuriakose Kuruvilla 			    cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
49027af88ac7SKuriakose Kuruvilla 			    cpi->cpi_xsave.xsav_hw_features_high,
49037af88ac7SKuriakose Kuruvilla 			    (int)cpi->cpi_xsave.xsav_max_size,
49047af88ac7SKuriakose Kuruvilla 			    (int)cpi->cpi_xsave.ymm_size,
49057af88ac7SKuriakose Kuruvilla 			    (int)cpi->cpi_xsave.ymm_offset);
49067af88ac7SKuriakose Kuruvilla 
49077af88ac7SKuriakose Kuruvilla 			if (xsave_state_size != 0) {
49087af88ac7SKuriakose Kuruvilla 				/*
49097af88ac7SKuriakose Kuruvilla 				 * This must be a non-boot CPU. We cannot
49107af88ac7SKuriakose Kuruvilla 				 * continue, because boot cpu has already
49117af88ac7SKuriakose Kuruvilla 				 * enabled XSAVE.
49127af88ac7SKuriakose Kuruvilla 				 */
49137af88ac7SKuriakose Kuruvilla 				ASSERT(cpu->cpu_id != 0);
49147af88ac7SKuriakose Kuruvilla 				cmn_err(CE_PANIC, "cpu%d: we have already "
49157af88ac7SKuriakose Kuruvilla 				    "enabled XSAVE on boot cpu, cannot "
49167af88ac7SKuriakose Kuruvilla 				    "continue.", cpu->cpu_id);
49177af88ac7SKuriakose Kuruvilla 			} else {
49187af88ac7SKuriakose Kuruvilla 				/*
4919dcf050afSRobert Mustacchi 				 * If we reached here on the boot CPU, it's also
4920dcf050afSRobert Mustacchi 				 * almost certain that we'll reach here on the
4921dcf050afSRobert Mustacchi 				 * non-boot CPUs. When we're here on a boot CPU
4922dcf050afSRobert Mustacchi 				 * we should disable the feature, on a non-boot
4923dcf050afSRobert Mustacchi 				 * CPU we need to confirm that we have.
49247af88ac7SKuriakose Kuruvilla 				 */
4925dcf050afSRobert Mustacchi 				if (cpu->cpu_id == 0) {
4926dcf050afSRobert Mustacchi 					remove_x86_feature(x86_featureset,
4927dcf050afSRobert Mustacchi 					    X86FSET_XSAVE);
4928dcf050afSRobert Mustacchi 					remove_x86_feature(x86_featureset,
4929dcf050afSRobert Mustacchi 					    X86FSET_AVX);
4930245ac945SRobert Mustacchi 					remove_x86_feature(x86_featureset,
4931245ac945SRobert Mustacchi 					    X86FSET_F16C);
4932245ac945SRobert Mustacchi 					remove_x86_feature(x86_featureset,
4933245ac945SRobert Mustacchi 					    X86FSET_BMI1);
4934245ac945SRobert Mustacchi 					remove_x86_feature(x86_featureset,
4935245ac945SRobert Mustacchi 					    X86FSET_BMI2);
4936245ac945SRobert Mustacchi 					remove_x86_feature(x86_featureset,
4937245ac945SRobert Mustacchi 					    X86FSET_FMA);
4938245ac945SRobert Mustacchi 					remove_x86_feature(x86_featureset,
4939245ac945SRobert Mustacchi 					    X86FSET_AVX2);
4940088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4941088d69f8SJerry Jelinek 					    X86FSET_MPX);
4942088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4943088d69f8SJerry Jelinek 					    X86FSET_AVX512F);
4944088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4945088d69f8SJerry Jelinek 					    X86FSET_AVX512DQ);
4946088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4947088d69f8SJerry Jelinek 					    X86FSET_AVX512PF);
4948088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4949088d69f8SJerry Jelinek 					    X86FSET_AVX512ER);
4950088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4951088d69f8SJerry Jelinek 					    X86FSET_AVX512CD);
4952088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4953088d69f8SJerry Jelinek 					    X86FSET_AVX512BW);
4954088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4955088d69f8SJerry Jelinek 					    X86FSET_AVX512VL);
4956088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4957088d69f8SJerry Jelinek 					    X86FSET_AVX512FMA);
4958088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4959088d69f8SJerry Jelinek 					    X86FSET_AVX512VBMI);
4960e4f6ce70SRobert Mustacchi 					remove_x86_feature(x86_featureset,
4961e4f6ce70SRobert Mustacchi 					    X86FSET_AVX512VNNI);
4962088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4963088d69f8SJerry Jelinek 					    X86FSET_AVX512VPOPCDQ);
4964088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4965088d69f8SJerry Jelinek 					    X86FSET_AVX512NNIW);
4966088d69f8SJerry Jelinek 					remove_x86_feature(x86_featureset,
4967088d69f8SJerry Jelinek 					    X86FSET_AVX512FMAPS);
49685edbd2feSRobert Mustacchi 					remove_x86_feature(x86_featureset,
49695edbd2feSRobert Mustacchi 					    X86FSET_VAES);
49705edbd2feSRobert Mustacchi 					remove_x86_feature(x86_featureset,
49715edbd2feSRobert Mustacchi 					    X86FSET_VPCLMULQDQ);
497256726c7eSRobert Mustacchi 					remove_x86_feature(x86_featureset,
497356726c7eSRobert Mustacchi 					    X86FSET_GFNI);
497456726c7eSRobert Mustacchi 					remove_x86_feature(x86_featureset,
497556726c7eSRobert Mustacchi 					    X86FSET_AVX512_VP2INT);
497656726c7eSRobert Mustacchi 					remove_x86_feature(x86_featureset,
497756726c7eSRobert Mustacchi 					    X86FSET_AVX512_BITALG);
497856726c7eSRobert Mustacchi 					remove_x86_feature(x86_featureset,
497956726c7eSRobert Mustacchi 					    X86FSET_AVX512_VBMI2);
498056726c7eSRobert Mustacchi 					remove_x86_feature(x86_featureset,
498156726c7eSRobert Mustacchi 					    X86FSET_AVX512_BF16);
4982088d69f8SJerry Jelinek 
4983dcf050afSRobert Mustacchi 					xsave_force_disable = B_TRUE;
4984dcf050afSRobert Mustacchi 				} else {
4985dcf050afSRobert Mustacchi 					VERIFY(is_x86_feature(x86_featureset,
4986dcf050afSRobert Mustacchi 					    X86FSET_XSAVE) == B_FALSE);
4987dcf050afSRobert Mustacchi 				}
49887af88ac7SKuriakose Kuruvilla 			}
49897af88ac7SKuriakose Kuruvilla 		}
49907af88ac7SKuriakose Kuruvilla 	}
49917af88ac7SKuriakose Kuruvilla 
49927af88ac7SKuriakose Kuruvilla 
49930ce813ffSRobert Mustacchi 	if ((cpi->cpi_xmaxeax & CPUID_LEAF_EXT_0) == 0)
4994ab5bb018SKeith M Wesolowski 		return;
49957c478bd9Sstevel@tonic-gate 
49960ce813ffSRobert Mustacchi 	if ((nmax = cpi->cpi_xmaxeax - CPUID_LEAF_EXT_0 + 1) > NMAX_CPI_EXTD)
49977c478bd9Sstevel@tonic-gate 		nmax = NMAX_CPI_EXTD;
49987c478bd9Sstevel@tonic-gate 	/*
4999651a12cbSRobert Mustacchi 	 * Copy the extended properties, fixing them as we go. While we start at
5000651a12cbSRobert Mustacchi 	 * 2 because we've already handled a few cases in the basic pass, the
5001651a12cbSRobert Mustacchi 	 * rest we let ourselves just grab again (e.g. 0x8, 0x21).
50027c478bd9Sstevel@tonic-gate 	 */
50037c478bd9Sstevel@tonic-gate 	iptr = (void *)cpi->cpi_brandstr;
50047c478bd9Sstevel@tonic-gate 	for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
50050ce813ffSRobert Mustacchi 		cp->cp_eax = CPUID_LEAF_EXT_0 + n;
50068949bcd6Sandrei 		(void) __cpuid_insn(cp);
50070ce813ffSRobert Mustacchi 		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_0 + n,
50080ce813ffSRobert Mustacchi 		    cp);
50097c478bd9Sstevel@tonic-gate 		switch (n) {
50107c478bd9Sstevel@tonic-gate 		case 2:
50117c478bd9Sstevel@tonic-gate 		case 3:
50127c478bd9Sstevel@tonic-gate 		case 4:
50137c478bd9Sstevel@tonic-gate 			/*
50147c478bd9Sstevel@tonic-gate 			 * Extract the brand string
50157c478bd9Sstevel@tonic-gate 			 */
50167c478bd9Sstevel@tonic-gate 			*iptr++ = cp->cp_eax;
50177c478bd9Sstevel@tonic-gate 			*iptr++ = cp->cp_ebx;
50187c478bd9Sstevel@tonic-gate 			*iptr++ = cp->cp_ecx;
50197c478bd9Sstevel@tonic-gate 			*iptr++ = cp->cp_edx;
50207c478bd9Sstevel@tonic-gate 			break;
50217c478bd9Sstevel@tonic-gate 		case 5:
50227c478bd9Sstevel@tonic-gate 			switch (cpi->cpi_vendor) {
50237c478bd9Sstevel@tonic-gate 			case X86_VENDOR_AMD:
50247c478bd9Sstevel@tonic-gate 				/*
50257c478bd9Sstevel@tonic-gate 				 * The Athlon and Duron were the first
50267c478bd9Sstevel@tonic-gate 				 * parts to report the sizes of the
50277c478bd9Sstevel@tonic-gate 				 * TLB for large pages. Before then,
50287c478bd9Sstevel@tonic-gate 				 * we don't trust the data.
50297c478bd9Sstevel@tonic-gate 				 */
50307c478bd9Sstevel@tonic-gate 				if (cpi->cpi_family < 6 ||
50317c478bd9Sstevel@tonic-gate 				    (cpi->cpi_family == 6 &&
50327c478bd9Sstevel@tonic-gate 				    cpi->cpi_model < 1))
50337c478bd9Sstevel@tonic-gate 					cp->cp_eax = 0;
50347c478bd9Sstevel@tonic-gate 				break;
50357c478bd9Sstevel@tonic-gate 			default:
50367c478bd9Sstevel@tonic-gate 				break;
50377c478bd9Sstevel@tonic-gate 			}
50387c478bd9Sstevel@tonic-gate 			break;
50397c478bd9Sstevel@tonic-gate 		case 6:
50407c478bd9Sstevel@tonic-gate 			switch (cpi->cpi_vendor) {
50417c478bd9Sstevel@tonic-gate 			case X86_VENDOR_AMD:
50427c478bd9Sstevel@tonic-gate 				/*
50437c478bd9Sstevel@tonic-gate 				 * The Athlon and Duron were the first
50447c478bd9Sstevel@tonic-gate 				 * AMD parts with L2 TLB's.
50457c478bd9Sstevel@tonic-gate 				 * Before then, don't trust the data.
50467c478bd9Sstevel@tonic-gate 				 */
50477c478bd9Sstevel@tonic-gate 				if (cpi->cpi_family < 6 ||
50483df2e8b2SRobert Mustacchi 				    (cpi->cpi_family == 6 &&
50493df2e8b2SRobert Mustacchi 				    cpi->cpi_model < 1))
50507c478bd9Sstevel@tonic-gate 					cp->cp_eax = cp->cp_ebx = 0;
50517c478bd9Sstevel@tonic-gate 				/*
50527c478bd9Sstevel@tonic-gate 				 * AMD Duron rev A0 reports L2
50537c478bd9Sstevel@tonic-gate 				 * cache size incorrectly as 1K
50547c478bd9Sstevel@tonic-gate 				 * when it is really 64K
50557c478bd9Sstevel@tonic-gate 				 */
50567c478bd9Sstevel@tonic-gate 				if (cpi->cpi_family == 6 &&
50577c478bd9Sstevel@tonic-gate 				    cpi->cpi_model == 3 &&
50587c478bd9Sstevel@tonic-gate 				    cpi->cpi_step == 0) {
50597c478bd9Sstevel@tonic-gate 					cp->cp_ecx &= 0xffff;
50607c478bd9Sstevel@tonic-gate 					cp->cp_ecx |= 0x400000;
50617c478bd9Sstevel@tonic-gate 				}
50627c478bd9Sstevel@tonic-gate 				break;
50637c478bd9Sstevel@tonic-gate 			case X86_VENDOR_Cyrix:	/* VIA C3 */
50647c478bd9Sstevel@tonic-gate 				/*
50657c478bd9Sstevel@tonic-gate 				 * VIA C3 processors are a bit messed
50667c478bd9Sstevel@tonic-gate 				 * up w.r.t. encoding cache sizes in %ecx
50677c478bd9Sstevel@tonic-gate 				 */
50687c478bd9Sstevel@tonic-gate 				if (cpi->cpi_family != 6)
50697c478bd9Sstevel@tonic-gate 					break;
50707c478bd9Sstevel@tonic-gate 				/*
50717c478bd9Sstevel@tonic-gate 				 * model 7 and 8 were incorrectly encoded
50727c478bd9Sstevel@tonic-gate 				 *
50737c478bd9Sstevel@tonic-gate 				 * xxx is model 8 really broken?
50747c478bd9Sstevel@tonic-gate 				 */
50757c478bd9Sstevel@tonic-gate 				if (cpi->cpi_model == 7 ||
50767c478bd9Sstevel@tonic-gate 				    cpi->cpi_model == 8)
50777c478bd9Sstevel@tonic-gate 					cp->cp_ecx =
50787c478bd9Sstevel@tonic-gate 					    BITX(cp->cp_ecx, 31, 24) << 16 |
50797c478bd9Sstevel@tonic-gate 					    BITX(cp->cp_ecx, 23, 16) << 12 |
50807c478bd9Sstevel@tonic-gate 					    BITX(cp->cp_ecx, 15, 8) << 8 |
50817c478bd9Sstevel@tonic-gate 					    BITX(cp->cp_ecx, 7, 0);
50827c478bd9Sstevel@tonic-gate 				/*
50837c478bd9Sstevel@tonic-gate 				 * model 9 stepping 1 has wrong associativity
50847c478bd9Sstevel@tonic-gate 				 */
50857c478bd9Sstevel@tonic-gate 				if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
50867c478bd9Sstevel@tonic-gate 					cp->cp_ecx |= 8 << 12;
50877c478bd9Sstevel@tonic-gate 				break;
50887c478bd9Sstevel@tonic-gate 			case X86_VENDOR_Intel:
50897c478bd9Sstevel@tonic-gate 				/*
50907c478bd9Sstevel@tonic-gate 				 * Extended L2 Cache features function.
50917c478bd9Sstevel@tonic-gate 				 * First appeared on Prescott.
50927c478bd9Sstevel@tonic-gate 				 */
50937c478bd9Sstevel@tonic-gate 			default:
50947c478bd9Sstevel@tonic-gate 				break;
50957c478bd9Sstevel@tonic-gate 			}
50967c478bd9Sstevel@tonic-gate 			break;
50977c478bd9Sstevel@tonic-gate 		default:
50987c478bd9Sstevel@tonic-gate 			break;
50997c478bd9Sstevel@tonic-gate 		}
51007c478bd9Sstevel@tonic-gate 	}
51017c478bd9Sstevel@tonic-gate }
51027c478bd9Sstevel@tonic-gate 
51037c478bd9Sstevel@tonic-gate static const char *
intel_cpubrand(const struct cpuid_info * cpi)51047c478bd9Sstevel@tonic-gate intel_cpubrand(const struct cpuid_info *cpi)
51057c478bd9Sstevel@tonic-gate {
51067c478bd9Sstevel@tonic-gate 	int i;
51077c478bd9Sstevel@tonic-gate 
5108ab5bb018SKeith M Wesolowski 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
51097c478bd9Sstevel@tonic-gate 
51107c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_family) {
51117c478bd9Sstevel@tonic-gate 	case 5:
51127c478bd9Sstevel@tonic-gate 		return ("Intel Pentium(r)");
51137c478bd9Sstevel@tonic-gate 	case 6:
51147c478bd9Sstevel@tonic-gate 		switch (cpi->cpi_model) {
51157c478bd9Sstevel@tonic-gate 			uint_t celeron, xeon;
51168949bcd6Sandrei 			const struct cpuid_regs *cp;
51177c478bd9Sstevel@tonic-gate 		case 0:
51187c478bd9Sstevel@tonic-gate 		case 1:
51197c478bd9Sstevel@tonic-gate 		case 2:
51207c478bd9Sstevel@tonic-gate 			return ("Intel Pentium(r) Pro");
51217c478bd9Sstevel@tonic-gate 		case 3:
51227c478bd9Sstevel@tonic-gate 		case 4:
51237c478bd9Sstevel@tonic-gate 			return ("Intel Pentium(r) II");
51247c478bd9Sstevel@tonic-gate 		case 6:
51257c478bd9Sstevel@tonic-gate 			return ("Intel Celeron(r)");
51267c478bd9Sstevel@tonic-gate 		case 5:
51277c478bd9Sstevel@tonic-gate 		case 7:
51287c478bd9Sstevel@tonic-gate 			celeron = xeon = 0;
51297c478bd9Sstevel@tonic-gate 			cp = &cpi->cpi_std[2];	/* cache info */
51307c478bd9Sstevel@tonic-gate 
513163d3f7dfSkk 			for (i = 1; i < 4; i++) {
51327c478bd9Sstevel@tonic-gate 				uint_t tmp;
51337c478bd9Sstevel@tonic-gate 
51347c478bd9Sstevel@tonic-gate 				tmp = (cp->cp_eax >> (8 * i)) & 0xff;
51357c478bd9Sstevel@tonic-gate 				if (tmp == 0x40)
51367c478bd9Sstevel@tonic-gate 					celeron++;
51377c478bd9Sstevel@tonic-gate 				if (tmp >= 0x44 && tmp <= 0x45)
51387c478bd9Sstevel@tonic-gate 					xeon++;
51397c478bd9Sstevel@tonic-gate 			}
51407c478bd9Sstevel@tonic-gate 
51417c478bd9Sstevel@tonic-gate 			for (i = 0; i < 2; i++) {
51427c478bd9Sstevel@tonic-gate 				uint_t tmp;
51437c478bd9Sstevel@tonic-gate 
51447c478bd9Sstevel@tonic-gate 				tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
51457c478bd9Sstevel@tonic-gate 				if (tmp == 0x40)
51467c478bd9Sstevel@tonic-gate 					celeron++;
51477c478bd9Sstevel@tonic-gate 				else if (tmp >= 0x44 && tmp <= 0x45)
51487c478bd9Sstevel@tonic-gate 					xeon++;
51497c478bd9Sstevel@tonic-gate 			}
51507c478bd9Sstevel@tonic-gate 
51517c478bd9Sstevel@tonic-gate 			for (i = 0; i < 4; i++) {
51527c478bd9Sstevel@tonic-gate 				uint_t tmp;
51537c478bd9Sstevel@tonic-gate 
51547c478bd9Sstevel@tonic-gate 				tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
51557c478bd9Sstevel@tonic-gate 				if (tmp == 0x40)
51567c478bd9Sstevel@tonic-gate 					celeron++;
51577c478bd9Sstevel@tonic-gate 				else if (tmp >= 0x44 && tmp <= 0x45)
51587c478bd9Sstevel@tonic-gate 					xeon++;
51597c478bd9Sstevel@tonic-gate 			}
51607c478bd9Sstevel@tonic-gate 
51617c478bd9Sstevel@tonic-gate 			for (i = 0; i < 4; i++) {
51627c478bd9Sstevel@tonic-gate 				uint_t tmp;
51637c478bd9Sstevel@tonic-gate 
51647c478bd9Sstevel@tonic-gate 				tmp = (cp->cp_edx >> (8 * i)) & 0xff;
51657c478bd9Sstevel@tonic-gate 				if (tmp == 0x40)
51667c478bd9Sstevel@tonic-gate 					celeron++;
51677c478bd9Sstevel@tonic-gate 				else if (tmp >= 0x44 && tmp <= 0x45)
51687c478bd9Sstevel@tonic-gate 					xeon++;
51697c478bd9Sstevel@tonic-gate 			}
51707c478bd9Sstevel@tonic-gate 
51717c478bd9Sstevel@tonic-gate 			if (celeron)
51727c478bd9Sstevel@tonic-gate 				return ("Intel Celeron(r)");
51737c478bd9Sstevel@tonic-gate 			if (xeon)
51747c478bd9Sstevel@tonic-gate 				return (cpi->cpi_model == 5 ?
51757c478bd9Sstevel@tonic-gate 				    "Intel Pentium(r) II Xeon(tm)" :
51767c478bd9Sstevel@tonic-gate 				    "Intel Pentium(r) III Xeon(tm)");
51777c478bd9Sstevel@tonic-gate 			return (cpi->cpi_model == 5 ?
51787c478bd9Sstevel@tonic-gate 			    "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
51797c478bd9Sstevel@tonic-gate 			    "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
51807c478bd9Sstevel@tonic-gate 		default:
51817c478bd9Sstevel@tonic-gate 			break;
51827c478bd9Sstevel@tonic-gate 		}
51837c478bd9Sstevel@tonic-gate 	default:
51847c478bd9Sstevel@tonic-gate 		break;
51857c478bd9Sstevel@tonic-gate 	}
51867c478bd9Sstevel@tonic-gate 
51875ff02082Sdmick 	/* BrandID is present if the field is nonzero */
51885ff02082Sdmick 	if (cpi->cpi_brandid != 0) {
51897c478bd9Sstevel@tonic-gate 		static const struct {
51907c478bd9Sstevel@tonic-gate 			uint_t bt_bid;
51917c478bd9Sstevel@tonic-gate 			const char *bt_str;
51927c478bd9Sstevel@tonic-gate 		} brand_tbl[] = {
51937c478bd9Sstevel@tonic-gate 			{ 0x1,	"Intel(r) Celeron(r)" },
51947c478bd9Sstevel@tonic-gate 			{ 0x2,	"Intel(r) Pentium(r) III" },
51957c478bd9Sstevel@tonic-gate 			{ 0x3,	"Intel(r) Pentium(r) III Xeon(tm)" },
51967c478bd9Sstevel@tonic-gate 			{ 0x4,	"Intel(r) Pentium(r) III" },
51977c478bd9Sstevel@tonic-gate 			{ 0x6,	"Mobile Intel(r) Pentium(r) III" },
51987c478bd9Sstevel@tonic-gate 			{ 0x7,	"Mobile Intel(r) Celeron(r)" },
51997c478bd9Sstevel@tonic-gate 			{ 0x8,	"Intel(r) Pentium(r) 4" },
52007c478bd9Sstevel@tonic-gate 			{ 0x9,	"Intel(r) Pentium(r) 4" },
52017c478bd9Sstevel@tonic-gate 			{ 0xa,	"Intel(r) Celeron(r)" },
52027c478bd9Sstevel@tonic-gate 			{ 0xb,	"Intel(r) Xeon(tm)" },
52037c478bd9Sstevel@tonic-gate 			{ 0xc,	"Intel(r) Xeon(tm) MP" },
52047c478bd9Sstevel@tonic-gate 			{ 0xe,	"Mobile Intel(r) Pentium(r) 4" },
52055ff02082Sdmick 			{ 0xf,	"Mobile Intel(r) Celeron(r)" },
52065ff02082Sdmick 			{ 0x11, "Mobile Genuine Intel(r)" },
52075ff02082Sdmick 			{ 0x12, "Intel(r) Celeron(r) M" },
52085ff02082Sdmick 			{ 0x13, "Mobile Intel(r) Celeron(r)" },
52095ff02082Sdmick 			{ 0x14, "Intel(r) Celeron(r)" },
52105ff02082Sdmick 			{ 0x15, "Mobile Genuine Intel(r)" },
52115ff02082Sdmick 			{ 0x16,	"Intel(r) Pentium(r) M" },
52125ff02082Sdmick 			{ 0x17, "Mobile Intel(r) Celeron(r)" }
52137c478bd9Sstevel@tonic-gate 		};
52147c478bd9Sstevel@tonic-gate 		uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
52157c478bd9Sstevel@tonic-gate 		uint_t sgn;
52167c478bd9Sstevel@tonic-gate 
52177c478bd9Sstevel@tonic-gate 		sgn = (cpi->cpi_family << 8) |
52187c478bd9Sstevel@tonic-gate 		    (cpi->cpi_model << 4) | cpi->cpi_step;
52197c478bd9Sstevel@tonic-gate 
52207c478bd9Sstevel@tonic-gate 		for (i = 0; i < btblmax; i++)
52217c478bd9Sstevel@tonic-gate 			if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
52227c478bd9Sstevel@tonic-gate 				break;
52237c478bd9Sstevel@tonic-gate 		if (i < btblmax) {
52247c478bd9Sstevel@tonic-gate 			if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
52257c478bd9Sstevel@tonic-gate 				return ("Intel(r) Celeron(r)");
52267c478bd9Sstevel@tonic-gate 			if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
52277c478bd9Sstevel@tonic-gate 				return ("Intel(r) Xeon(tm) MP");
52287c478bd9Sstevel@tonic-gate 			if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
52297c478bd9Sstevel@tonic-gate 				return ("Intel(r) Xeon(tm)");
52307c478bd9Sstevel@tonic-gate 			return (brand_tbl[i].bt_str);
52317c478bd9Sstevel@tonic-gate 		}
52327c478bd9Sstevel@tonic-gate 	}
52337c478bd9Sstevel@tonic-gate 
52347c478bd9Sstevel@tonic-gate 	return (NULL);
52357c478bd9Sstevel@tonic-gate }
52367c478bd9Sstevel@tonic-gate 
52377c478bd9Sstevel@tonic-gate static const char *
amd_cpubrand(const struct cpuid_info * cpi)52387c478bd9Sstevel@tonic-gate amd_cpubrand(const struct cpuid_info *cpi)
52397c478bd9Sstevel@tonic-gate {
5240ab5bb018SKeith M Wesolowski 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
52417c478bd9Sstevel@tonic-gate 
52427c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_family) {
52437c478bd9Sstevel@tonic-gate 	case 5:
52447c478bd9Sstevel@tonic-gate 		switch (cpi->cpi_model) {
52457c478bd9Sstevel@tonic-gate 		case 0:
52467c478bd9Sstevel@tonic-gate 		case 1:
52477c478bd9Sstevel@tonic-gate 		case 2:
52487c478bd9Sstevel@tonic-gate 		case 3:
52497c478bd9Sstevel@tonic-gate 		case 4:
52507c478bd9Sstevel@tonic-gate 		case 5:
52517c478bd9Sstevel@tonic-gate 			return ("AMD-K5(r)");
52527c478bd9Sstevel@tonic-gate 		case 6:
52537c478bd9Sstevel@tonic-gate 		case 7:
52547c478bd9Sstevel@tonic-gate 			return ("AMD-K6(r)");
52557c478bd9Sstevel@tonic-gate 		case 8:
52567c478bd9Sstevel@tonic-gate 			return ("AMD-K6(r)-2");
52577c478bd9Sstevel@tonic-gate 		case 9:
52587c478bd9Sstevel@tonic-gate 			return ("AMD-K6(r)-III");
52597c478bd9Sstevel@tonic-gate 		default:
52607c478bd9Sstevel@tonic-gate 			return ("AMD (family 5)");
52617c478bd9Sstevel@tonic-gate 		}
52627c478bd9Sstevel@tonic-gate 	case 6:
52637c478bd9Sstevel@tonic-gate 		switch (cpi->cpi_model) {
52647c478bd9Sstevel@tonic-gate 		case 1:
52657c478bd9Sstevel@tonic-gate 			return ("AMD-K7(tm)");
52667c478bd9Sstevel@tonic-gate 		case 0:
52677c478bd9Sstevel@tonic-gate 		case 2:
52687c478bd9Sstevel@tonic-gate 		case 4:
52697c478bd9Sstevel@tonic-gate 			return ("AMD Athlon(tm)");
52707c478bd9Sstevel@tonic-gate 		case 3:
52717c478bd9Sstevel@tonic-gate 		case 7:
52727c478bd9Sstevel@tonic-gate 			return ("AMD Duron(tm)");
52737c478bd9Sstevel@tonic-gate 		case 6:
52747c478bd9Sstevel@tonic-gate 		case 8:
52757c478bd9Sstevel@tonic-gate 		case 10:
52767c478bd9Sstevel@tonic-gate 			/*
52777c478bd9Sstevel@tonic-gate 			 * Use the L2 cache size to distinguish
52787c478bd9Sstevel@tonic-gate 			 */
52797c478bd9Sstevel@tonic-gate 			return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
52807c478bd9Sstevel@tonic-gate 			    "AMD Athlon(tm)" : "AMD Duron(tm)");
52817c478bd9Sstevel@tonic-gate 		default:
52827c478bd9Sstevel@tonic-gate 			return ("AMD (family 6)");
52837c478bd9Sstevel@tonic-gate 		}
52847c478bd9Sstevel@tonic-gate 	default:
52857c478bd9Sstevel@tonic-gate 		break;
52867c478bd9Sstevel@tonic-gate 	}
52877c478bd9Sstevel@tonic-gate 
52887c478bd9Sstevel@tonic-gate 	if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
52897c478bd9Sstevel@tonic-gate 	    cpi->cpi_brandid != 0) {
52907c478bd9Sstevel@tonic-gate 		switch (BITX(cpi->cpi_brandid, 7, 5)) {
52917c478bd9Sstevel@tonic-gate 		case 3:
52927c478bd9Sstevel@tonic-gate 			return ("AMD Opteron(tm) UP 1xx");
52937c478bd9Sstevel@tonic-gate 		case 4:
52947c478bd9Sstevel@tonic-gate 			return ("AMD Opteron(tm) DP 2xx");
52957c478bd9Sstevel@tonic-gate 		case 5:
52967c478bd9Sstevel@tonic-gate 			return ("AMD Opteron(tm) MP 8xx");
52977c478bd9Sstevel@tonic-gate 		default:
52987c478bd9Sstevel@tonic-gate 			return ("AMD Opteron(tm)");
52997c478bd9Sstevel@tonic-gate 		}
53007c478bd9Sstevel@tonic-gate 	}
53017c478bd9Sstevel@tonic-gate 
53027c478bd9Sstevel@tonic-gate 	return (NULL);
53037c478bd9Sstevel@tonic-gate }
53047c478bd9Sstevel@tonic-gate 
53057c478bd9Sstevel@tonic-gate static const char *
cyrix_cpubrand(struct cpuid_info * cpi,uint_t type)53067c478bd9Sstevel@tonic-gate cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
53077c478bd9Sstevel@tonic-gate {
5308ab5bb018SKeith M Wesolowski 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
53097c478bd9Sstevel@tonic-gate 
53107c478bd9Sstevel@tonic-gate 	switch (type) {
53117c478bd9Sstevel@tonic-gate 	case X86_TYPE_CYRIX_6x86:
53127c478bd9Sstevel@tonic-gate 		return ("Cyrix 6x86");
53137c478bd9Sstevel@tonic-gate 	case X86_TYPE_CYRIX_6x86L:
53147c478bd9Sstevel@tonic-gate 		return ("Cyrix 6x86L");
53157c478bd9Sstevel@tonic-gate 	case X86_TYPE_CYRIX_6x86MX:
53167c478bd9Sstevel@tonic-gate 		return ("Cyrix 6x86MX");
53177c478bd9Sstevel@tonic-gate 	case X86_TYPE_CYRIX_GXm:
53187c478bd9Sstevel@tonic-gate 		return ("Cyrix GXm");
53197c478bd9Sstevel@tonic-gate 	case X86_TYPE_CYRIX_MediaGX:
53207c478bd9Sstevel@tonic-gate 		return ("Cyrix MediaGX");
53217c478bd9Sstevel@tonic-gate 	case X86_TYPE_CYRIX_MII:
53227c478bd9Sstevel@tonic-gate 		return ("Cyrix M2");
53237c478bd9Sstevel@tonic-gate 	case X86_TYPE_VIA_CYRIX_III:
53247c478bd9Sstevel@tonic-gate 		return ("VIA Cyrix M3");
53257c478bd9Sstevel@tonic-gate 	default:
53267c478bd9Sstevel@tonic-gate 		/*
53277c478bd9Sstevel@tonic-gate 		 * Have another wild guess ..
53287c478bd9Sstevel@tonic-gate 		 */
53297c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
53307c478bd9Sstevel@tonic-gate 			return ("Cyrix 5x86");
53317c478bd9Sstevel@tonic-gate 		else if (cpi->cpi_family == 5) {
53327c478bd9Sstevel@tonic-gate 			switch (cpi->cpi_model) {
53337c478bd9Sstevel@tonic-gate 			case 2:
53347c478bd9Sstevel@tonic-gate 				return ("Cyrix 6x86");	/* Cyrix M1 */
53357c478bd9Sstevel@tonic-gate 			case 4:
53367c478bd9Sstevel@tonic-gate 				return ("Cyrix MediaGX");
53377c478bd9Sstevel@tonic-gate 			default:
53387c478bd9Sstevel@tonic-gate 				break;
53397c478bd9Sstevel@tonic-gate 			}
53407c478bd9Sstevel@tonic-gate 		} else if (cpi->cpi_family == 6) {
53417c478bd9Sstevel@tonic-gate 			switch (cpi->cpi_model) {
53427c478bd9Sstevel@tonic-gate 			case 0:
53437c478bd9Sstevel@tonic-gate 				return ("Cyrix 6x86MX"); /* Cyrix M2? */
53447c478bd9Sstevel@tonic-gate 			case 5:
53457c478bd9Sstevel@tonic-gate 			case 6:
53467c478bd9Sstevel@tonic-gate 			case 7:
53477c478bd9Sstevel@tonic-gate 			case 8:
53487c478bd9Sstevel@tonic-gate 			case 9:
53497c478bd9Sstevel@tonic-gate 				return ("VIA C3");
53507c478bd9Sstevel@tonic-gate 			default:
53517c478bd9Sstevel@tonic-gate 				break;
53527c478bd9Sstevel@tonic-gate 			}
53537c478bd9Sstevel@tonic-gate 		}
53547c478bd9Sstevel@tonic-gate 		break;
53557c478bd9Sstevel@tonic-gate 	}
53567c478bd9Sstevel@tonic-gate 	return (NULL);
53577c478bd9Sstevel@tonic-gate }
53587c478bd9Sstevel@tonic-gate 
53597c478bd9Sstevel@tonic-gate /*
53607c478bd9Sstevel@tonic-gate  * This only gets called in the case that the CPU extended
53617c478bd9Sstevel@tonic-gate  * feature brand string (0x80000002, 0x80000003, 0x80000004)
53627c478bd9Sstevel@tonic-gate  * aren't available, or contain null bytes for some reason.
53637c478bd9Sstevel@tonic-gate  */
53647c478bd9Sstevel@tonic-gate static void
fabricate_brandstr(struct cpuid_info * cpi)53657c478bd9Sstevel@tonic-gate fabricate_brandstr(struct cpuid_info *cpi)
53667c478bd9Sstevel@tonic-gate {
53677c478bd9Sstevel@tonic-gate 	const char *brand = NULL;
53687c478bd9Sstevel@tonic-gate 
53697c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
53707c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
53717c478bd9Sstevel@tonic-gate 		brand = intel_cpubrand(cpi);
53727c478bd9Sstevel@tonic-gate 		break;
53737c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
53747c478bd9Sstevel@tonic-gate 		brand = amd_cpubrand(cpi);
53757c478bd9Sstevel@tonic-gate 		break;
53767c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Cyrix:
53777c478bd9Sstevel@tonic-gate 		brand = cyrix_cpubrand(cpi, x86_type);
53787c478bd9Sstevel@tonic-gate 		break;
53797c478bd9Sstevel@tonic-gate 	case X86_VENDOR_NexGen:
53807c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
53817c478bd9Sstevel@tonic-gate 			brand = "NexGen Nx586";
53827c478bd9Sstevel@tonic-gate 		break;
53837c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Centaur:
53847c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5)
53857c478bd9Sstevel@tonic-gate 			switch (cpi->cpi_model) {
53867c478bd9Sstevel@tonic-gate 			case 4:
53877c478bd9Sstevel@tonic-gate 				brand = "Centaur C6";
53887c478bd9Sstevel@tonic-gate 				break;
53897c478bd9Sstevel@tonic-gate 			case 8:
53907c478bd9Sstevel@tonic-gate 				brand = "Centaur C2";
53917c478bd9Sstevel@tonic-gate 				break;
53927c478bd9Sstevel@tonic-gate 			case 9:
53937c478bd9Sstevel@tonic-gate 				brand = "Centaur C3";
53947c478bd9Sstevel@tonic-gate 				break;
53957c478bd9Sstevel@tonic-gate 			default:
53967c478bd9Sstevel@tonic-gate 				break;
53977c478bd9Sstevel@tonic-gate 			}
53987c478bd9Sstevel@tonic-gate 		break;
53997c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Rise:
54007c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5 &&
54017c478bd9Sstevel@tonic-gate 		    (cpi->cpi_model == 0 || cpi->cpi_model == 2))
54027c478bd9Sstevel@tonic-gate 			brand = "Rise mP6";
54037c478bd9Sstevel@tonic-gate 		break;
54047c478bd9Sstevel@tonic-gate 	case X86_VENDOR_SiS:
54057c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
54067c478bd9Sstevel@tonic-gate 			brand = "SiS 55x";
54077c478bd9Sstevel@tonic-gate 		break;
54087c478bd9Sstevel@tonic-gate 	case X86_VENDOR_TM:
54097c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
54107c478bd9Sstevel@tonic-gate 			brand = "Transmeta Crusoe TM3x00 or TM5x00";
54117c478bd9Sstevel@tonic-gate 		break;
54127c478bd9Sstevel@tonic-gate 	case X86_VENDOR_NSC:
54137c478bd9Sstevel@tonic-gate 	case X86_VENDOR_UMC:
54147c478bd9Sstevel@tonic-gate 	default:
54157c478bd9Sstevel@tonic-gate 		break;
54167c478bd9Sstevel@tonic-gate 	}
54177c478bd9Sstevel@tonic-gate 	if (brand) {
54187c478bd9Sstevel@tonic-gate 		(void) strcpy((char *)cpi->cpi_brandstr, brand);
54197c478bd9Sstevel@tonic-gate 		return;
54207c478bd9Sstevel@tonic-gate 	}
54217c478bd9Sstevel@tonic-gate 
54227c478bd9Sstevel@tonic-gate 	/*
54237c478bd9Sstevel@tonic-gate 	 * If all else fails ...
54247c478bd9Sstevel@tonic-gate 	 */
54257c478bd9Sstevel@tonic-gate 	(void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
54267c478bd9Sstevel@tonic-gate 	    "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
54277c478bd9Sstevel@tonic-gate 	    cpi->cpi_model, cpi->cpi_step);
54287c478bd9Sstevel@tonic-gate }
54297c478bd9Sstevel@tonic-gate 
54307c478bd9Sstevel@tonic-gate /*
54317c478bd9Sstevel@tonic-gate  * This routine is called just after kernel memory allocation
54327c478bd9Sstevel@tonic-gate  * becomes available on cpu0, and as part of mp_startup() on
54337c478bd9Sstevel@tonic-gate  * the other cpus.
54347c478bd9Sstevel@tonic-gate  *
5435d129bde2Sesaxe  * Fixup the brand string, and collect any information from cpuid
543679ec9da8SYuri Pankov  * that requires dynamically allocated storage to represent.
54377c478bd9Sstevel@tonic-gate  */
5438ab5bb018SKeith M Wesolowski 
5439ab5bb018SKeith M Wesolowski static void
cpuid_pass_dynamic(cpu_t * cpu,void * _arg __unused)5440ab5bb018SKeith M Wesolowski cpuid_pass_dynamic(cpu_t *cpu, void *_arg __unused)
54417c478bd9Sstevel@tonic-gate {
5442d129bde2Sesaxe 	int	i, max, shft, level, size;
5443d129bde2Sesaxe 	struct cpuid_regs regs;
5444d129bde2Sesaxe 	struct cpuid_regs *cp;
54457c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
54467c478bd9Sstevel@tonic-gate 
54477c478bd9Sstevel@tonic-gate 	/*
54480ce813ffSRobert Mustacchi 	 * Deterministic cache parameters
5449d129bde2Sesaxe 	 *
54500ce813ffSRobert Mustacchi 	 * Intel uses leaf 0x4 for this, while AMD uses leaf 0x8000001d. The
54510ce813ffSRobert Mustacchi 	 * values that are present are currently defined to be the same. This
54520ce813ffSRobert Mustacchi 	 * means we can use the same logic to parse it as long as we use the
54530ce813ffSRobert Mustacchi 	 * appropriate leaf to get the data. If you're updating this, make sure
54540ce813ffSRobert Mustacchi 	 * you're careful about which vendor supports which aspect.
54550ce813ffSRobert Mustacchi 	 *
54560ce813ffSRobert Mustacchi 	 * Take this opportunity to detect the number of threads sharing the
54570ce813ffSRobert Mustacchi 	 * last level cache, and construct a corresponding cache id. The
54580ce813ffSRobert Mustacchi 	 * respective cpuid_info members are initialized to the default case of
54590ce813ffSRobert Mustacchi 	 * "no last level cache sharing".
54607c478bd9Sstevel@tonic-gate 	 */
5461d129bde2Sesaxe 	cpi->cpi_ncpu_shr_last_cache = 1;
5462d129bde2Sesaxe 	cpi->cpi_last_lvl_cacheid = cpu->cpu_id;
5463d129bde2Sesaxe 
54640ce813ffSRobert Mustacchi 	if ((cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) ||
54659b0429a1SPu Wen 	    ((cpi->cpi_vendor == X86_VENDOR_AMD ||
54669b0429a1SPu Wen 	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
54670ce813ffSRobert Mustacchi 	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1d &&
54680ce813ffSRobert Mustacchi 	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT))) {
54690ce813ffSRobert Mustacchi 		uint32_t leaf;
54700ce813ffSRobert Mustacchi 
54710ce813ffSRobert Mustacchi 		if (cpi->cpi_vendor == X86_VENDOR_Intel) {
54720ce813ffSRobert Mustacchi 			leaf = 4;
54730ce813ffSRobert Mustacchi 		} else {
54740ce813ffSRobert Mustacchi 			leaf = CPUID_LEAF_EXT_1d;
54750ce813ffSRobert Mustacchi 		}
54767c478bd9Sstevel@tonic-gate 
54777c478bd9Sstevel@tonic-gate 		/*
54780ce813ffSRobert Mustacchi 		 * Find the # of elements (size) returned by the leaf and along
5479d129bde2Sesaxe 		 * the way detect last level cache sharing details.
54807c478bd9Sstevel@tonic-gate 		 */
5481d129bde2Sesaxe 		bzero(&regs, sizeof (regs));
5482d129bde2Sesaxe 		cp = &regs;
5483d129bde2Sesaxe 		for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
54840ce813ffSRobert Mustacchi 			cp->cp_eax = leaf;
5485d129bde2Sesaxe 			cp->cp_ecx = i;
5486d129bde2Sesaxe 
5487d129bde2Sesaxe 			(void) __cpuid_insn(cp);
5488d129bde2Sesaxe 
5489d129bde2Sesaxe 			if (CPI_CACHE_TYPE(cp) == 0)
5490d129bde2Sesaxe 				break;
5491d129bde2Sesaxe 			level = CPI_CACHE_LVL(cp);
5492d129bde2Sesaxe 			if (level > max) {
5493d129bde2Sesaxe 				max = level;
5494d129bde2Sesaxe 				cpi->cpi_ncpu_shr_last_cache =
5495d129bde2Sesaxe 				    CPI_NTHR_SHR_CACHE(cp) + 1;
5496d129bde2Sesaxe 			}
5497d129bde2Sesaxe 		}
54980ce813ffSRobert Mustacchi 		cpi->cpi_cache_leaf_size = size = i;
54997c478bd9Sstevel@tonic-gate 
55007c478bd9Sstevel@tonic-gate 		/*
55010ce813ffSRobert Mustacchi 		 * Allocate the cpi_cache_leaves array. The first element
55020ce813ffSRobert Mustacchi 		 * references the regs for the corresponding leaf with %ecx set
5503ab5bb018SKeith M Wesolowski 		 * to 0. This was gathered in cpuid_pass_extended().
55047c478bd9Sstevel@tonic-gate 		 */
5505d129bde2Sesaxe 		if (size > 0) {
55060ce813ffSRobert Mustacchi 			cpi->cpi_cache_leaves =
5507d129bde2Sesaxe 			    kmem_alloc(size * sizeof (cp), KM_SLEEP);
55080ce813ffSRobert Mustacchi 			if (cpi->cpi_vendor == X86_VENDOR_Intel) {
55090ce813ffSRobert Mustacchi 				cpi->cpi_cache_leaves[0] = &cpi->cpi_std[4];
55100ce813ffSRobert Mustacchi 			} else {
55110ce813ffSRobert Mustacchi 				cpi->cpi_cache_leaves[0] = &cpi->cpi_extd[0x1d];
55120ce813ffSRobert Mustacchi 			}
5513d129bde2Sesaxe 
5514d129bde2Sesaxe 			/*
5515d129bde2Sesaxe 			 * Allocate storage to hold the additional regs
55160ce813ffSRobert Mustacchi 			 * for the leaf, %ecx == 1 .. cpi_cache_leaf_size.
5517d129bde2Sesaxe 			 *
55180ce813ffSRobert Mustacchi 			 * The regs for the leaf, %ecx == 0 has already
5519d129bde2Sesaxe 			 * been allocated as indicated above.
5520d129bde2Sesaxe 			 */
5521d129bde2Sesaxe 			for (i = 1; i < size; i++) {
55220ce813ffSRobert Mustacchi 				cp = cpi->cpi_cache_leaves[i] =
5523d129bde2Sesaxe 				    kmem_zalloc(sizeof (regs), KM_SLEEP);
55240ce813ffSRobert Mustacchi 				cp->cp_eax = leaf;
5525d129bde2Sesaxe 				cp->cp_ecx = i;
5526d129bde2Sesaxe 
5527d129bde2Sesaxe 				(void) __cpuid_insn(cp);
55287c478bd9Sstevel@tonic-gate 			}
55297c478bd9Sstevel@tonic-gate 		}
55307c478bd9Sstevel@tonic-gate 		/*
5531d129bde2Sesaxe 		 * Determine the number of bits needed to represent
5532d129bde2Sesaxe 		 * the number of CPUs sharing the last level cache.
5533d129bde2Sesaxe 		 *
5534d129bde2Sesaxe 		 * Shift off that number of bits from the APIC id to
5535d129bde2Sesaxe 		 * derive the cache id.
55367c478bd9Sstevel@tonic-gate 		 */
5537d129bde2Sesaxe 		shft = 0;
5538d129bde2Sesaxe 		for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
5539d129bde2Sesaxe 			shft++;
5540b6917abeSmishra 		cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
5541d129bde2Sesaxe 	}
5542d129bde2Sesaxe 
5543d129bde2Sesaxe 	/*
5544d129bde2Sesaxe 	 * Now fixup the brand string
5545d129bde2Sesaxe 	 */
55460ce813ffSRobert Mustacchi 	if ((cpi->cpi_xmaxeax & CPUID_LEAF_EXT_0) == 0) {
55477c478bd9Sstevel@tonic-gate 		fabricate_brandstr(cpi);
5548d129bde2Sesaxe 	} else {
5549d129bde2Sesaxe 
5550d129bde2Sesaxe 		/*
5551d129bde2Sesaxe 		 * If we successfully extracted a brand string from the cpuid
5552d129bde2Sesaxe 		 * instruction, clean it up by removing leading spaces and
5553d129bde2Sesaxe 		 * similar junk.
5554d129bde2Sesaxe 		 */
5555d129bde2Sesaxe 		if (cpi->cpi_brandstr[0]) {
5556d129bde2Sesaxe 			size_t maxlen = sizeof (cpi->cpi_brandstr);
5557d129bde2Sesaxe 			char *src, *dst;
55587c478bd9Sstevel@tonic-gate 
5559d129bde2Sesaxe 			dst = src = (char *)cpi->cpi_brandstr;
5560d129bde2Sesaxe 			src[maxlen - 1] = '\0';
5561d129bde2Sesaxe 			/*
5562d129bde2Sesaxe 			 * strip leading spaces
5563d129bde2Sesaxe 			 */
5564d129bde2Sesaxe 			while (*src == ' ')
5565d129bde2Sesaxe 				src++;
5566d129bde2Sesaxe 			/*
5567d129bde2Sesaxe 			 * Remove any 'Genuine' or "Authentic" prefixes
5568d129bde2Sesaxe 			 */
5569d129bde2Sesaxe 			if (strncmp(src, "Genuine ", 8) == 0)
5570d129bde2Sesaxe 				src += 8;
5571d129bde2Sesaxe 			if (strncmp(src, "Authentic ", 10) == 0)
5572d129bde2Sesaxe 				src += 10;
5573d129bde2Sesaxe 
5574d129bde2Sesaxe 			/*
5575d129bde2Sesaxe 			 * Now do an in-place copy.
5576d129bde2Sesaxe 			 * Map (R) to (r) and (TM) to (tm).
5577d129bde2Sesaxe 			 * The era of teletypes is long gone, and there's
5578d129bde2Sesaxe 			 * -really- no need to shout.
5579d129bde2Sesaxe 			 */
5580d129bde2Sesaxe 			while (*src != '\0') {
5581d129bde2Sesaxe 				if (src[0] == '(') {
5582d129bde2Sesaxe 					if (strncmp(src + 1, "R)", 2) == 0) {
5583d129bde2Sesaxe 						(void) strncpy(dst, "(r)", 3);
5584d129bde2Sesaxe 						src += 3;
5585d129bde2Sesaxe 						dst += 3;
5586d129bde2Sesaxe 						continue;
5587d129bde2Sesaxe 					}
5588d129bde2Sesaxe 					if (strncmp(src + 1, "TM)", 3) == 0) {
5589d129bde2Sesaxe 						(void) strncpy(dst, "(tm)", 4);
5590d129bde2Sesaxe 						src += 4;
5591d129bde2Sesaxe 						dst += 4;
5592d129bde2Sesaxe 						continue;
5593d129bde2Sesaxe 					}
5594d129bde2Sesaxe 				}
5595d129bde2Sesaxe 				*dst++ = *src++;
5596d129bde2Sesaxe 			}
5597d129bde2Sesaxe 			*dst = '\0';
5598d129bde2Sesaxe 
5599d129bde2Sesaxe 			/*
5600d129bde2Sesaxe 			 * Finally, remove any trailing spaces
5601d129bde2Sesaxe 			 */
5602d129bde2Sesaxe 			while (--dst > cpi->cpi_brandstr)
5603d129bde2Sesaxe 				if (*dst == ' ')
5604d129bde2Sesaxe 					*dst = '\0';
5605d129bde2Sesaxe 				else
5606d129bde2Sesaxe 					break;
5607d129bde2Sesaxe 		} else
5608d129bde2Sesaxe 			fabricate_brandstr(cpi);
5609d129bde2Sesaxe 	}
56107c478bd9Sstevel@tonic-gate }
56117c478bd9Sstevel@tonic-gate 
561256726c7eSRobert Mustacchi typedef struct {
561356726c7eSRobert Mustacchi 	uint32_t avm_av;
561456726c7eSRobert Mustacchi 	uint32_t avm_feat;
561556726c7eSRobert Mustacchi } av_feat_map_t;
561656726c7eSRobert Mustacchi 
561756726c7eSRobert Mustacchi /*
561856726c7eSRobert Mustacchi  * These arrays are used to map features that we should add based on x86
561956726c7eSRobert Mustacchi  * features that are present. As a large number depend on kernel features,
562056726c7eSRobert Mustacchi  * rather than rechecking and clearing CPUID everywhere, we simply map these.
562156726c7eSRobert Mustacchi  * There is an array of these for each hwcap word. Some features aren't tracked
562256726c7eSRobert Mustacchi  * in the kernel x86 featureset and that's ok. They will not show up in here.
562356726c7eSRobert Mustacchi  */
562456726c7eSRobert Mustacchi static const av_feat_map_t x86fset_to_av1[] = {
562556726c7eSRobert Mustacchi 	{ AV_386_CX8, X86FSET_CX8 },
562656726c7eSRobert Mustacchi 	{ AV_386_SEP, X86FSET_SEP },
562756726c7eSRobert Mustacchi 	{ AV_386_AMD_SYSC, X86FSET_ASYSC },
562856726c7eSRobert Mustacchi 	{ AV_386_CMOV, X86FSET_CMOV },
562956726c7eSRobert Mustacchi 	{ AV_386_FXSR, X86FSET_SSE },
563056726c7eSRobert Mustacchi 	{ AV_386_SSE, X86FSET_SSE },
563156726c7eSRobert Mustacchi 	{ AV_386_SSE2, X86FSET_SSE2 },
563256726c7eSRobert Mustacchi 	{ AV_386_SSE3, X86FSET_SSE3 },
563356726c7eSRobert Mustacchi 	{ AV_386_CX16, X86FSET_CX16 },
563456726c7eSRobert Mustacchi 	{ AV_386_TSCP, X86FSET_TSCP },
563556726c7eSRobert Mustacchi 	{ AV_386_AMD_SSE4A, X86FSET_SSE4A },
563656726c7eSRobert Mustacchi 	{ AV_386_SSSE3, X86FSET_SSSE3 },
563756726c7eSRobert Mustacchi 	{ AV_386_SSE4_1, X86FSET_SSE4_1 },
563856726c7eSRobert Mustacchi 	{ AV_386_SSE4_2, X86FSET_SSE4_2 },
563956726c7eSRobert Mustacchi 	{ AV_386_AES, X86FSET_AES },
564056726c7eSRobert Mustacchi 	{ AV_386_PCLMULQDQ, X86FSET_PCLMULQDQ },
564156726c7eSRobert Mustacchi 	{ AV_386_XSAVE, X86FSET_XSAVE },
564256726c7eSRobert Mustacchi 	{ AV_386_AVX, X86FSET_AVX },
564356726c7eSRobert Mustacchi 	{ AV_386_VMX, X86FSET_VMX },
564456726c7eSRobert Mustacchi 	{ AV_386_AMD_SVM, X86FSET_SVM }
564556726c7eSRobert Mustacchi };
564656726c7eSRobert Mustacchi 
564756726c7eSRobert Mustacchi static const av_feat_map_t x86fset_to_av2[] = {
564856726c7eSRobert Mustacchi 	{ AV_386_2_F16C, X86FSET_F16C },
564956726c7eSRobert Mustacchi 	{ AV_386_2_RDRAND, X86FSET_RDRAND },
565056726c7eSRobert Mustacchi 	{ AV_386_2_BMI1, X86FSET_BMI1 },
565156726c7eSRobert Mustacchi 	{ AV_386_2_BMI2, X86FSET_BMI2 },
565256726c7eSRobert Mustacchi 	{ AV_386_2_FMA, X86FSET_FMA },
565356726c7eSRobert Mustacchi 	{ AV_386_2_AVX2, X86FSET_AVX2 },
565456726c7eSRobert Mustacchi 	{ AV_386_2_ADX, X86FSET_ADX },
565556726c7eSRobert Mustacchi 	{ AV_386_2_RDSEED, X86FSET_RDSEED },
565656726c7eSRobert Mustacchi 	{ AV_386_2_AVX512F, X86FSET_AVX512F },
565756726c7eSRobert Mustacchi 	{ AV_386_2_AVX512DQ, X86FSET_AVX512DQ },
565856726c7eSRobert Mustacchi 	{ AV_386_2_AVX512IFMA, X86FSET_AVX512FMA },
565956726c7eSRobert Mustacchi 	{ AV_386_2_AVX512PF, X86FSET_AVX512PF },
566056726c7eSRobert Mustacchi 	{ AV_386_2_AVX512ER, X86FSET_AVX512ER },
566156726c7eSRobert Mustacchi 	{ AV_386_2_AVX512CD, X86FSET_AVX512CD },
566256726c7eSRobert Mustacchi 	{ AV_386_2_AVX512BW, X86FSET_AVX512BW },
566356726c7eSRobert Mustacchi 	{ AV_386_2_AVX512VL, X86FSET_AVX512VL },
566456726c7eSRobert Mustacchi 	{ AV_386_2_AVX512VBMI, X86FSET_AVX512VBMI },
566556726c7eSRobert Mustacchi 	{ AV_386_2_AVX512VPOPCDQ, X86FSET_AVX512VPOPCDQ },
566656726c7eSRobert Mustacchi 	{ AV_386_2_SHA, X86FSET_SHA },
566756726c7eSRobert Mustacchi 	{ AV_386_2_FSGSBASE, X86FSET_FSGSBASE },
566856726c7eSRobert Mustacchi 	{ AV_386_2_CLFLUSHOPT, X86FSET_CLFLUSHOPT },
566956726c7eSRobert Mustacchi 	{ AV_386_2_CLWB, X86FSET_CLWB },
567056726c7eSRobert Mustacchi 	{ AV_386_2_MONITORX, X86FSET_MONITORX },
567156726c7eSRobert Mustacchi 	{ AV_386_2_CLZERO, X86FSET_CLZERO },
567256726c7eSRobert Mustacchi 	{ AV_386_2_AVX512_VNNI, X86FSET_AVX512VNNI },
567356726c7eSRobert Mustacchi 	{ AV_386_2_VPCLMULQDQ, X86FSET_VPCLMULQDQ },
567456726c7eSRobert Mustacchi 	{ AV_386_2_VAES, X86FSET_VAES },
567556726c7eSRobert Mustacchi 	{ AV_386_2_GFNI, X86FSET_GFNI },
567656726c7eSRobert Mustacchi 	{ AV_386_2_AVX512_VP2INT, X86FSET_AVX512_VP2INT },
567756726c7eSRobert Mustacchi 	{ AV_386_2_AVX512_BITALG, X86FSET_AVX512_BITALG }
567856726c7eSRobert Mustacchi };
567956726c7eSRobert Mustacchi 
568056726c7eSRobert Mustacchi static const av_feat_map_t x86fset_to_av3[] = {
568156726c7eSRobert Mustacchi 	{ AV_386_3_AVX512_VBMI2, X86FSET_AVX512_VBMI2 },
568256726c7eSRobert Mustacchi 	{ AV_386_3_AVX512_BF16, X86FSET_AVX512_BF16 }
568356726c7eSRobert Mustacchi };
568456726c7eSRobert Mustacchi 
56857c478bd9Sstevel@tonic-gate /*
56867c478bd9Sstevel@tonic-gate  * This routine is called out of bind_hwcap() much later in the life
56877c478bd9Sstevel@tonic-gate  * of the kernel (post_startup()).  The job of this routine is to resolve
56887c478bd9Sstevel@tonic-gate  * the hardware feature support and kernel support for those features into
56897c478bd9Sstevel@tonic-gate  * what we're actually going to tell applications via the aux vector.
569056726c7eSRobert Mustacchi  *
569156726c7eSRobert Mustacchi  * Most of the aux vector is derived from the x86_featureset array vector where
569256726c7eSRobert Mustacchi  * a given feature indicates that an aux vector should be plumbed through. This
569356726c7eSRobert Mustacchi  * allows the kernel to use one tracking mechanism for these based on whether or
569456726c7eSRobert Mustacchi  * not it has the required hardware support (most often xsave). Most newer
569556726c7eSRobert Mustacchi  * features are added there in case we need them in the kernel. Otherwise,
569656726c7eSRobert Mustacchi  * features are evaluated based on looking at the cpuid features that remain. If
569756726c7eSRobert Mustacchi  * you find yourself wanting to clear out cpuid features for some reason, they
569856726c7eSRobert Mustacchi  * should instead be driven by the feature set so we have a consistent view.
56997c478bd9Sstevel@tonic-gate  */
5700ab5bb018SKeith M Wesolowski 
5701ab5bb018SKeith M Wesolowski static void
cpuid_pass_resolve(cpu_t * cpu,void * arg)5702ab5bb018SKeith M Wesolowski cpuid_pass_resolve(cpu_t *cpu, void *arg)
57037c478bd9Sstevel@tonic-gate {
5704ab5bb018SKeith M Wesolowski 	uint_t *hwcap_out = (uint_t *)arg;
57057c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi;
570656726c7eSRobert Mustacchi 	uint_t hwcap_flags = 0, hwcap_flags_2 = 0, hwcap_flags_3 = 0;
57077c478bd9Sstevel@tonic-gate 
57087c478bd9Sstevel@tonic-gate 	cpi = cpu->cpu_m.mcpu_cpi;
57097c478bd9Sstevel@tonic-gate 
571056726c7eSRobert Mustacchi 	for (uint_t i = 0; i < ARRAY_SIZE(x86fset_to_av1); i++) {
571156726c7eSRobert Mustacchi 		if (is_x86_feature(x86_featureset,
571256726c7eSRobert Mustacchi 		    x86fset_to_av1[i].avm_feat)) {
571356726c7eSRobert Mustacchi 			hwcap_flags |= x86fset_to_av1[i].avm_av;
571456726c7eSRobert Mustacchi 		}
571556726c7eSRobert Mustacchi 	}
571656726c7eSRobert Mustacchi 
571756726c7eSRobert Mustacchi 	for (uint_t i = 0; i < ARRAY_SIZE(x86fset_to_av2); i++) {
571856726c7eSRobert Mustacchi 		if (is_x86_feature(x86_featureset,
571956726c7eSRobert Mustacchi 		    x86fset_to_av2[i].avm_feat)) {
572056726c7eSRobert Mustacchi 			hwcap_flags_2 |= x86fset_to_av2[i].avm_av;
572156726c7eSRobert Mustacchi 		}
572256726c7eSRobert Mustacchi 	}
572356726c7eSRobert Mustacchi 
572456726c7eSRobert Mustacchi 	for (uint_t i = 0; i < ARRAY_SIZE(x86fset_to_av3); i++) {
572556726c7eSRobert Mustacchi 		if (is_x86_feature(x86_featureset,
572656726c7eSRobert Mustacchi 		    x86fset_to_av3[i].avm_feat)) {
572756726c7eSRobert Mustacchi 			hwcap_flags_3 |= x86fset_to_av3[i].avm_av;
572856726c7eSRobert Mustacchi 		}
572956726c7eSRobert Mustacchi 	}
573056726c7eSRobert Mustacchi 
573156726c7eSRobert Mustacchi 	/*
573256726c7eSRobert Mustacchi 	 * From here on out we're working through features that don't have
573356726c7eSRobert Mustacchi 	 * corresponding kernel feature flags for various reasons that are
573456726c7eSRobert Mustacchi 	 * mostly just due to the historical implementation.
573556726c7eSRobert Mustacchi 	 */
57367c478bd9Sstevel@tonic-gate 	if (cpi->cpi_maxeax >= 1) {
57377c478bd9Sstevel@tonic-gate 		uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
57387c478bd9Sstevel@tonic-gate 		uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];
57397c478bd9Sstevel@tonic-gate 
57407c478bd9Sstevel@tonic-gate 		*edx = CPI_FEATURES_EDX(cpi);
57417c478bd9Sstevel@tonic-gate 		*ecx = CPI_FEATURES_ECX(cpi);
5742d0f8ff6eSkk 
57437c478bd9Sstevel@tonic-gate 		/*
57447c478bd9Sstevel@tonic-gate 		 * [no explicit support required beyond x87 fp context]
57457c478bd9Sstevel@tonic-gate 		 */
57467c478bd9Sstevel@tonic-gate 		if (!fpu_exists)
57477c478bd9Sstevel@tonic-gate 			*edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);
57487c478bd9Sstevel@tonic-gate 
57497c478bd9Sstevel@tonic-gate 		/*
57507c478bd9Sstevel@tonic-gate 		 * Now map the supported feature vector to things that we
57517c478bd9Sstevel@tonic-gate 		 * think userland will care about.
57527c478bd9Sstevel@tonic-gate 		 */
575363408480SHans Rosenfeld 		if (*ecx & CPUID_INTC_ECX_MOVBE)
575463408480SHans Rosenfeld 			hwcap_flags |= AV_386_MOVBE;
575556726c7eSRobert Mustacchi 
5756f8801251Skk 		if (*ecx & CPUID_INTC_ECX_POPCNT)
5757f8801251Skk 			hwcap_flags |= AV_386_POPCNT;
57587c478bd9Sstevel@tonic-gate 		if (*edx & CPUID_INTC_EDX_FPU)
57597c478bd9Sstevel@tonic-gate 			hwcap_flags |= AV_386_FPU;
57607c478bd9Sstevel@tonic-gate 		if (*edx & CPUID_INTC_EDX_MMX)
57617c478bd9Sstevel@tonic-gate 			hwcap_flags |= AV_386_MMX;
57627c478bd9Sstevel@tonic-gate 		if (*edx & CPUID_INTC_EDX_TSC)
57637c478bd9Sstevel@tonic-gate 			hwcap_flags |= AV_386_TSC;
57647c478bd9Sstevel@tonic-gate 	}
576556726c7eSRobert Mustacchi 
5766cff040f3SRobert Mustacchi 	/*
576756726c7eSRobert Mustacchi 	 * Check a few miscellaneous features.
5768cff040f3SRobert Mustacchi 	 */
57697c478bd9Sstevel@tonic-gate 	if (cpi->cpi_xmaxeax < 0x80000001)
5770ab5bb018SKeith M Wesolowski 		goto resolve_done;
57717c478bd9Sstevel@tonic-gate 
57727c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
5773ae115bc7Smrj 		uint32_t *edx, *ecx;
5774ae115bc7Smrj 
5775ae115bc7Smrj 	case X86_VENDOR_Intel:
5776ae115bc7Smrj 		/*
5777ae115bc7Smrj 		 * Seems like Intel duplicated what we necessary
5778ae115bc7Smrj 		 * here to make the initial crop of 64-bit OS's work.
5779ae115bc7Smrj 		 * Hopefully, those are the only "extended" bits
5780ae115bc7Smrj 		 * they'll add.
5781ae115bc7Smrj 		 */
5782ae115bc7Smrj 		/*FALLTHROUGH*/
57837c478bd9Sstevel@tonic-gate 
57847c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
57859b0429a1SPu Wen 	case X86_VENDOR_HYGON:
57867c478bd9Sstevel@tonic-gate 		edx = &cpi->cpi_support[AMD_EDX_FEATURES];
5787ae115bc7Smrj 		ecx = &cpi->cpi_support[AMD_ECX_FEATURES];
57887c478bd9Sstevel@tonic-gate 
57897c478bd9Sstevel@tonic-gate 		*edx = CPI_FEATURES_XTD_EDX(cpi);
5790ae115bc7Smrj 		*ecx = CPI_FEATURES_XTD_ECX(cpi);
5791ae115bc7Smrj 
57927c478bd9Sstevel@tonic-gate 		/*
57937c478bd9Sstevel@tonic-gate 		 * [no explicit support required beyond
57947c478bd9Sstevel@tonic-gate 		 * x87 fp context and exception handlers]
57957c478bd9Sstevel@tonic-gate 		 */
57967c478bd9Sstevel@tonic-gate 		if (!fpu_exists)
57977c478bd9Sstevel@tonic-gate 			*edx &= ~(CPUID_AMD_EDX_MMXamd |
57987c478bd9Sstevel@tonic-gate 			    CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);
57997c478bd9Sstevel@tonic-gate 
58007c478bd9Sstevel@tonic-gate 		/*
58017c478bd9Sstevel@tonic-gate 		 * Now map the supported feature vector to
58027c478bd9Sstevel@tonic-gate 		 * things that we think userland will care about.
58037c478bd9Sstevel@tonic-gate 		 */
58047c478bd9Sstevel@tonic-gate 		if (*edx & CPUID_AMD_EDX_MMXamd)
58057c478bd9Sstevel@tonic-gate 			hwcap_flags |= AV_386_AMD_MMX;
58067c478bd9Sstevel@tonic-gate 		if (*edx & CPUID_AMD_EDX_3DNow)
58077c478bd9Sstevel@tonic-gate 			hwcap_flags |= AV_386_AMD_3DNow;
58087c478bd9Sstevel@tonic-gate 		if (*edx & CPUID_AMD_EDX_3DNowx)
58097c478bd9Sstevel@tonic-gate 			hwcap_flags |= AV_386_AMD_3DNowx;
5810ae115bc7Smrj 
5811ae115bc7Smrj 		switch (cpi->cpi_vendor) {
5812ae115bc7Smrj 		case X86_VENDOR_AMD:
58139b0429a1SPu Wen 		case X86_VENDOR_HYGON:
5814ae115bc7Smrj 			if (*ecx & CPUID_AMD_ECX_AHF64)
5815ae115bc7Smrj 				hwcap_flags |= AV_386_AHF;
5816f8801251Skk 			if (*ecx & CPUID_AMD_ECX_LZCNT)
5817f8801251Skk 				hwcap_flags |= AV_386_AMD_LZCNT;
5818ae115bc7Smrj 			break;
5819ae115bc7Smrj 
5820ae115bc7Smrj 		case X86_VENDOR_Intel:
58211567de9cSRobert Mustacchi 			if (*ecx & CPUID_AMD_ECX_LZCNT)
58221567de9cSRobert Mustacchi 				hwcap_flags |= AV_386_AMD_LZCNT;
5823ae115bc7Smrj 			/*
5824ae115bc7Smrj 			 * Aarrgh.
5825ae115bc7Smrj 			 * Intel uses a different bit in the same word.
5826ae115bc7Smrj 			 */
5827ae115bc7Smrj 			if (*ecx & CPUID_INTC_ECX_AHF64)
5828ae115bc7Smrj 				hwcap_flags |= AV_386_AHF;
5829ae115bc7Smrj 			break;
5830ae115bc7Smrj 		default:
5831ae115bc7Smrj 			break;
5832ae115bc7Smrj 		}
58337c478bd9Sstevel@tonic-gate 		break;
58347c478bd9Sstevel@tonic-gate 
58357c478bd9Sstevel@tonic-gate 	default:
58367c478bd9Sstevel@tonic-gate 		break;
58377c478bd9Sstevel@tonic-gate 	}
58387c478bd9Sstevel@tonic-gate 
5839ab5bb018SKeith M Wesolowski resolve_done:
5840ebb8ac07SRobert Mustacchi 	if (hwcap_out != NULL) {
5841ebb8ac07SRobert Mustacchi 		hwcap_out[0] = hwcap_flags;
5842ebb8ac07SRobert Mustacchi 		hwcap_out[1] = hwcap_flags_2;
584356726c7eSRobert Mustacchi 		hwcap_out[2] = hwcap_flags_3;
5844ebb8ac07SRobert Mustacchi 	}
58457c478bd9Sstevel@tonic-gate }
58467c478bd9Sstevel@tonic-gate 
58477c478bd9Sstevel@tonic-gate 
58487c478bd9Sstevel@tonic-gate /*
58497c478bd9Sstevel@tonic-gate  * Simulate the cpuid instruction using the data we previously
58507c478bd9Sstevel@tonic-gate  * captured about this CPU.  We try our best to return the truth
58517c478bd9Sstevel@tonic-gate  * about the hardware, independently of kernel support.
58527c478bd9Sstevel@tonic-gate  */
58537c478bd9Sstevel@tonic-gate uint32_t
cpuid_insn(cpu_t * cpu,struct cpuid_regs * cp)58548949bcd6Sandrei cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
58557c478bd9Sstevel@tonic-gate {
58567c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi;
58578949bcd6Sandrei 	struct cpuid_regs *xcp;
58587c478bd9Sstevel@tonic-gate 
58597c478bd9Sstevel@tonic-gate 	if (cpu == NULL)
58607c478bd9Sstevel@tonic-gate 		cpu = CPU;
58617c478bd9Sstevel@tonic-gate 	cpi = cpu->cpu_m.mcpu_cpi;
58627c478bd9Sstevel@tonic-gate 
5863ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));
58647c478bd9Sstevel@tonic-gate 
58657c478bd9Sstevel@tonic-gate 	/*
58667c478bd9Sstevel@tonic-gate 	 * CPUID data is cached in two separate places: cpi_std for standard
58670ce813ffSRobert Mustacchi 	 * CPUID leaves , and cpi_extd for extended CPUID leaves.
58687c478bd9Sstevel@tonic-gate 	 */
58690ce813ffSRobert Mustacchi 	if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD) {
58708949bcd6Sandrei 		xcp = &cpi->cpi_std[cp->cp_eax];
58710ce813ffSRobert Mustacchi 	} else if (cp->cp_eax >= CPUID_LEAF_EXT_0 &&
58720ce813ffSRobert Mustacchi 	    cp->cp_eax <= cpi->cpi_xmaxeax &&
58730ce813ffSRobert Mustacchi 	    cp->cp_eax < CPUID_LEAF_EXT_0 + NMAX_CPI_EXTD) {
58740ce813ffSRobert Mustacchi 		xcp = &cpi->cpi_extd[cp->cp_eax - CPUID_LEAF_EXT_0];
58750ce813ffSRobert Mustacchi 	} else {
58767c478bd9Sstevel@tonic-gate 		/*
58777c478bd9Sstevel@tonic-gate 		 * The caller is asking for data from an input parameter which
58787c478bd9Sstevel@tonic-gate 		 * the kernel has not cached.  In this case we go fetch from
58797c478bd9Sstevel@tonic-gate 		 * the hardware and return the data directly to the user.
58807c478bd9Sstevel@tonic-gate 		 */
58818949bcd6Sandrei 		return (__cpuid_insn(cp));
58820ce813ffSRobert Mustacchi 	}
58838949bcd6Sandrei 
58848949bcd6Sandrei 	cp->cp_eax = xcp->cp_eax;
58858949bcd6Sandrei 	cp->cp_ebx = xcp->cp_ebx;
58868949bcd6Sandrei 	cp->cp_ecx = xcp->cp_ecx;
58878949bcd6Sandrei 	cp->cp_edx = xcp->cp_edx;
58887c478bd9Sstevel@tonic-gate 	return (cp->cp_eax);
58897c478bd9Sstevel@tonic-gate }
58907c478bd9Sstevel@tonic-gate 
5891ab5bb018SKeith M Wesolowski boolean_t
cpuid_checkpass(const cpu_t * const cpu,const cpuid_pass_t pass)5892ab5bb018SKeith M Wesolowski cpuid_checkpass(const cpu_t *const cpu, const cpuid_pass_t pass)
58937c478bd9Sstevel@tonic-gate {
58947c478bd9Sstevel@tonic-gate 	return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
58957c478bd9Sstevel@tonic-gate 	    cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
58967c478bd9Sstevel@tonic-gate }
58977c478bd9Sstevel@tonic-gate 
58987c478bd9Sstevel@tonic-gate int
cpuid_getbrandstr(cpu_t * cpu,char * s,size_t n)58997c478bd9Sstevel@tonic-gate cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
59007c478bd9Sstevel@tonic-gate {
5901ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));
59027c478bd9Sstevel@tonic-gate 
59037c478bd9Sstevel@tonic-gate 	return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
59047c478bd9Sstevel@tonic-gate }
59057c478bd9Sstevel@tonic-gate 
59067c478bd9Sstevel@tonic-gate int
cpuid_is_cmt(cpu_t * cpu)59078949bcd6Sandrei cpuid_is_cmt(cpu_t *cpu)
59087c478bd9Sstevel@tonic-gate {
59097c478bd9Sstevel@tonic-gate 	if (cpu == NULL)
59107c478bd9Sstevel@tonic-gate 		cpu = CPU;
59117c478bd9Sstevel@tonic-gate 
5912ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
59137c478bd9Sstevel@tonic-gate 
59147c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
59157c478bd9Sstevel@tonic-gate }
59167c478bd9Sstevel@tonic-gate 
59177c478bd9Sstevel@tonic-gate /*
59187c478bd9Sstevel@tonic-gate  * AMD and Intel both implement the 64-bit variant of the syscall
59197c478bd9Sstevel@tonic-gate  * instruction (syscallq), so if there's -any- support for syscall,
59207c478bd9Sstevel@tonic-gate  * cpuid currently says "yes, we support this".
59217c478bd9Sstevel@tonic-gate  *
59227c478bd9Sstevel@tonic-gate  * However, Intel decided to -not- implement the 32-bit variant of the
59237c478bd9Sstevel@tonic-gate  * syscall instruction, so we provide a predicate to allow our caller
59247c478bd9Sstevel@tonic-gate  * to test that subtlety here.
5925843e1988Sjohnlev  *
5926843e1988Sjohnlev  * XXPV	Currently, 32-bit syscall instructions don't work via the hypervisor,
5927843e1988Sjohnlev  *	even in the case where the hardware would in fact support it.
59287c478bd9Sstevel@tonic-gate  */
59297c478bd9Sstevel@tonic-gate /*ARGSUSED*/
59307c478bd9Sstevel@tonic-gate int
cpuid_syscall32_insn(cpu_t * cpu)59317c478bd9Sstevel@tonic-gate cpuid_syscall32_insn(cpu_t *cpu)
59327c478bd9Sstevel@tonic-gate {
5933ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), CPUID_PASS_BASIC));
59347c478bd9Sstevel@tonic-gate 
5935843e1988Sjohnlev #if !defined(__xpv)
5936ae115bc7Smrj 	if (cpu == NULL)
5937ae115bc7Smrj 		cpu = CPU;
5938ae115bc7Smrj 
5939ae115bc7Smrj 	/*CSTYLED*/
5940ae115bc7Smrj 	{
5941ae115bc7Smrj 		struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
5942ae115bc7Smrj 
59439b0429a1SPu Wen 		if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
59449b0429a1SPu Wen 		    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
5945ae115bc7Smrj 		    cpi->cpi_xmaxeax >= 0x80000001 &&
5946ae115bc7Smrj 		    (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
5947ae115bc7Smrj 			return (1);
5948ae115bc7Smrj 	}
5949843e1988Sjohnlev #endif
59507c478bd9Sstevel@tonic-gate 	return (0);
59517c478bd9Sstevel@tonic-gate }
59527c478bd9Sstevel@tonic-gate 
59537c478bd9Sstevel@tonic-gate int
cpuid_getidstr(cpu_t * cpu,char * s,size_t n)59547c478bd9Sstevel@tonic-gate cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
59557c478bd9Sstevel@tonic-gate {
59567c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
59577c478bd9Sstevel@tonic-gate 
59587c478bd9Sstevel@tonic-gate 	static const char fmt[] =
5959ecfa43a5Sdmick 	    "x86 (%s %X family %d model %d step %d clock %d MHz)";
59607c478bd9Sstevel@tonic-gate 	static const char fmt_ht[] =
5961ecfa43a5Sdmick 	    "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";
59627c478bd9Sstevel@tonic-gate 
5963ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
59647c478bd9Sstevel@tonic-gate 
59658949bcd6Sandrei 	if (cpuid_is_cmt(cpu))
59667c478bd9Sstevel@tonic-gate 		return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
5967ecfa43a5Sdmick 		    cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
5968ecfa43a5Sdmick 		    cpi->cpi_family, cpi->cpi_model,
59697c478bd9Sstevel@tonic-gate 		    cpi->cpi_step, cpu->cpu_type_info.pi_clock));
59707c478bd9Sstevel@tonic-gate 	return (snprintf(s, n, fmt,
5971ecfa43a5Sdmick 	    cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
5972ecfa43a5Sdmick 	    cpi->cpi_family, cpi->cpi_model,
59737c478bd9Sstevel@tonic-gate 	    cpi->cpi_step, cpu->cpu_type_info.pi_clock));
59747c478bd9Sstevel@tonic-gate }
59757c478bd9Sstevel@tonic-gate 
59767c478bd9Sstevel@tonic-gate const char *
cpuid_getvendorstr(cpu_t * cpu)59777c478bd9Sstevel@tonic-gate cpuid_getvendorstr(cpu_t *cpu)
59787c478bd9Sstevel@tonic-gate {
5979ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
59807c478bd9Sstevel@tonic-gate 	return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
59817c478bd9Sstevel@tonic-gate }
59827c478bd9Sstevel@tonic-gate 
59837c478bd9Sstevel@tonic-gate uint_t
cpuid_getvendor(cpu_t * cpu)59847c478bd9Sstevel@tonic-gate cpuid_getvendor(cpu_t *cpu)
59857c478bd9Sstevel@tonic-gate {
5986ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
59877c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
59887c478bd9Sstevel@tonic-gate }
59897c478bd9Sstevel@tonic-gate 
59907c478bd9Sstevel@tonic-gate uint_t
cpuid_getfamily(cpu_t * cpu)59917c478bd9Sstevel@tonic-gate cpuid_getfamily(cpu_t *cpu)
59927c478bd9Sstevel@tonic-gate {
5993ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
59947c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_family);
59957c478bd9Sstevel@tonic-gate }
59967c478bd9Sstevel@tonic-gate 
59977c478bd9Sstevel@tonic-gate uint_t
cpuid_getmodel(cpu_t * cpu)59987c478bd9Sstevel@tonic-gate cpuid_getmodel(cpu_t *cpu)
59997c478bd9Sstevel@tonic-gate {
6000ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
60017c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_model);
60027c478bd9Sstevel@tonic-gate }
60037c478bd9Sstevel@tonic-gate 
60047c478bd9Sstevel@tonic-gate uint_t
cpuid_get_ncpu_per_chip(cpu_t * cpu)60057c478bd9Sstevel@tonic-gate cpuid_get_ncpu_per_chip(cpu_t *cpu)
60067c478bd9Sstevel@tonic-gate {
6007ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
60087c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
60097c478bd9Sstevel@tonic-gate }
60107c478bd9Sstevel@tonic-gate 
60118949bcd6Sandrei uint_t
cpuid_get_ncore_per_chip(cpu_t * cpu)60128949bcd6Sandrei cpuid_get_ncore_per_chip(cpu_t *cpu)
60138949bcd6Sandrei {
6014ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
60158949bcd6Sandrei 	return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
60168949bcd6Sandrei }
60178949bcd6Sandrei 
6018d129bde2Sesaxe uint_t
cpuid_get_ncpu_sharing_last_cache(cpu_t * cpu)6019d129bde2Sesaxe cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
6020d129bde2Sesaxe {
6021ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_EXTENDED));
6022d129bde2Sesaxe 	return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
6023d129bde2Sesaxe }
6024d129bde2Sesaxe 
6025d129bde2Sesaxe id_t
cpuid_get_last_lvl_cacheid(cpu_t * cpu)6026d129bde2Sesaxe cpuid_get_last_lvl_cacheid(cpu_t *cpu)
6027d129bde2Sesaxe {
6028ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_EXTENDED));
6029d129bde2Sesaxe 	return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
6030d129bde2Sesaxe }
6031d129bde2Sesaxe 
60327c478bd9Sstevel@tonic-gate uint_t
cpuid_getstep(cpu_t * cpu)60337c478bd9Sstevel@tonic-gate cpuid_getstep(cpu_t *cpu)
60347c478bd9Sstevel@tonic-gate {
6035ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
60367c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_step);
60377c478bd9Sstevel@tonic-gate }
60387c478bd9Sstevel@tonic-gate 
60392449e17fSsherrym uint_t
cpuid_getsig(struct cpu * cpu)60402449e17fSsherrym cpuid_getsig(struct cpu *cpu)
60412449e17fSsherrym {
6042ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
60432449e17fSsherrym 	return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
60442449e17fSsherrym }
60452449e17fSsherrym 
60468a40a695Sgavinm uint32_t
cpuid_getchiprev(struct cpu * cpu)60478a40a695Sgavinm cpuid_getchiprev(struct cpu *cpu)
60488a40a695Sgavinm {
6049ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
60508a40a695Sgavinm 	return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
60518a40a695Sgavinm }
60528a40a695Sgavinm 
60538a40a695Sgavinm const char *
cpuid_getchiprevstr(struct cpu * cpu)60548a40a695Sgavinm cpuid_getchiprevstr(struct cpu *cpu)
60558a40a695Sgavinm {
6056ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
60578a40a695Sgavinm 	return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
60588a40a695Sgavinm }
60598a40a695Sgavinm 
60608a40a695Sgavinm uint32_t
cpuid_getsockettype(struct cpu * cpu)60618a40a695Sgavinm cpuid_getsockettype(struct cpu *cpu)
60628a40a695Sgavinm {
6063ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
60648a40a695Sgavinm 	return (cpu->cpu_m.mcpu_cpi->cpi_socket);
60658a40a695Sgavinm }
60668a40a695Sgavinm 
606789e921d5SKuriakose Kuruvilla const char *
cpuid_getsocketstr(cpu_t * cpu)606889e921d5SKuriakose Kuruvilla cpuid_getsocketstr(cpu_t *cpu)
606989e921d5SKuriakose Kuruvilla {
607089e921d5SKuriakose Kuruvilla 	static const char *socketstr = NULL;
607189e921d5SKuriakose Kuruvilla 	struct cpuid_info *cpi;
607289e921d5SKuriakose Kuruvilla 
6073ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
607489e921d5SKuriakose Kuruvilla 	cpi = cpu->cpu_m.mcpu_cpi;
607589e921d5SKuriakose Kuruvilla 
607689e921d5SKuriakose Kuruvilla 	/* Assume that socket types are the same across the system */
607789e921d5SKuriakose Kuruvilla 	if (socketstr == NULL)
607889e921d5SKuriakose Kuruvilla 		socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
607989e921d5SKuriakose Kuruvilla 		    cpi->cpi_model, cpi->cpi_step);
608089e921d5SKuriakose Kuruvilla 
608189e921d5SKuriakose Kuruvilla 
608289e921d5SKuriakose Kuruvilla 	return (socketstr);
608389e921d5SKuriakose Kuruvilla }
608489e921d5SKuriakose Kuruvilla 
608522e4c3acSKeith M Wesolowski x86_uarchrev_t
cpuid_getuarchrev(cpu_t * cpu)608622e4c3acSKeith M Wesolowski cpuid_getuarchrev(cpu_t *cpu)
608722e4c3acSKeith M Wesolowski {
608822e4c3acSKeith M Wesolowski 	return (cpu->cpu_m.mcpu_cpi->cpi_uarchrev);
608922e4c3acSKeith M Wesolowski }
609022e4c3acSKeith M Wesolowski 
6091fb2f18f8Sesaxe int
cpuid_get_chipid(cpu_t * cpu)6092fb2f18f8Sesaxe cpuid_get_chipid(cpu_t *cpu)
60937c478bd9Sstevel@tonic-gate {
6094ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
60957c478bd9Sstevel@tonic-gate 
60968949bcd6Sandrei 	if (cpuid_is_cmt(cpu))
60977c478bd9Sstevel@tonic-gate 		return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
60987c478bd9Sstevel@tonic-gate 	return (cpu->cpu_id);
60997c478bd9Sstevel@tonic-gate }
61007c478bd9Sstevel@tonic-gate 
61018949bcd6Sandrei id_t
cpuid_get_coreid(cpu_t * cpu)6102fb2f18f8Sesaxe cpuid_get_coreid(cpu_t *cpu)
61038949bcd6Sandrei {
6104ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61058949bcd6Sandrei 	return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
61068949bcd6Sandrei }
61078949bcd6Sandrei 
610810569901Sgavinm int
cpuid_get_pkgcoreid(cpu_t * cpu)610910569901Sgavinm cpuid_get_pkgcoreid(cpu_t *cpu)
611010569901Sgavinm {
6111ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
611210569901Sgavinm 	return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
611310569901Sgavinm }
611410569901Sgavinm 
61157c478bd9Sstevel@tonic-gate int
cpuid_get_clogid(cpu_t * cpu)6116fb2f18f8Sesaxe cpuid_get_clogid(cpu_t *cpu)
61177c478bd9Sstevel@tonic-gate {
6118ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61197c478bd9Sstevel@tonic-gate 	return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
61207c478bd9Sstevel@tonic-gate }
61217c478bd9Sstevel@tonic-gate 
6122b885580bSAlexander Kolbasov int
cpuid_get_cacheid(cpu_t * cpu)6123b885580bSAlexander Kolbasov cpuid_get_cacheid(cpu_t *cpu)
6124b885580bSAlexander Kolbasov {
6125ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
6126b885580bSAlexander Kolbasov 	return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
6127b885580bSAlexander Kolbasov }
6128b885580bSAlexander Kolbasov 
61298031591dSSrihari Venkatesan uint_t
cpuid_get_procnodeid(cpu_t * cpu)61308031591dSSrihari Venkatesan cpuid_get_procnodeid(cpu_t *cpu)
61318031591dSSrihari Venkatesan {
6132ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61338031591dSSrihari Venkatesan 	return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
61348031591dSSrihari Venkatesan }
61358031591dSSrihari Venkatesan 
61368031591dSSrihari Venkatesan uint_t
cpuid_get_procnodes_per_pkg(cpu_t * cpu)61378031591dSSrihari Venkatesan cpuid_get_procnodes_per_pkg(cpu_t *cpu)
61388031591dSSrihari Venkatesan {
6139ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61408031591dSSrihari Venkatesan 	return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
61418031591dSSrihari Venkatesan }
61428031591dSSrihari Venkatesan 
61437660e73fSHans Rosenfeld uint_t
cpuid_get_compunitid(cpu_t * cpu)61447660e73fSHans Rosenfeld cpuid_get_compunitid(cpu_t *cpu)
61457660e73fSHans Rosenfeld {
6146ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61477660e73fSHans Rosenfeld 	return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
61487660e73fSHans Rosenfeld }
61497660e73fSHans Rosenfeld 
61507660e73fSHans Rosenfeld uint_t
cpuid_get_cores_per_compunit(cpu_t * cpu)61517660e73fSHans Rosenfeld cpuid_get_cores_per_compunit(cpu_t *cpu)
61527660e73fSHans Rosenfeld {
6153ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61547660e73fSHans Rosenfeld 	return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
61557660e73fSHans Rosenfeld }
61567660e73fSHans Rosenfeld 
6157fa96bd91SMichael Corcoran uint32_t
cpuid_get_apicid(cpu_t * cpu)6158fa96bd91SMichael Corcoran cpuid_get_apicid(cpu_t *cpu)
6159fa96bd91SMichael Corcoran {
6160ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
6161fa96bd91SMichael Corcoran 	if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
6162fa96bd91SMichael Corcoran 		return (UINT32_MAX);
6163fa96bd91SMichael Corcoran 	} else {
6164fa96bd91SMichael Corcoran 		return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
6165fa96bd91SMichael Corcoran 	}
6166fa96bd91SMichael Corcoran }
6167fa96bd91SMichael Corcoran 
61687c478bd9Sstevel@tonic-gate void
cpuid_get_addrsize(cpu_t * cpu,uint_t * pabits,uint_t * vabits)61697c478bd9Sstevel@tonic-gate cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
61707c478bd9Sstevel@tonic-gate {
61717c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi;
61727c478bd9Sstevel@tonic-gate 
61737c478bd9Sstevel@tonic-gate 	if (cpu == NULL)
61747c478bd9Sstevel@tonic-gate 		cpu = CPU;
61757c478bd9Sstevel@tonic-gate 	cpi = cpu->cpu_m.mcpu_cpi;
61767c478bd9Sstevel@tonic-gate 
6177ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
61787c478bd9Sstevel@tonic-gate 
61797c478bd9Sstevel@tonic-gate 	if (pabits)
61807c478bd9Sstevel@tonic-gate 		*pabits = cpi->cpi_pabits;
61817c478bd9Sstevel@tonic-gate 	if (vabits)
61827c478bd9Sstevel@tonic-gate 		*vabits = cpi->cpi_vabits;
61837c478bd9Sstevel@tonic-gate }
61847c478bd9Sstevel@tonic-gate 
6185088d69f8SJerry Jelinek size_t
cpuid_get_xsave_size(void)6186ed093b41SRobert Mustacchi cpuid_get_xsave_size(void)
6187088d69f8SJerry Jelinek {
6188088d69f8SJerry Jelinek 	return (MAX(cpuid_info0.cpi_xsave.xsav_max_size,
6189088d69f8SJerry Jelinek 	    sizeof (struct xsave_state)));
6190088d69f8SJerry Jelinek }
6191088d69f8SJerry Jelinek 
6192ed093b41SRobert Mustacchi /*
6193ed093b41SRobert Mustacchi  * Export information about known offsets to the kernel. We only care about
6194ed093b41SRobert Mustacchi  * things we have actually enabled support for in %xcr0.
6195ed093b41SRobert Mustacchi  */
6196ed093b41SRobert Mustacchi void
cpuid_get_xsave_info(uint64_t bit,size_t * sizep,size_t * offp)6197ed093b41SRobert Mustacchi cpuid_get_xsave_info(uint64_t bit, size_t *sizep, size_t *offp)
6198ed093b41SRobert Mustacchi {
6199ed093b41SRobert Mustacchi 	size_t size, off;
6200ed093b41SRobert Mustacchi 
6201ed093b41SRobert Mustacchi 	VERIFY3U(bit & xsave_bv_all, !=, 0);
6202ed093b41SRobert Mustacchi 
6203ed093b41SRobert Mustacchi 	if (sizep == NULL)
6204ed093b41SRobert Mustacchi 		sizep = &size;
6205ed093b41SRobert Mustacchi 	if (offp == NULL)
6206ed093b41SRobert Mustacchi 		offp = &off;
6207ed093b41SRobert Mustacchi 
6208ed093b41SRobert Mustacchi 	switch (bit) {
6209ed093b41SRobert Mustacchi 	case XFEATURE_LEGACY_FP:
6210ed093b41SRobert Mustacchi 	case XFEATURE_SSE:
6211ed093b41SRobert Mustacchi 		*sizep = sizeof (struct fxsave_state);
6212ed093b41SRobert Mustacchi 		*offp = 0;
6213ed093b41SRobert Mustacchi 		break;
6214ed093b41SRobert Mustacchi 	case XFEATURE_AVX:
6215ed093b41SRobert Mustacchi 		*sizep = cpuid_info0.cpi_xsave.ymm_size;
6216ed093b41SRobert Mustacchi 		*offp = cpuid_info0.cpi_xsave.ymm_offset;
6217ed093b41SRobert Mustacchi 		break;
6218ed093b41SRobert Mustacchi 	case XFEATURE_AVX512_OPMASK:
6219ed093b41SRobert Mustacchi 		*sizep = cpuid_info0.cpi_xsave.opmask_size;
6220ed093b41SRobert Mustacchi 		*offp = cpuid_info0.cpi_xsave.opmask_offset;
6221ed093b41SRobert Mustacchi 		break;
6222ed093b41SRobert Mustacchi 	case XFEATURE_AVX512_ZMM:
6223ed093b41SRobert Mustacchi 		*sizep = cpuid_info0.cpi_xsave.zmmlo_size;
6224ed093b41SRobert Mustacchi 		*offp = cpuid_info0.cpi_xsave.zmmlo_offset;
6225ed093b41SRobert Mustacchi 		break;
6226ed093b41SRobert Mustacchi 	case XFEATURE_AVX512_HI_ZMM:
6227ed093b41SRobert Mustacchi 		*sizep = cpuid_info0.cpi_xsave.zmmhi_size;
6228ed093b41SRobert Mustacchi 		*offp = cpuid_info0.cpi_xsave.zmmhi_offset;
6229ed093b41SRobert Mustacchi 		break;
6230ed093b41SRobert Mustacchi 	default:
6231ed093b41SRobert Mustacchi 		panic("asked for unsupported xsave feature: 0x%lx", bit);
6232ed093b41SRobert Mustacchi 	}
6233ed093b41SRobert Mustacchi }
6234ed093b41SRobert Mustacchi 
6235088d69f8SJerry Jelinek /*
6236088d69f8SJerry Jelinek  * Return true if the CPUs on this system require 'pointer clearing' for the
6237088d69f8SJerry Jelinek  * floating point error pointer exception handling. In the past, this has been
6238088d69f8SJerry Jelinek  * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
6239088d69f8SJerry Jelinek  * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
624074ac317bSRobert Mustacchi  * feature bit and is reflected in the cpi_fp_amd_save member.
6241088d69f8SJerry Jelinek  */
6242088d69f8SJerry Jelinek boolean_t
cpuid_need_fp_excp_handling(void)6243ed093b41SRobert Mustacchi cpuid_need_fp_excp_handling(void)
6244088d69f8SJerry Jelinek {
624574ac317bSRobert Mustacchi 	return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD &&
624674ac317bSRobert Mustacchi 	    cpuid_info0.cpi_fp_amd_save != 0);
6247088d69f8SJerry Jelinek }
6248088d69f8SJerry Jelinek 
62497c478bd9Sstevel@tonic-gate /*
62507c478bd9Sstevel@tonic-gate  * Returns the number of data TLB entries for a corresponding
62517c478bd9Sstevel@tonic-gate  * pagesize.  If it can't be computed, or isn't known, the
62527c478bd9Sstevel@tonic-gate  * routine returns zero.  If you ask about an architecturally
62537c478bd9Sstevel@tonic-gate  * impossible pagesize, the routine will panic (so that the
62547c478bd9Sstevel@tonic-gate  * hat implementor knows that things are inconsistent.)
62557c478bd9Sstevel@tonic-gate  */
62567c478bd9Sstevel@tonic-gate uint_t
cpuid_get_dtlb_nent(cpu_t * cpu,size_t pagesize)62577c478bd9Sstevel@tonic-gate cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
62587c478bd9Sstevel@tonic-gate {
62597c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi;
62607c478bd9Sstevel@tonic-gate 	uint_t dtlb_nent = 0;
62617c478bd9Sstevel@tonic-gate 
62627c478bd9Sstevel@tonic-gate 	if (cpu == NULL)
62637c478bd9Sstevel@tonic-gate 		cpu = CPU;
62647c478bd9Sstevel@tonic-gate 	cpi = cpu->cpu_m.mcpu_cpi;
62657c478bd9Sstevel@tonic-gate 
6266ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
62677c478bd9Sstevel@tonic-gate 
62687c478bd9Sstevel@tonic-gate 	/*
62697c478bd9Sstevel@tonic-gate 	 * Check the L2 TLB info
62707c478bd9Sstevel@tonic-gate 	 */
62717c478bd9Sstevel@tonic-gate 	if (cpi->cpi_xmaxeax >= 0x80000006) {
62728949bcd6Sandrei 		struct cpuid_regs *cp = &cpi->cpi_extd[6];
62737c478bd9Sstevel@tonic-gate 
62747c478bd9Sstevel@tonic-gate 		switch (pagesize) {
62757c478bd9Sstevel@tonic-gate 
62767c478bd9Sstevel@tonic-gate 		case 4 * 1024:
62777c478bd9Sstevel@tonic-gate 			/*
62787c478bd9Sstevel@tonic-gate 			 * All zero in the top 16 bits of the register
62797c478bd9Sstevel@tonic-gate 			 * indicates a unified TLB. Size is in low 16 bits.
62807c478bd9Sstevel@tonic-gate 			 */
62817c478bd9Sstevel@tonic-gate 			if ((cp->cp_ebx & 0xffff0000) == 0)
62827c478bd9Sstevel@tonic-gate 				dtlb_nent = cp->cp_ebx & 0x0000ffff;
62837c478bd9Sstevel@tonic-gate 			else
62847c478bd9Sstevel@tonic-gate 				dtlb_nent = BITX(cp->cp_ebx, 27, 16);
62857c478bd9Sstevel@tonic-gate 			break;
62867c478bd9Sstevel@tonic-gate 
62877c478bd9Sstevel@tonic-gate 		case 2 * 1024 * 1024:
62887c478bd9Sstevel@tonic-gate 			if ((cp->cp_eax & 0xffff0000) == 0)
62897c478bd9Sstevel@tonic-gate 				dtlb_nent = cp->cp_eax & 0x0000ffff;
62907c478bd9Sstevel@tonic-gate 			else
62917c478bd9Sstevel@tonic-gate 				dtlb_nent = BITX(cp->cp_eax, 27, 16);
62927c478bd9Sstevel@tonic-gate 			break;
62937c478bd9Sstevel@tonic-gate 
62947c478bd9Sstevel@tonic-gate 		default:
62957c478bd9Sstevel@tonic-gate 			panic("unknown L2 pagesize");
62967c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
62977c478bd9Sstevel@tonic-gate 		}
62987c478bd9Sstevel@tonic-gate 	}
62997c478bd9Sstevel@tonic-gate 
63007c478bd9Sstevel@tonic-gate 	if (dtlb_nent != 0)
63017c478bd9Sstevel@tonic-gate 		return (dtlb_nent);
63027c478bd9Sstevel@tonic-gate 
63037c478bd9Sstevel@tonic-gate 	/*
63047c478bd9Sstevel@tonic-gate 	 * No L2 TLB support for this size, try L1.
63057c478bd9Sstevel@tonic-gate 	 */
63067c478bd9Sstevel@tonic-gate 	if (cpi->cpi_xmaxeax >= 0x80000005) {
63078949bcd6Sandrei 		struct cpuid_regs *cp = &cpi->cpi_extd[5];
63087c478bd9Sstevel@tonic-gate 
63097c478bd9Sstevel@tonic-gate 		switch (pagesize) {
63107c478bd9Sstevel@tonic-gate 		case 4 * 1024:
63117c478bd9Sstevel@tonic-gate 			dtlb_nent = BITX(cp->cp_ebx, 23, 16);
63127c478bd9Sstevel@tonic-gate 			break;
63137c478bd9Sstevel@tonic-gate 		case 2 * 1024 * 1024:
63147c478bd9Sstevel@tonic-gate 			dtlb_nent = BITX(cp->cp_eax, 23, 16);
63157c478bd9Sstevel@tonic-gate 			break;
63167c478bd9Sstevel@tonic-gate 		default:
63177c478bd9Sstevel@tonic-gate 			panic("unknown L1 d-TLB pagesize");
63187c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
63197c478bd9Sstevel@tonic-gate 		}
63207c478bd9Sstevel@tonic-gate 	}
63217c478bd9Sstevel@tonic-gate 
63227c478bd9Sstevel@tonic-gate 	return (dtlb_nent);
63237c478bd9Sstevel@tonic-gate }
63247c478bd9Sstevel@tonic-gate 
63257c478bd9Sstevel@tonic-gate /*
63267c478bd9Sstevel@tonic-gate  * Return 0 if the erratum is not present or not applicable, positive
63277c478bd9Sstevel@tonic-gate  * if it is, and negative if the status of the erratum is unknown.
63287c478bd9Sstevel@tonic-gate  *
63297c478bd9Sstevel@tonic-gate  * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
63302201b277Skucharsk  * Processors" #25759, Rev 3.57, August 2005
63317c478bd9Sstevel@tonic-gate  */
63327c478bd9Sstevel@tonic-gate int
cpuid_opteron_erratum(cpu_t * cpu,uint_t erratum)63337c478bd9Sstevel@tonic-gate cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
63347c478bd9Sstevel@tonic-gate {
63357c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
63368949bcd6Sandrei 	uint_t eax;
63377c478bd9Sstevel@tonic-gate 
6338ea99987eSsethg 	/*
6339ea99987eSsethg 	 * Bail out if this CPU isn't an AMD CPU, or if it's
6340ea99987eSsethg 	 * a legacy (32-bit) AMD CPU.
6341ea99987eSsethg 	 */
6342ea99987eSsethg 	if (cpi->cpi_vendor != X86_VENDOR_AMD ||
6343875b116eSkchow 	    cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
6344bf9b145bSRobert Mustacchi 	    cpi->cpi_family == 6) {
63457c478bd9Sstevel@tonic-gate 		return (0);
6346bf9b145bSRobert Mustacchi 	}
63477c478bd9Sstevel@tonic-gate 
63487c478bd9Sstevel@tonic-gate 	eax = cpi->cpi_std[1].cp_eax;
63497c478bd9Sstevel@tonic-gate 
63507c478bd9Sstevel@tonic-gate #define	SH_B0(eax)	(eax == 0xf40 || eax == 0xf50)
635158b49504SHans Rosenfeld #define	SH_B3(eax)	(eax == 0xf51)
6352ee88d2b9Skchow #define	B(eax)		(SH_B0(eax) || SH_B3(eax))
63537c478bd9Sstevel@tonic-gate 
63547c478bd9Sstevel@tonic-gate #define	SH_C0(eax)	(eax == 0xf48 || eax == 0xf58)
63557c478bd9Sstevel@tonic-gate 
63567c478bd9Sstevel@tonic-gate #define	SH_CG(eax)	(eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
63577c478bd9Sstevel@tonic-gate #define	DH_CG(eax)	(eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
63587c478bd9Sstevel@tonic-gate #define	CH_CG(eax)	(eax == 0xf82 || eax == 0xfb2)
6359ee88d2b9Skchow #define	CG(eax)		(SH_CG(eax) || DH_CG(eax) || CH_CG(eax))
63607c478bd9Sstevel@tonic-gate 
63617c478bd9Sstevel@tonic-gate #define	SH_D0(eax)	(eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
63627c478bd9Sstevel@tonic-gate #define	DH_D0(eax)	(eax == 0x10fc0 || eax == 0x10ff0)
63637c478bd9Sstevel@tonic-gate #define	CH_D0(eax)	(eax == 0x10f80 || eax == 0x10fb0)
6364ee88d2b9Skchow #define	D0(eax)		(SH_D0(eax) || DH_D0(eax) || CH_D0(eax))
63657c478bd9Sstevel@tonic-gate 
63667c478bd9Sstevel@tonic-gate #define	SH_E0(eax)	(eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
63677c478bd9Sstevel@tonic-gate #define	JH_E1(eax)	(eax == 0x20f10)	/* JH8_E0 had 0x20f30 */
63687c478bd9Sstevel@tonic-gate #define	DH_E3(eax)	(eax == 0x20fc0 || eax == 0x20ff0)
63697c478bd9Sstevel@tonic-gate #define	SH_E4(eax)	(eax == 0x20f51 || eax == 0x20f71)
63707c478bd9Sstevel@tonic-gate #define	BH_E4(eax)	(eax == 0x20fb1)
63717c478bd9Sstevel@tonic-gate #define	SH_E5(eax)	(eax == 0x20f42)
63727c478bd9Sstevel@tonic-gate #define	DH_E6(eax)	(eax == 0x20ff2 || eax == 0x20fc2)
63737c478bd9Sstevel@tonic-gate #define	JH_E6(eax)	(eax == 0x20f12 || eax == 0x20f32)
6374ee88d2b9Skchow #define	EX(eax)		(SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
6375ee88d2b9Skchow 			    SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
6376ee88d2b9Skchow 			    DH_E6(eax) || JH_E6(eax))
63777c478bd9Sstevel@tonic-gate 
6378512cf780Skchow #define	DR_AX(eax)	(eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
6379512cf780Skchow #define	DR_B0(eax)	(eax == 0x100f20)
6380512cf780Skchow #define	DR_B1(eax)	(eax == 0x100f21)
6381512cf780Skchow #define	DR_BA(eax)	(eax == 0x100f2a)
6382512cf780Skchow #define	DR_B2(eax)	(eax == 0x100f22)
6383512cf780Skchow #define	DR_B3(eax)	(eax == 0x100f23)
6384512cf780Skchow #define	RB_C0(eax)	(eax == 0x100f40)
6385512cf780Skchow 
63867c478bd9Sstevel@tonic-gate 	switch (erratum) {
63877c478bd9Sstevel@tonic-gate 	case 1:
6388875b116eSkchow 		return (cpi->cpi_family < 0x10);
63897c478bd9Sstevel@tonic-gate 	case 51:	/* what does the asterisk mean? */
63907c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax));
63917c478bd9Sstevel@tonic-gate 	case 52:
63927c478bd9Sstevel@tonic-gate 		return (B(eax));
63937c478bd9Sstevel@tonic-gate 	case 57:
6394512cf780Skchow 		return (cpi->cpi_family <= 0x11);
63957c478bd9Sstevel@tonic-gate 	case 58:
63967c478bd9Sstevel@tonic-gate 		return (B(eax));
63977c478bd9Sstevel@tonic-gate 	case 60:
6398512cf780Skchow 		return (cpi->cpi_family <= 0x11);
63997c478bd9Sstevel@tonic-gate 	case 61:
64007c478bd9Sstevel@tonic-gate 	case 62:
64017c478bd9Sstevel@tonic-gate 	case 63:
64027c478bd9Sstevel@tonic-gate 	case 64:
64037c478bd9Sstevel@tonic-gate 	case 65:
64047c478bd9Sstevel@tonic-gate 	case 66:
64057c478bd9Sstevel@tonic-gate 	case 68:
64067c478bd9Sstevel@tonic-gate 	case 69:
64077c478bd9Sstevel@tonic-gate 	case 70:
64087c478bd9Sstevel@tonic-gate 	case 71:
64097c478bd9Sstevel@tonic-gate 		return (B(eax));
64107c478bd9Sstevel@tonic-gate 	case 72:
64117c478bd9Sstevel@tonic-gate 		return (SH_B0(eax));
64127c478bd9Sstevel@tonic-gate 	case 74:
64137c478bd9Sstevel@tonic-gate 		return (B(eax));
64147c478bd9Sstevel@tonic-gate 	case 75:
6415875b116eSkchow 		return (cpi->cpi_family < 0x10);
64167c478bd9Sstevel@tonic-gate 	case 76:
64177c478bd9Sstevel@tonic-gate 		return (B(eax));
64187c478bd9Sstevel@tonic-gate 	case 77:
6419512cf780Skchow 		return (cpi->cpi_family <= 0x11);
64207c478bd9Sstevel@tonic-gate 	case 78:
64217c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax));
64227c478bd9Sstevel@tonic-gate 	case 79:
64237c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
64247c478bd9Sstevel@tonic-gate 	case 80:
64257c478bd9Sstevel@tonic-gate 	case 81:
64267c478bd9Sstevel@tonic-gate 	case 82:
64277c478bd9Sstevel@tonic-gate 		return (B(eax));
64287c478bd9Sstevel@tonic-gate 	case 83:
64297c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax));
64307c478bd9Sstevel@tonic-gate 	case 85:
6431875b116eSkchow 		return (cpi->cpi_family < 0x10);
64327c478bd9Sstevel@tonic-gate 	case 86:
64337c478bd9Sstevel@tonic-gate 		return (SH_C0(eax) || CG(eax));
64347c478bd9Sstevel@tonic-gate 	case 88:
64357c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax));
64367c478bd9Sstevel@tonic-gate 	case 89:
6437875b116eSkchow 		return (cpi->cpi_family < 0x10);
64387c478bd9Sstevel@tonic-gate 	case 90:
64397c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax));
64407c478bd9Sstevel@tonic-gate 	case 91:
64417c478bd9Sstevel@tonic-gate 	case 92:
64427c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax));
64437c478bd9Sstevel@tonic-gate 	case 93:
64447c478bd9Sstevel@tonic-gate 		return (SH_C0(eax));
64457c478bd9Sstevel@tonic-gate 	case 94:
64467c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax));
64477c478bd9Sstevel@tonic-gate 	case 95:
64487c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax));
64497c478bd9Sstevel@tonic-gate 	case 96:
64507c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax));
64517c478bd9Sstevel@tonic-gate 	case 97:
64527c478bd9Sstevel@tonic-gate 	case 98:
64537c478bd9Sstevel@tonic-gate 		return (SH_C0(eax) || CG(eax));
64547c478bd9Sstevel@tonic-gate 	case 99:
64557c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
64567c478bd9Sstevel@tonic-gate 	case 100:
64577c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax));
64587c478bd9Sstevel@tonic-gate 	case 101:
64597c478bd9Sstevel@tonic-gate 	case 103:
64607c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
64617c478bd9Sstevel@tonic-gate 	case 104:
64627c478bd9Sstevel@tonic-gate 		return (SH_C0(eax) || CG(eax) || D0(eax));
64637c478bd9Sstevel@tonic-gate 	case 105:
64647c478bd9Sstevel@tonic-gate 	case 106:
64657c478bd9Sstevel@tonic-gate 	case 107:
64667c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
64677c478bd9Sstevel@tonic-gate 	case 108:
64687c478bd9Sstevel@tonic-gate 		return (DH_CG(eax));
64697c478bd9Sstevel@tonic-gate 	case 109:
64707c478bd9Sstevel@tonic-gate 		return (SH_C0(eax) || CG(eax) || D0(eax));
64717c478bd9Sstevel@tonic-gate 	case 110:
64727c478bd9Sstevel@tonic-gate 		return (D0(eax) || EX(eax));
64737c478bd9Sstevel@tonic-gate 	case 111:
64747c478bd9Sstevel@tonic-gate 		return (CG(eax));
64757c478bd9Sstevel@tonic-gate 	case 112:
64767c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
64777c478bd9Sstevel@tonic-gate 	case 113:
64787c478bd9Sstevel@tonic-gate 		return (eax == 0x20fc0);
64797c478bd9Sstevel@tonic-gate 	case 114:
64807c478bd9Sstevel@tonic-gate 		return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
64817c478bd9Sstevel@tonic-gate 	case 115:
64827c478bd9Sstevel@tonic-gate 		return (SH_E0(eax) || JH_E1(eax));
64837c478bd9Sstevel@tonic-gate 	case 116:
64847c478bd9Sstevel@tonic-gate 		return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
64857c478bd9Sstevel@tonic-gate 	case 117:
64867c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
64877c478bd9Sstevel@tonic-gate 	case 118:
64887c478bd9Sstevel@tonic-gate 		return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
64897c478bd9Sstevel@tonic-gate 		    JH_E6(eax));
64907c478bd9Sstevel@tonic-gate 	case 121:
64917c478bd9Sstevel@tonic-gate 		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
64927c478bd9Sstevel@tonic-gate 	case 122:
6493512cf780Skchow 		return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
64947c478bd9Sstevel@tonic-gate 	case 123:
64957c478bd9Sstevel@tonic-gate 		return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
64962201b277Skucharsk 	case 131:
6497875b116eSkchow 		return (cpi->cpi_family < 0x10);
6498ef50d8c0Sesaxe 	case 6336786:
6499bf9b145bSRobert Mustacchi 
6500ef50d8c0Sesaxe 		/*
6501ef50d8c0Sesaxe 		 * Test for AdvPowerMgmtInfo.TscPStateInvariant
6502bf9b145bSRobert Mustacchi 		 * if this is a K8 family or newer processor. We're testing for
6503bf9b145bSRobert Mustacchi 		 * this 'erratum' to determine whether or not we have a constant
6504bf9b145bSRobert Mustacchi 		 * TSC.
6505bf9b145bSRobert Mustacchi 		 *
6506bf9b145bSRobert Mustacchi 		 * Our current fix for this is to disable the C1-Clock ramping.
6507bf9b145bSRobert Mustacchi 		 * However, this doesn't work on newer processor families nor
6508bf9b145bSRobert Mustacchi 		 * does it work when virtualized as those devices don't exist.
6509ef50d8c0Sesaxe 		 */
6510bf9b145bSRobert Mustacchi 		if (cpi->cpi_family >= 0x12 || get_hwenv() != HW_NATIVE) {
6511bf9b145bSRobert Mustacchi 			return (0);
6512bf9b145bSRobert Mustacchi 		}
6513bf9b145bSRobert Mustacchi 
6514ef50d8c0Sesaxe 		if (CPI_FAMILY(cpi) == 0xf) {
65158949bcd6Sandrei 			struct cpuid_regs regs;
65168949bcd6Sandrei 			regs.cp_eax = 0x80000007;
65178949bcd6Sandrei 			(void) __cpuid_insn(&regs);
65188949bcd6Sandrei 			return (!(regs.cp_edx & 0x100));
6519ef50d8c0Sesaxe 		}
6520ef50d8c0Sesaxe 		return (0);
6521ee6ee36aSPatrick Mooney 	case 147:
6522bf9b145bSRobert Mustacchi 		/*
6523bf9b145bSRobert Mustacchi 		 * This erratum (K8 #147) is not present on family 10 and newer.
6524bf9b145bSRobert Mustacchi 		 */
6525bf9b145bSRobert Mustacchi 		if (cpi->cpi_family >= 0x10) {
6526bf9b145bSRobert Mustacchi 			return (0);
6527bf9b145bSRobert Mustacchi 		}
6528ee88d2b9Skchow 		return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
6529ee88d2b9Skchow 		    (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);
6530ee88d2b9Skchow 
6531512cf780Skchow 	case 6671130:
6532512cf780Skchow 		/*
6533512cf780Skchow 		 * check for processors (pre-Shanghai) that do not provide
6534512cf780Skchow 		 * optimal management of 1gb ptes in its tlb.
6535512cf780Skchow 		 */
6536512cf780Skchow 		return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);
6537512cf780Skchow 
6538512cf780Skchow 	case 298:
6539512cf780Skchow 		return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
6540512cf780Skchow 		    DR_B2(eax) || RB_C0(eax));
6541512cf780Skchow 
65425e54b56dSHans Rosenfeld 	case 721:
65435e54b56dSHans Rosenfeld 		return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);
65445e54b56dSHans Rosenfeld 
6545512cf780Skchow 	default:
6546512cf780Skchow 		return (-1);
6547512cf780Skchow 
6548512cf780Skchow 	}
6549512cf780Skchow }
6550512cf780Skchow 
6551512cf780Skchow /*
6552512cf780Skchow  * Determine if specified erratum is present via OSVW (OS Visible Workaround).
6553512cf780Skchow  * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
6554512cf780Skchow  */
6555512cf780Skchow int
osvw_opteron_erratum(cpu_t * cpu,uint_t erratum)6556512cf780Skchow osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
6557512cf780Skchow {
6558512cf780Skchow 	struct cpuid_info	*cpi;
6559512cf780Skchow 	uint_t			osvwid;
6560512cf780Skchow 	static int		osvwfeature = -1;
6561512cf780Skchow 	uint64_t		osvwlength;
6562512cf780Skchow 
6563512cf780Skchow 
6564512cf780Skchow 	cpi = cpu->cpu_m.mcpu_cpi;
6565512cf780Skchow 
6566512cf780Skchow 	/* confirm OSVW supported */
6567512cf780Skchow 	if (osvwfeature == -1) {
6568512cf780Skchow 		osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
6569512cf780Skchow 	} else {
6570512cf780Skchow 		/* assert that osvw feature setting is consistent on all cpus */
6571512cf780Skchow 		ASSERT(osvwfeature ==
6572512cf780Skchow 		    (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
6573512cf780Skchow 	}
6574512cf780Skchow 	if (!osvwfeature)
6575512cf780Skchow 		return (-1);
6576512cf780Skchow 
6577512cf780Skchow 	osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;
6578512cf780Skchow 
6579512cf780Skchow 	switch (erratum) {
6580512cf780Skchow 	case 298:	/* osvwid is 0 */
6581512cf780Skchow 		osvwid = 0;
6582512cf780Skchow 		if (osvwlength <= (uint64_t)osvwid) {
6583512cf780Skchow 			/* osvwid 0 is unknown */
6584512cf780Skchow 			return (-1);
6585512cf780Skchow 		}
6586512cf780Skchow 
6587512cf780Skchow 		/*
6588512cf780Skchow 		 * Check the OSVW STATUS MSR to determine the state
6589512cf780Skchow 		 * of the erratum where:
6590512cf780Skchow 		 *   0 - fixed by HW
6591512cf780Skchow 		 *   1 - BIOS has applied the workaround when BIOS
6592512cf780Skchow 		 *   workaround is available. (Or for other errata,
6593512cf780Skchow 		 *   OS workaround is required.)
6594512cf780Skchow 		 * For a value of 1, caller will confirm that the
6595512cf780Skchow 		 * erratum 298 workaround has indeed been applied by BIOS.
6596512cf780Skchow 		 *
6597512cf780Skchow 		 * A 1 may be set in cpus that have a HW fix
6598512cf780Skchow 		 * in a mixed cpu system. Regarding erratum 298:
6599512cf780Skchow 		 *   In a multiprocessor platform, the workaround above
6600512cf780Skchow 		 *   should be applied to all processors regardless of
6601512cf780Skchow 		 *   silicon revision when an affected processor is
6602512cf780Skchow 		 *   present.
6603512cf780Skchow 		 */
6604512cf780Skchow 
6605512cf780Skchow 		return (rdmsr(MSR_AMD_OSVW_STATUS +
6606512cf780Skchow 		    (osvwid / OSVW_ID_CNT_PER_MSR)) &
6607512cf780Skchow 		    (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));
6608512cf780Skchow 
66097c478bd9Sstevel@tonic-gate 	default:
66107c478bd9Sstevel@tonic-gate 		return (-1);
66117c478bd9Sstevel@tonic-gate 	}
66127c478bd9Sstevel@tonic-gate }
66137c478bd9Sstevel@tonic-gate 
66147c478bd9Sstevel@tonic-gate static const char assoc_str[] = "associativity";
66157c478bd9Sstevel@tonic-gate static const char line_str[] = "line-size";
66167c478bd9Sstevel@tonic-gate static const char size_str[] = "size";
66177c478bd9Sstevel@tonic-gate 
66187c478bd9Sstevel@tonic-gate static void
add_cache_prop(dev_info_t * devi,const char * label,const char * type,uint32_t val)66197c478bd9Sstevel@tonic-gate add_cache_prop(dev_info_t *devi, const char *label, const char *type,
66207c478bd9Sstevel@tonic-gate     uint32_t val)
66217c478bd9Sstevel@tonic-gate {
66227c478bd9Sstevel@tonic-gate 	char buf[128];
66237c478bd9Sstevel@tonic-gate 
66247c478bd9Sstevel@tonic-gate 	/*
66257c478bd9Sstevel@tonic-gate 	 * ndi_prop_update_int() is used because it is desirable for
66267c478bd9Sstevel@tonic-gate 	 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
66277c478bd9Sstevel@tonic-gate 	 */
66287c478bd9Sstevel@tonic-gate 	if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
66297c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
66307c478bd9Sstevel@tonic-gate }
66317c478bd9Sstevel@tonic-gate 
66327c478bd9Sstevel@tonic-gate /*
66337c478bd9Sstevel@tonic-gate  * Intel-style cache/tlb description
66347c478bd9Sstevel@tonic-gate  *
66357c478bd9Sstevel@tonic-gate  * Standard cpuid level 2 gives a randomly ordered
66367c478bd9Sstevel@tonic-gate  * selection of tags that index into a table that describes
66377c478bd9Sstevel@tonic-gate  * cache and tlb properties.
66387c478bd9Sstevel@tonic-gate  */
66397c478bd9Sstevel@tonic-gate 
66407c478bd9Sstevel@tonic-gate static const char l1_icache_str[] = "l1-icache";
66417c478bd9Sstevel@tonic-gate static const char l1_dcache_str[] = "l1-dcache";
66427c478bd9Sstevel@tonic-gate static const char l2_cache_str[] = "l2-cache";
6643ae115bc7Smrj static const char l3_cache_str[] = "l3-cache";
66447c478bd9Sstevel@tonic-gate static const char itlb4k_str[] = "itlb-4K";
66457c478bd9Sstevel@tonic-gate static const char dtlb4k_str[] = "dtlb-4K";
6646824e4fecSvd static const char itlb2M_str[] = "itlb-2M";
66477c478bd9Sstevel@tonic-gate static const char itlb4M_str[] = "itlb-4M";
66487c478bd9Sstevel@tonic-gate static const char dtlb4M_str[] = "dtlb-4M";
664925dfb062Sksadhukh static const char dtlb24_str[] = "dtlb0-2M-4M";
66507c478bd9Sstevel@tonic-gate static const char itlb424_str[] = "itlb-4K-2M-4M";
665125dfb062Sksadhukh static const char itlb24_str[] = "itlb-2M-4M";
66527c478bd9Sstevel@tonic-gate static const char dtlb44_str[] = "dtlb-4K-4M";
66537c478bd9Sstevel@tonic-gate static const char sl1_dcache_str[] = "sectored-l1-dcache";
66547c478bd9Sstevel@tonic-gate static const char sl2_cache_str[] = "sectored-l2-cache";
66557c478bd9Sstevel@tonic-gate static const char itrace_str[] = "itrace-cache";
66567c478bd9Sstevel@tonic-gate static const char sl3_cache_str[] = "sectored-l3-cache";
665725dfb062Sksadhukh static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";
66587c478bd9Sstevel@tonic-gate 
66597c478bd9Sstevel@tonic-gate static const struct cachetab {
6660c7749d0fSJohn Levon 	uint8_t		ct_code;
66617c478bd9Sstevel@tonic-gate 	uint8_t		ct_assoc;
666258b49504SHans Rosenfeld 	uint16_t	ct_line_size;
66637c478bd9Sstevel@tonic-gate 	size_t		ct_size;
66647c478bd9Sstevel@tonic-gate 	const char	*ct_label;
66657c478bd9Sstevel@tonic-gate } intel_ctab[] = {
6666824e4fecSvd 	/*
6667824e4fecSvd 	 * maintain descending order!
6668824e4fecSvd 	 *
6669824e4fecSvd 	 * Codes ignored - Reason
6670824e4fecSvd 	 * ----------------------
6671824e4fecSvd 	 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
6672824e4fecSvd 	 * f0H/f1H - Currently we do not interpret prefetch size by design
6673824e4fecSvd 	 */
667425dfb062Sksadhukh 	{ 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
667525dfb062Sksadhukh 	{ 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
667625dfb062Sksadhukh 	{ 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
667725dfb062Sksadhukh 	{ 0xde, 12, 64, 6*1024*1024, l3_cache_str},
667825dfb062Sksadhukh 	{ 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
667925dfb062Sksadhukh 	{ 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
668025dfb062Sksadhukh 	{ 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
668125dfb062Sksadhukh 	{ 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
668225dfb062Sksadhukh 	{ 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
668325dfb062Sksadhukh 	{ 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
668425dfb062Sksadhukh 	{ 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
668525dfb062Sksadhukh 	{ 0xd0, 4, 64, 512*1024, l3_cache_str},
668625dfb062Sksadhukh 	{ 0xca, 4, 0, 512, sh_l2_tlb4k_str},
6687824e4fecSvd 	{ 0xc0, 4, 0, 8, dtlb44_str },
6688824e4fecSvd 	{ 0xba, 4, 0, 64, dtlb4k_str },
6689ae115bc7Smrj 	{ 0xb4, 4, 0, 256, dtlb4k_str },
66907c478bd9Sstevel@tonic-gate 	{ 0xb3, 4, 0, 128, dtlb4k_str },
669125dfb062Sksadhukh 	{ 0xb2, 4, 0, 64, itlb4k_str },
66927c478bd9Sstevel@tonic-gate 	{ 0xb0, 4, 0, 128, itlb4k_str },
66937c478bd9Sstevel@tonic-gate 	{ 0x87, 8, 64, 1024*1024, l2_cache_str},
66947c478bd9Sstevel@tonic-gate 	{ 0x86, 4, 64, 512*1024, l2_cache_str},
66957c478bd9Sstevel@tonic-gate 	{ 0x85, 8, 32, 2*1024*1024, l2_cache_str},
66967c478bd9Sstevel@tonic-gate 	{ 0x84, 8, 32, 1024*1024, l2_cache_str},
66977c478bd9Sstevel@tonic-gate 	{ 0x83, 8, 32, 512*1024, l2_cache_str},
66987c478bd9Sstevel@tonic-gate 	{ 0x82, 8, 32, 256*1024, l2_cache_str},
6699824e4fecSvd 	{ 0x80, 8, 64, 512*1024, l2_cache_str},
67007c478bd9Sstevel@tonic-gate 	{ 0x7f, 2, 64, 512*1024, l2_cache_str},
67017c478bd9Sstevel@tonic-gate 	{ 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
67027c478bd9Sstevel@tonic-gate 	{ 0x7c, 8, 64, 1024*1024, sl2_cache_str},
67037c478bd9Sstevel@tonic-gate 	{ 0x7b, 8, 64, 512*1024, sl2_cache_str},
67047c478bd9Sstevel@tonic-gate 	{ 0x7a, 8, 64, 256*1024, sl2_cache_str},
67057c478bd9Sstevel@tonic-gate 	{ 0x79, 8, 64, 128*1024, sl2_cache_str},
67067c478bd9Sstevel@tonic-gate 	{ 0x78, 8, 64, 1024*1024, l2_cache_str},
6707ae115bc7Smrj 	{ 0x73, 8, 0, 64*1024, itrace_str},
67087c478bd9Sstevel@tonic-gate 	{ 0x72, 8, 0, 32*1024, itrace_str},
67097c478bd9Sstevel@tonic-gate 	{ 0x71, 8, 0, 16*1024, itrace_str},
67107c478bd9Sstevel@tonic-gate 	{ 0x70, 8, 0, 12*1024, itrace_str},
67117c478bd9Sstevel@tonic-gate 	{ 0x68, 4, 64, 32*1024, sl1_dcache_str},
67127c478bd9Sstevel@tonic-gate 	{ 0x67, 4, 64, 16*1024, sl1_dcache_str},
67137c478bd9Sstevel@tonic-gate 	{ 0x66, 4, 64, 8*1024, sl1_dcache_str},
67147c478bd9Sstevel@tonic-gate 	{ 0x60, 8, 64, 16*1024, sl1_dcache_str},
67157c478bd9Sstevel@tonic-gate 	{ 0x5d, 0, 0, 256, dtlb44_str},
67167c478bd9Sstevel@tonic-gate 	{ 0x5c, 0, 0, 128, dtlb44_str},
67177c478bd9Sstevel@tonic-gate 	{ 0x5b, 0, 0, 64, dtlb44_str},
671825dfb062Sksadhukh 	{ 0x5a, 4, 0, 32, dtlb24_str},
6719824e4fecSvd 	{ 0x59, 0, 0, 16, dtlb4k_str},
6720824e4fecSvd 	{ 0x57, 4, 0, 16, dtlb4k_str},
6721824e4fecSvd 	{ 0x56, 4, 0, 16, dtlb4M_str},
672225dfb062Sksadhukh 	{ 0x55, 0, 0, 7, itlb24_str},
67237c478bd9Sstevel@tonic-gate 	{ 0x52, 0, 0, 256, itlb424_str},
67247c478bd9Sstevel@tonic-gate 	{ 0x51, 0, 0, 128, itlb424_str},
67257c478bd9Sstevel@tonic-gate 	{ 0x50, 0, 0, 64, itlb424_str},
6726824e4fecSvd 	{ 0x4f, 0, 0, 32, itlb4k_str},
6727824e4fecSvd 	{ 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
6728ae115bc7Smrj 	{ 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
6729ae115bc7Smrj 	{ 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
6730ae115bc7Smrj 	{ 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
6731ae115bc7Smrj 	{ 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
6732ae115bc7Smrj 	{ 0x49, 16, 64, 4*1024*1024, l3_cache_str},
6733824e4fecSvd 	{ 0x48, 12, 64, 3*1024*1024, l2_cache_str},
6734ae115bc7Smrj 	{ 0x47, 8, 64, 8*1024*1024, l3_cache_str},
6735ae115bc7Smrj 	{ 0x46, 4, 64, 4*1024*1024, l3_cache_str},
67367c478bd9Sstevel@tonic-gate 	{ 0x45, 4, 32, 2*1024*1024, l2_cache_str},
67377c478bd9Sstevel@tonic-gate 	{ 0x44, 4, 32, 1024*1024, l2_cache_str},
67387c478bd9Sstevel@tonic-gate 	{ 0x43, 4, 32, 512*1024, l2_cache_str},
67397c478bd9Sstevel@tonic-gate 	{ 0x42, 4, 32, 256*1024, l2_cache_str},
67407c478bd9Sstevel@tonic-gate 	{ 0x41, 4, 32, 128*1024, l2_cache_str},
6741ae115bc7Smrj 	{ 0x3e, 4, 64, 512*1024, sl2_cache_str},
6742ae115bc7Smrj 	{ 0x3d, 6, 64, 384*1024, sl2_cache_str},
67437c478bd9Sstevel@tonic-gate 	{ 0x3c, 4, 64, 256*1024, sl2_cache_str},
67447c478bd9Sstevel@tonic-gate 	{ 0x3b, 2, 64, 128*1024, sl2_cache_str},
6745ae115bc7Smrj 	{ 0x3a, 6, 64, 192*1024, sl2_cache_str},
67467c478bd9Sstevel@tonic-gate 	{ 0x39, 4, 64, 128*1024, sl2_cache_str},
67477c478bd9Sstevel@tonic-gate 	{ 0x30, 8, 64, 32*1024, l1_icache_str},
67487c478bd9Sstevel@tonic-gate 	{ 0x2c, 8, 64, 32*1024, l1_dcache_str},
67497c478bd9Sstevel@tonic-gate 	{ 0x29, 8, 64, 4096*1024, sl3_cache_str},
67507c478bd9Sstevel@tonic-gate 	{ 0x25, 8, 64, 2048*1024, sl3_cache_str},
67517c478bd9Sstevel@tonic-gate 	{ 0x23, 8, 64, 1024*1024, sl3_cache_str},
67527c478bd9Sstevel@tonic-gate 	{ 0x22, 4, 64, 512*1024, sl3_cache_str},
6753824e4fecSvd 	{ 0x0e, 6, 64, 24*1024, l1_dcache_str},
675425dfb062Sksadhukh 	{ 0x0d, 4, 32, 16*1024, l1_dcache_str},
67557c478bd9Sstevel@tonic-gate 	{ 0x0c, 4, 32, 16*1024, l1_dcache_str},
6756ae115bc7Smrj 	{ 0x0b, 4, 0, 4, itlb4M_str},
67577c478bd9Sstevel@tonic-gate 	{ 0x0a, 2, 32, 8*1024, l1_dcache_str},
67587c478bd9Sstevel@tonic-gate 	{ 0x08, 4, 32, 16*1024, l1_icache_str},
67597c478bd9Sstevel@tonic-gate 	{ 0x06, 4, 32, 8*1024, l1_icache_str},
6760824e4fecSvd 	{ 0x05, 4, 0, 32, dtlb4M_str},
67617c478bd9Sstevel@tonic-gate 	{ 0x04, 4, 0, 8, dtlb4M_str},
67627c478bd9Sstevel@tonic-gate 	{ 0x03, 4, 0, 64, dtlb4k_str},
67637c478bd9Sstevel@tonic-gate 	{ 0x02, 4, 0, 2, itlb4M_str},
67647c478bd9Sstevel@tonic-gate 	{ 0x01, 4, 0, 32, itlb4k_str},
67657c478bd9Sstevel@tonic-gate 	{ 0 }
67667c478bd9Sstevel@tonic-gate };
67677c478bd9Sstevel@tonic-gate 
67687c478bd9Sstevel@tonic-gate static const struct cachetab cyrix_ctab[] = {
67697c478bd9Sstevel@tonic-gate 	{ 0x70, 4, 0, 32, "tlb-4K" },
67707c478bd9Sstevel@tonic-gate 	{ 0x80, 4, 16, 16*1024, "l1-cache" },
67717c478bd9Sstevel@tonic-gate 	{ 0 }
67727c478bd9Sstevel@tonic-gate };
67737c478bd9Sstevel@tonic-gate 
67747c478bd9Sstevel@tonic-gate /*
67757c478bd9Sstevel@tonic-gate  * Search a cache table for a matching entry
67767c478bd9Sstevel@tonic-gate  */
67777c478bd9Sstevel@tonic-gate static const struct cachetab *
find_cacheent(const struct cachetab * ct,uint_t code)67787c478bd9Sstevel@tonic-gate find_cacheent(const struct cachetab *ct, uint_t code)
67797c478bd9Sstevel@tonic-gate {
67807c478bd9Sstevel@tonic-gate 	if (code != 0) {
67817c478bd9Sstevel@tonic-gate 		for (; ct->ct_code != 0; ct++)
67827c478bd9Sstevel@tonic-gate 			if (ct->ct_code <= code)
67837c478bd9Sstevel@tonic-gate 				break;
67847c478bd9Sstevel@tonic-gate 		if (ct->ct_code == code)
67857c478bd9Sstevel@tonic-gate 			return (ct);
67867c478bd9Sstevel@tonic-gate 	}
67877c478bd9Sstevel@tonic-gate 	return (NULL);
67887c478bd9Sstevel@tonic-gate }
67897c478bd9Sstevel@tonic-gate 
67907dee861bSksadhukh /*
67917dee861bSksadhukh  * Populate cachetab entry with L2 or L3 cache-information using
67927dee861bSksadhukh  * cpuid function 4. This function is called from intel_walk_cacheinfo()
67937dee861bSksadhukh  * when descriptor 0x49 is encountered. It returns 0 if no such cache
67947dee861bSksadhukh  * information is found.
67957dee861bSksadhukh  */
67967dee861bSksadhukh static int
intel_cpuid_4_cache_info(struct cachetab * ct,struct cpuid_info * cpi)67977dee861bSksadhukh intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
67987dee861bSksadhukh {
67997dee861bSksadhukh 	uint32_t level, i;
68007dee861bSksadhukh 	int ret = 0;
68017dee861bSksadhukh 
68020ce813ffSRobert Mustacchi 	for (i = 0; i < cpi->cpi_cache_leaf_size; i++) {
68030ce813ffSRobert Mustacchi 		level = CPI_CACHE_LVL(cpi->cpi_cache_leaves[i]);
68047dee861bSksadhukh 
68057dee861bSksadhukh 		if (level == 2 || level == 3) {
68060ce813ffSRobert Mustacchi 			ct->ct_assoc =
68070ce813ffSRobert Mustacchi 			    CPI_CACHE_WAYS(cpi->cpi_cache_leaves[i]) + 1;
68087dee861bSksadhukh 			ct->ct_line_size =
68090ce813ffSRobert Mustacchi 			    CPI_CACHE_COH_LN_SZ(cpi->cpi_cache_leaves[i]) + 1;
68107dee861bSksadhukh 			ct->ct_size = ct->ct_assoc *
68110ce813ffSRobert Mustacchi 			    (CPI_CACHE_PARTS(cpi->cpi_cache_leaves[i]) + 1) *
68127dee861bSksadhukh 			    ct->ct_line_size *
68130ce813ffSRobert Mustacchi 			    (cpi->cpi_cache_leaves[i]->cp_ecx + 1);
68147dee861bSksadhukh 
68157dee861bSksadhukh 			if (level == 2) {
68167dee861bSksadhukh 				ct->ct_label = l2_cache_str;
68177dee861bSksadhukh 			} else if (level == 3) {
68187dee861bSksadhukh 				ct->ct_label = l3_cache_str;
68197dee861bSksadhukh 			}
68207dee861bSksadhukh 			ret = 1;
68217dee861bSksadhukh 		}
68227dee861bSksadhukh 	}
68237dee861bSksadhukh 
68247dee861bSksadhukh 	return (ret);
68257dee861bSksadhukh }
68267dee861bSksadhukh 
68277c478bd9Sstevel@tonic-gate /*
68287c478bd9Sstevel@tonic-gate  * Walk the cacheinfo descriptor, applying 'func' to every valid element
68297c478bd9Sstevel@tonic-gate  * The walk is terminated if the walker returns non-zero.
68307c478bd9Sstevel@tonic-gate  */
68317c478bd9Sstevel@tonic-gate static void
intel_walk_cacheinfo(struct cpuid_info * cpi,void * arg,int (* func)(void *,const struct cachetab *))68327c478bd9Sstevel@tonic-gate intel_walk_cacheinfo(struct cpuid_info *cpi,
68337c478bd9Sstevel@tonic-gate     void *arg, int (*func)(void *, const struct cachetab *))
68347c478bd9Sstevel@tonic-gate {
68357c478bd9Sstevel@tonic-gate 	const struct cachetab *ct;
6836824e4fecSvd 	struct cachetab des_49_ct, des_b1_ct;
68377c478bd9Sstevel@tonic-gate 	uint8_t *dp;
68387c478bd9Sstevel@tonic-gate 	int i;
68397c478bd9Sstevel@tonic-gate 
68407c478bd9Sstevel@tonic-gate 	if ((dp = cpi->cpi_cacheinfo) == NULL)
68417c478bd9Sstevel@tonic-gate 		return;
6842f1d742a9Sksadhukh 	for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
6843f1d742a9Sksadhukh 		/*
6844f1d742a9Sksadhukh 		 * For overloaded descriptor 0x49 we use cpuid function 4
68457dee861bSksadhukh 		 * if supported by the current processor, to create
6846f1d742a9Sksadhukh 		 * cache information.
6847824e4fecSvd 		 * For overloaded descriptor 0xb1 we use X86_PAE flag
6848824e4fecSvd 		 * to disambiguate the cache information.
6849f1d742a9Sksadhukh 		 */
68507dee861bSksadhukh 		if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
68517dee861bSksadhukh 		    intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
68527dee861bSksadhukh 				ct = &des_49_ct;
6853824e4fecSvd 		} else if (*dp == 0xb1) {
6854824e4fecSvd 			des_b1_ct.ct_code = 0xb1;
6855824e4fecSvd 			des_b1_ct.ct_assoc = 4;
6856824e4fecSvd 			des_b1_ct.ct_line_size = 0;
68577417cfdeSKuriakose Kuruvilla 			if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
6858824e4fecSvd 				des_b1_ct.ct_size = 8;
6859824e4fecSvd 				des_b1_ct.ct_label = itlb2M_str;
6860824e4fecSvd 			} else {
6861824e4fecSvd 				des_b1_ct.ct_size = 4;
6862824e4fecSvd 				des_b1_ct.ct_label = itlb4M_str;
6863824e4fecSvd 			}
6864824e4fecSvd 			ct = &des_b1_ct;
68657dee861bSksadhukh 		} else {
68667dee861bSksadhukh 			if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
68677dee861bSksadhukh 				continue;
68687dee861bSksadhukh 			}
6869f1d742a9Sksadhukh 		}
6870f1d742a9Sksadhukh 
68717dee861bSksadhukh 		if (func(arg, ct) != 0) {
68727dee861bSksadhukh 			break;
68737c478bd9Sstevel@tonic-gate 		}
6874f1d742a9Sksadhukh 	}
68757c478bd9Sstevel@tonic-gate }
68767c478bd9Sstevel@tonic-gate 
68777c478bd9Sstevel@tonic-gate /*
68787c478bd9Sstevel@tonic-gate  * (Like the Intel one, except for Cyrix CPUs)
68797c478bd9Sstevel@tonic-gate  */
68807c478bd9Sstevel@tonic-gate static void
cyrix_walk_cacheinfo(struct cpuid_info * cpi,void * arg,int (* func)(void *,const struct cachetab *))68817c478bd9Sstevel@tonic-gate cyrix_walk_cacheinfo(struct cpuid_info *cpi,
68827c478bd9Sstevel@tonic-gate     void *arg, int (*func)(void *, const struct cachetab *))
68837c478bd9Sstevel@tonic-gate {
68847c478bd9Sstevel@tonic-gate 	const struct cachetab *ct;
68857c478bd9Sstevel@tonic-gate 	uint8_t *dp;
68867c478bd9Sstevel@tonic-gate 	int i;
68877c478bd9Sstevel@tonic-gate 
68887c478bd9Sstevel@tonic-gate 	if ((dp = cpi->cpi_cacheinfo) == NULL)
68897c478bd9Sstevel@tonic-gate 		return;
68907c478bd9Sstevel@tonic-gate 	for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
68917c478bd9Sstevel@tonic-gate 		/*
68927c478bd9Sstevel@tonic-gate 		 * Search Cyrix-specific descriptor table first ..
68937c478bd9Sstevel@tonic-gate 		 */
68947c478bd9Sstevel@tonic-gate 		if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
68957c478bd9Sstevel@tonic-gate 			if (func(arg, ct) != 0)
68967c478bd9Sstevel@tonic-gate 				break;
68977c478bd9Sstevel@tonic-gate 			continue;
68987c478bd9Sstevel@tonic-gate 		}
68997c478bd9Sstevel@tonic-gate 		/*
69007c478bd9Sstevel@tonic-gate 		 * .. else fall back to the Intel one
69017c478bd9Sstevel@tonic-gate 		 */
69027c478bd9Sstevel@tonic-gate 		if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
69037c478bd9Sstevel@tonic-gate 			if (func(arg, ct) != 0)
69047c478bd9Sstevel@tonic-gate 				break;
69057c478bd9Sstevel@tonic-gate 			continue;
69067c478bd9Sstevel@tonic-gate 		}
69077c478bd9Sstevel@tonic-gate 	}
69087c478bd9Sstevel@tonic-gate }
69097c478bd9Sstevel@tonic-gate 
69107c478bd9Sstevel@tonic-gate /*
69117c478bd9Sstevel@tonic-gate  * A cacheinfo walker that adds associativity, line-size, and size properties
69127c478bd9Sstevel@tonic-gate  * to the devinfo node it is passed as an argument.
69137c478bd9Sstevel@tonic-gate  */
69147c478bd9Sstevel@tonic-gate static int
add_cacheent_props(void * arg,const struct cachetab * ct)69157c478bd9Sstevel@tonic-gate add_cacheent_props(void *arg, const struct cachetab *ct)
69167c478bd9Sstevel@tonic-gate {
69177c478bd9Sstevel@tonic-gate 	dev_info_t *devi = arg;
69187c478bd9Sstevel@tonic-gate 
69197c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
69207c478bd9Sstevel@tonic-gate 	if (ct->ct_line_size != 0)
69217c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, ct->ct_label, line_str,
69227c478bd9Sstevel@tonic-gate 		    ct->ct_line_size);
69237c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
69247c478bd9Sstevel@tonic-gate 	return (0);
69257c478bd9Sstevel@tonic-gate }
69267c478bd9Sstevel@tonic-gate 
6927f1d742a9Sksadhukh 
69287c478bd9Sstevel@tonic-gate static const char fully_assoc[] = "fully-associative?";
69297c478bd9Sstevel@tonic-gate 
69307c478bd9Sstevel@tonic-gate /*
69317c478bd9Sstevel@tonic-gate  * AMD style cache/tlb description
69327c478bd9Sstevel@tonic-gate  *
69337c478bd9Sstevel@tonic-gate  * Extended functions 5 and 6 directly describe properties of
69347c478bd9Sstevel@tonic-gate  * tlbs and various cache levels.
69357c478bd9Sstevel@tonic-gate  */
69367c478bd9Sstevel@tonic-gate static void
add_amd_assoc(dev_info_t * devi,const char * label,uint_t assoc)69377c478bd9Sstevel@tonic-gate add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
69387c478bd9Sstevel@tonic-gate {
69397c478bd9Sstevel@tonic-gate 	switch (assoc) {
69407c478bd9Sstevel@tonic-gate 	case 0:	/* reserved; ignore */
69417c478bd9Sstevel@tonic-gate 		break;
69427c478bd9Sstevel@tonic-gate 	default:
69437c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, assoc_str, assoc);
69447c478bd9Sstevel@tonic-gate 		break;
69457c478bd9Sstevel@tonic-gate 	case 0xff:
69467c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, fully_assoc, 1);
69477c478bd9Sstevel@tonic-gate 		break;
69487c478bd9Sstevel@tonic-gate 	}
69497c478bd9Sstevel@tonic-gate }
69507c478bd9Sstevel@tonic-gate 
69517c478bd9Sstevel@tonic-gate static void
add_amd_tlb(dev_info_t * devi,const char * label,uint_t assoc,uint_t size)69527c478bd9Sstevel@tonic-gate add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
69537c478bd9Sstevel@tonic-gate {
69547c478bd9Sstevel@tonic-gate 	if (size == 0)
69557c478bd9Sstevel@tonic-gate 		return;
69567c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, label, size_str, size);
69577c478bd9Sstevel@tonic-gate 	add_amd_assoc(devi, label, assoc);
69587c478bd9Sstevel@tonic-gate }
69597c478bd9Sstevel@tonic-gate 
69607c478bd9Sstevel@tonic-gate static void
add_amd_cache(dev_info_t * devi,const char * label,uint_t size,uint_t assoc,uint_t lines_per_tag,uint_t line_size)69617c478bd9Sstevel@tonic-gate add_amd_cache(dev_info_t *devi, const char *label,
69627c478bd9Sstevel@tonic-gate     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
69637c478bd9Sstevel@tonic-gate {
69647c478bd9Sstevel@tonic-gate 	if (size == 0 || line_size == 0)
69657c478bd9Sstevel@tonic-gate 		return;
69667c478bd9Sstevel@tonic-gate 	add_amd_assoc(devi, label, assoc);
69677c478bd9Sstevel@tonic-gate 	/*
69687c478bd9Sstevel@tonic-gate 	 * Most AMD parts have a sectored cache. Multiple cache lines are
69697c478bd9Sstevel@tonic-gate 	 * associated with each tag. A sector consists of all cache lines
69707c478bd9Sstevel@tonic-gate 	 * associated with a tag. For example, the AMD K6-III has a sector
69717c478bd9Sstevel@tonic-gate 	 * size of 2 cache lines per tag.
69727c478bd9Sstevel@tonic-gate 	 */
69737c478bd9Sstevel@tonic-gate 	if (lines_per_tag != 0)
69747c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
69757c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, label, line_str, line_size);
69767c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, label, size_str, size * 1024);
69777c478bd9Sstevel@tonic-gate }
69787c478bd9Sstevel@tonic-gate 
69797c478bd9Sstevel@tonic-gate static void
add_amd_l2_assoc(dev_info_t * devi,const char * label,uint_t assoc)69807c478bd9Sstevel@tonic-gate add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
69817c478bd9Sstevel@tonic-gate {
69827c478bd9Sstevel@tonic-gate 	switch (assoc) {
69837c478bd9Sstevel@tonic-gate 	case 0:	/* off */
69847c478bd9Sstevel@tonic-gate 		break;
69857c478bd9Sstevel@tonic-gate 	case 1:
69867c478bd9Sstevel@tonic-gate 	case 2:
69877c478bd9Sstevel@tonic-gate 	case 4:
69887c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, assoc_str, assoc);
69897c478bd9Sstevel@tonic-gate 		break;
69907c478bd9Sstevel@tonic-gate 	case 6:
69917c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, assoc_str, 8);
69927c478bd9Sstevel@tonic-gate 		break;
69937c478bd9Sstevel@tonic-gate 	case 8:
69947c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, assoc_str, 16);
69957c478bd9Sstevel@tonic-gate 		break;
69967c478bd9Sstevel@tonic-gate 	case 0xf:
69977c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, fully_assoc, 1);
69987c478bd9Sstevel@tonic-gate 		break;
69997c478bd9Sstevel@tonic-gate 	default: /* reserved; ignore */
70007c478bd9Sstevel@tonic-gate 		break;
70017c478bd9Sstevel@tonic-gate 	}
70027c478bd9Sstevel@tonic-gate }
70037c478bd9Sstevel@tonic-gate 
70047c478bd9Sstevel@tonic-gate static void
add_amd_l2_tlb(dev_info_t * devi,const char * label,uint_t assoc,uint_t size)70057c478bd9Sstevel@tonic-gate add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
70067c478bd9Sstevel@tonic-gate {
70077c478bd9Sstevel@tonic-gate 	if (size == 0 || assoc == 0)
70087c478bd9Sstevel@tonic-gate 		return;
70097c478bd9Sstevel@tonic-gate 	add_amd_l2_assoc(devi, label, assoc);
70107c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, label, size_str, size);
70117c478bd9Sstevel@tonic-gate }
70127c478bd9Sstevel@tonic-gate 
70137c478bd9Sstevel@tonic-gate static void
add_amd_l2_cache(dev_info_t * devi,const char * label,uint_t size,uint_t assoc,uint_t lines_per_tag,uint_t line_size)70147c478bd9Sstevel@tonic-gate add_amd_l2_cache(dev_info_t *devi, const char *label,
70157c478bd9Sstevel@tonic-gate     uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
70167c478bd9Sstevel@tonic-gate {
70177c478bd9Sstevel@tonic-gate 	if (size == 0 || assoc == 0 || line_size == 0)
70187c478bd9Sstevel@tonic-gate 		return;
70197c478bd9Sstevel@tonic-gate 	add_amd_l2_assoc(devi, label, assoc);
70207c478bd9Sstevel@tonic-gate 	if (lines_per_tag != 0)
70217c478bd9Sstevel@tonic-gate 		add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
70227c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, label, line_str, line_size);
70237c478bd9Sstevel@tonic-gate 	add_cache_prop(devi, label, size_str, size * 1024);
70247c478bd9Sstevel@tonic-gate }
70257c478bd9Sstevel@tonic-gate 
70267c478bd9Sstevel@tonic-gate static void
amd_cache_info(struct cpuid_info * cpi,dev_info_t * devi)70277c478bd9Sstevel@tonic-gate amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
70287c478bd9Sstevel@tonic-gate {
70298949bcd6Sandrei 	struct cpuid_regs *cp;
70307c478bd9Sstevel@tonic-gate 
70317c478bd9Sstevel@tonic-gate 	if (cpi->cpi_xmaxeax < 0x80000005)
70327c478bd9Sstevel@tonic-gate 		return;
70337c478bd9Sstevel@tonic-gate 	cp = &cpi->cpi_extd[5];
70347c478bd9Sstevel@tonic-gate 
70357c478bd9Sstevel@tonic-gate 	/*
70367c478bd9Sstevel@tonic-gate 	 * 4M/2M L1 TLB configuration
70377c478bd9Sstevel@tonic-gate 	 *
70387c478bd9Sstevel@tonic-gate 	 * We report the size for 2M pages because AMD uses two
70397c478bd9Sstevel@tonic-gate 	 * TLB entries for one 4M page.
70407c478bd9Sstevel@tonic-gate 	 */
70417c478bd9Sstevel@tonic-gate 	add_amd_tlb(devi, "dtlb-2M",
70427c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
70437c478bd9Sstevel@tonic-gate 	add_amd_tlb(devi, "itlb-2M",
70447c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));
70457c478bd9Sstevel@tonic-gate 
70467c478bd9Sstevel@tonic-gate 	/*
70477c478bd9Sstevel@tonic-gate 	 * 4K L1 TLB configuration
70487c478bd9Sstevel@tonic-gate 	 */
70497c478bd9Sstevel@tonic-gate 
70507c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
70517c478bd9Sstevel@tonic-gate 		uint_t nentries;
70527c478bd9Sstevel@tonic-gate 	case X86_VENDOR_TM:
70537c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family >= 5) {
70547c478bd9Sstevel@tonic-gate 			/*
70557c478bd9Sstevel@tonic-gate 			 * Crusoe processors have 256 TLB entries, but
70567c478bd9Sstevel@tonic-gate 			 * cpuid data format constrains them to only
70577c478bd9Sstevel@tonic-gate 			 * reporting 255 of them.
70587c478bd9Sstevel@tonic-gate 			 */
70597c478bd9Sstevel@tonic-gate 			if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
70607c478bd9Sstevel@tonic-gate 				nentries = 256;
70617c478bd9Sstevel@tonic-gate 			/*
70627c478bd9Sstevel@tonic-gate 			 * Crusoe processors also have a unified TLB
70637c478bd9Sstevel@tonic-gate 			 */
70647c478bd9Sstevel@tonic-gate 			add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
70657c478bd9Sstevel@tonic-gate 			    nentries);
70667c478bd9Sstevel@tonic-gate 			break;
70677c478bd9Sstevel@tonic-gate 		}
70687c478bd9Sstevel@tonic-gate 		/*FALLTHROUGH*/
70697c478bd9Sstevel@tonic-gate 	default:
70707c478bd9Sstevel@tonic-gate 		add_amd_tlb(devi, itlb4k_str,
70717c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
70727c478bd9Sstevel@tonic-gate 		add_amd_tlb(devi, dtlb4k_str,
70737c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
70747c478bd9Sstevel@tonic-gate 		break;
70757c478bd9Sstevel@tonic-gate 	}
70767c478bd9Sstevel@tonic-gate 
70777c478bd9Sstevel@tonic-gate 	/*
70787c478bd9Sstevel@tonic-gate 	 * data L1 cache configuration
70797c478bd9Sstevel@tonic-gate 	 */
70807c478bd9Sstevel@tonic-gate 
70817c478bd9Sstevel@tonic-gate 	add_amd_cache(devi, l1_dcache_str,
70827c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
70837c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));
70847c478bd9Sstevel@tonic-gate 
70857c478bd9Sstevel@tonic-gate 	/*
70867c478bd9Sstevel@tonic-gate 	 * code L1 cache configuration
70877c478bd9Sstevel@tonic-gate 	 */
70887c478bd9Sstevel@tonic-gate 
70897c478bd9Sstevel@tonic-gate 	add_amd_cache(devi, l1_icache_str,
70907c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
70917c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));
70927c478bd9Sstevel@tonic-gate 
70937c478bd9Sstevel@tonic-gate 	if (cpi->cpi_xmaxeax < 0x80000006)
70947c478bd9Sstevel@tonic-gate 		return;
70957c478bd9Sstevel@tonic-gate 	cp = &cpi->cpi_extd[6];
70967c478bd9Sstevel@tonic-gate 
70977c478bd9Sstevel@tonic-gate 	/* Check for a unified L2 TLB for large pages */
70987c478bd9Sstevel@tonic-gate 
70997c478bd9Sstevel@tonic-gate 	if (BITX(cp->cp_eax, 31, 16) == 0)
71007c478bd9Sstevel@tonic-gate 		add_amd_l2_tlb(devi, "l2-tlb-2M",
71017c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
71027c478bd9Sstevel@tonic-gate 	else {
71037c478bd9Sstevel@tonic-gate 		add_amd_l2_tlb(devi, "l2-dtlb-2M",
71047c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
71057c478bd9Sstevel@tonic-gate 		add_amd_l2_tlb(devi, "l2-itlb-2M",
71067c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
71077c478bd9Sstevel@tonic-gate 	}
71087c478bd9Sstevel@tonic-gate 
71097c478bd9Sstevel@tonic-gate 	/* Check for a unified L2 TLB for 4K pages */
71107c478bd9Sstevel@tonic-gate 
71117c478bd9Sstevel@tonic-gate 	if (BITX(cp->cp_ebx, 31, 16) == 0) {
71127c478bd9Sstevel@tonic-gate 		add_amd_l2_tlb(devi, "l2-tlb-4K",
71137c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
71147c478bd9Sstevel@tonic-gate 	} else {
71157c478bd9Sstevel@tonic-gate 		add_amd_l2_tlb(devi, "l2-dtlb-4K",
71167c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
71177c478bd9Sstevel@tonic-gate 		add_amd_l2_tlb(devi, "l2-itlb-4K",
71187c478bd9Sstevel@tonic-gate 		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
71197c478bd9Sstevel@tonic-gate 	}
71207c478bd9Sstevel@tonic-gate 
71217c478bd9Sstevel@tonic-gate 	add_amd_l2_cache(devi, l2_cache_str,
71227c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
71237c478bd9Sstevel@tonic-gate 	    BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
71247c478bd9Sstevel@tonic-gate }
71257c478bd9Sstevel@tonic-gate 
71267c478bd9Sstevel@tonic-gate /*
71277c478bd9Sstevel@tonic-gate  * There are two basic ways that the x86 world describes it cache
71287c478bd9Sstevel@tonic-gate  * and tlb architecture - Intel's way and AMD's way.
71297c478bd9Sstevel@tonic-gate  *
71307c478bd9Sstevel@tonic-gate  * Return which flavor of cache architecture we should use
71317c478bd9Sstevel@tonic-gate  */
71327c478bd9Sstevel@tonic-gate static int
x86_which_cacheinfo(struct cpuid_info * cpi)71337c478bd9Sstevel@tonic-gate x86_which_cacheinfo(struct cpuid_info *cpi)
71347c478bd9Sstevel@tonic-gate {
71357c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
71367c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
71377c478bd9Sstevel@tonic-gate 		if (cpi->cpi_maxeax >= 2)
71387c478bd9Sstevel@tonic-gate 			return (X86_VENDOR_Intel);
71397c478bd9Sstevel@tonic-gate 		break;
71407c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
71417c478bd9Sstevel@tonic-gate 		/*
71427c478bd9Sstevel@tonic-gate 		 * The K5 model 1 was the first part from AMD that reported
71437c478bd9Sstevel@tonic-gate 		 * cache sizes via extended cpuid functions.
71447c478bd9Sstevel@tonic-gate 		 */
71457c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family > 5 ||
71467c478bd9Sstevel@tonic-gate 		    (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
71477c478bd9Sstevel@tonic-gate 			return (X86_VENDOR_AMD);
71487c478bd9Sstevel@tonic-gate 		break;
71499b0429a1SPu Wen 	case X86_VENDOR_HYGON:
71509b0429a1SPu Wen 		return (X86_VENDOR_AMD);
71517c478bd9Sstevel@tonic-gate 	case X86_VENDOR_TM:
71527c478bd9Sstevel@tonic-gate 		if (cpi->cpi_family >= 5)
71537c478bd9Sstevel@tonic-gate 			return (X86_VENDOR_AMD);
71547c478bd9Sstevel@tonic-gate 		/*FALLTHROUGH*/
71557c478bd9Sstevel@tonic-gate 	default:
71567c478bd9Sstevel@tonic-gate 		/*
71577c478bd9Sstevel@tonic-gate 		 * If they have extended CPU data for 0x80000005
71587c478bd9Sstevel@tonic-gate 		 * then we assume they have AMD-format cache
71597c478bd9Sstevel@tonic-gate 		 * information.
71607c478bd9Sstevel@tonic-gate 		 *
71617c478bd9Sstevel@tonic-gate 		 * If not, and the vendor happens to be Cyrix,
71627c478bd9Sstevel@tonic-gate 		 * then try our-Cyrix specific handler.
71637c478bd9Sstevel@tonic-gate 		 *
71647c478bd9Sstevel@tonic-gate 		 * If we're not Cyrix, then assume we're using Intel's
71657c478bd9Sstevel@tonic-gate 		 * table-driven format instead.
71667c478bd9Sstevel@tonic-gate 		 */
71677c478bd9Sstevel@tonic-gate 		if (cpi->cpi_xmaxeax >= 0x80000005)
71687c478bd9Sstevel@tonic-gate 			return (X86_VENDOR_AMD);
71697c478bd9Sstevel@tonic-gate 		else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
71707c478bd9Sstevel@tonic-gate 			return (X86_VENDOR_Cyrix);
71717c478bd9Sstevel@tonic-gate 		else if (cpi->cpi_maxeax >= 2)
71727c478bd9Sstevel@tonic-gate 			return (X86_VENDOR_Intel);
71737c478bd9Sstevel@tonic-gate 		break;
71747c478bd9Sstevel@tonic-gate 	}
71757c478bd9Sstevel@tonic-gate 	return (-1);
71767c478bd9Sstevel@tonic-gate }
71777c478bd9Sstevel@tonic-gate 
71787c478bd9Sstevel@tonic-gate void
cpuid_set_cpu_properties(void * dip,processorid_t cpu_id,struct cpuid_info * cpi)7179fa96bd91SMichael Corcoran cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
7180fa96bd91SMichael Corcoran     struct cpuid_info *cpi)
71817c478bd9Sstevel@tonic-gate {
71827c478bd9Sstevel@tonic-gate 	dev_info_t *cpu_devi;
71837c478bd9Sstevel@tonic-gate 	int create;
71847c478bd9Sstevel@tonic-gate 
7185fa96bd91SMichael Corcoran 	cpu_devi = (dev_info_t *)dip;
71867c478bd9Sstevel@tonic-gate 
71877c478bd9Sstevel@tonic-gate 	/* device_type */
71887c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
71897c478bd9Sstevel@tonic-gate 	    "device_type", "cpu");
71907c478bd9Sstevel@tonic-gate 
71917c478bd9Sstevel@tonic-gate 	/* reg */
71927c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
71937c478bd9Sstevel@tonic-gate 	    "reg", cpu_id);
71947c478bd9Sstevel@tonic-gate 
71957c478bd9Sstevel@tonic-gate 	/* cpu-mhz, and clock-frequency */
71967c478bd9Sstevel@tonic-gate 	if (cpu_freq > 0) {
71977c478bd9Sstevel@tonic-gate 		long long mul;
71987c478bd9Sstevel@tonic-gate 
71997c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
72007c478bd9Sstevel@tonic-gate 		    "cpu-mhz", cpu_freq);
72017c478bd9Sstevel@tonic-gate 		if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
72027c478bd9Sstevel@tonic-gate 			(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
72037c478bd9Sstevel@tonic-gate 			    "clock-frequency", (int)mul);
72047c478bd9Sstevel@tonic-gate 	}
72057c478bd9Sstevel@tonic-gate 
7206ab5bb018SKeith M Wesolowski 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
72077c478bd9Sstevel@tonic-gate 
72087c478bd9Sstevel@tonic-gate 	/* vendor-id */
72097c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
7210f98fbcecSbholler 	    "vendor-id", cpi->cpi_vendorstr);
72117c478bd9Sstevel@tonic-gate 
72127c478bd9Sstevel@tonic-gate 	if (cpi->cpi_maxeax == 0) {
72137c478bd9Sstevel@tonic-gate 		return;
72147c478bd9Sstevel@tonic-gate 	}
72157c478bd9Sstevel@tonic-gate 
72167c478bd9Sstevel@tonic-gate 	/*
72177c478bd9Sstevel@tonic-gate 	 * family, model, and step
72187c478bd9Sstevel@tonic-gate 	 */
72197c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7220f98fbcecSbholler 	    "family", CPI_FAMILY(cpi));
72217c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7222f98fbcecSbholler 	    "cpu-model", CPI_MODEL(cpi));
72237c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7224f98fbcecSbholler 	    "stepping-id", CPI_STEP(cpi));
72257c478bd9Sstevel@tonic-gate 
72267c478bd9Sstevel@tonic-gate 	/* type */
72277c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
72287c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
72297c478bd9Sstevel@tonic-gate 		create = 1;
72307c478bd9Sstevel@tonic-gate 		break;
72317c478bd9Sstevel@tonic-gate 	default:
72327c478bd9Sstevel@tonic-gate 		create = 0;
72337c478bd9Sstevel@tonic-gate 		break;
72347c478bd9Sstevel@tonic-gate 	}
72357c478bd9Sstevel@tonic-gate 	if (create)
72367c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7237f98fbcecSbholler 		    "type", CPI_TYPE(cpi));
72387c478bd9Sstevel@tonic-gate 
72397c478bd9Sstevel@tonic-gate 	/* ext-family */
72407c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
72417c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
72427c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
72437c478bd9Sstevel@tonic-gate 		create = cpi->cpi_family >= 0xf;
72447c478bd9Sstevel@tonic-gate 		break;
72459b0429a1SPu Wen 	case X86_VENDOR_HYGON:
72469b0429a1SPu Wen 		create = 1;
72479b0429a1SPu Wen 		break;
72487c478bd9Sstevel@tonic-gate 	default:
72497c478bd9Sstevel@tonic-gate 		create = 0;
72507c478bd9Sstevel@tonic-gate 		break;
72517c478bd9Sstevel@tonic-gate 	}
72527c478bd9Sstevel@tonic-gate 	if (create)
72537c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
72547c478bd9Sstevel@tonic-gate 		    "ext-family", CPI_FAMILY_XTD(cpi));
72557c478bd9Sstevel@tonic-gate 
72567c478bd9Sstevel@tonic-gate 	/* ext-model */
72577c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
72587c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
725963d3f7dfSkk 		create = IS_EXTENDED_MODEL_INTEL(cpi);
726068c91426Sdmick 		break;
72617c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
7262ee88d2b9Skchow 		create = CPI_FAMILY(cpi) == 0xf;
72637c478bd9Sstevel@tonic-gate 		break;
72649b0429a1SPu Wen 	case X86_VENDOR_HYGON:
72659b0429a1SPu Wen 		create = 1;
72669b0429a1SPu Wen 		break;
72677c478bd9Sstevel@tonic-gate 	default:
72687c478bd9Sstevel@tonic-gate 		create = 0;
72697c478bd9Sstevel@tonic-gate 		break;
72707c478bd9Sstevel@tonic-gate 	}
72717c478bd9Sstevel@tonic-gate 	if (create)
72727c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7273f98fbcecSbholler 		    "ext-model", CPI_MODEL_XTD(cpi));
72747c478bd9Sstevel@tonic-gate 
72757c478bd9Sstevel@tonic-gate 	/* generation */
72767c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
72777c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
72789b0429a1SPu Wen 	case X86_VENDOR_HYGON:
72797c478bd9Sstevel@tonic-gate 		/*
72807c478bd9Sstevel@tonic-gate 		 * AMD K5 model 1 was the first part to support this
72817c478bd9Sstevel@tonic-gate 		 */
72827c478bd9Sstevel@tonic-gate 		create = cpi->cpi_xmaxeax >= 0x80000001;
72837c478bd9Sstevel@tonic-gate 		break;
72847c478bd9Sstevel@tonic-gate 	default:
72857c478bd9Sstevel@tonic-gate 		create = 0;
72867c478bd9Sstevel@tonic-gate 		break;
72877c478bd9Sstevel@tonic-gate 	}
72887c478bd9Sstevel@tonic-gate 	if (create)
72897c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
72907c478bd9Sstevel@tonic-gate 		    "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));
72917c478bd9Sstevel@tonic-gate 
72927c478bd9Sstevel@tonic-gate 	/* brand-id */
72937c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
72947c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
72957c478bd9Sstevel@tonic-gate 		/*
72967c478bd9Sstevel@tonic-gate 		 * brand id first appeared on Pentium III Xeon model 8,
72977c478bd9Sstevel@tonic-gate 		 * and Celeron model 8 processors and Opteron
72987c478bd9Sstevel@tonic-gate 		 */
72997c478bd9Sstevel@tonic-gate 		create = cpi->cpi_family > 6 ||
73007c478bd9Sstevel@tonic-gate 		    (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
73017c478bd9Sstevel@tonic-gate 		break;
73027c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
73037c478bd9Sstevel@tonic-gate 		create = cpi->cpi_family >= 0xf;
73047c478bd9Sstevel@tonic-gate 		break;
73059b0429a1SPu Wen 	case X86_VENDOR_HYGON:
73069b0429a1SPu Wen 		create = 1;
73079b0429a1SPu Wen 		break;
73087c478bd9Sstevel@tonic-gate 	default:
73097c478bd9Sstevel@tonic-gate 		create = 0;
73107c478bd9Sstevel@tonic-gate 		break;
73117c478bd9Sstevel@tonic-gate 	}
73127c478bd9Sstevel@tonic-gate 	if (create && cpi->cpi_brandid != 0) {
73137c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
73147c478bd9Sstevel@tonic-gate 		    "brand-id", cpi->cpi_brandid);
73157c478bd9Sstevel@tonic-gate 	}
73167c478bd9Sstevel@tonic-gate 
73177c478bd9Sstevel@tonic-gate 	/* chunks, and apic-id */
73187c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
73197c478bd9Sstevel@tonic-gate 		/*
73207c478bd9Sstevel@tonic-gate 		 * first available on Pentium IV and Opteron (K8)
73217c478bd9Sstevel@tonic-gate 		 */
73225ff02082Sdmick 	case X86_VENDOR_Intel:
73235ff02082Sdmick 		create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
73245ff02082Sdmick 		break;
73255ff02082Sdmick 	case X86_VENDOR_AMD:
73267c478bd9Sstevel@tonic-gate 		create = cpi->cpi_family >= 0xf;
73277c478bd9Sstevel@tonic-gate 		break;
73289b0429a1SPu Wen 	case X86_VENDOR_HYGON:
73299b0429a1SPu Wen 		create = 1;
73309b0429a1SPu Wen 		break;
73317c478bd9Sstevel@tonic-gate 	default:
73327c478bd9Sstevel@tonic-gate 		create = 0;
73337c478bd9Sstevel@tonic-gate 		break;
73347c478bd9Sstevel@tonic-gate 	}
73357c478bd9Sstevel@tonic-gate 	if (create) {
73367c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7337f98fbcecSbholler 		    "chunks", CPI_CHUNKS(cpi));
73387c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7339b6917abeSmishra 		    "apic-id", cpi->cpi_apicid);
73407aec1d6eScindi 		if (cpi->cpi_chipid >= 0) {
73417c478bd9Sstevel@tonic-gate 			(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
73427c478bd9Sstevel@tonic-gate 			    "chip#", cpi->cpi_chipid);
73437aec1d6eScindi 			(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
73447aec1d6eScindi 			    "clog#", cpi->cpi_clogid);
73457aec1d6eScindi 		}
73467c478bd9Sstevel@tonic-gate 	}
73477c478bd9Sstevel@tonic-gate 
73487c478bd9Sstevel@tonic-gate 	/* cpuid-features */
73497c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
73507c478bd9Sstevel@tonic-gate 	    "cpuid-features", CPI_FEATURES_EDX(cpi));
73517c478bd9Sstevel@tonic-gate 
73527c478bd9Sstevel@tonic-gate 
73537c478bd9Sstevel@tonic-gate 	/* cpuid-features-ecx */
73547c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
73557c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
73565ff02082Sdmick 		create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
73577c478bd9Sstevel@tonic-gate 		break;
735863408480SHans Rosenfeld 	case X86_VENDOR_AMD:
735963408480SHans Rosenfeld 		create = cpi->cpi_family >= 0xf;
736063408480SHans Rosenfeld 		break;
73619b0429a1SPu Wen 	case X86_VENDOR_HYGON:
73629b0429a1SPu Wen 		create = 1;
73639b0429a1SPu Wen 		break;
73647c478bd9Sstevel@tonic-gate 	default:
73657c478bd9Sstevel@tonic-gate 		create = 0;
73667c478bd9Sstevel@tonic-gate 		break;
73677c478bd9Sstevel@tonic-gate 	}
73687c478bd9Sstevel@tonic-gate 	if (create)
73697c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
73707c478bd9Sstevel@tonic-gate 		    "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));
73717c478bd9Sstevel@tonic-gate 
73727c478bd9Sstevel@tonic-gate 	/* ext-cpuid-features */
73737c478bd9Sstevel@tonic-gate 	switch (cpi->cpi_vendor) {
73745ff02082Sdmick 	case X86_VENDOR_Intel:
73757c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
73769b0429a1SPu Wen 	case X86_VENDOR_HYGON:
73777c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Cyrix:
73787c478bd9Sstevel@tonic-gate 	case X86_VENDOR_TM:
73797c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Centaur:
73807c478bd9Sstevel@tonic-gate 		create = cpi->cpi_xmaxeax >= 0x80000001;
73817c478bd9Sstevel@tonic-gate 		break;
73827c478bd9Sstevel@tonic-gate 	default:
73837c478bd9Sstevel@tonic-gate 		create = 0;
73847c478bd9Sstevel@tonic-gate 		break;
73857c478bd9Sstevel@tonic-gate 	}
73865ff02082Sdmick 	if (create) {
73877c478bd9Sstevel@tonic-gate 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7388f98fbcecSbholler 		    "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
73895ff02082Sdmick 		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
7390f98fbcecSbholler 		    "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
73915ff02082Sdmick 	}
73927c478bd9Sstevel@tonic-gate 
73937c478bd9Sstevel@tonic-gate 	/*
73947c478bd9Sstevel@tonic-gate 	 * Brand String first appeared in Intel Pentium IV, AMD K5
73957c478bd9Sstevel@tonic-gate 	 * model 1, and Cyrix GXm.  On earlier models we try and
73967c478bd9Sstevel@tonic-gate 	 * simulate something similar .. so this string should always
73977c478bd9Sstevel@tonic-gate 	 * same -something- about the processor, however lame.
73987c478bd9Sstevel@tonic-gate 	 */
73997c478bd9Sstevel@tonic-gate 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
74007c478bd9Sstevel@tonic-gate 	    "brand-string", cpi->cpi_brandstr);
74017c478bd9Sstevel@tonic-gate 
74027c478bd9Sstevel@tonic-gate 	/*
74037c478bd9Sstevel@tonic-gate 	 * Finally, cache and tlb information
74047c478bd9Sstevel@tonic-gate 	 */
74057c478bd9Sstevel@tonic-gate 	switch (x86_which_cacheinfo(cpi)) {
74067c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
74077c478bd9Sstevel@tonic-gate 		intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
74087c478bd9Sstevel@tonic-gate 		break;
74097c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Cyrix:
74107c478bd9Sstevel@tonic-gate 		cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
74117c478bd9Sstevel@tonic-gate 		break;
74127c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
74137c478bd9Sstevel@tonic-gate 		amd_cache_info(cpi, cpu_devi);
74147c478bd9Sstevel@tonic-gate 		break;
74157c478bd9Sstevel@tonic-gate 	default:
74167c478bd9Sstevel@tonic-gate 		break;
74177c478bd9Sstevel@tonic-gate 	}
74187c478bd9Sstevel@tonic-gate }
74197c478bd9Sstevel@tonic-gate 
74207c478bd9Sstevel@tonic-gate struct l2info {
74217c478bd9Sstevel@tonic-gate 	int *l2i_csz;
74227c478bd9Sstevel@tonic-gate 	int *l2i_lsz;
74237c478bd9Sstevel@tonic-gate 	int *l2i_assoc;
74247c478bd9Sstevel@tonic-gate 	int l2i_ret;
74257c478bd9Sstevel@tonic-gate };
74267c478bd9Sstevel@tonic-gate 
74277c478bd9Sstevel@tonic-gate /*
74287c478bd9Sstevel@tonic-gate  * A cacheinfo walker that fetches the size, line-size and associativity
74297c478bd9Sstevel@tonic-gate  * of the L2 cache
74307c478bd9Sstevel@tonic-gate  */
74317c478bd9Sstevel@tonic-gate static int
intel_l2cinfo(void * arg,const struct cachetab * ct)74327c478bd9Sstevel@tonic-gate intel_l2cinfo(void *arg, const struct cachetab *ct)
74337c478bd9Sstevel@tonic-gate {
74347c478bd9Sstevel@tonic-gate 	struct l2info *l2i = arg;
74357c478bd9Sstevel@tonic-gate 	int *ip;
74367c478bd9Sstevel@tonic-gate 
74377c478bd9Sstevel@tonic-gate 	if (ct->ct_label != l2_cache_str &&
74387c478bd9Sstevel@tonic-gate 	    ct->ct_label != sl2_cache_str)
74397c478bd9Sstevel@tonic-gate 		return (0);	/* not an L2 -- keep walking */
74407c478bd9Sstevel@tonic-gate 
74417c478bd9Sstevel@tonic-gate 	if ((ip = l2i->l2i_csz) != NULL)
74427c478bd9Sstevel@tonic-gate 		*ip = ct->ct_size;
74437c478bd9Sstevel@tonic-gate 	if ((ip = l2i->l2i_lsz) != NULL)
74447c478bd9Sstevel@tonic-gate 		*ip = ct->ct_line_size;
74457c478bd9Sstevel@tonic-gate 	if ((ip = l2i->l2i_assoc) != NULL)
74467c478bd9Sstevel@tonic-gate 		*ip = ct->ct_assoc;
74477c478bd9Sstevel@tonic-gate 	l2i->l2i_ret = ct->ct_size;
74487c478bd9Sstevel@tonic-gate 	return (1);		/* was an L2 -- terminate walk */
74497c478bd9Sstevel@tonic-gate }
74507c478bd9Sstevel@tonic-gate 
7451606303c9Skchow /*
7452606303c9Skchow  * AMD L2/L3 Cache and TLB Associativity Field Definition:
7453606303c9Skchow  *
7454606303c9Skchow  *	Unlike the associativity for the L1 cache and tlb where the 8 bit
7455606303c9Skchow  *	value is the associativity, the associativity for the L2 cache and
7456606303c9Skchow  *	tlb is encoded in the following table. The 4 bit L2 value serves as
7457606303c9Skchow  *	an index into the amd_afd[] array to determine the associativity.
7458606303c9Skchow  *	-1 is undefined. 0 is fully associative.
7459606303c9Skchow  */
7460606303c9Skchow 
7461606303c9Skchow static int amd_afd[] =
7462606303c9Skchow 	{-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};
7463606303c9Skchow 
74647c478bd9Sstevel@tonic-gate static void
amd_l2cacheinfo(struct cpuid_info * cpi,struct l2info * l2i)74657c478bd9Sstevel@tonic-gate amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
74667c478bd9Sstevel@tonic-gate {
74678949bcd6Sandrei 	struct cpuid_regs *cp;
74687c478bd9Sstevel@tonic-gate 	uint_t size, assoc;
7469606303c9Skchow 	int i;
74707c478bd9Sstevel@tonic-gate 	int *ip;
74717c478bd9Sstevel@tonic-gate 
74727c478bd9Sstevel@tonic-gate 	if (cpi->cpi_xmaxeax < 0x80000006)
74737c478bd9Sstevel@tonic-gate 		return;
74747c478bd9Sstevel@tonic-gate 	cp = &cpi->cpi_extd[6];
74757c478bd9Sstevel@tonic-gate 
7476606303c9Skchow 	if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
74777c478bd9Sstevel@tonic-gate 	    (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
74787c478bd9Sstevel@tonic-gate 		uint_t cachesz = size * 1024;
7479606303c9Skchow 		assoc = amd_afd[i];
74807c478bd9Sstevel@tonic-gate 
7481606303c9Skchow 		ASSERT(assoc != -1);
74827c478bd9Sstevel@tonic-gate 
74837c478bd9Sstevel@tonic-gate 		if ((ip = l2i->l2i_csz) != NULL)
74847c478bd9Sstevel@tonic-gate 			*ip = cachesz;
74857c478bd9Sstevel@tonic-gate 		if ((ip = l2i->l2i_lsz) != NULL)
74867c478bd9Sstevel@tonic-gate 			*ip = BITX(cp->cp_ecx, 7, 0);
74877c478bd9Sstevel@tonic-gate 		if ((ip = l2i->l2i_assoc) != NULL)
74887c478bd9Sstevel@tonic-gate 			*ip = assoc;
74897c478bd9Sstevel@tonic-gate 		l2i->l2i_ret = cachesz;
74907c478bd9Sstevel@tonic-gate 	}
74917c478bd9Sstevel@tonic-gate }
74927c478bd9Sstevel@tonic-gate 
74937c478bd9Sstevel@tonic-gate int
getl2cacheinfo(cpu_t * cpu,int * csz,int * lsz,int * assoc)74947c478bd9Sstevel@tonic-gate getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
74957c478bd9Sstevel@tonic-gate {
74967c478bd9Sstevel@tonic-gate 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
74977c478bd9Sstevel@tonic-gate 	struct l2info __l2info, *l2i = &__l2info;
74987c478bd9Sstevel@tonic-gate 
74997c478bd9Sstevel@tonic-gate 	l2i->l2i_csz = csz;
75007c478bd9Sstevel@tonic-gate 	l2i->l2i_lsz = lsz;
75017c478bd9Sstevel@tonic-gate 	l2i->l2i_assoc = assoc;
75027c478bd9Sstevel@tonic-gate 	l2i->l2i_ret = -1;
75037c478bd9Sstevel@tonic-gate 
75047c478bd9Sstevel@tonic-gate 	switch (x86_which_cacheinfo(cpi)) {
75057c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Intel:
75067c478bd9Sstevel@tonic-gate 		intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
75077c478bd9Sstevel@tonic-gate 		break;
75087c478bd9Sstevel@tonic-gate 	case X86_VENDOR_Cyrix:
75097c478bd9Sstevel@tonic-gate 		cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
75107c478bd9Sstevel@tonic-gate 		break;
75117c478bd9Sstevel@tonic-gate 	case X86_VENDOR_AMD:
75127c478bd9Sstevel@tonic-gate 		amd_l2cacheinfo(cpi, l2i);
75137c478bd9Sstevel@tonic-gate 		break;
75147c478bd9Sstevel@tonic-gate 	default:
75157c478bd9Sstevel@tonic-gate 		break;
75167c478bd9Sstevel@tonic-gate 	}
75177c478bd9Sstevel@tonic-gate 	return (l2i->l2i_ret);
75187c478bd9Sstevel@tonic-gate }
7519f98fbcecSbholler 
7520843e1988Sjohnlev #if !defined(__xpv)
7521843e1988Sjohnlev 
75225b8a6efeSbholler uint32_t *
cpuid_mwait_alloc(cpu_t * cpu)75235b8a6efeSbholler cpuid_mwait_alloc(cpu_t *cpu)
7524f98fbcecSbholler {
75255b8a6efeSbholler 	uint32_t	*ret;
75265b8a6efeSbholler 	size_t		mwait_size;
75275b8a6efeSbholler 
7528ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_EXTENDED));
75295b8a6efeSbholler 
7530a3114836SGerry Liu 	mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
75315b8a6efeSbholler 	if (mwait_size == 0)
75325b8a6efeSbholler 		return (NULL);
75335b8a6efeSbholler 
75345b8a6efeSbholler 	/*
75355b8a6efeSbholler 	 * kmem_alloc() returns cache line size aligned data for mwait_size
75365b8a6efeSbholler 	 * allocations.  mwait_size is currently cache line sized.  Neither
75375b8a6efeSbholler 	 * of these implementation details are guarantied to be true in the
75385b8a6efeSbholler 	 * future.
75395b8a6efeSbholler 	 *
75405b8a6efeSbholler 	 * First try allocating mwait_size as kmem_alloc() currently returns
75415b8a6efeSbholler 	 * correctly aligned memory.  If kmem_alloc() does not return
75425b8a6efeSbholler 	 * mwait_size aligned memory, then use mwait_size ROUNDUP.
75435b8a6efeSbholler 	 *
75445b8a6efeSbholler 	 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
75455b8a6efeSbholler 	 * decide to free this memory.
75465b8a6efeSbholler 	 */
75475b8a6efeSbholler 	ret = kmem_zalloc(mwait_size, KM_SLEEP);
75485b8a6efeSbholler 	if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
75495b8a6efeSbholler 		cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
75505b8a6efeSbholler 		cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
75515b8a6efeSbholler 		*ret = MWAIT_RUNNING;
75525b8a6efeSbholler 		return (ret);
75535b8a6efeSbholler 	} else {
75545b8a6efeSbholler 		kmem_free(ret, mwait_size);
75555b8a6efeSbholler 		ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
75565b8a6efeSbholler 		cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
75575b8a6efeSbholler 		cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
75585b8a6efeSbholler 		ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
75595b8a6efeSbholler 		*ret = MWAIT_RUNNING;
75605b8a6efeSbholler 		return (ret);
75615b8a6efeSbholler 	}
75625b8a6efeSbholler }
75635b8a6efeSbholler 
75645b8a6efeSbholler void
cpuid_mwait_free(cpu_t * cpu)75655b8a6efeSbholler cpuid_mwait_free(cpu_t *cpu)
75665b8a6efeSbholler {
7567a3114836SGerry Liu 	if (cpu->cpu_m.mcpu_cpi == NULL) {
7568a3114836SGerry Liu 		return;
7569a3114836SGerry Liu 	}
75705b8a6efeSbholler 
75715b8a6efeSbholler 	if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
75725b8a6efeSbholler 	    cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
75735b8a6efeSbholler 		kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
75745b8a6efeSbholler 		    cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
75755b8a6efeSbholler 	}
75765b8a6efeSbholler 
75775b8a6efeSbholler 	cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
75785b8a6efeSbholler 	cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
7579f98fbcecSbholler }
7580843e1988Sjohnlev 
7581247dbb3dSsudheer void
patch_tsc_read(int flag)7582247dbb3dSsudheer patch_tsc_read(int flag)
7583247dbb3dSsudheer {
7584247dbb3dSsudheer 	size_t cnt;
7585e4b86885SCheng Sean Ye 
7586247dbb3dSsudheer 	switch (flag) {
75872428aad8SPatrick Mooney 	case TSC_NONE:
7588247dbb3dSsudheer 		cnt = &_no_rdtsc_end - &_no_rdtsc_start;
75892b0bcb26Ssudheer 		(void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
7590247dbb3dSsudheer 		break;
75912428aad8SPatrick Mooney 	case TSC_RDTSC_LFENCE:
759215363b27Ssudheer 		cnt = &_tsc_lfence_end - &_tsc_lfence_start;
759315363b27Ssudheer 		(void) memcpy((void *)tsc_read,
759415363b27Ssudheer 		    (void *)&_tsc_lfence_start, cnt);
759515363b27Ssudheer 		break;
75962428aad8SPatrick Mooney 	case TSC_TSCP:
75972428aad8SPatrick Mooney 		cnt = &_tscp_end - &_tscp_start;
75982428aad8SPatrick Mooney 		(void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
75992428aad8SPatrick Mooney 		break;
7600247dbb3dSsudheer 	default:
76012428aad8SPatrick Mooney 		/* Bail for unexpected TSC types. (TSC_NONE covers 0) */
76022428aad8SPatrick Mooney 		cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
7603247dbb3dSsudheer 		break;
7604247dbb3dSsudheer 	}
76052428aad8SPatrick Mooney 	tsc_type = flag;
7606247dbb3dSsudheer }
7607247dbb3dSsudheer 
76080e751525SEric Saxe int
cpuid_deep_cstates_supported(void)76090e751525SEric Saxe cpuid_deep_cstates_supported(void)
76100e751525SEric Saxe {
76110e751525SEric Saxe 	struct cpuid_info *cpi;
76120e751525SEric Saxe 	struct cpuid_regs regs;
76130e751525SEric Saxe 
7614ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
7615ab5bb018SKeith M Wesolowski 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
76160e751525SEric Saxe 
76170e751525SEric Saxe 	cpi = CPU->cpu_m.mcpu_cpi;
76180e751525SEric Saxe 
76190e751525SEric Saxe 	switch (cpi->cpi_vendor) {
76200e751525SEric Saxe 	case X86_VENDOR_Intel:
76210e751525SEric Saxe 		if (cpi->cpi_xmaxeax < 0x80000007)
76220e751525SEric Saxe 			return (0);
76230e751525SEric Saxe 
76240e751525SEric Saxe 		/*
7625ab5bb018SKeith M Wesolowski 		 * Does TSC run at a constant rate in all C-states?
76260e751525SEric Saxe 		 */
76270e751525SEric Saxe 		regs.cp_eax = 0x80000007;
76280e751525SEric Saxe 		(void) __cpuid_insn(&regs);
76290e751525SEric Saxe 		return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);
76300e751525SEric Saxe 
76310e751525SEric Saxe 	default:
76320e751525SEric Saxe 		return (0);
76330e751525SEric Saxe 	}
76340e751525SEric Saxe }
76350e751525SEric Saxe 
7636e774b42bSBill Holler #endif	/* !__xpv */
7637e774b42bSBill Holler 
7638e774b42bSBill Holler void
post_startup_cpu_fixups(void)7639e774b42bSBill Holler post_startup_cpu_fixups(void)
7640e774b42bSBill Holler {
7641e774b42bSBill Holler #ifndef __xpv
7642e774b42bSBill Holler 	/*
7643e774b42bSBill Holler 	 * Some AMD processors support C1E state. Entering this state will
7644e774b42bSBill Holler 	 * cause the local APIC timer to stop, which we can't deal with at
7645e774b42bSBill Holler 	 * this time.
7646e774b42bSBill Holler 	 */
7647e774b42bSBill Holler 	if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
7648e774b42bSBill Holler 		on_trap_data_t otd;
7649e774b42bSBill Holler 		uint64_t reg;
7650e774b42bSBill Holler 
7651e774b42bSBill Holler 		if (!on_trap(&otd, OT_DATA_ACCESS)) {
7652e774b42bSBill Holler 			reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
7653e774b42bSBill Holler 			/* Disable C1E state if it is enabled by BIOS */
7654e774b42bSBill Holler 			if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
7655e774b42bSBill Holler 			    AMD_ACTONCMPHALT_MASK) {
7656e774b42bSBill Holler 				reg &= ~(AMD_ACTONCMPHALT_MASK <<
7657e774b42bSBill Holler 				    AMD_ACTONCMPHALT_SHIFT);
7658e774b42bSBill Holler 				wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
7659e774b42bSBill Holler 			}
7660e774b42bSBill Holler 		}
7661e774b42bSBill Holler 		no_trap();
7662e774b42bSBill Holler 	}
7663e774b42bSBill Holler #endif	/* !__xpv */
7664e774b42bSBill Holler }
7665e774b42bSBill Holler 
766674ecdb51SJohn Levon void
enable_pcid(void)766774ecdb51SJohn Levon enable_pcid(void)
766874ecdb51SJohn Levon {
766974ecdb51SJohn Levon 	if (x86_use_pcid == -1)
767074ecdb51SJohn Levon 		x86_use_pcid = is_x86_feature(x86_featureset, X86FSET_PCID);
767174ecdb51SJohn Levon 
767274ecdb51SJohn Levon 	if (x86_use_invpcid == -1) {
767374ecdb51SJohn Levon 		x86_use_invpcid = is_x86_feature(x86_featureset,
767474ecdb51SJohn Levon 		    X86FSET_INVPCID);
767574ecdb51SJohn Levon 	}
767674ecdb51SJohn Levon 
767774ecdb51SJohn Levon 	if (!x86_use_pcid)
767874ecdb51SJohn Levon 		return;
767974ecdb51SJohn Levon 
768074ecdb51SJohn Levon 	/*
768174ecdb51SJohn Levon 	 * Intel say that on setting PCIDE, it immediately starts using the PCID
768274ecdb51SJohn Levon 	 * bits; better make sure there's nothing there.
768374ecdb51SJohn Levon 	 */
768474ecdb51SJohn Levon 	ASSERT((getcr3() & MMU_PAGEOFFSET) == PCID_NONE);
768574ecdb51SJohn Levon 
768674ecdb51SJohn Levon 	setcr4(getcr4() | CR4_PCIDE);
768774ecdb51SJohn Levon }
768874ecdb51SJohn Levon 
76897af88ac7SKuriakose Kuruvilla /*
76907af88ac7SKuriakose Kuruvilla  * Setup necessary registers to enable XSAVE feature on this processor.
76917af88ac7SKuriakose Kuruvilla  * This function needs to be called early enough, so that no xsave/xrstor
76927af88ac7SKuriakose Kuruvilla  * ops will execute on the processor before the MSRs are properly set up.
76937af88ac7SKuriakose Kuruvilla  *
76947af88ac7SKuriakose Kuruvilla  * Current implementation has the following assumption:
7695ab5bb018SKeith M Wesolowski  * - cpuid_pass_basic() is done, so that X86 features are known.
76967af88ac7SKuriakose Kuruvilla  * - fpu_probe() is done, so that fp_save_mech is chosen.
76977af88ac7SKuriakose Kuruvilla  */
76987af88ac7SKuriakose Kuruvilla void
xsave_setup_msr(cpu_t * cpu)76997af88ac7SKuriakose Kuruvilla xsave_setup_msr(cpu_t *cpu)
77007af88ac7SKuriakose Kuruvilla {
7701ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
77027af88ac7SKuriakose Kuruvilla 	ASSERT(fp_save_mech == FP_XSAVE);
77037af88ac7SKuriakose Kuruvilla 	ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
77047af88ac7SKuriakose Kuruvilla 
77057af88ac7SKuriakose Kuruvilla 	/* Enable OSXSAVE in CR4. */
77067af88ac7SKuriakose Kuruvilla 	setcr4(getcr4() | CR4_OSXSAVE);
77077af88ac7SKuriakose Kuruvilla 	/*
77087af88ac7SKuriakose Kuruvilla 	 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
77097af88ac7SKuriakose Kuruvilla 	 * correct value.
77107af88ac7SKuriakose Kuruvilla 	 */
77117af88ac7SKuriakose Kuruvilla 	cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
77127af88ac7SKuriakose Kuruvilla 	setup_xfem();
77137af88ac7SKuriakose Kuruvilla }
77147af88ac7SKuriakose Kuruvilla 
7715cef70d2cSBill Holler /*
7716cef70d2cSBill Holler  * Starting with the Westmere processor the local
7717cef70d2cSBill Holler  * APIC timer will continue running in all C-states,
7718cef70d2cSBill Holler  * including the deepest C-states.
7719cef70d2cSBill Holler  */
7720cef70d2cSBill Holler int
cpuid_arat_supported(void)7721cef70d2cSBill Holler cpuid_arat_supported(void)
7722cef70d2cSBill Holler {
7723cef70d2cSBill Holler 	struct cpuid_info *cpi;
7724cef70d2cSBill Holler 	struct cpuid_regs regs;
7725cef70d2cSBill Holler 
7726ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
77277417cfdeSKuriakose Kuruvilla 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
7728cef70d2cSBill Holler 
7729cef70d2cSBill Holler 	cpi = CPU->cpu_m.mcpu_cpi;
7730cef70d2cSBill Holler 
7731cef70d2cSBill Holler 	switch (cpi->cpi_vendor) {
7732cef70d2cSBill Holler 	case X86_VENDOR_Intel:
7733cef70d2cSBill Holler 		/*
7734cef70d2cSBill Holler 		 * Always-running Local APIC Timer is
7735cef70d2cSBill Holler 		 * indicated by CPUID.6.EAX[2].
7736cef70d2cSBill Holler 		 */
7737cef70d2cSBill Holler 		if (cpi->cpi_maxeax >= 6) {
7738cef70d2cSBill Holler 			regs.cp_eax = 6;
7739cef70d2cSBill Holler 			(void) cpuid_insn(NULL, &regs);
7740f2dbfd32SRobert Mustacchi 			return (regs.cp_eax & CPUID_INTC_EAX_ARAT);
7741cef70d2cSBill Holler 		} else {
7742cef70d2cSBill Holler 			return (0);
7743cef70d2cSBill Holler 		}
7744cef70d2cSBill Holler 	default:
7745cef70d2cSBill Holler 		return (0);
7746cef70d2cSBill Holler 	}
7747cef70d2cSBill Holler }
7748cef70d2cSBill Holler 
7749f21ed392Saubrey.li@intel.com /*
7750f21ed392Saubrey.li@intel.com  * Check support for Intel ENERGY_PERF_BIAS feature
7751f21ed392Saubrey.li@intel.com  */
7752f21ed392Saubrey.li@intel.com int
cpuid_iepb_supported(struct cpu * cp)7753f21ed392Saubrey.li@intel.com cpuid_iepb_supported(struct cpu *cp)
7754f21ed392Saubrey.li@intel.com {
7755f21ed392Saubrey.li@intel.com 	struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
7756f21ed392Saubrey.li@intel.com 	struct cpuid_regs regs;
7757f21ed392Saubrey.li@intel.com 
7758ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cp, CPUID_PASS_BASIC));
7759ab5bb018SKeith M Wesolowski 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
7760f21ed392Saubrey.li@intel.com 
7761ab5bb018SKeith M Wesolowski 	if (!(is_x86_feature(x86_featureset, X86FSET_MSR))) {
7762f21ed392Saubrey.li@intel.com 		return (0);
7763f21ed392Saubrey.li@intel.com 	}
7764f21ed392Saubrey.li@intel.com 
7765f21ed392Saubrey.li@intel.com 	/*
7766f21ed392Saubrey.li@intel.com 	 * Intel ENERGY_PERF_BIAS MSR is indicated by
7767f21ed392Saubrey.li@intel.com 	 * capability bit CPUID.6.ECX.3
7768f21ed392Saubrey.li@intel.com 	 */
7769f21ed392Saubrey.li@intel.com 	if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
7770f21ed392Saubrey.li@intel.com 		return (0);
7771f21ed392Saubrey.li@intel.com 
7772f21ed392Saubrey.li@intel.com 	regs.cp_eax = 0x6;
7773f21ed392Saubrey.li@intel.com 	(void) cpuid_insn(NULL, &regs);
7774f2dbfd32SRobert Mustacchi 	return (regs.cp_ecx & CPUID_INTC_ECX_PERFBIAS);
7775f21ed392Saubrey.li@intel.com }
7776f21ed392Saubrey.li@intel.com 
777741afdfa7SKrishnendu Sadhukhan - Sun Microsystems /*
777841afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * Check support for TSC deadline timer
777941afdfa7SKrishnendu Sadhukhan - Sun Microsystems  *
778041afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * TSC deadline timer provides a superior software programming
778141afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * model over local APIC timer that eliminates "time drifts".
778241afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * Instead of specifying a relative time, software specifies an
778341afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * absolute time as the target at which the processor should
778441afdfa7SKrishnendu Sadhukhan - Sun Microsystems  * generate a timer event.
778541afdfa7SKrishnendu Sadhukhan - Sun Microsystems  */
778641afdfa7SKrishnendu Sadhukhan - Sun Microsystems int
cpuid_deadline_tsc_supported(void)778741afdfa7SKrishnendu Sadhukhan - Sun Microsystems cpuid_deadline_tsc_supported(void)
778841afdfa7SKrishnendu Sadhukhan - Sun Microsystems {
778941afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
779041afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	struct cpuid_regs regs;
779141afdfa7SKrishnendu Sadhukhan - Sun Microsystems 
7792ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
779341afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));
779441afdfa7SKrishnendu Sadhukhan - Sun Microsystems 
779541afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	switch (cpi->cpi_vendor) {
779641afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	case X86_VENDOR_Intel:
779741afdfa7SKrishnendu Sadhukhan - Sun Microsystems 		if (cpi->cpi_maxeax >= 1) {
779841afdfa7SKrishnendu Sadhukhan - Sun Microsystems 			regs.cp_eax = 1;
779941afdfa7SKrishnendu Sadhukhan - Sun Microsystems 			(void) cpuid_insn(NULL, &regs);
780041afdfa7SKrishnendu Sadhukhan - Sun Microsystems 			return (regs.cp_ecx & CPUID_DEADLINE_TSC);
780141afdfa7SKrishnendu Sadhukhan - Sun Microsystems 		} else {
780241afdfa7SKrishnendu Sadhukhan - Sun Microsystems 			return (0);
780341afdfa7SKrishnendu Sadhukhan - Sun Microsystems 		}
780441afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	default:
780541afdfa7SKrishnendu Sadhukhan - Sun Microsystems 		return (0);
780641afdfa7SKrishnendu Sadhukhan - Sun Microsystems 	}
780741afdfa7SKrishnendu Sadhukhan - Sun Microsystems }
780841afdfa7SKrishnendu Sadhukhan - Sun Microsystems 
780986ef0a63SRichard Lowe #if !defined(__xpv)
781022cc0e45SBill Holler /*
781122cc0e45SBill Holler  * Patch in versions of bcopy for high performance Intel Nhm processors
781222cc0e45SBill Holler  * and later...
781322cc0e45SBill Holler  */
781422cc0e45SBill Holler void
patch_memops(uint_t vendor)781522cc0e45SBill Holler patch_memops(uint_t vendor)
781622cc0e45SBill Holler {
781722cc0e45SBill Holler 	size_t cnt, i;
781822cc0e45SBill Holler 	caddr_t to, from;
781922cc0e45SBill Holler 
78207417cfdeSKuriakose Kuruvilla 	if ((vendor == X86_VENDOR_Intel) &&
78217417cfdeSKuriakose Kuruvilla 	    is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
782222cc0e45SBill Holler 		cnt = &bcopy_patch_end - &bcopy_patch_start;
782322cc0e45SBill Holler 		to = &bcopy_ck_size;
782422cc0e45SBill Holler 		from = &bcopy_patch_start;
782522cc0e45SBill Holler 		for (i = 0; i < cnt; i++) {
782622cc0e45SBill Holler 			*to++ = *from++;
782722cc0e45SBill Holler 		}
782822cc0e45SBill Holler 	}
782922cc0e45SBill Holler }
783086ef0a63SRichard Lowe #endif  /*  !__xpv */
78312d2efdc6SVuong Nguyen 
78322d2efdc6SVuong Nguyen /*
78330ce813ffSRobert Mustacchi  * We're being asked to tell the system how many bits are required to represent
7834d6517bbdSRobert Mustacchi  * the various thread and strand IDs. While it's tempting to derive this based
7835d6517bbdSRobert Mustacchi  * on the values in cpi_ncore_per_chip and cpi_ncpu_per_chip, that isn't quite
7836d6517bbdSRobert Mustacchi  * correct. Instead, this needs to be based on the number of bits that the APIC
7837d6517bbdSRobert Mustacchi  * allows for these different configurations. We only update these to a larger
7838d6517bbdSRobert Mustacchi  * value if we find one.
78392d2efdc6SVuong Nguyen  */
78402d2efdc6SVuong Nguyen void
cpuid_get_ext_topo(cpu_t * cpu,uint_t * core_nbits,uint_t * strand_nbits)78410ce813ffSRobert Mustacchi cpuid_get_ext_topo(cpu_t *cpu, uint_t *core_nbits, uint_t *strand_nbits)
78422d2efdc6SVuong Nguyen {
78430ce813ffSRobert Mustacchi 	struct cpuid_info *cpi;
78442d2efdc6SVuong Nguyen 
7845ab5bb018SKeith M Wesolowski 	VERIFY(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
78460ce813ffSRobert Mustacchi 	cpi = cpu->cpu_m.mcpu_cpi;
78472d2efdc6SVuong Nguyen 
7848d6517bbdSRobert Mustacchi 	if (cpi->cpi_ncore_bits > *core_nbits) {
7849d6517bbdSRobert Mustacchi 		*core_nbits = cpi->cpi_ncore_bits;
7850d6517bbdSRobert Mustacchi 	}
7851d6517bbdSRobert Mustacchi 
7852d6517bbdSRobert Mustacchi 	if (cpi->cpi_nthread_bits > *strand_nbits) {
7853d6517bbdSRobert Mustacchi 		*strand_nbits = cpi->cpi_nthread_bits;
7854d6517bbdSRobert Mustacchi 	}
78552d2efdc6SVuong Nguyen }
785601add34aSRobert Mustacchi 
785701add34aSRobert Mustacchi void
cpuid_pass_ucode(cpu_t * cpu,uchar_t * fset)785801add34aSRobert Mustacchi cpuid_pass_ucode(cpu_t *cpu, uchar_t *fset)
785901add34aSRobert Mustacchi {
786001add34aSRobert Mustacchi 	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
786101add34aSRobert Mustacchi 	struct cpuid_regs cp;
786201add34aSRobert Mustacchi 
786301add34aSRobert Mustacchi 	/*
786401add34aSRobert Mustacchi 	 * Reread the CPUID portions that we need for various security
786501add34aSRobert Mustacchi 	 * information.
786601add34aSRobert Mustacchi 	 */
786701add34aSRobert Mustacchi 	if (cpi->cpi_vendor == X86_VENDOR_Intel) {
786801add34aSRobert Mustacchi 		/*
786901add34aSRobert Mustacchi 		 * Check if we now have leaf 7 available to us.
787001add34aSRobert Mustacchi 		 */
787101add34aSRobert Mustacchi 		if (cpi->cpi_maxeax < 7) {
787201add34aSRobert Mustacchi 			bzero(&cp, sizeof (cp));
787301add34aSRobert Mustacchi 			cp.cp_eax = 0;
787401add34aSRobert Mustacchi 			cpi->cpi_maxeax = __cpuid_insn(&cp);
787501add34aSRobert Mustacchi 			if (cpi->cpi_maxeax < 7)
787601add34aSRobert Mustacchi 				return;
787701add34aSRobert Mustacchi 		}
787801add34aSRobert Mustacchi 
787901add34aSRobert Mustacchi 		bzero(&cp, sizeof (cp));
788001add34aSRobert Mustacchi 		cp.cp_eax = 7;
788101add34aSRobert Mustacchi 		cp.cp_ecx = 0;
788201add34aSRobert Mustacchi 		(void) __cpuid_insn(&cp);
788301add34aSRobert Mustacchi 		cpi->cpi_std[7] = cp;
78849b0429a1SPu Wen 	} else if (cpi->cpi_vendor == X86_VENDOR_AMD ||
78859b0429a1SPu Wen 	    cpi->cpi_vendor == X86_VENDOR_HYGON) {
788601add34aSRobert Mustacchi 		/* No xcpuid support */
788701add34aSRobert Mustacchi 		if (cpi->cpi_family < 5 ||
788801add34aSRobert Mustacchi 		    (cpi->cpi_family == 5 && cpi->cpi_model < 1))
788901add34aSRobert Mustacchi 			return;
789001add34aSRobert Mustacchi 
78910ce813ffSRobert Mustacchi 		if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_8) {
789201add34aSRobert Mustacchi 			bzero(&cp, sizeof (cp));
78930ce813ffSRobert Mustacchi 			cp.cp_eax = CPUID_LEAF_EXT_0;
789401add34aSRobert Mustacchi 			cpi->cpi_xmaxeax = __cpuid_insn(&cp);
78950ce813ffSRobert Mustacchi 			if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_8) {
789601add34aSRobert Mustacchi 				return;
789701add34aSRobert Mustacchi 			}
789801add34aSRobert Mustacchi 		}
789901add34aSRobert Mustacchi 
7900651a12cbSRobert Mustacchi 		/*
7901651a12cbSRobert Mustacchi 		 * Most AMD features are in leaf 8. Automatic IBRS was added in
7902651a12cbSRobert Mustacchi 		 * leaf 0x21. So we also check that.
7903651a12cbSRobert Mustacchi 		 */
790401add34aSRobert Mustacchi 		bzero(&cp, sizeof (cp));
79050ce813ffSRobert Mustacchi 		cp.cp_eax = CPUID_LEAF_EXT_8;
790601add34aSRobert Mustacchi 		(void) __cpuid_insn(&cp);
79070ce813ffSRobert Mustacchi 		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_8, &cp);
790801add34aSRobert Mustacchi 		cpi->cpi_extd[8] = cp;
7909651a12cbSRobert Mustacchi 
7910651a12cbSRobert Mustacchi 		if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_21) {
7911651a12cbSRobert Mustacchi 			return;
7912651a12cbSRobert Mustacchi 		}
7913651a12cbSRobert Mustacchi 
7914651a12cbSRobert Mustacchi 		bzero(&cp, sizeof (cp));
7915651a12cbSRobert Mustacchi 		cp.cp_eax = CPUID_LEAF_EXT_21;
7916651a12cbSRobert Mustacchi 		(void) __cpuid_insn(&cp);
7917651a12cbSRobert Mustacchi 		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_21, &cp);
7918651a12cbSRobert Mustacchi 		cpi->cpi_extd[0x21] = cp;
791901add34aSRobert Mustacchi 	} else {
792001add34aSRobert Mustacchi 		/*
792101add34aSRobert Mustacchi 		 * Nothing to do here. Return an empty set which has already
792201add34aSRobert Mustacchi 		 * been zeroed for us.
792301add34aSRobert Mustacchi 		 */
792401add34aSRobert Mustacchi 		return;
792501add34aSRobert Mustacchi 	}
792601add34aSRobert Mustacchi 	cpuid_scan_security(cpu, fset);
792701add34aSRobert Mustacchi }
792801add34aSRobert Mustacchi 
792901add34aSRobert Mustacchi /* ARGSUSED */
793001add34aSRobert Mustacchi static int
cpuid_post_ucodeadm_xc(xc_arg_t arg0,xc_arg_t arg1,xc_arg_t arg2)793101add34aSRobert Mustacchi cpuid_post_ucodeadm_xc(xc_arg_t arg0, xc_arg_t arg1, xc_arg_t arg2)
793201add34aSRobert Mustacchi {
793301add34aSRobert Mustacchi 	uchar_t *fset;
793465f20420SRobert Mustacchi 	boolean_t first_pass = (boolean_t)arg1;
793501add34aSRobert Mustacchi 
793601add34aSRobert Mustacchi 	fset = (uchar_t *)(arg0 + sizeof (x86_featureset) * CPU->cpu_id);
793765f20420SRobert Mustacchi 	if (first_pass && CPU->cpu_id != 0)
793865f20420SRobert Mustacchi 		return (0);
793965f20420SRobert Mustacchi 	if (!first_pass && CPU->cpu_id == 0)
794065f20420SRobert Mustacchi 		return (0);
794101add34aSRobert Mustacchi 	cpuid_pass_ucode(CPU, fset);
794201add34aSRobert Mustacchi 
794301add34aSRobert Mustacchi 	return (0);
794401add34aSRobert Mustacchi }
794501add34aSRobert Mustacchi 
794601add34aSRobert Mustacchi /*
794701add34aSRobert Mustacchi  * After a microcode update where the version has changed, then we need to
794801add34aSRobert Mustacchi  * rescan CPUID. To do this we check every CPU to make sure that they have the
794901add34aSRobert Mustacchi  * same microcode. Then we perform a cross call to all such CPUs. It's the
795001add34aSRobert Mustacchi  * caller's job to make sure that no one else can end up doing an update while
795101add34aSRobert Mustacchi  * this is going on.
795201add34aSRobert Mustacchi  *
795301add34aSRobert Mustacchi  * We assume that the system is microcode capable if we're called.
795401add34aSRobert Mustacchi  */
795501add34aSRobert Mustacchi void
cpuid_post_ucodeadm(void)795601add34aSRobert Mustacchi cpuid_post_ucodeadm(void)
795701add34aSRobert Mustacchi {
795801add34aSRobert Mustacchi 	uint32_t rev;
795901add34aSRobert Mustacchi 	int i;
796001add34aSRobert Mustacchi 	struct cpu *cpu;
796101add34aSRobert Mustacchi 	cpuset_t cpuset;
796201add34aSRobert Mustacchi 	void *argdata;
796301add34aSRobert Mustacchi 	uchar_t *f0;
796401add34aSRobert Mustacchi 
796501add34aSRobert Mustacchi 	argdata = kmem_zalloc(sizeof (x86_featureset) * NCPU, KM_SLEEP);
796601add34aSRobert Mustacchi 
796701add34aSRobert Mustacchi 	mutex_enter(&cpu_lock);
796801add34aSRobert Mustacchi 	cpu = cpu_get(0);
796901add34aSRobert Mustacchi 	rev = cpu->cpu_m.mcpu_ucode_info->cui_rev;
797001add34aSRobert Mustacchi 	CPUSET_ONLY(cpuset, 0);
797101add34aSRobert Mustacchi 	for (i = 1; i < max_ncpus; i++) {
797201add34aSRobert Mustacchi 		if ((cpu = cpu_get(i)) == NULL)
797301add34aSRobert Mustacchi 			continue;
797401add34aSRobert Mustacchi 
797501add34aSRobert Mustacchi 		if (cpu->cpu_m.mcpu_ucode_info->cui_rev != rev) {
797601add34aSRobert Mustacchi 			panic("post microcode update CPU %d has differing "
797701add34aSRobert Mustacchi 			    "microcode revision (%u) from CPU 0 (%u)",
797801add34aSRobert Mustacchi 			    i, cpu->cpu_m.mcpu_ucode_info->cui_rev, rev);
797901add34aSRobert Mustacchi 		}
798001add34aSRobert Mustacchi 		CPUSET_ADD(cpuset, i);
798101add34aSRobert Mustacchi 	}
798201add34aSRobert Mustacchi 
798365f20420SRobert Mustacchi 	/*
798465f20420SRobert Mustacchi 	 * We do the cross calls in two passes. The first pass is only for the
798565f20420SRobert Mustacchi 	 * boot CPU. The second pass is for all of the other CPUs. This allows
798665f20420SRobert Mustacchi 	 * the boot CPU to go through and change behavior related to patching or
798765f20420SRobert Mustacchi 	 * whether or not Enhanced IBRS needs to be enabled and then allow all
798865f20420SRobert Mustacchi 	 * other CPUs to follow suit.
798965f20420SRobert Mustacchi 	 */
799001add34aSRobert Mustacchi 	kpreempt_disable();
799165f20420SRobert Mustacchi 	xc_sync((xc_arg_t)argdata, B_TRUE, 0, CPUSET2BV(cpuset),
799265f20420SRobert Mustacchi 	    cpuid_post_ucodeadm_xc);
799365f20420SRobert Mustacchi 	xc_sync((xc_arg_t)argdata, B_FALSE, 0, CPUSET2BV(cpuset),
799401add34aSRobert Mustacchi 	    cpuid_post_ucodeadm_xc);
799501add34aSRobert Mustacchi 	kpreempt_enable();
799601add34aSRobert Mustacchi 
799701add34aSRobert Mustacchi 	/*
799801add34aSRobert Mustacchi 	 * OK, now look at each CPU and see if their feature sets are equal.
799901add34aSRobert Mustacchi 	 */
800001add34aSRobert Mustacchi 	f0 = argdata;
800101add34aSRobert Mustacchi 	for (i = 1; i < max_ncpus; i++) {
800201add34aSRobert Mustacchi 		uchar_t *fset;
800301add34aSRobert Mustacchi 		if (!CPU_IN_SET(cpuset, i))
800401add34aSRobert Mustacchi 			continue;
800501add34aSRobert Mustacchi 
800601add34aSRobert Mustacchi 		fset = (uchar_t *)((uintptr_t)argdata +
800701add34aSRobert Mustacchi 		    sizeof (x86_featureset) * i);
800801add34aSRobert Mustacchi 
800901add34aSRobert Mustacchi 		if (!compare_x86_featureset(f0, fset)) {
801001add34aSRobert Mustacchi 			panic("Post microcode update CPU %d has "
801101add34aSRobert Mustacchi 			    "differing security feature (%p) set from CPU 0 "
801201add34aSRobert Mustacchi 			    "(%p), not appending to feature set", i,
801301add34aSRobert Mustacchi 			    (void *)fset, (void *)f0);
801401add34aSRobert Mustacchi 		}
801501add34aSRobert Mustacchi 	}
801601add34aSRobert Mustacchi 
801701add34aSRobert Mustacchi 	mutex_exit(&cpu_lock);
801801add34aSRobert Mustacchi 
801901add34aSRobert Mustacchi 	for (i = 0; i < NUM_X86_FEATURES; i++) {
802001add34aSRobert Mustacchi 		cmn_err(CE_CONT, "?post-ucode x86_feature: %s\n",
802101add34aSRobert Mustacchi 		    x86_feature_names[i]);
802201add34aSRobert Mustacchi 		if (is_x86_feature(f0, i)) {
802301add34aSRobert Mustacchi 			add_x86_feature(x86_featureset, i);
802401add34aSRobert Mustacchi 		}
802501add34aSRobert Mustacchi 	}
802601add34aSRobert Mustacchi 	kmem_free(argdata, sizeof (x86_featureset) * NCPU);
802701add34aSRobert Mustacchi }
8028ab5bb018SKeith M Wesolowski 
8029ab5bb018SKeith M Wesolowski typedef void (*cpuid_pass_f)(cpu_t *, void *);
8030ab5bb018SKeith M Wesolowski 
8031ab5bb018SKeith M Wesolowski typedef struct cpuid_pass_def {
8032ab5bb018SKeith M Wesolowski 	cpuid_pass_t cpd_pass;
8033ab5bb018SKeith M Wesolowski 	cpuid_pass_f cpd_func;
8034ab5bb018SKeith M Wesolowski } cpuid_pass_def_t;
8035ab5bb018SKeith M Wesolowski 
8036ab5bb018SKeith M Wesolowski /*
8037ab5bb018SKeith M Wesolowski  * See block comment at the top; note that cpuid_pass_ucode is not a pass in the
8038ab5bb018SKeith M Wesolowski  * normal sense and should not appear here.
8039ab5bb018SKeith M Wesolowski  */
8040ab5bb018SKeith M Wesolowski static const cpuid_pass_def_t cpuid_pass_defs[] = {
8041ab5bb018SKeith M Wesolowski 	{ CPUID_PASS_PRELUDE, cpuid_pass_prelude },
8042ab5bb018SKeith M Wesolowski 	{ CPUID_PASS_IDENT, cpuid_pass_ident },
8043ab5bb018SKeith M Wesolowski 	{ CPUID_PASS_BASIC, cpuid_pass_basic },
8044ab5bb018SKeith M Wesolowski 	{ CPUID_PASS_EXTENDED, cpuid_pass_extended },
8045ab5bb018SKeith M Wesolowski 	{ CPUID_PASS_DYNAMIC, cpuid_pass_dynamic },
8046ab5bb018SKeith M Wesolowski 	{ CPUID_PASS_RESOLVE, cpuid_pass_resolve },
8047ab5bb018SKeith M Wesolowski };
8048ab5bb018SKeith M Wesolowski 
8049ab5bb018SKeith M Wesolowski void
cpuid_execpass(cpu_t * cp,cpuid_pass_t pass,void * arg)8050ab5bb018SKeith M Wesolowski cpuid_execpass(cpu_t *cp, cpuid_pass_t pass, void *arg)
8051ab5bb018SKeith M Wesolowski {
8052ab5bb018SKeith M Wesolowski 	VERIFY3S(pass, !=, CPUID_PASS_NONE);
8053ab5bb018SKeith M Wesolowski 
8054ab5bb018SKeith M Wesolowski 	if (cp == NULL)
8055ab5bb018SKeith M Wesolowski 		cp = CPU;
8056ab5bb018SKeith M Wesolowski 
8057ab5bb018SKeith M Wesolowski 	/*
8058ab5bb018SKeith M Wesolowski 	 * Space statically allocated for BSP, ensure pointer is set
8059ab5bb018SKeith M Wesolowski 	 */
8060ab5bb018SKeith M Wesolowski 	if (cp->cpu_id == 0 && cp->cpu_m.mcpu_cpi == NULL)
8061ab5bb018SKeith M Wesolowski 		cp->cpu_m.mcpu_cpi = &cpuid_info0;
8062ab5bb018SKeith M Wesolowski 
8063ab5bb018SKeith M Wesolowski 	ASSERT(cpuid_checkpass(cp, pass - 1));
8064ab5bb018SKeith M Wesolowski 
8065ab5bb018SKeith M Wesolowski 	for (uint_t i = 0; i < ARRAY_SIZE(cpuid_pass_defs); i++) {
8066ab5bb018SKeith M Wesolowski 		if (cpuid_pass_defs[i].cpd_pass == pass) {
8067ab5bb018SKeith M Wesolowski 			cpuid_pass_defs[i].cpd_func(cp, arg);
8068ab5bb018SKeith M Wesolowski 			cp->cpu_m.mcpu_cpi->cpi_pass = pass;
8069ab5bb018SKeith M Wesolowski 			return;
8070ab5bb018SKeith M Wesolowski 		}
8071ab5bb018SKeith M Wesolowski 	}
8072ab5bb018SKeith M Wesolowski 
8073ab5bb018SKeith M Wesolowski 	panic("unable to execute invalid cpuid pass %d on cpu%d\n",
8074ab5bb018SKeith M Wesolowski 	    pass, cp->cpu_id);
8075ab5bb018SKeith M Wesolowski }
807622e4c3acSKeith M Wesolowski 
807722e4c3acSKeith M Wesolowski /*
807822e4c3acSKeith M Wesolowski  * Extract the processor family from a chiprev.  Processor families are not the
807922e4c3acSKeith M Wesolowski  * same as cpuid families; see comments above and in x86_archext.h.
808022e4c3acSKeith M Wesolowski  */
808122e4c3acSKeith M Wesolowski x86_processor_family_t
chiprev_family(const x86_chiprev_t cr)808222e4c3acSKeith M Wesolowski chiprev_family(const x86_chiprev_t cr)
808322e4c3acSKeith M Wesolowski {
808422e4c3acSKeith M Wesolowski 	return ((x86_processor_family_t)_X86_CHIPREV_FAMILY(cr));
808522e4c3acSKeith M Wesolowski }
808622e4c3acSKeith M Wesolowski 
808722e4c3acSKeith M Wesolowski /*
808822e4c3acSKeith M Wesolowski  * A chiprev matches its template if the vendor and family are identical and the
808922e4c3acSKeith M Wesolowski  * revision of the chiprev matches one of the bits set in the template.  Callers
809022e4c3acSKeith M Wesolowski  * may bitwise-OR together chiprevs of the same vendor and family to form the
809122e4c3acSKeith M Wesolowski  * template, or use the _ANY variant.  It is not possible to match chiprevs of
809222e4c3acSKeith M Wesolowski  * multiple vendors or processor families with a single call.  Note that this
809322e4c3acSKeith M Wesolowski  * function operates on processor families, not cpuid families.
809422e4c3acSKeith M Wesolowski  */
809522e4c3acSKeith M Wesolowski boolean_t
chiprev_matches(const x86_chiprev_t cr,const x86_chiprev_t template)809622e4c3acSKeith M Wesolowski chiprev_matches(const x86_chiprev_t cr, const x86_chiprev_t template)
809722e4c3acSKeith M Wesolowski {
809822e4c3acSKeith M Wesolowski 	return (_X86_CHIPREV_VENDOR(cr) == _X86_CHIPREV_VENDOR(template) &&
809922e4c3acSKeith M Wesolowski 	    _X86_CHIPREV_FAMILY(cr) == _X86_CHIPREV_FAMILY(template) &&
810022e4c3acSKeith M Wesolowski 	    (_X86_CHIPREV_REV(cr) & _X86_CHIPREV_REV(template)) != 0);
810122e4c3acSKeith M Wesolowski }
810222e4c3acSKeith M Wesolowski 
810322e4c3acSKeith M Wesolowski /*
810422e4c3acSKeith M Wesolowski  * A chiprev is at least min if the vendor and family are identical and the
810522e4c3acSKeith M Wesolowski  * revision of the chiprev is at least as recent as that of min.  Processor
810622e4c3acSKeith M Wesolowski  * families are considered unordered and cannot be compared using this function.
810722e4c3acSKeith M Wesolowski  * Note that this function operates on processor families, not cpuid families.
810822e4c3acSKeith M Wesolowski  * Use of the _ANY chiprev variant with this function is not useful; it will
810922e4c3acSKeith M Wesolowski  * always return B_FALSE if the _ANY variant is supplied as the minimum
811022e4c3acSKeith M Wesolowski  * revision.  To determine only whether a chiprev is of a given processor
811122e4c3acSKeith M Wesolowski  * family, test the return value of chiprev_family() instead.
811222e4c3acSKeith M Wesolowski  */
811322e4c3acSKeith M Wesolowski boolean_t
chiprev_at_least(const x86_chiprev_t cr,const x86_chiprev_t min)811422e4c3acSKeith M Wesolowski chiprev_at_least(const x86_chiprev_t cr, const x86_chiprev_t min)
811522e4c3acSKeith M Wesolowski {
811622e4c3acSKeith M Wesolowski 	return (_X86_CHIPREV_VENDOR(cr) == _X86_CHIPREV_VENDOR(min) &&
811722e4c3acSKeith M Wesolowski 	    _X86_CHIPREV_FAMILY(cr) == _X86_CHIPREV_FAMILY(min) &&
811822e4c3acSKeith M Wesolowski 	    _X86_CHIPREV_REV(cr) >= _X86_CHIPREV_REV(min));
811922e4c3acSKeith M Wesolowski }
812022e4c3acSKeith M Wesolowski 
812122e4c3acSKeith M Wesolowski /*
812222e4c3acSKeith M Wesolowski  * The uarch functions operate in a manner similar to the chiprev functions
812322e4c3acSKeith M Wesolowski  * above.  While it is tempting to allow these to operate on microarchitectures
812422e4c3acSKeith M Wesolowski  * produced by a specific vendor in an ordered fashion (e.g., ZEN3 is "newer"
812522e4c3acSKeith M Wesolowski  * than ZEN2), we elect not to do so because a manufacturer may supply
812622e4c3acSKeith M Wesolowski  * processors of multiple different microarchitecture families each of which may
812722e4c3acSKeith M Wesolowski  * be internally ordered but unordered with respect to those of other families.
812822e4c3acSKeith M Wesolowski  */
812922e4c3acSKeith M Wesolowski x86_uarch_t
uarchrev_uarch(const x86_uarchrev_t ur)813022e4c3acSKeith M Wesolowski uarchrev_uarch(const x86_uarchrev_t ur)
813122e4c3acSKeith M Wesolowski {
813222e4c3acSKeith M Wesolowski 	return ((x86_uarch_t)_X86_UARCHREV_UARCH(ur));
813322e4c3acSKeith M Wesolowski }
813422e4c3acSKeith M Wesolowski 
813522e4c3acSKeith M Wesolowski boolean_t
uarchrev_matches(const x86_uarchrev_t ur,const x86_uarchrev_t template)813622e4c3acSKeith M Wesolowski uarchrev_matches(const x86_uarchrev_t ur, const x86_uarchrev_t template)
813722e4c3acSKeith M Wesolowski {
813822e4c3acSKeith M Wesolowski 	return (_X86_UARCHREV_VENDOR(ur) == _X86_UARCHREV_VENDOR(template) &&
813922e4c3acSKeith M Wesolowski 	    _X86_UARCHREV_UARCH(ur) == _X86_UARCHREV_UARCH(template) &&
814022e4c3acSKeith M Wesolowski 	    (_X86_UARCHREV_REV(ur) & _X86_UARCHREV_REV(template)) != 0);
814122e4c3acSKeith M Wesolowski }
814222e4c3acSKeith M Wesolowski 
814322e4c3acSKeith M Wesolowski boolean_t
uarchrev_at_least(const x86_uarchrev_t ur,const x86_uarchrev_t min)814422e4c3acSKeith M Wesolowski uarchrev_at_least(const x86_uarchrev_t ur, const x86_uarchrev_t min)
814522e4c3acSKeith M Wesolowski {
814622e4c3acSKeith M Wesolowski 	return (_X86_UARCHREV_VENDOR(ur) == _X86_UARCHREV_VENDOR(min) &&
814722e4c3acSKeith M Wesolowski 	    _X86_UARCHREV_UARCH(ur) == _X86_UARCHREV_UARCH(min) &&
814822e4c3acSKeith M Wesolowski 	    _X86_UARCHREV_REV(ur) >= _X86_UARCHREV_REV(min));
814922e4c3acSKeith M Wesolowski }
8150dd23d762SRobert Mustacchi 
8151dd23d762SRobert Mustacchi /*
8152dd23d762SRobert Mustacchi  * Topology cache related information. This is yet another cache interface that
8153dd23d762SRobert Mustacchi  * we're exposing out intended to be used when we have either Intel Leaf 4 or
8154dd23d762SRobert Mustacchi  * AMD Leaf 8x1D (introduced with Zen 1).
8155dd23d762SRobert Mustacchi  */
8156dd23d762SRobert Mustacchi static boolean_t
cpuid_cache_topo_sup(const struct cpuid_info * cpi)8157dd23d762SRobert Mustacchi cpuid_cache_topo_sup(const struct cpuid_info *cpi)
8158dd23d762SRobert Mustacchi {
8159dd23d762SRobert Mustacchi 	switch (cpi->cpi_vendor) {
8160dd23d762SRobert Mustacchi 	case X86_VENDOR_Intel:
8161dd23d762SRobert Mustacchi 		if (cpi->cpi_maxeax >= 4) {
8162dd23d762SRobert Mustacchi 			return (B_TRUE);
8163dd23d762SRobert Mustacchi 		}
8164dd23d762SRobert Mustacchi 		break;
8165dd23d762SRobert Mustacchi 	case X86_VENDOR_AMD:
8166dd23d762SRobert Mustacchi 	case X86_VENDOR_HYGON:
8167dd23d762SRobert Mustacchi 		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1d &&
8168dd23d762SRobert Mustacchi 		    is_x86_feature(x86_featureset, X86FSET_TOPOEXT)) {
8169dd23d762SRobert Mustacchi 			return (B_TRUE);
8170dd23d762SRobert Mustacchi 		}
8171dd23d762SRobert Mustacchi 		break;
8172dd23d762SRobert Mustacchi 	default:
8173dd23d762SRobert Mustacchi 		break;
8174dd23d762SRobert Mustacchi 	}
8175dd23d762SRobert Mustacchi 
8176dd23d762SRobert Mustacchi 	return (B_FALSE);
8177dd23d762SRobert Mustacchi }
8178dd23d762SRobert Mustacchi 
8179dd23d762SRobert Mustacchi int
cpuid_getncaches(struct cpu * cpu,uint32_t * ncache)8180dd23d762SRobert Mustacchi cpuid_getncaches(struct cpu *cpu, uint32_t *ncache)
8181dd23d762SRobert Mustacchi {
8182dd23d762SRobert Mustacchi 	const struct cpuid_info *cpi;
8183dd23d762SRobert Mustacchi 
8184dd23d762SRobert Mustacchi 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));
8185dd23d762SRobert Mustacchi 	cpi = cpu->cpu_m.mcpu_cpi;
8186dd23d762SRobert Mustacchi 
8187dd23d762SRobert Mustacchi 	if (!cpuid_cache_topo_sup(cpi)) {
8188dd23d762SRobert Mustacchi 		return (ENOTSUP);
8189dd23d762SRobert Mustacchi 	}
8190dd23d762SRobert Mustacchi 
8191dd23d762SRobert Mustacchi 	*ncache = cpi->cpi_cache_leaf_size;
8192dd23d762SRobert Mustacchi 	return (0);
8193dd23d762SRobert Mustacchi }
8194dd23d762SRobert Mustacchi 
8195dd23d762SRobert Mustacchi int
cpuid_getcache(struct cpu * cpu,uint32_t cno,x86_cache_t * cache)8196dd23d762SRobert Mustacchi cpuid_getcache(struct cpu *cpu, uint32_t cno, x86_cache_t *cache)
8197dd23d762SRobert Mustacchi {
8198dd23d762SRobert Mustacchi 	const struct cpuid_info *cpi;
8199dd23d762SRobert Mustacchi 	const struct cpuid_regs *cp;
8200dd23d762SRobert Mustacchi 
8201dd23d762SRobert Mustacchi 	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));
8202dd23d762SRobert Mustacchi 	cpi = cpu->cpu_m.mcpu_cpi;
8203dd23d762SRobert Mustacchi 
8204dd23d762SRobert Mustacchi 	if (!cpuid_cache_topo_sup(cpi)) {
8205dd23d762SRobert Mustacchi 		return (ENOTSUP);
8206dd23d762SRobert Mustacchi 	}
8207dd23d762SRobert Mustacchi 
8208dd23d762SRobert Mustacchi 	if (cno >= cpi->cpi_cache_leaf_size) {
8209dd23d762SRobert Mustacchi 		return (EINVAL);
8210dd23d762SRobert Mustacchi 	}
8211dd23d762SRobert Mustacchi 
8212dd23d762SRobert Mustacchi 	bzero(cache, sizeof (cache));
8213dd23d762SRobert Mustacchi 	cp = cpi->cpi_cache_leaves[cno];
8214dd23d762SRobert Mustacchi 	switch (CPI_CACHE_TYPE(cp)) {
8215dd23d762SRobert Mustacchi 	case CPI_CACHE_TYPE_DATA:
8216dd23d762SRobert Mustacchi 		cache->xc_type = X86_CACHE_TYPE_DATA;
8217dd23d762SRobert Mustacchi 		break;
8218dd23d762SRobert Mustacchi 	case CPI_CACHE_TYPE_INSTR:
8219dd23d762SRobert Mustacchi 		cache->xc_type = X86_CACHE_TYPE_INST;
8220dd23d762SRobert Mustacchi 		break;
8221dd23d762SRobert Mustacchi 	case CPI_CACHE_TYPE_UNIFIED:
8222dd23d762SRobert Mustacchi 		cache->xc_type = X86_CACHE_TYPE_UNIFIED;
8223dd23d762SRobert Mustacchi 		break;
8224dd23d762SRobert Mustacchi 	case CPI_CACHE_TYPE_DONE:
8225dd23d762SRobert Mustacchi 	default:
8226dd23d762SRobert Mustacchi 		return (EINVAL);
8227dd23d762SRobert Mustacchi 	}
8228dd23d762SRobert Mustacchi 	cache->xc_level = CPI_CACHE_LVL(cp);
8229dd23d762SRobert Mustacchi 	if (CPI_FULL_ASSOC_CACHE(cp) != 0) {
8230dd23d762SRobert Mustacchi 		cache->xc_flags |= X86_CACHE_F_FULL_ASSOC;
8231dd23d762SRobert Mustacchi 	}
8232dd23d762SRobert Mustacchi 	cache->xc_nparts = CPI_CACHE_PARTS(cp) + 1;
8233dd23d762SRobert Mustacchi 	/*
8234dd23d762SRobert Mustacchi 	 * The number of sets is reserved on AMD if the CPU is tagged as fully
8235dd23d762SRobert Mustacchi 	 * associative, where as it is considered valid on Intel.
8236dd23d762SRobert Mustacchi 	 */
8237dd23d762SRobert Mustacchi 	if (cpi->cpi_vendor == X86_VENDOR_AMD &&
8238dd23d762SRobert Mustacchi 	    CPI_FULL_ASSOC_CACHE(cp) != 0) {
8239dd23d762SRobert Mustacchi 		cache->xc_nsets = 1;
8240dd23d762SRobert Mustacchi 	} else {
8241dd23d762SRobert Mustacchi 		cache->xc_nsets = CPI_CACHE_SETS(cp) + 1;
8242dd23d762SRobert Mustacchi 	}
8243dd23d762SRobert Mustacchi 	cache->xc_nways = CPI_CACHE_WAYS(cp) + 1;
8244dd23d762SRobert Mustacchi 	cache->xc_line_size = CPI_CACHE_COH_LN_SZ(cp) + 1;
8245dd23d762SRobert Mustacchi 	cache->xc_size = cache->xc_nparts * cache->xc_nsets * cache->xc_nways *
8246dd23d762SRobert Mustacchi 	    cache->xc_line_size;
8247dd23d762SRobert Mustacchi 	/*
8248dd23d762SRobert Mustacchi 	 * We're looking for the number of bits to cover the number of CPUs that
8249dd23d762SRobert Mustacchi 	 * are being shared. Normally this would be the value - 1, but the CPUID
8250dd23d762SRobert Mustacchi 	 * value is encoded as the actual value minus one, so we don't modify
8251dd23d762SRobert Mustacchi 	 * this at all.
8252dd23d762SRobert Mustacchi 	 */
8253dd23d762SRobert Mustacchi 	cache->xc_apic_shift = highbit(CPI_NTHR_SHR_CACHE(cp));
8254dd23d762SRobert Mustacchi 
8255dd23d762SRobert Mustacchi 	/*
8256dd23d762SRobert Mustacchi 	 * To construct a unique ID we construct a uint64_t that looks as
8257dd23d762SRobert Mustacchi 	 * follows:
8258dd23d762SRobert Mustacchi 	 *
8259dd23d762SRobert Mustacchi 	 * [47:40] cache level
8260dd23d762SRobert Mustacchi 	 * [39:32] CPUID cache type
8261dd23d762SRobert Mustacchi 	 * [31:00] shifted APIC ID
8262dd23d762SRobert Mustacchi 	 *
8263dd23d762SRobert Mustacchi 	 * The shifted APIC ID gives us a guarantee that a given cache entry is
8264dd23d762SRobert Mustacchi 	 * unique within its peers. The other two numbers give us something that
8265dd23d762SRobert Mustacchi 	 * ensures that something is unique within the CPU. If we just had the
8266dd23d762SRobert Mustacchi 	 * APIC ID shifted over by the indicated number of bits we'd end up with
8267dd23d762SRobert Mustacchi 	 * an ID of zero for the L1I, L1D, L2, and L3.
8268dd23d762SRobert Mustacchi 	 *
8269dd23d762SRobert Mustacchi 	 * The format of this ID is private to the system and can change across
8270dd23d762SRobert Mustacchi 	 * a reboot for the time being.
8271dd23d762SRobert Mustacchi 	 */
8272dd23d762SRobert Mustacchi 	cache->xc_id = (uint64_t)cache->xc_level << 40;
8273dd23d762SRobert Mustacchi 	cache->xc_id |= (uint64_t)cache->xc_type << 32;
8274dd23d762SRobert Mustacchi 	cache->xc_id |= (uint64_t)cpi->cpi_apicid >> cache->xc_apic_shift;
8275dd23d762SRobert Mustacchi 
8276dd23d762SRobert Mustacchi 	return (0);
8277dd23d762SRobert Mustacchi }
8278