17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
54179143andrei * Common Development and Distribution License (the "License").
64179143andrei * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
220d92875Gary Mills * Copyright (c) 2012 Gary Mills
230d92875Gary Mills *
247417cfdKuriakose Kuruvilla * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
25cfe84b8Matt Amdur * Copyright (c) 2011 by Delphix. All rights reserved.
26c3377eeJohn Levon * Copyright 2019 Joyent, Inc.
277c478bdstevel@tonic-gate */
28a311483Gerry Liu/*
29a311483Gerry Liu * Copyright (c) 2010, Intel Corporation.
30a311483Gerry Liu * All rights reserved.
31a311483Gerry Liu */
327c478bdstevel@tonic-gate
337c478bdstevel@tonic-gate#include <sys/types.h>
34ae115bcmrj#include <sys/sysmacros.h>
357c478bdstevel@tonic-gate#include <sys/disp.h>
367c478bdstevel@tonic-gate#include <sys/promif.h>
377c478bdstevel@tonic-gate#include <sys/clock.h>
387c478bdstevel@tonic-gate#include <sys/cpuvar.h>
397c478bdstevel@tonic-gate#include <sys/stack.h>
407c478bdstevel@tonic-gate#include <vm/as.h>
417c478bdstevel@tonic-gate#include <vm/hat.h>
427c478bdstevel@tonic-gate#include <sys/reboot.h>
437c478bdstevel@tonic-gate#include <sys/avintr.h>
447c478bdstevel@tonic-gate#include <sys/vtrace.h>
457c478bdstevel@tonic-gate#include <sys/proc.h>
467c478bdstevel@tonic-gate#include <sys/thread.h>
477c478bdstevel@tonic-gate#include <sys/cpupart.h>
487c478bdstevel@tonic-gate#include <sys/pset.h>
497c478bdstevel@tonic-gate#include <sys/copyops.h>
50fb2f18fesaxe#include <sys/pg.h>
517c478bdstevel@tonic-gate#include <sys/disp.h>
527c478bdstevel@tonic-gate#include <sys/debug.h>
537c478bdstevel@tonic-gate#include <sys/sunddi.h>
547c478bdstevel@tonic-gate#include <sys/x86_archext.h>
557c478bdstevel@tonic-gate#include <sys/privregs.h>
567c478bdstevel@tonic-gate#include <sys/machsystm.h>
577c478bdstevel@tonic-gate#include <sys/ontrap.h>
587c478bdstevel@tonic-gate#include <sys/bootconf.h>
599db7147Sherry Moore#include <sys/boot_console.h>
60ae115bcmrj#include <sys/kdi_machimpl.h>
617c478bdstevel@tonic-gate#include <sys/archsystm.h>
627c478bdstevel@tonic-gate#include <sys/promif.h>
63c88420bdmick#include <sys/pci_cfgspace.h>
640c26abfJohn Levon#include <sys/apic.h>
650c26abfJohn Levon#include <sys/apic_common.h>
660181461Keith M Wesolowski#include <sys/bootvfs.h>
672428aadPatrick Mooney#include <sys/tsc.h>
68c3377eeJohn Levon#include <sys/smt.h>
69843e198johnlev#ifdef __xpv
70843e198johnlev#include <sys/hypervisor.h>
71349b53dStuart Maybee#else
72349b53dStuart Maybee#include <sys/xpv_support.h>
73843e198johnlev#endif
747c478bdstevel@tonic-gate
757c478bdstevel@tonic-gate/*
767c478bdstevel@tonic-gate * some globals for patching the result of cpuid
777c478bdstevel@tonic-gate * to solve problems w/ creative cpu vendors
787c478bdstevel@tonic-gate */
797c478bdstevel@tonic-gate
807c478bdstevel@tonic-gateextern uint32_t cpuid_feature_ecx_include;
817c478bdstevel@tonic-gateextern uint32_t cpuid_feature_ecx_exclude;
827c478bdstevel@tonic-gateextern uint32_t cpuid_feature_edx_include;
837c478bdstevel@tonic-gateextern uint32_t cpuid_feature_edx_exclude;
847c478bdstevel@tonic-gate
850c26abfJohn Levonnmi_action_t nmi_action = NMI_ACTION_UNSET;
860c26abfJohn Levon
877c478bdstevel@tonic-gate/*
889db7147Sherry Moore * Set console mode
899db7147Sherry Moore */
909db7147Sherry Moorestatic void
919db7147Sherry Mooreset_console_mode(uint8_t val)
929db7147Sherry Moore{
939db7147Sherry Moore	struct bop_regs rp = {0};
949db7147Sherry Moore
959db7147Sherry Moore	rp.eax.byte.ah = 0x0;
969db7147Sherry Moore	rp.eax.byte.al = val;
979db7147Sherry Moore	rp.ebx.word.bx = 0x0;
989db7147Sherry Moore
999db7147Sherry Moore	BOP_DOINT(bootops, 0x10, &rp);
1009db7147Sherry Moore}
1019db7147Sherry Moore
1027c478bdstevel@tonic-gate
1037c478bdstevel@tonic-gate/*
1047c478bdstevel@tonic-gate * Setup routine called right before main(). Interposing this function
1057c478bdstevel@tonic-gate * before main() allows us to call it in a machine-independent fashion.
1067c478bdstevel@tonic-gate */
1077c478bdstevel@tonic-gatevoid
1087c478bdstevel@tonic-gatemlsetup(struct regs *rp)
1097c478bdstevel@tonic-gate{
1102baa66aJonathan Chew	u_longlong_t prop_value;
1110c26abfJohn Levon	char prop_str[BP_MAX_STRLEN];
1127c478bdstevel@tonic-gate	extern struct classfuncs sys_classfuncs;
1137c478bdstevel@tonic-gate	extern disp_t cpu0_disp;
1147c478bdstevel@tonic-gate	extern char t0stack[];
1159db7147Sherry Moore	extern int post_fastreboot;
116a311483Gerry Liu	extern uint64_t plat_dr_options;
1177c478bdstevel@tonic-gate
1187c478bdstevel@tonic-gate	ASSERT_STACK_ALIGNED();
1197c478bdstevel@tonic-gate
1207c478bdstevel@tonic-gate	/*
1217c478bdstevel@tonic-gate	 * initialize cpu_self
1227c478bdstevel@tonic-gate	 */
1237c478bdstevel@tonic-gate	cpu[0]->cpu_self = cpu[0];
1247c478bdstevel@tonic-gate
125843e198johnlev#if defined(__xpv)
126843e198johnlev	/*
127843e198johnlev	 * Point at the hypervisor's virtual cpu structure
128843e198johnlev	 */
129843e198johnlev	cpu[0]->cpu_m.mcpu_vcpu_info = &HYPERVISOR_shared_info->vcpu_info[0];
130843e198johnlev#endif
131843e198johnlev
1327c478bdstevel@tonic-gate	/*
1337c478bdstevel@tonic-gate	 * check if we've got special bits to clear or set
1347c478bdstevel@tonic-gate	 * when checking cpu features
1357c478bdstevel@tonic-gate	 */
1367c478bdstevel@tonic-gate
1372baa66aJonathan Chew	if (bootprop_getval("cpuid_feature_ecx_include", &prop_value) != 0)
1382baa66aJonathan Chew		cpuid_feature_ecx_include = 0;
1392baa66aJonathan Chew	else
1402baa66aJonathan Chew		cpuid_feature_ecx_include = (uint32_t)prop_value;
1412baa66aJonathan Chew
1422baa66aJonathan Chew	if (bootprop_getval("cpuid_feature_ecx_exclude", &prop_value) != 0)
1432baa66aJonathan Chew		cpuid_feature_ecx_exclude = 0;
1442baa66aJonathan Chew	else
1452baa66aJonathan Chew		cpuid_feature_ecx_exclude = (uint32_t)prop_value;
1462baa66aJonathan Chew
1472baa66aJonathan Chew	if (bootprop_getval("cpuid_feature_edx_include", &prop_value) != 0)
1482baa66aJonathan Chew		cpuid_feature_edx_include = 0;
1492baa66aJonathan Chew	else
1502baa66aJonathan Chew		cpuid_feature_edx_include = (uint32_t)prop_value;
1512baa66aJonathan Chew
1522baa66aJonathan Chew	if (bootprop_getval("cpuid_feature_edx_exclude", &prop_value) != 0)
1532baa66aJonathan Chew		cpuid_feature_edx_exclude = 0;
1542baa66aJonathan Chew	else
1552baa66aJonathan Chew		cpuid_feature_edx_exclude = (uint32_t)prop_value;
1567c478bdstevel@tonic-gate
15774ecdb5John Levon#if !defined(__xpv)
1580c26abfJohn Levon	if (bootprop_getstr("nmi", prop_str, sizeof (prop_str)) == 0) {
1590c26abfJohn Levon		if (strcmp(prop_str, "ignore") == 0) {
1600c26abfJohn Levon			nmi_action = NMI_ACTION_IGNORE;
1610c26abfJohn Levon		} else if (strcmp(prop_str, "panic") == 0) {
1620c26abfJohn Levon			nmi_action = NMI_ACTION_PANIC;
1630c26abfJohn Levon		} else if (strcmp(prop_str, "kmdb") == 0) {
1640c26abfJohn Levon			nmi_action = NMI_ACTION_KMDB;
1650c26abfJohn Levon		} else {
1660c26abfJohn Levon			prom_printf("unix: ignoring unknown nmi=%s\n",
1670c26abfJohn Levon			    prop_str);
1680c26abfJohn Levon		}
1690c26abfJohn Levon	}
1700c26abfJohn Levon
17174ecdb5John Levon	/*
17274ecdb5John Levon	 * Check to see if KPTI has been explicitly enabled or disabled.
17374ecdb5John Levon	 * We have to check this before init_desctbls().
17474ecdb5John Levon	 */
17574ecdb5John Levon	if (bootprop_getval("kpti", &prop_value) == 0) {
17674ecdb5John Levon		kpti_enable = (uint64_t)(prop_value == 1);
17774ecdb5John Levon		prom_printf("unix: forcing kpti to %s due to boot argument\n",
17874ecdb5John Levon		    (kpti_enable == 1) ? "ON" : "OFF");
17974ecdb5John Levon	} else {
18074ecdb5John Levon		kpti_enable = 1;
18174ecdb5John Levon	}
18274ecdb5John Levon
18374ecdb5John Levon	if (bootprop_getval("pcid", &prop_value) == 0 && prop_value == 0) {
18474ecdb5John Levon		prom_printf("unix: forcing pcid to OFF due to boot argument\n");
18574ecdb5John Levon		x86_use_pcid = 0;
18674ecdb5John Levon	} else if (kpti_enable != 1) {
18774ecdb5John Levon		x86_use_pcid = 0;
18874ecdb5John Levon	}
189c3377eeJohn Levon
190c3377eeJohn Levon	/*
191c3377eeJohn Levon	 * While we don't need to check this until later, we might as well do it
192c3377eeJohn Levon	 * here.
193c3377eeJohn Levon	 */
194c3377eeJohn Levon	if (bootprop_getstr("smt_enabled", prop_str, sizeof (prop_str)) == 0) {
195c3377eeJohn Levon		if (strcasecmp(prop_str, "false") == 0 ||
196c3377eeJohn Levon		    strcmp(prop_str, "0") == 0)
197c3377eeJohn Levon			smt_boot_disable = 1;
198c3377eeJohn Levon	}
199c3377eeJohn Levon
20074ecdb5John Levon#endif
20174ecdb5John Levon
2027c478bdstevel@tonic-gate	/*
20345e032fDan Mick	 * Initialize idt0, gdt0, ldt0_default, ktss0 and dftss.
20445e032fDan Mick	 */
20545e032fDan Mick	init_desctbls();
20645e032fDan Mick
20745e032fDan Mick	/*
20845e032fDan Mick	 * lgrp_init() and possibly cpuid_pass1() need PCI config
20945e032fDan Mick	 * space access
21045e032fDan Mick	 */
21145e032fDan Mick#if defined(__xpv)
21245e032fDan Mick	if (DOMAIN_IS_INITDOMAIN(xen_info))
21345e032fDan Mick		pci_cfgspace_init();
21445e032fDan Mick#else
21545e032fDan Mick	pci_cfgspace_init();
216cfe84b8Matt Amdur	/*
217cfe84b8Matt Amdur	 * Initialize the platform type from CPU 0 to ensure that
218cfe84b8Matt Amdur	 * determine_platform() is only ever called once.
219cfe84b8Matt Amdur	 */
220cfe84b8Matt Amdur	determine_platform();
22145e032fDan Mick#endif
22245e032fDan Mick
22345e032fDan Mick	/*
2247c478bdstevel@tonic-gate	 * The first lightweight pass (pass0) through the cpuid data
2257c478bdstevel@tonic-gate	 * was done in locore before mlsetup was called.  Do the next
2267c478bdstevel@tonic-gate	 * pass in C code.
2277c478bdstevel@tonic-gate	 *
2287417cfdKuriakose Kuruvilla	 * The x86_featureset is initialized here based on the capabilities
2297c478bdstevel@tonic-gate	 * of the boot CPU.  Note that if we choose to support CPUs that have
2307c478bdstevel@tonic-gate	 * different feature sets (at which point we would almost certainly
2317c478bdstevel@tonic-gate	 * want to set the feature bits to correspond to the feature
2327c478bdstevel@tonic-gate	 * minimum) this value may be altered.
2337c478bdstevel@tonic-gate	 */
234dfea898Kuriakose Kuruvilla	cpuid_pass1(cpu[0], x86_featureset);
2357c478bdstevel@tonic-gate
236247dbb3sudheer#if !defined(__xpv)
23779ec9daYuri Pankov	if ((get_hwenv() & HW_XEN_HVM) != 0)
238349b53dStuart Maybee		xen_hvm_init();
239349b53dStuart Maybee
240247dbb3sudheer	/*
2414948216Keith M Wesolowski	 * Before we do anything with the TSCs, we need to work around
2424948216Keith M Wesolowski	 * Intel erratum BT81.  On some CPUs, warm reset does not
2434948216Keith M Wesolowski	 * clear the TSC.  If we are on such a CPU, we will clear TSC ourselves
2444948216Keith M Wesolowski	 * here.  Other CPUs will clear it when we boot them later, and the
2454948216Keith M Wesolowski	 * resulting skew will be handled by tsc_sync_master()/_slave();
2464948216Keith M Wesolowski	 * note that such skew already exists and has to be handled anyway.
2474948216Keith M Wesolowski	 *
2484948216Keith M Wesolowski	 * We do this only on metal.  This same problem can occur with a
2494948216Keith M Wesolowski	 * hypervisor that does not happen to virtualise a TSC that starts from
2504948216Keith M Wesolowski	 * zero, regardless of CPU type; however, we do not expect hypervisors
2514948216Keith M Wesolowski	 * that do not virtualise TSC that way to handle writes to TSC
2524948216Keith M Wesolowski	 * correctly, either.
2534948216Keith M Wesolowski	 */
2544948216Keith M Wesolowski	if (get_hwenv() == HW_NATIVE &&
2554948216Keith M Wesolowski	    cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
2564948216Keith M Wesolowski	    cpuid_getfamily(CPU) == 6 &&
2574948216Keith M Wesolowski	    (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) &&
2584948216Keith M Wesolowski	    is_x86_feature(x86_featureset, X86FSET_TSC)) {
2594948216Keith M Wesolowski		(void) wrmsr(REG_TSC, 0UL);
2604948216Keith M Wesolowski	}
2614948216Keith M Wesolowski
2624948216Keith M Wesolowski	/*
263247dbb3sudheer	 * Patch the tsc_read routine with appropriate set of instructions,
264247dbb3sudheer	 * depending on the processor family and architecure, to read the
265247dbb3sudheer	 * time-stamp counter while ensuring no out-of-order execution.
266247dbb3sudheer	 * Patch it while the kernel text is still writable.
267247dbb3sudheer	 *
268247dbb3sudheer	 * Note: tsc_read is not patched for intel processors whose family
269247dbb3sudheer	 * is >6 and for amd whose family >f (in case they don't support rdtscp
270247dbb3sudheer	 * instruction, unlikely). By default tsc_read will use cpuid for
271247dbb3sudheer	 * serialization in such cases. The following code needs to be
272247dbb3sudheer	 * revisited if intel processors of family >= f retains the
273247dbb3sudheer	 * instruction serialization nature of mfence instruction.
2742b0bcb2sudheer	 * Note: tsc_read is not patched for x86 processors which do
2752b0bcb2sudheer	 * not support "mfence". By default tsc_read will use cpuid for
2762b0bcb2sudheer	 * serialization in such cases.
277551bc2amrj	 *
278551bc2amrj	 * The Xen hypervisor does not correctly report whether rdtscp is
279551bc2amrj	 * supported or not, so we must assume that it is not.
280247dbb3sudheer	 */
28179ec9daYuri Pankov	if ((get_hwenv() & HW_XEN_HVM) == 0 &&
2827417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_TSCP))
2832428aadPatrick Mooney		patch_tsc_read(TSC_TSCP);
284247dbb3sudheer	else if (cpuid_getvendor(CPU) == X86_VENDOR_AMD &&
2857417cfdKuriakose Kuruvilla	    cpuid_getfamily(CPU) <= 0xf &&
2867417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_SSE2))
2872428aadPatrick Mooney		patch_tsc_read(TSC_RDTSC_MFENCE);
288247dbb3sudheer	else if (cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
2897417cfdKuriakose Kuruvilla	    cpuid_getfamily(CPU) <= 6 &&
2907417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_SSE2))
2912428aadPatrick Mooney		patch_tsc_read(TSC_RDTSC_LFENCE);
292247dbb3sudheer
293247dbb3sudheer#endif	/* !__xpv */
2947c478bdstevel@tonic-gate
295843e198johnlev#if defined(__i386) && !defined(__xpv)
2967c478bdstevel@tonic-gate	/*
2977c478bdstevel@tonic-gate	 * Some i386 processors do not implement the rdtsc instruction,
298247dbb3sudheer	 * or at least they do not implement it correctly. Patch them to
299247dbb3sudheer	 * return 0.
3007c478bdstevel@tonic-gate	 */
3017417cfdKuriakose Kuruvilla	if (!is_x86_feature(x86_featureset, X86FSET_TSC))
3022428aadPatrick Mooney		patch_tsc_read(TSC_NONE);
303843e198johnlev#endif	/* __i386 && !__xpv */
304843e198johnlev
30522cc0e4Bill Holler#if defined(__amd64) && !defined(__xpv)
30622cc0e4Bill Holler	patch_memops(cpuid_getvendor(CPU));
30722cc0e4Bill Holler#endif	/* __amd64 && !__xpv */
30822cc0e4Bill Holler
309843e198johnlev#if !defined(__xpv)
310843e198johnlev	/* XXPV	what, if anything, should be dorked with here under xen? */
311ae115bcmrj
312ae115bcmrj	/*
313ae115bcmrj	 * While we're thinking about the TSC, let's set up %cr4 so that
314ae115bcmrj	 * userland can issue rdtsc, and initialize the TSC_AUX value
315ae115bcmrj	 * (the cpuid) for the rdtscp instruction on appropriately
316ae115bcmrj	 * capable hardware.
317ae115bcmrj	 */
3187417cfdKuriakose Kuruvilla	if (is_x86_feature(x86_featureset, X86FSET_TSC))
319ae115bcmrj		setcr4(getcr4() & ~CR4_TSD);
320ae115bcmrj
3217417cfdKuriakose Kuruvilla	if (is_x86_feature(x86_featureset, X86FSET_TSCP))
322ae115bcmrj		(void) wrmsr(MSR_AMD_TSCAUX, 0);
323