17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 541791439Sandrei * Common Development and Distribution License (the "License"). 641791439Sandrei * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 220d928757SGary Mills * Copyright (c) 2012 Gary Mills 230d928757SGary Mills * 247417cfdeSKuriakose Kuruvilla * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. 25cfe84b82SMatt Amdur * Copyright (c) 2011 by Delphix. All rights reserved. 26*c3377ee9SJohn Levon * Copyright 2019 Joyent, Inc. 277c478bd9Sstevel@tonic-gate */ 28a3114836SGerry Liu /* 29a3114836SGerry Liu * Copyright (c) 2010, Intel Corporation. 30a3114836SGerry Liu * All rights reserved. 31a3114836SGerry Liu */ 327c478bd9Sstevel@tonic-gate 337c478bd9Sstevel@tonic-gate #include <sys/types.h> 34ae115bc7Smrj #include <sys/sysmacros.h> 357c478bd9Sstevel@tonic-gate #include <sys/disp.h> 367c478bd9Sstevel@tonic-gate #include <sys/promif.h> 377c478bd9Sstevel@tonic-gate #include <sys/clock.h> 387c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 397c478bd9Sstevel@tonic-gate #include <sys/stack.h> 407c478bd9Sstevel@tonic-gate #include <vm/as.h> 417c478bd9Sstevel@tonic-gate #include <vm/hat.h> 427c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 437c478bd9Sstevel@tonic-gate #include <sys/avintr.h> 447c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 457c478bd9Sstevel@tonic-gate #include <sys/proc.h> 467c478bd9Sstevel@tonic-gate #include <sys/thread.h> 477c478bd9Sstevel@tonic-gate #include <sys/cpupart.h> 487c478bd9Sstevel@tonic-gate #include <sys/pset.h> 497c478bd9Sstevel@tonic-gate #include <sys/copyops.h> 50fb2f18f8Sesaxe #include <sys/pg.h> 517c478bd9Sstevel@tonic-gate #include <sys/disp.h> 527c478bd9Sstevel@tonic-gate #include <sys/debug.h> 537c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 547c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 557c478bd9Sstevel@tonic-gate #include <sys/privregs.h> 567c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 577c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 587c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 599db7147eSSherry Moore #include <sys/boot_console.h> 60ae115bc7Smrj #include <sys/kdi_machimpl.h> 617c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 627c478bd9Sstevel@tonic-gate #include <sys/promif.h> 63c88420b3Sdmick #include <sys/pci_cfgspace.h> 640c26abfeSJohn Levon #include <sys/apic.h> 650c26abfeSJohn Levon #include <sys/apic_common.h> 660181461bSKeith M Wesolowski #include <sys/bootvfs.h> 672428aad8SPatrick Mooney #include <sys/tsc.h> 68*c3377ee9SJohn Levon #include <sys/smt.h> 69843e1988Sjohnlev #ifdef __xpv 70843e1988Sjohnlev #include <sys/hypervisor.h> 71349b53ddSStuart Maybee #else 72349b53ddSStuart Maybee #include <sys/xpv_support.h> 73843e1988Sjohnlev #endif 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate /* 767c478bd9Sstevel@tonic-gate * some globals for patching the result of cpuid 777c478bd9Sstevel@tonic-gate * to solve problems w/ creative cpu vendors 787c478bd9Sstevel@tonic-gate */ 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate extern uint32_t cpuid_feature_ecx_include; 817c478bd9Sstevel@tonic-gate extern uint32_t cpuid_feature_ecx_exclude; 827c478bd9Sstevel@tonic-gate extern uint32_t cpuid_feature_edx_include; 837c478bd9Sstevel@tonic-gate extern uint32_t cpuid_feature_edx_exclude; 847c478bd9Sstevel@tonic-gate 850c26abfeSJohn Levon nmi_action_t nmi_action = NMI_ACTION_UNSET; 860c26abfeSJohn Levon 879db7147eSSherry Moore /* 889db7147eSSherry Moore * Set console mode 899db7147eSSherry Moore */ 909db7147eSSherry Moore static void 919db7147eSSherry Moore set_console_mode(uint8_t val) 929db7147eSSherry Moore { 939db7147eSSherry Moore struct bop_regs rp = {0}; 949db7147eSSherry Moore 959db7147eSSherry Moore rp.eax.byte.ah = 0x0; 969db7147eSSherry Moore rp.eax.byte.al = val; 979db7147eSSherry Moore rp.ebx.word.bx = 0x0; 989db7147eSSherry Moore 999db7147eSSherry Moore BOP_DOINT(bootops, 0x10, &rp); 1009db7147eSSherry Moore } 1019db7147eSSherry Moore 1027c478bd9Sstevel@tonic-gate 1037c478bd9Sstevel@tonic-gate /* 1047c478bd9Sstevel@tonic-gate * Setup routine called right before main(). Interposing this function 1057c478bd9Sstevel@tonic-gate * before main() allows us to call it in a machine-independent fashion. 1067c478bd9Sstevel@tonic-gate */ 1077c478bd9Sstevel@tonic-gate void 1087c478bd9Sstevel@tonic-gate mlsetup(struct regs *rp) 1097c478bd9Sstevel@tonic-gate { 1102baa66a0SJonathan Chew u_longlong_t prop_value; 1110c26abfeSJohn Levon char prop_str[BP_MAX_STRLEN]; 1127c478bd9Sstevel@tonic-gate extern struct classfuncs sys_classfuncs; 1137c478bd9Sstevel@tonic-gate extern disp_t cpu0_disp; 1147c478bd9Sstevel@tonic-gate extern char t0stack[]; 1159db7147eSSherry Moore extern int post_fastreboot; 116a3114836SGerry Liu extern uint64_t plat_dr_options; 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED(); 1197c478bd9Sstevel@tonic-gate 1207c478bd9Sstevel@tonic-gate /* 1217c478bd9Sstevel@tonic-gate * initialize cpu_self 1227c478bd9Sstevel@tonic-gate */ 1237c478bd9Sstevel@tonic-gate cpu[0]->cpu_self = cpu[0]; 1247c478bd9Sstevel@tonic-gate 125843e1988Sjohnlev #if defined(__xpv) 126843e1988Sjohnlev /* 127843e1988Sjohnlev * Point at the hypervisor's virtual cpu structure 128843e1988Sjohnlev */ 129843e1988Sjohnlev cpu[0]->cpu_m.mcpu_vcpu_info = &HYPERVISOR_shared_info->vcpu_info[0]; 130843e1988Sjohnlev #endif 131843e1988Sjohnlev 1327c478bd9Sstevel@tonic-gate /* 1337c478bd9Sstevel@tonic-gate * check if we've got special bits to clear or set 1347c478bd9Sstevel@tonic-gate * when checking cpu features 1357c478bd9Sstevel@tonic-gate */ 1367c478bd9Sstevel@tonic-gate 1372baa66a0SJonathan Chew if (bootprop_getval("cpuid_feature_ecx_include", &prop_value) != 0) 1382baa66a0SJonathan Chew cpuid_feature_ecx_include = 0; 1392baa66a0SJonathan Chew else 1402baa66a0SJonathan Chew cpuid_feature_ecx_include = (uint32_t)prop_value; 1412baa66a0SJonathan Chew 1422baa66a0SJonathan Chew if (bootprop_getval("cpuid_feature_ecx_exclude", &prop_value) != 0) 1432baa66a0SJonathan Chew cpuid_feature_ecx_exclude = 0; 1442baa66a0SJonathan Chew else 1452baa66a0SJonathan Chew cpuid_feature_ecx_exclude = (uint32_t)prop_value; 1462baa66a0SJonathan Chew 1472baa66a0SJonathan Chew if (bootprop_getval("cpuid_feature_edx_include", &prop_value) != 0) 1482baa66a0SJonathan Chew cpuid_feature_edx_include = 0; 1492baa66a0SJonathan Chew else 1502baa66a0SJonathan Chew cpuid_feature_edx_include = (uint32_t)prop_value; 1512baa66a0SJonathan Chew 1522baa66a0SJonathan Chew if (bootprop_getval("cpuid_feature_edx_exclude", &prop_value) != 0) 1532baa66a0SJonathan Chew cpuid_feature_edx_exclude = 0; 1542baa66a0SJonathan Chew else 1552baa66a0SJonathan Chew cpuid_feature_edx_exclude = (uint32_t)prop_value; 1567c478bd9Sstevel@tonic-gate 15774ecdb51SJohn Levon #if !defined(__xpv) 1580c26abfeSJohn Levon if (bootprop_getstr("nmi", prop_str, sizeof (prop_str)) == 0) { 1590c26abfeSJohn Levon if (strcmp(prop_str, "ignore") == 0) { 1600c26abfeSJohn Levon nmi_action = NMI_ACTION_IGNORE; 1610c26abfeSJohn Levon } else if (strcmp(prop_str, "panic") == 0) { 1620c26abfeSJohn Levon nmi_action = NMI_ACTION_PANIC; 1630c26abfeSJohn Levon } else if (strcmp(prop_str, "kmdb") == 0) { 1640c26abfeSJohn Levon nmi_action = NMI_ACTION_KMDB; 1650c26abfeSJohn Levon } else { 1660c26abfeSJohn Levon prom_printf("unix: ignoring unknown nmi=%s\n", 1670c26abfeSJohn Levon prop_str); 1680c26abfeSJohn Levon } 1690c26abfeSJohn Levon } 1700c26abfeSJohn Levon 17174ecdb51SJohn Levon /* 17274ecdb51SJohn Levon * Check to see if KPTI has been explicitly enabled or disabled. 17374ecdb51SJohn Levon * We have to check this before init_desctbls(). 17474ecdb51SJohn Levon */ 17574ecdb51SJohn Levon if (bootprop_getval("kpti", &prop_value) == 0) { 17674ecdb51SJohn Levon kpti_enable = (uint64_t)(prop_value == 1); 17774ecdb51SJohn Levon prom_printf("unix: forcing kpti to %s due to boot argument\n", 17874ecdb51SJohn Levon (kpti_enable == 1) ? "ON" : "OFF"); 17974ecdb51SJohn Levon } else { 18074ecdb51SJohn Levon kpti_enable = 1; 18174ecdb51SJohn Levon } 18274ecdb51SJohn Levon 18374ecdb51SJohn Levon if (bootprop_getval("pcid", &prop_value) == 0 && prop_value == 0) { 18474ecdb51SJohn Levon prom_printf("unix: forcing pcid to OFF due to boot argument\n"); 18574ecdb51SJohn Levon x86_use_pcid = 0; 18674ecdb51SJohn Levon } else if (kpti_enable != 1) { 18774ecdb51SJohn Levon x86_use_pcid = 0; 18874ecdb51SJohn Levon } 189*c3377ee9SJohn Levon 190*c3377ee9SJohn Levon /* 191*c3377ee9SJohn Levon * While we don't need to check this until later, we might as well do it 192*c3377ee9SJohn Levon * here. 193*c3377ee9SJohn Levon */ 194*c3377ee9SJohn Levon if (bootprop_getstr("smt_enabled", prop_str, sizeof (prop_str)) == 0) { 195*c3377ee9SJohn Levon if (strcasecmp(prop_str, "false") == 0 || 196*c3377ee9SJohn Levon strcmp(prop_str, "0") == 0) 197*c3377ee9SJohn Levon smt_boot_disable = 1; 198*c3377ee9SJohn Levon } 199*c3377ee9SJohn Levon 20074ecdb51SJohn Levon #endif 20174ecdb51SJohn Levon 20245e032f7SDan Mick /* 20345e032f7SDan Mick * Initialize idt0, gdt0, ldt0_default, ktss0 and dftss. 20445e032f7SDan Mick */ 20545e032f7SDan Mick init_desctbls(); 20645e032f7SDan Mick 20745e032f7SDan Mick /* 20845e032f7SDan Mick * lgrp_init() and possibly cpuid_pass1() need PCI config 20945e032f7SDan Mick * space access 21045e032f7SDan Mick */ 21145e032f7SDan Mick #if defined(__xpv) 21245e032f7SDan Mick if (DOMAIN_IS_INITDOMAIN(xen_info)) 21345e032f7SDan Mick pci_cfgspace_init(); 21445e032f7SDan Mick #else 21545e032f7SDan Mick pci_cfgspace_init(); 216cfe84b82SMatt Amdur /* 217cfe84b82SMatt Amdur * Initialize the platform type from CPU 0 to ensure that 218cfe84b82SMatt Amdur * determine_platform() is only ever called once. 219cfe84b82SMatt Amdur */ 220cfe84b82SMatt Amdur determine_platform(); 22145e032f7SDan Mick #endif 22245e032f7SDan Mick 2237c478bd9Sstevel@tonic-gate /* 2247c478bd9Sstevel@tonic-gate * The first lightweight pass (pass0) through the cpuid data 2257c478bd9Sstevel@tonic-gate * was done in locore before mlsetup was called. Do the next 2267c478bd9Sstevel@tonic-gate * pass in C code. 2277c478bd9Sstevel@tonic-gate * 2287417cfdeSKuriakose Kuruvilla * The x86_featureset is initialized here based on the capabilities 2297c478bd9Sstevel@tonic-gate * of the boot CPU. Note that if we choose to support CPUs that have 2307c478bd9Sstevel@tonic-gate * different feature sets (at which point we would almost certainly 2317c478bd9Sstevel@tonic-gate * want to set the feature bits to correspond to the feature 2327c478bd9Sstevel@tonic-gate * minimum) this value may be altered. 2337c478bd9Sstevel@tonic-gate */ 234dfea898aSKuriakose Kuruvilla cpuid_pass1(cpu[0], x86_featureset); 2357c478bd9Sstevel@tonic-gate 236247dbb3dSsudheer #if !defined(__xpv) 23779ec9da8SYuri Pankov if ((get_hwenv() & HW_XEN_HVM) != 0) 238349b53ddSStuart Maybee xen_hvm_init(); 239349b53ddSStuart Maybee 2404948216cSKeith M Wesolowski /* 2414948216cSKeith M Wesolowski * Before we do anything with the TSCs, we need to work around 2424948216cSKeith M Wesolowski * Intel erratum BT81. On some CPUs, warm reset does not 2434948216cSKeith M Wesolowski * clear the TSC. If we are on such a CPU, we will clear TSC ourselves 2444948216cSKeith M Wesolowski * here. Other CPUs will clear it when we boot them later, and the 2454948216cSKeith M Wesolowski * resulting skew will be handled by tsc_sync_master()/_slave(); 2464948216cSKeith M Wesolowski * note that such skew already exists and has to be handled anyway. 2474948216cSKeith M Wesolowski * 2484948216cSKeith M Wesolowski * We do this only on metal. This same problem can occur with a 2494948216cSKeith M Wesolowski * hypervisor that does not happen to virtualise a TSC that starts from 2504948216cSKeith M Wesolowski * zero, regardless of CPU type; however, we do not expect hypervisors 2514948216cSKeith M Wesolowski * that do not virtualise TSC that way to handle writes to TSC 2524948216cSKeith M Wesolowski * correctly, either. 2534948216cSKeith M Wesolowski */ 2544948216cSKeith M Wesolowski if (get_hwenv() == HW_NATIVE && 2554948216cSKeith M Wesolowski cpuid_getvendor(CPU) == X86_VENDOR_Intel && 2564948216cSKeith M Wesolowski cpuid_getfamily(CPU) == 6 && 2574948216cSKeith M Wesolowski (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) && 2584948216cSKeith M Wesolowski is_x86_feature(x86_featureset, X86FSET_TSC)) { 2594948216cSKeith M Wesolowski (void) wrmsr(REG_TSC, 0UL); 2604948216cSKeith M Wesolowski } 2614948216cSKeith M Wesolowski 262247dbb3dSsudheer /* 263247dbb3dSsudheer * Patch the tsc_read routine with appropriate set of instructions, 264247dbb3dSsudheer * depending on the processor family and architecure, to read the 265247dbb3dSsudheer * time-stamp counter while ensuring no out-of-order execution. 266247dbb3dSsudheer * Patch it while the kernel text is still writable. 267247dbb3dSsudheer * 268247dbb3dSsudheer * Note: tsc_read is not patched for intel processors whose family 269247dbb3dSsudheer * is >6 and for amd whose family >f (in case they don't support rdtscp 270247dbb3dSsudheer * instruction, unlikely). By default tsc_read will use cpuid for 271247dbb3dSsudheer * serialization in such cases. The following code needs to be 272247dbb3dSsudheer * revisited if intel processors of family >= f retains the 273247dbb3dSsudheer * instruction serialization nature of mfence instruction. 2742b0bcb26Ssudheer * Note: tsc_read is not patched for x86 processors which do 2752b0bcb26Ssudheer * not support "mfence". By default tsc_read will use cpuid for 2762b0bcb26Ssudheer * serialization in such cases. 277551bc2a6Smrj * 278551bc2a6Smrj * The Xen hypervisor does not correctly report whether rdtscp is 279551bc2a6Smrj * supported or not, so we must assume that it is not. 280247dbb3dSsudheer */ 28179ec9da8SYuri Pankov if ((get_hwenv() & HW_XEN_HVM) == 0 && 2827417cfdeSKuriakose Kuruvilla is_x86_feature(x86_featureset, X86FSET_TSCP)) 2832428aad8SPatrick Mooney patch_tsc_read(TSC_TSCP); 284247dbb3dSsudheer else if (cpuid_getvendor(CPU) == X86_VENDOR_AMD && 2857417cfdeSKuriakose Kuruvilla cpuid_getfamily(CPU) <= 0xf && 2867417cfdeSKuriakose Kuruvilla is_x86_feature(x86_featureset, X86FSET_SSE2)) 2872428aad8SPatrick Mooney patch_tsc_read(TSC_RDTSC_MFENCE); 288247dbb3dSsudheer else if (cpuid_getvendor(CPU) == X86_VENDOR_Intel && 2897417cfdeSKuriakose Kuruvilla cpuid_getfamily(CPU) <= 6 && 2907417cfdeSKuriakose Kuruvilla is_x86_feature(x86_featureset, X86FSET_SSE2)) 2912428aad8SPatrick Mooney patch_tsc_read(TSC_RDTSC_LFENCE); 292247dbb3dSsudheer 293247dbb3dSsudheer #endif /* !__xpv */ 2947c478bd9Sstevel@tonic-gate 295843e1988Sjohnlev #if defined(__i386) && !defined(__xpv) 2967c478bd9Sstevel@tonic-gate /* 2977c478bd9Sstevel@tonic-gate * Some i386 processors do not implement the rdtsc instruction, 298247dbb3dSsudheer * or at least they do not implement it correctly. Patch them to 299247dbb3dSsudheer * return 0. 3007c478bd9Sstevel@tonic-gate */ 3017417cfdeSKuriakose Kuruvilla if (!is_x86_feature(x86_featureset, X86FSET_TSC)) 3022428aad8SPatrick Mooney patch_tsc_read(TSC_NONE); 303843e1988Sjohnlev #endif /* __i386 && !__xpv */ 304843e1988Sjohnlev 30522cc0e45SBill Holler #if defined(__amd64) && !defined(__xpv) 30622cc0e45SBill Holler patch_memops(cpuid_getvendor(CPU)); 30722cc0e45SBill Holler #endif /* __amd64 && !__xpv */ 30822cc0e45SBill Holler 309843e1988Sjohnlev #if !defined(__xpv) 310843e1988Sjohnlev /* XXPV what, if anything, should be dorked with here under xen? */ 311ae115bc7Smrj 312ae115bc7Smrj /* 313ae115bc7Smrj * While we're thinking about the TSC, let's set up %cr4 so that 314ae115bc7Smrj * userland can issue rdtsc, and initialize the TSC_AUX value 315ae115bc7Smrj * (the cpuid) for the rdtscp instruction on appropriately 316ae115bc7Smrj * capable hardware. 317ae115bc7Smrj */ 3187417cfdeSKuriakose Kuruvilla if (is_x86_feature(x86_featureset, X86FSET_TSC)) 319ae115bc7Smrj setcr4(getcr4() & ~CR4_TSD); 320ae115bc7Smrj 3217417cfdeSKuriakose Kuruvilla if (is_x86_feature(x86_featureset, X86FSET_TSCP)) 322ae115bc7Smrj (void) wrmsr(MSR_AMD_TSCAUX, 0); 323ae115bc7Smrj 324799823bbSRobert Mustacchi /* 3253ce2fcdcSRobert Mustacchi * Let's get the other %cr4 stuff while we're here. Note, we defer 3263ce2fcdcSRobert Mustacchi * enabling CR4_SMAP until startup_end(); however, that's importantly 3273ce2fcdcSRobert Mustacchi * before we start other CPUs. That ensures that it will be synced out 3283ce2fcdcSRobert Mustacchi * to other CPUs. 329799823bbSRobert Mustacchi */ 3307417cfdeSKuriakose Kuruvilla if (is_x86_feature(x86_featureset, X86FSET_DE)) 331ae115bc7Smrj setcr4(getcr4() | CR4_DE); 332799823bbSRobert Mustacchi 333799823bbSRobert Mustacchi if (is_x86_feature(x86_featureset, X86FSET_SMEP)) 334799823bbSRobert Mustacchi setcr4(getcr4() | CR4_SMEP); 335843e1988Sjohnlev #endif /* __xpv */ 3367c478bd9Sstevel@tonic-gate 3377c478bd9Sstevel@tonic-gate /* 3387c478bd9Sstevel@tonic-gate * initialize t0 3397c478bd9Sstevel@tonic-gate */ 3407c478bd9Sstevel@tonic-gate t0.t_stk = (caddr_t)rp - MINFRAME; 3417c478bd9Sstevel@tonic-gate t0.t_stkbase = t0stack; 3427c478bd9Sstevel@tonic-gate t0.t_pri = maxclsyspri - 3; 3437c478bd9Sstevel@tonic-gate t0.t_schedflag = TS_LOAD | TS_DONT_SWAP; 3447c478bd9Sstevel@tonic-gate t0.t_procp = &p0; 3457c478bd9Sstevel@tonic-gate t0.t_plockp = &p0lock.pl_lock; 3467c478bd9Sstevel@tonic-gate t0.t_lwp = &lwp0; 3477c478bd9Sstevel@tonic-gate t0.t_forw = &t0; 3487c478bd9Sstevel@tonic-gate t0.t_back = &t0; 3497c478bd9Sstevel@tonic-gate t0.t_next = &t0; 3507c478bd9Sstevel@tonic-gate t0.t_prev = &t0; 3517c478bd9Sstevel@tonic-gate t0.t_cpu = cpu[0]; 3527c478bd9Sstevel@tonic-gate t0.t_disp_queue = &cpu0_disp; 3537c478bd9Sstevel@tonic-gate t0.t_bind_cpu = PBIND_NONE; 3547c478bd9Sstevel@tonic-gate t0.t_bind_pset = PS_NONE; 3550b70c467Sakolb t0.t_bindflag = (uchar_t)default_binding_mode; 3567c478bd9Sstevel@tonic-gate t0.t_cpupart = &cp_default; 3577c478bd9Sstevel@tonic-gate t0.t_clfuncs = &sys_classfuncs.thread; 3587c478bd9Sstevel@tonic-gate t0.t_copyops = NULL; 3597c478bd9Sstevel@tonic-gate THREAD_ONPROC(&t0, CPU); 3607c478bd9Sstevel@tonic-gate 3617c478bd9Sstevel@tonic-gate lwp0.lwp_thread = &t0; 362ae115bc7Smrj lwp0.lwp_regs = (void *)rp; 3637c478bd9Sstevel@tonic-gate lwp0.lwp_procp = &p0; 3647c478bd9Sstevel@tonic-gate t0.t_tid = p0.p_lwpcnt = p0.p_lwprcnt = p0.p_lwpid = 1; 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate p0.p_exec = NULL; 3677c478bd9Sstevel@tonic-gate p0.p_stat = SRUN; 3687c478bd9Sstevel@tonic-gate p0.p_flag = SSYS; 3697c478bd9Sstevel@tonic-gate p0.p_tlist = &t0; 3707c478bd9Sstevel@tonic-gate p0.p_stksize = 2*PAGESIZE; 3717c478bd9Sstevel@tonic-gate p0.p_stkpageszc = 0; 3727c478bd9Sstevel@tonic-gate p0.p_as = &kas; 3737c478bd9Sstevel@tonic-gate p0.p_lockp = &p0lock; 3747c478bd9Sstevel@tonic-gate p0.p_brkpageszc = 0; 3752cb27123Saguzovsk p0.p_t1_lgrpid = LGRP_NONE; 3762cb27123Saguzovsk p0.p_tr_lgrpid = LGRP_NONE; 377d2a70789SRichard Lowe psecflags_default(&p0.p_secflags); 378d2a70789SRichard Lowe 3797c478bd9Sstevel@tonic-gate sigorset(&p0.p_ignore, &ignoredefault); 3807c478bd9Sstevel@tonic-gate 3817c478bd9Sstevel@tonic-gate CPU->cpu_thread = &t0; 3827c478bd9Sstevel@tonic-gate bzero(&cpu0_disp, sizeof (disp_t)); 3837c478bd9Sstevel@tonic-gate CPU->cpu_disp = &cpu0_disp; 3847c478bd9Sstevel@tonic-gate CPU->cpu_disp->disp_cpu = CPU; 3857c478bd9Sstevel@tonic-gate CPU->cpu_dispthread = &t0; 3867c478bd9Sstevel@tonic-gate CPU->cpu_idle_thread = &t0; 3877c478bd9Sstevel@tonic-gate CPU->cpu_flags = CPU_READY | CPU_RUNNING | CPU_EXISTS | CPU_ENABLE; 3887c478bd9Sstevel@tonic-gate CPU->cpu_dispatch_pri = t0.t_pri; 3897c478bd9Sstevel@tonic-gate 3907c478bd9Sstevel@tonic-gate CPU->cpu_id = 0; 3917c478bd9Sstevel@tonic-gate 3927c478bd9Sstevel@tonic-gate CPU->cpu_pri = 12; /* initial PIL for the boot CPU */ 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate /* 395ae115bc7Smrj * Initialize thread/cpu microstate accounting 3967c478bd9Sstevel@tonic-gate */ 3977c478bd9Sstevel@tonic-gate init_mstate(&t0, LMS_SYSTEM); 3987c478bd9Sstevel@tonic-gate init_cpu_mstate(CPU, CMS_SYSTEM); 3997c478bd9Sstevel@tonic-gate 4007c478bd9Sstevel@tonic-gate /* 4017c478bd9Sstevel@tonic-gate * Initialize lists of available and active CPUs. 4027c478bd9Sstevel@tonic-gate */ 4037c478bd9Sstevel@tonic-gate cpu_list_init(CPU); 4040e751525SEric Saxe 4050e751525SEric Saxe pg_cpu_bootstrap(CPU); 4067c478bd9Sstevel@tonic-gate 407ae115bc7Smrj /* 408ae115bc7Smrj * Now that we have taken over the GDT, IDT and have initialized 409ae115bc7Smrj * active CPU list it's time to inform kmdb if present. 410ae115bc7Smrj */ 411ae115bc7Smrj if (boothowto & RB_DEBUG) 412ae115bc7Smrj kdi_idt_sync(); 413ae115bc7Smrj 414c66b891eSToomas Soome if (BOP_GETPROPLEN(bootops, "efi-systab") < 0) { 415c66b891eSToomas Soome /* 416c66b891eSToomas Soome * In BIOS system, explicitly set console to text mode (0x3) 417c66b891eSToomas Soome * if this is a boot post Fast Reboot, and the console is set 418c66b891eSToomas Soome * to CONS_SCREEN_TEXT. 419c66b891eSToomas Soome */ 420c66b891eSToomas Soome if (post_fastreboot && 421c66b891eSToomas Soome boot_console_type(NULL) == CONS_SCREEN_TEXT) { 422c66b891eSToomas Soome set_console_mode(0x3); 423c66b891eSToomas Soome } 424c66b891eSToomas Soome } 4259db7147eSSherry Moore 426ae115bc7Smrj /* 427ae115bc7Smrj * If requested (boot -d) drop into kmdb. 428ae115bc7Smrj * 429ae115bc7Smrj * This must be done after cpu_list_init() on the 64-bit kernel 430ae115bc7Smrj * since taking a trap requires that we re-compute gsbase based 431ae115bc7Smrj * on the cpu list. 432ae115bc7Smrj */ 433ae115bc7Smrj if (boothowto & RB_DEBUGENTER) 434ae115bc7Smrj kmdb_enter(); 435ae115bc7Smrj 436affbd3ccSkchow cpu_vm_data_init(CPU); 437affbd3ccSkchow 4387c478bd9Sstevel@tonic-gate rp->r_fp = 0; /* terminate kernel stack traces! */ 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate prom_init("kernel", (void *)NULL); 4417c478bd9Sstevel@tonic-gate 442a3114836SGerry Liu /* User-set option overrides firmware value. */ 443a3114836SGerry Liu if (bootprop_getval(PLAT_DR_OPTIONS_NAME, &prop_value) == 0) { 444a3114836SGerry Liu plat_dr_options = (uint64_t)prop_value; 445a3114836SGerry Liu } 446a3114836SGerry Liu #if defined(__xpv) 447a3114836SGerry Liu /* No support of DR operations on xpv */ 448a3114836SGerry Liu plat_dr_options = 0; 449a3114836SGerry Liu #else /* __xpv */ 450a3114836SGerry Liu /* Flag PLAT_DR_FEATURE_ENABLED should only be set by DR driver. */ 451a3114836SGerry Liu plat_dr_options &= ~PLAT_DR_FEATURE_ENABLED; 452a3114836SGerry Liu #ifndef __amd64 453a3114836SGerry Liu /* Only enable CPU/memory DR on 64 bits kernel. */ 454a3114836SGerry Liu plat_dr_options &= ~PLAT_DR_FEATURE_MEMORY; 455a3114836SGerry Liu plat_dr_options &= ~PLAT_DR_FEATURE_CPU; 456a3114836SGerry Liu #endif /* __amd64 */ 457a3114836SGerry Liu #endif /* __xpv */ 458a3114836SGerry Liu 459a3114836SGerry Liu /* 460a3114836SGerry Liu * Get value of "plat_dr_physmax" boot option. 461a3114836SGerry Liu * It overrides values calculated from MSCT or SRAT table. 462a3114836SGerry Liu */ 463a3114836SGerry Liu if (bootprop_getval(PLAT_DR_PHYSMAX_NAME, &prop_value) == 0) { 464a3114836SGerry Liu plat_dr_physmax = ((uint64_t)prop_value) >> PAGESHIFT; 465a3114836SGerry Liu } 466a3114836SGerry Liu 467a3114836SGerry Liu /* Get value of boot_ncpus. */ 468a3114836SGerry Liu if (bootprop_getval(BOOT_NCPUS_NAME, &prop_value) != 0) { 469ae115bc7Smrj boot_ncpus = NCPU; 470a3114836SGerry Liu } else { 4712baa66a0SJonathan Chew boot_ncpus = (int)prop_value; 4722baa66a0SJonathan Chew if (boot_ncpus <= 0 || boot_ncpus > NCPU) 4732baa66a0SJonathan Chew boot_ncpus = NCPU; 4742baa66a0SJonathan Chew } 47541791439Sandrei 476a3114836SGerry Liu /* 477a3114836SGerry Liu * Set max_ncpus and boot_max_ncpus to boot_ncpus if platform doesn't 478a3114836SGerry Liu * support CPU DR operations. 479a3114836SGerry Liu */ 480a3114836SGerry Liu if (plat_dr_support_cpu() == 0) { 481a3114836SGerry Liu max_ncpus = boot_max_ncpus = boot_ncpus; 482a3114836SGerry Liu } else { 483a3114836SGerry Liu if (bootprop_getval(PLAT_MAX_NCPUS_NAME, &prop_value) != 0) { 484a3114836SGerry Liu max_ncpus = NCPU; 485a3114836SGerry Liu } else { 486a3114836SGerry Liu max_ncpus = (int)prop_value; 487a3114836SGerry Liu if (max_ncpus <= 0 || max_ncpus > NCPU) { 488a3114836SGerry Liu max_ncpus = NCPU; 489a3114836SGerry Liu } 490a3114836SGerry Liu if (boot_ncpus > max_ncpus) { 491a3114836SGerry Liu boot_ncpus = max_ncpus; 492a3114836SGerry Liu } 493a3114836SGerry Liu } 494a3114836SGerry Liu 495a3114836SGerry Liu if (bootprop_getval(BOOT_MAX_NCPUS_NAME, &prop_value) != 0) { 496a3114836SGerry Liu boot_max_ncpus = boot_ncpus; 497a3114836SGerry Liu } else { 498a3114836SGerry Liu boot_max_ncpus = (int)prop_value; 499a3114836SGerry Liu if (boot_max_ncpus <= 0 || boot_max_ncpus > NCPU) { 500a3114836SGerry Liu boot_max_ncpus = boot_ncpus; 501a3114836SGerry Liu } else if (boot_max_ncpus > max_ncpus) { 502a3114836SGerry Liu boot_max_ncpus = max_ncpus; 503a3114836SGerry Liu } 504a3114836SGerry Liu } 505a3114836SGerry Liu } 50641791439Sandrei 5072e2c009bSjjc /* 5082e2c009bSjjc * Initialize the lgrp framework 5092e2c009bSjjc */ 510d5d7cf4eSJonathan Chew lgrp_init(LGRP_INIT_STAGE1); 5112e2c009bSjjc 5127c478bd9Sstevel@tonic-gate if (boothowto & RB_HALT) { 5137c478bd9Sstevel@tonic-gate prom_printf("unix: kernel halted by -h flag\n"); 5147c478bd9Sstevel@tonic-gate prom_enter_mon(); 5157c478bd9Sstevel@tonic-gate } 5167c478bd9Sstevel@tonic-gate 5177c478bd9Sstevel@tonic-gate ASSERT_STACK_ALIGNED(); 5187c478bd9Sstevel@tonic-gate 5192449e17fSsherrym /* 5202449e17fSsherrym * Fill out cpu_ucode_info. Update microcode if necessary. 5212449e17fSsherrym */ 5222449e17fSsherrym ucode_check(CPU); 52301add34aSRobert Mustacchi cpuid_pass_ucode(CPU, x86_featureset); 5242449e17fSsherrym 5257c478bd9Sstevel@tonic-gate if (workaround_errata(CPU) != 0) 5267c478bd9Sstevel@tonic-gate panic("critical workaround(s) missing for boot cpu"); 5277c478bd9Sstevel@tonic-gate } 528986fd29aSsetje 529986fd29aSsetje 530986fd29aSsetje void 531986fd29aSsetje mach_modpath(char *path, const char *filename) 532986fd29aSsetje { 533986fd29aSsetje /* 534986fd29aSsetje * Construct the directory path from the filename. 535986fd29aSsetje */ 536986fd29aSsetje 537986fd29aSsetje int len; 538986fd29aSsetje char *p; 539986fd29aSsetje const char isastr[] = "/amd64"; 540986fd29aSsetje size_t isalen = strlen(isastr); 541986fd29aSsetje 5420181461bSKeith M Wesolowski len = strlen(SYSTEM_BOOT_PATH "/kernel"); 5430181461bSKeith M Wesolowski (void) strcpy(path, SYSTEM_BOOT_PATH "/kernel "); 5440181461bSKeith M Wesolowski path += len + 1; 5450181461bSKeith M Wesolowski 546986fd29aSsetje if ((p = strrchr(filename, '/')) == NULL) 547986fd29aSsetje return; 548986fd29aSsetje 549986fd29aSsetje while (p > filename && *(p - 1) == '/') 550986fd29aSsetje p--; /* remove trailing '/' characters */ 551986fd29aSsetje if (p == filename) 552986fd29aSsetje p++; /* so "/" -is- the modpath in this case */ 553986fd29aSsetje 554986fd29aSsetje /* 555986fd29aSsetje * Remove optional isa-dependent directory name - the module 556986fd29aSsetje * subsystem will put this back again (!) 557986fd29aSsetje */ 558986fd29aSsetje len = p - filename; 559986fd29aSsetje if (len > isalen && 560986fd29aSsetje strncmp(&filename[len - isalen], isastr, isalen) == 0) 561986fd29aSsetje p -= isalen; 562986fd29aSsetje 563986fd29aSsetje /* 564986fd29aSsetje * "/platform/mumblefrotz" + " " + MOD_DEFPATH 565986fd29aSsetje */ 566986fd29aSsetje len += (p - filename) + 1 + strlen(MOD_DEFPATH) + 1; 567986fd29aSsetje (void) strncpy(path, filename, p - filename); 568986fd29aSsetje } 569