17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 23b4b46911Skchow * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/types.h> 307c478bd9Sstevel@tonic-gate #include <sys/thread.h> 317c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 327c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 337c478bd9Sstevel@tonic-gate #include <sys/param.h> 347c478bd9Sstevel@tonic-gate #include <sys/proc.h> 357c478bd9Sstevel@tonic-gate #include <sys/disp.h> 367c478bd9Sstevel@tonic-gate #include <sys/mmu.h> 377c478bd9Sstevel@tonic-gate #include <sys/class.h> 387c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 397c478bd9Sstevel@tonic-gate #include <sys/debug.h> 407c478bd9Sstevel@tonic-gate #include <sys/asm_linkage.h> 417c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 427c478bd9Sstevel@tonic-gate #include <sys/systm.h> 437c478bd9Sstevel@tonic-gate #include <sys/var.h> 447c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 457c478bd9Sstevel@tonic-gate #include <vm/hat.h> 467c478bd9Sstevel@tonic-gate #include <sys/mmu.h> 477c478bd9Sstevel@tonic-gate #include <vm/as.h> 487c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 497c478bd9Sstevel@tonic-gate #include <sys/segments.h> 507c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 517c478bd9Sstevel@tonic-gate #include <sys/stack.h> 527c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h> 537c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 547c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 557c478bd9Sstevel@tonic-gate #include <sys/traptrace.h> 567c478bd9Sstevel@tonic-gate #include <sys/clock.h> 577c478bd9Sstevel@tonic-gate #include <sys/cpc_impl.h> 587c478bd9Sstevel@tonic-gate #include <sys/chip.h> 597c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 607c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 617c478bd9Sstevel@tonic-gate #include <sys/fp.h> 627c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 637c478bd9Sstevel@tonic-gate #include <sys/kdi.h> 647c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h> 657c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 66ef50d8c0Sesaxe #include <sys/pci_cfgspace.h> 67*7aec1d6eScindi #include <sys/cpu_module.h> 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate struct cpu cpus[1]; /* CPU data */ 707c478bd9Sstevel@tonic-gate struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 717c478bd9Sstevel@tonic-gate cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 727c478bd9Sstevel@tonic-gate 737c478bd9Sstevel@tonic-gate /* 747c478bd9Sstevel@tonic-gate * Useful for disabling MP bring-up for an MP capable kernel 757c478bd9Sstevel@tonic-gate * (a kernel that was built with MP defined) 767c478bd9Sstevel@tonic-gate */ 777c478bd9Sstevel@tonic-gate int use_mp = 1; 787c478bd9Sstevel@tonic-gate 797c478bd9Sstevel@tonic-gate int mp_cpus = 0x1; /* to be set by platform specific module */ 807c478bd9Sstevel@tonic-gate 817c478bd9Sstevel@tonic-gate /* 827c478bd9Sstevel@tonic-gate * This variable is used by the hat layer to decide whether or not 837c478bd9Sstevel@tonic-gate * critical sections are needed to prevent race conditions. For sun4m, 847c478bd9Sstevel@tonic-gate * this variable is set once enough MP initialization has been done in 857c478bd9Sstevel@tonic-gate * order to allow cross calls. 867c478bd9Sstevel@tonic-gate */ 877c478bd9Sstevel@tonic-gate int flushes_require_xcalls = 0; 887c478bd9Sstevel@tonic-gate ulong_t cpu_ready_set = 1; 897c478bd9Sstevel@tonic-gate 907c478bd9Sstevel@tonic-gate extern void real_mode_start(void); 917c478bd9Sstevel@tonic-gate extern void real_mode_end(void); 927c478bd9Sstevel@tonic-gate static void mp_startup(void); 937c478bd9Sstevel@tonic-gate 947c478bd9Sstevel@tonic-gate static void cpu_sep_enable(void); 957c478bd9Sstevel@tonic-gate static void cpu_sep_disable(void); 967c478bd9Sstevel@tonic-gate static void cpu_asysc_enable(void); 977c478bd9Sstevel@tonic-gate static void cpu_asysc_disable(void); 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate extern int tsc_gethrtime_enable; 1007c478bd9Sstevel@tonic-gate 1017c478bd9Sstevel@tonic-gate /* 1027c478bd9Sstevel@tonic-gate * Init CPU info - get CPU type info for processor_info system call. 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate void 1057c478bd9Sstevel@tonic-gate init_cpu_info(struct cpu *cp) 1067c478bd9Sstevel@tonic-gate { 1077c478bd9Sstevel@tonic-gate processor_info_t *pi = &cp->cpu_type_info; 1087c478bd9Sstevel@tonic-gate char buf[CPU_IDSTRLEN]; 1097c478bd9Sstevel@tonic-gate 1107c478bd9Sstevel@tonic-gate /* 1117c478bd9Sstevel@tonic-gate * Get clock-frequency property for the CPU. 1127c478bd9Sstevel@tonic-gate */ 1137c478bd9Sstevel@tonic-gate pi->pi_clock = cpu_freq; 1147c478bd9Sstevel@tonic-gate 1157c478bd9Sstevel@tonic-gate (void) strcpy(pi->pi_processor_type, "i386"); 1167c478bd9Sstevel@tonic-gate if (fpu_exists) 1177c478bd9Sstevel@tonic-gate (void) strcpy(pi->pi_fputypes, "i387 compatible"); 1187c478bd9Sstevel@tonic-gate 1197c478bd9Sstevel@tonic-gate (void) cpuid_getidstr(cp, buf, sizeof (buf)); 1207c478bd9Sstevel@tonic-gate 1217c478bd9Sstevel@tonic-gate cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 1227c478bd9Sstevel@tonic-gate (void) strcpy(cp->cpu_idstr, buf); 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 1277c478bd9Sstevel@tonic-gate cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 1287c478bd9Sstevel@tonic-gate (void) strcpy(cp->cpu_brandstr, buf); 1297c478bd9Sstevel@tonic-gate 1307c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 1317c478bd9Sstevel@tonic-gate } 1327c478bd9Sstevel@tonic-gate 1337c478bd9Sstevel@tonic-gate /* 1347c478bd9Sstevel@tonic-gate * Configure syscall support on this CPU. 1357c478bd9Sstevel@tonic-gate */ 1367c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1377c478bd9Sstevel@tonic-gate static void 1387c478bd9Sstevel@tonic-gate init_cpu_syscall(struct cpu *cp) 1397c478bd9Sstevel@tonic-gate { 1407c478bd9Sstevel@tonic-gate kpreempt_disable(); 1417c478bd9Sstevel@tonic-gate 1427c478bd9Sstevel@tonic-gate #if defined(__amd64) 1437c478bd9Sstevel@tonic-gate if (x86_feature & X86_ASYSC) { 1447c478bd9Sstevel@tonic-gate 1457c478bd9Sstevel@tonic-gate #if !defined(__lint) 1467c478bd9Sstevel@tonic-gate /* 1477c478bd9Sstevel@tonic-gate * The syscall instruction imposes a certain ordering on 1487c478bd9Sstevel@tonic-gate * segment selectors, so we double-check that ordering 1497c478bd9Sstevel@tonic-gate * here. 1507c478bd9Sstevel@tonic-gate */ 1517c478bd9Sstevel@tonic-gate ASSERT(KDS_SEL == KCS_SEL + 8); 1527c478bd9Sstevel@tonic-gate ASSERT(UDS_SEL == U32CS_SEL + 8); 1537c478bd9Sstevel@tonic-gate ASSERT(UCS_SEL == U32CS_SEL + 16); 1547c478bd9Sstevel@tonic-gate #endif 1557c478bd9Sstevel@tonic-gate /* 1567c478bd9Sstevel@tonic-gate * Turn syscall/sysret extensions on. 1577c478bd9Sstevel@tonic-gate */ 1587c478bd9Sstevel@tonic-gate cpu_asysc_enable(); 1597c478bd9Sstevel@tonic-gate 1607c478bd9Sstevel@tonic-gate /* 1617c478bd9Sstevel@tonic-gate * Program the magic registers .. 1627c478bd9Sstevel@tonic-gate */ 1630ac7d7d8Skucharsk wrmsr(MSR_AMD_STAR, ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 1640ac7d7d8Skucharsk 32); 1650ac7d7d8Skucharsk wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall); 1660ac7d7d8Skucharsk wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32); 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate /* 1697c478bd9Sstevel@tonic-gate * This list of flags is masked off the incoming 1707c478bd9Sstevel@tonic-gate * %rfl when we enter the kernel. 1717c478bd9Sstevel@tonic-gate */ 1720ac7d7d8Skucharsk wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T)); 1737c478bd9Sstevel@tonic-gate } 1747c478bd9Sstevel@tonic-gate #endif 1757c478bd9Sstevel@tonic-gate 1767c478bd9Sstevel@tonic-gate /* 1777c478bd9Sstevel@tonic-gate * On 32-bit kernels, we use sysenter/sysexit because it's too 1787c478bd9Sstevel@tonic-gate * hard to use syscall/sysret, and it is more portable anyway. 1797c478bd9Sstevel@tonic-gate * 1807c478bd9Sstevel@tonic-gate * On 64-bit kernels on Nocona machines, the 32-bit syscall 1817c478bd9Sstevel@tonic-gate * variant isn't available to 32-bit applications, but sysenter is. 1827c478bd9Sstevel@tonic-gate */ 1837c478bd9Sstevel@tonic-gate if (x86_feature & X86_SEP) { 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate #if !defined(__lint) 1867c478bd9Sstevel@tonic-gate /* 1877c478bd9Sstevel@tonic-gate * The sysenter instruction imposes a certain ordering on 1887c478bd9Sstevel@tonic-gate * segment selectors, so we double-check that ordering 1897c478bd9Sstevel@tonic-gate * here. See "sysenter" in Intel document 245471-012, "IA-32 1907c478bd9Sstevel@tonic-gate * Intel Architecture Software Developer's Manual Volume 2: 1917c478bd9Sstevel@tonic-gate * Instruction Set Reference" 1927c478bd9Sstevel@tonic-gate */ 1937c478bd9Sstevel@tonic-gate ASSERT(KDS_SEL == KCS_SEL + 8); 1947c478bd9Sstevel@tonic-gate 1957c478bd9Sstevel@tonic-gate ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 1967c478bd9Sstevel@tonic-gate ASSERT32(UDS_SEL == UCS_SEL + 8); 1977c478bd9Sstevel@tonic-gate 1987c478bd9Sstevel@tonic-gate ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 1997c478bd9Sstevel@tonic-gate ASSERT64(UDS_SEL == U32CS_SEL + 8); 2007c478bd9Sstevel@tonic-gate #endif 2017c478bd9Sstevel@tonic-gate 2027c478bd9Sstevel@tonic-gate cpu_sep_enable(); 2037c478bd9Sstevel@tonic-gate 2047c478bd9Sstevel@tonic-gate /* 2057c478bd9Sstevel@tonic-gate * resume() sets this value to the base of the threads stack 2067c478bd9Sstevel@tonic-gate * via a context handler. 2077c478bd9Sstevel@tonic-gate */ 2080ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_ESP, 0ULL); 2090ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter); 2107c478bd9Sstevel@tonic-gate } 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate kpreempt_enable(); 2137c478bd9Sstevel@tonic-gate } 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate /* 2167c478bd9Sstevel@tonic-gate * Multiprocessor initialization. 2177c478bd9Sstevel@tonic-gate * 2187c478bd9Sstevel@tonic-gate * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 2197c478bd9Sstevel@tonic-gate * startup and idle threads for the specified CPU. 2207c478bd9Sstevel@tonic-gate */ 2217c478bd9Sstevel@tonic-gate static void 2227c478bd9Sstevel@tonic-gate mp_startup_init(int cpun) 2237c478bd9Sstevel@tonic-gate { 2247c478bd9Sstevel@tonic-gate #if defined(__amd64) 2257c478bd9Sstevel@tonic-gate extern void *long_mode_64(void); 2267c478bd9Sstevel@tonic-gate #endif /* __amd64 */ 2277c478bd9Sstevel@tonic-gate 2287c478bd9Sstevel@tonic-gate struct cpu *cp; 2297c478bd9Sstevel@tonic-gate struct tss *ntss; 2307c478bd9Sstevel@tonic-gate kthread_id_t tp; 2317c478bd9Sstevel@tonic-gate caddr_t sp; 2327c478bd9Sstevel@tonic-gate int size; 2337c478bd9Sstevel@tonic-gate proc_t *procp; 2347c478bd9Sstevel@tonic-gate extern void idle(); 2357c478bd9Sstevel@tonic-gate extern void init_intr_threads(struct cpu *); 2367c478bd9Sstevel@tonic-gate 2377c478bd9Sstevel@tonic-gate struct cpu_tables *tablesp; 2387c478bd9Sstevel@tonic-gate rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 2397c478bd9Sstevel@tonic-gate 2407c478bd9Sstevel@tonic-gate #ifdef TRAPTRACE 2417c478bd9Sstevel@tonic-gate trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 2427c478bd9Sstevel@tonic-gate #endif 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate ASSERT(cpun < NCPU && cpu[cpun] == NULL); 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 2477c478bd9Sstevel@tonic-gate panic("mp_startup_init: cpu%d: " 2487c478bd9Sstevel@tonic-gate "no memory for cpu structure", cpun); 2497c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 2507c478bd9Sstevel@tonic-gate } 2517c478bd9Sstevel@tonic-gate procp = curthread->t_procp; 2527c478bd9Sstevel@tonic-gate 2537c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 2547c478bd9Sstevel@tonic-gate /* 2557c478bd9Sstevel@tonic-gate * Initialize the dispatcher first. 2567c478bd9Sstevel@tonic-gate */ 2577c478bd9Sstevel@tonic-gate disp_cpu_init(cp); 2587c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2597c478bd9Sstevel@tonic-gate 260affbd3ccSkchow cpu_vm_data_init(cp); 261affbd3ccSkchow 2627c478bd9Sstevel@tonic-gate /* 2637c478bd9Sstevel@tonic-gate * Allocate and initialize the startup thread for this CPU. 2647c478bd9Sstevel@tonic-gate * Interrupt and process switch stacks get allocated later 2657c478bd9Sstevel@tonic-gate * when the CPU starts running. 2667c478bd9Sstevel@tonic-gate */ 2677c478bd9Sstevel@tonic-gate tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 2687c478bd9Sstevel@tonic-gate TS_STOPPED, maxclsyspri); 2697c478bd9Sstevel@tonic-gate 2707c478bd9Sstevel@tonic-gate /* 2717c478bd9Sstevel@tonic-gate * Set state to TS_ONPROC since this thread will start running 2727c478bd9Sstevel@tonic-gate * as soon as the CPU comes online. 2737c478bd9Sstevel@tonic-gate * 2747c478bd9Sstevel@tonic-gate * All the other fields of the thread structure are setup by 2757c478bd9Sstevel@tonic-gate * thread_create(). 2767c478bd9Sstevel@tonic-gate */ 2777c478bd9Sstevel@tonic-gate THREAD_ONPROC(tp, cp); 2787c478bd9Sstevel@tonic-gate tp->t_preempt = 1; 2797c478bd9Sstevel@tonic-gate tp->t_bound_cpu = cp; 2807c478bd9Sstevel@tonic-gate tp->t_affinitycnt = 1; 2817c478bd9Sstevel@tonic-gate tp->t_cpu = cp; 2827c478bd9Sstevel@tonic-gate tp->t_disp_queue = cp->cpu_disp; 2837c478bd9Sstevel@tonic-gate 2847c478bd9Sstevel@tonic-gate /* 2857c478bd9Sstevel@tonic-gate * Setup thread to start in mp_startup. 2867c478bd9Sstevel@tonic-gate */ 2877c478bd9Sstevel@tonic-gate sp = tp->t_stk; 2887c478bd9Sstevel@tonic-gate tp->t_pc = (uintptr_t)mp_startup; 2897c478bd9Sstevel@tonic-gate tp->t_sp = (uintptr_t)(sp - MINFRAME); 2907c478bd9Sstevel@tonic-gate 2917c478bd9Sstevel@tonic-gate cp->cpu_id = cpun; 2927c478bd9Sstevel@tonic-gate cp->cpu_self = cp; 2937c478bd9Sstevel@tonic-gate cp->cpu_mask = 1 << cpun; 2947c478bd9Sstevel@tonic-gate cp->cpu_thread = tp; 2957c478bd9Sstevel@tonic-gate cp->cpu_lwp = NULL; 2967c478bd9Sstevel@tonic-gate cp->cpu_dispthread = tp; 2977c478bd9Sstevel@tonic-gate cp->cpu_dispatch_pri = DISP_PRIO(tp); 2987c478bd9Sstevel@tonic-gate 2997c478bd9Sstevel@tonic-gate /* 3007c478bd9Sstevel@tonic-gate * Now, initialize per-CPU idle thread for this CPU. 3017c478bd9Sstevel@tonic-gate */ 3027c478bd9Sstevel@tonic-gate tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate cp->cpu_idle_thread = tp; 3057c478bd9Sstevel@tonic-gate 3067c478bd9Sstevel@tonic-gate tp->t_preempt = 1; 3077c478bd9Sstevel@tonic-gate tp->t_bound_cpu = cp; 3087c478bd9Sstevel@tonic-gate tp->t_affinitycnt = 1; 3097c478bd9Sstevel@tonic-gate tp->t_cpu = cp; 3107c478bd9Sstevel@tonic-gate tp->t_disp_queue = cp->cpu_disp; 3117c478bd9Sstevel@tonic-gate 312394b433dSesaxe /* 313394b433dSesaxe * Bootstrap the CPU for CMT aware scheduling 314394b433dSesaxe * The rest of the initialization will happen from 315394b433dSesaxe * mp_startup() 316394b433dSesaxe */ 317394b433dSesaxe chip_bootstrap_cpu(cp); 318394b433dSesaxe 3197c478bd9Sstevel@tonic-gate /* 3207c478bd9Sstevel@tonic-gate * Perform CPC intialization on the new CPU. 3217c478bd9Sstevel@tonic-gate */ 3227c478bd9Sstevel@tonic-gate kcpc_hw_init(cp); 3237c478bd9Sstevel@tonic-gate 3247c478bd9Sstevel@tonic-gate /* 3257c478bd9Sstevel@tonic-gate * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 3267c478bd9Sstevel@tonic-gate * for each CPU. 3277c478bd9Sstevel@tonic-gate */ 3287c478bd9Sstevel@tonic-gate 3297c478bd9Sstevel@tonic-gate setup_vaddr_for_ppcopy(cp); 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate /* 3327c478bd9Sstevel@tonic-gate * Allocate space for page directory, stack, tss, gdt and idt. 3337c478bd9Sstevel@tonic-gate * This assumes that kmem_alloc will return memory which is aligned 3347c478bd9Sstevel@tonic-gate * to the next higher power of 2 or a page(if size > MAXABIG) 3357c478bd9Sstevel@tonic-gate * If this assumption goes wrong at any time due to change in 3367c478bd9Sstevel@tonic-gate * kmem alloc, things may not work as the page directory has to be 3377c478bd9Sstevel@tonic-gate * page aligned 3387c478bd9Sstevel@tonic-gate */ 3397c478bd9Sstevel@tonic-gate if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 3407c478bd9Sstevel@tonic-gate panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 3437c478bd9Sstevel@tonic-gate kmem_free(tablesp, sizeof (struct cpu_tables)); 3447c478bd9Sstevel@tonic-gate size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 3457c478bd9Sstevel@tonic-gate tablesp = kmem_zalloc(size, KM_NOSLEEP); 3467c478bd9Sstevel@tonic-gate tablesp = (struct cpu_tables *) 3477c478bd9Sstevel@tonic-gate (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 3487c478bd9Sstevel@tonic-gate MMU_STD_PAGEMASK); 3497c478bd9Sstevel@tonic-gate } 3507c478bd9Sstevel@tonic-gate 3517c478bd9Sstevel@tonic-gate ntss = cp->cpu_tss = &tablesp->ct_tss; 3527c478bd9Sstevel@tonic-gate cp->cpu_gdt = tablesp->ct_gdt; 3537c478bd9Sstevel@tonic-gate bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate #if defined(__amd64) 3567c478bd9Sstevel@tonic-gate 3577c478bd9Sstevel@tonic-gate /* 3587c478bd9Sstevel@tonic-gate * #DF (double fault). 3597c478bd9Sstevel@tonic-gate */ 3607c478bd9Sstevel@tonic-gate ntss->tss_ist1 = 3617c478bd9Sstevel@tonic-gate (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate #elif defined(__i386) 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 3667c478bd9Sstevel@tonic-gate (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 3677c478bd9Sstevel@tonic-gate 3687c478bd9Sstevel@tonic-gate ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate ntss->tss_eip = (uint32_t)mp_startup; 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate ntss->tss_cs = KCS_SEL; 3737c478bd9Sstevel@tonic-gate ntss->tss_fs = KFS_SEL; 3747c478bd9Sstevel@tonic-gate ntss->tss_gs = KGS_SEL; 3757c478bd9Sstevel@tonic-gate 3767c478bd9Sstevel@tonic-gate /* 3777c478bd9Sstevel@tonic-gate * setup kernel %gs. 3787c478bd9Sstevel@tonic-gate */ 3797c478bd9Sstevel@tonic-gate set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 3807c478bd9Sstevel@tonic-gate SEL_KPL, 0, 1); 3817c478bd9Sstevel@tonic-gate 3827c478bd9Sstevel@tonic-gate #endif /* __i386 */ 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate /* 3857c478bd9Sstevel@tonic-gate * Set I/O bit map offset equal to size of TSS segment limit 3867c478bd9Sstevel@tonic-gate * for no I/O permission map. This will cause all user I/O 3877c478bd9Sstevel@tonic-gate * instructions to generate #gp fault. 3887c478bd9Sstevel@tonic-gate */ 3897c478bd9Sstevel@tonic-gate ntss->tss_bitmapbase = sizeof (*ntss); 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate /* 3927c478bd9Sstevel@tonic-gate * setup kernel tss. 3937c478bd9Sstevel@tonic-gate */ 3947c478bd9Sstevel@tonic-gate set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 3957c478bd9Sstevel@tonic-gate sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 3967c478bd9Sstevel@tonic-gate 3977c478bd9Sstevel@tonic-gate /* 3987c478bd9Sstevel@tonic-gate * If we have more than one node, each cpu gets a copy of IDT 3997c478bd9Sstevel@tonic-gate * local to its node. If this is a Pentium box, we use cpu 0's 4007c478bd9Sstevel@tonic-gate * IDT. cpu 0's IDT has been made read-only to workaround the 4017c478bd9Sstevel@tonic-gate * cmpxchgl register bug 4027c478bd9Sstevel@tonic-gate */ 4037c478bd9Sstevel@tonic-gate cp->cpu_idt = CPU->cpu_idt; 4047c478bd9Sstevel@tonic-gate if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 4057c478bd9Sstevel@tonic-gate cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 4067c478bd9Sstevel@tonic-gate bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate /* 4107c478bd9Sstevel@tonic-gate * Get interrupt priority data from cpu 0 4117c478bd9Sstevel@tonic-gate */ 4127c478bd9Sstevel@tonic-gate cp->cpu_pri_data = CPU->cpu_pri_data; 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate hat_cpu_online(cp); 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate /* Should remove all entries for the current process/thread here */ 4177c478bd9Sstevel@tonic-gate 4187c478bd9Sstevel@tonic-gate /* 4197c478bd9Sstevel@tonic-gate * Fill up the real mode platter to make it easy for real mode code to 4207c478bd9Sstevel@tonic-gate * kick it off. This area should really be one passed by boot to kernel 4217c478bd9Sstevel@tonic-gate * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 4227c478bd9Sstevel@tonic-gate * have identical physical and virtual address in paged mode. 4237c478bd9Sstevel@tonic-gate */ 4247c478bd9Sstevel@tonic-gate real_mode_platter->rm_idt_base = cp->cpu_idt; 4257c478bd9Sstevel@tonic-gate real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 4267c478bd9Sstevel@tonic-gate real_mode_platter->rm_gdt_base = cp->cpu_gdt; 4277c478bd9Sstevel@tonic-gate real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 4287c478bd9Sstevel@tonic-gate real_mode_platter->rm_pdbr = getcr3(); 4297c478bd9Sstevel@tonic-gate real_mode_platter->rm_cpu = cpun; 4307c478bd9Sstevel@tonic-gate real_mode_platter->rm_x86feature = x86_feature; 4317c478bd9Sstevel@tonic-gate real_mode_platter->rm_cr4 = cr4_value; 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate #if defined(__amd64) 4347c478bd9Sstevel@tonic-gate if (getcr3() > 0xffffffffUL) 4357c478bd9Sstevel@tonic-gate panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 4367c478bd9Sstevel@tonic-gate "located above 4G in physical memory (@ 0x%llx).", 4377c478bd9Sstevel@tonic-gate (unsigned long long)getcr3()); 4387c478bd9Sstevel@tonic-gate 4397c478bd9Sstevel@tonic-gate /* 4407c478bd9Sstevel@tonic-gate * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 4417c478bd9Sstevel@tonic-gate * by code in real_mode_start(): 4427c478bd9Sstevel@tonic-gate * 4437c478bd9Sstevel@tonic-gate * GDT[0]: NULL selector 4447c478bd9Sstevel@tonic-gate * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 4457c478bd9Sstevel@tonic-gate * 4467c478bd9Sstevel@tonic-gate * Clear the IDT as interrupts will be off and a limit of 0 will cause 4477c478bd9Sstevel@tonic-gate * the CPU to triple fault and reset on an NMI, seemingly as reasonable 4487c478bd9Sstevel@tonic-gate * a course of action as any other, though it may cause the entire 4497c478bd9Sstevel@tonic-gate * platform to reset in some cases... 4507c478bd9Sstevel@tonic-gate */ 4517c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt[0] = 0ULL; 4527c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 4537c478bd9Sstevel@tonic-gate 4547c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt_lim = (ushort_t) 4557c478bd9Sstevel@tonic-gate (sizeof (real_mode_platter->rm_temp_gdt) - 1); 4567c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 4577c478bd9Sstevel@tonic-gate (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_idt_lim = 0; 4607c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_idt_base = 0; 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate /* 4637c478bd9Sstevel@tonic-gate * Since the CPU needs to jump to protected mode using an identity 4647c478bd9Sstevel@tonic-gate * mapped address, we need to calculate it here. 4657c478bd9Sstevel@tonic-gate */ 4667c478bd9Sstevel@tonic-gate real_mode_platter->rm_longmode64_addr = rm_platter_pa + 4677c478bd9Sstevel@tonic-gate ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 4687c478bd9Sstevel@tonic-gate #endif /* __amd64 */ 4697c478bd9Sstevel@tonic-gate 4707c478bd9Sstevel@tonic-gate #ifdef TRAPTRACE 4717c478bd9Sstevel@tonic-gate /* 4727c478bd9Sstevel@tonic-gate * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 4737c478bd9Sstevel@tonic-gate * CPU. 4747c478bd9Sstevel@tonic-gate */ 4757c478bd9Sstevel@tonic-gate ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 4767c478bd9Sstevel@tonic-gate ttc->ttc_next = ttc->ttc_first; 4777c478bd9Sstevel@tonic-gate ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 4787c478bd9Sstevel@tonic-gate #endif 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate /* 4817c478bd9Sstevel@tonic-gate * Record that we have another CPU. 4827c478bd9Sstevel@tonic-gate */ 4837c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 4847c478bd9Sstevel@tonic-gate /* 4857c478bd9Sstevel@tonic-gate * Initialize the interrupt threads for this CPU 4867c478bd9Sstevel@tonic-gate */ 4877c478bd9Sstevel@tonic-gate init_intr_threads(cp); 4887c478bd9Sstevel@tonic-gate /* 4897c478bd9Sstevel@tonic-gate * Add CPU to list of available CPUs. It'll be on the active list 4907c478bd9Sstevel@tonic-gate * after mp_startup(). 4917c478bd9Sstevel@tonic-gate */ 4927c478bd9Sstevel@tonic-gate cpu_add_unit(cp); 4937c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 4947c478bd9Sstevel@tonic-gate } 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate /* 4977c478bd9Sstevel@tonic-gate * Apply workarounds for known errata, and warn about those that are absent. 4987c478bd9Sstevel@tonic-gate * 4997c478bd9Sstevel@tonic-gate * System vendors occasionally create configurations which contain different 5007c478bd9Sstevel@tonic-gate * revisions of the CPUs that are almost but not exactly the same. At the 5017c478bd9Sstevel@tonic-gate * time of writing, this meant that their clock rates were the same, their 5027c478bd9Sstevel@tonic-gate * feature sets were the same, but the required workaround were -not- 5037c478bd9Sstevel@tonic-gate * necessarily the same. So, this routine is invoked on -every- CPU soon 5047c478bd9Sstevel@tonic-gate * after starting to make sure that the resulting system contains the most 5057c478bd9Sstevel@tonic-gate * pessimal set of workarounds needed to cope with *any* of the CPUs in the 5067c478bd9Sstevel@tonic-gate * system. 5077c478bd9Sstevel@tonic-gate * 508ef50d8c0Sesaxe * workaround_errata is invoked early in mlsetup() for CPU 0, and in 509ef50d8c0Sesaxe * mp_startup() for all slave CPUs. Slaves process workaround_errata prior 510ef50d8c0Sesaxe * to acknowledging their readiness to the master, so this routine will 511ef50d8c0Sesaxe * never be executed by multiple CPUs in parallel, thus making updates to 512ef50d8c0Sesaxe * global data safe. 513ef50d8c0Sesaxe * 5142201b277Skucharsk * These workarounds are based on Rev 3.57 of the Revision Guide for 5152201b277Skucharsk * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005. 5167c478bd9Sstevel@tonic-gate */ 5177c478bd9Sstevel@tonic-gate 5187c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_91) 5197c478bd9Sstevel@tonic-gate int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 5207c478bd9Sstevel@tonic-gate #endif 5217c478bd9Sstevel@tonic-gate 5227c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_93) 5237c478bd9Sstevel@tonic-gate int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 5247c478bd9Sstevel@tonic-gate #endif 5257c478bd9Sstevel@tonic-gate 5267c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_100) 5277c478bd9Sstevel@tonic-gate int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 5287c478bd9Sstevel@tonic-gate #endif 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109) 5317c478bd9Sstevel@tonic-gate int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 5327c478bd9Sstevel@tonic-gate #endif 5337c478bd9Sstevel@tonic-gate 5347c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_121) 5357c478bd9Sstevel@tonic-gate int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 5367c478bd9Sstevel@tonic-gate #endif 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_122) 5397c478bd9Sstevel@tonic-gate int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 5407c478bd9Sstevel@tonic-gate #endif 5417c478bd9Sstevel@tonic-gate 5427c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_123) 5437c478bd9Sstevel@tonic-gate int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 5447c478bd9Sstevel@tonic-gate #endif 5457c478bd9Sstevel@tonic-gate 5462201b277Skucharsk #if defined(OPTERON_ERRATUM_131) 5472201b277Skucharsk int opteron_erratum_131; /* if non-zero -> at least one cpu has it */ 5482201b277Skucharsk #endif 5497c478bd9Sstevel@tonic-gate 550ef50d8c0Sesaxe #if defined(OPTERON_WORKAROUND_6336786) 551ef50d8c0Sesaxe int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */ 552ef50d8c0Sesaxe int opteron_workaround_6336786_UP = 0; /* Not needed for UP */ 553ef50d8c0Sesaxe #endif 554ef50d8c0Sesaxe 5557c478bd9Sstevel@tonic-gate #define WARNING(cpu, n) \ 5567c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 5577c478bd9Sstevel@tonic-gate (cpu)->cpu_id, (n)) 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate uint_t 5607c478bd9Sstevel@tonic-gate workaround_errata(struct cpu *cpu) 5617c478bd9Sstevel@tonic-gate { 5627c478bd9Sstevel@tonic-gate uint_t missing = 0; 5637c478bd9Sstevel@tonic-gate 5647c478bd9Sstevel@tonic-gate ASSERT(cpu == CPU); 5657c478bd9Sstevel@tonic-gate 5667c478bd9Sstevel@tonic-gate /*LINTED*/ 5677c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 88) > 0) { 5687c478bd9Sstevel@tonic-gate /* 5697c478bd9Sstevel@tonic-gate * SWAPGS May Fail To Read Correct GS Base 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_88) 5727c478bd9Sstevel@tonic-gate /* 5737c478bd9Sstevel@tonic-gate * The workaround is an mfence in the relevant assembler code 5747c478bd9Sstevel@tonic-gate */ 5757c478bd9Sstevel@tonic-gate #else 5767c478bd9Sstevel@tonic-gate WARNING(cpu, 88); 5777c478bd9Sstevel@tonic-gate missing++; 5787c478bd9Sstevel@tonic-gate #endif 5797c478bd9Sstevel@tonic-gate } 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 91) > 0) { 5827c478bd9Sstevel@tonic-gate /* 5837c478bd9Sstevel@tonic-gate * Software Prefetches May Report A Page Fault 5847c478bd9Sstevel@tonic-gate */ 5857c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_91) 5867c478bd9Sstevel@tonic-gate /* 5877c478bd9Sstevel@tonic-gate * fix is in trap.c 5887c478bd9Sstevel@tonic-gate */ 5897c478bd9Sstevel@tonic-gate opteron_erratum_91++; 5907c478bd9Sstevel@tonic-gate #else 5917c478bd9Sstevel@tonic-gate WARNING(cpu, 91); 5927c478bd9Sstevel@tonic-gate missing++; 5937c478bd9Sstevel@tonic-gate #endif 5947c478bd9Sstevel@tonic-gate } 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 93) > 0) { 5977c478bd9Sstevel@tonic-gate /* 5987c478bd9Sstevel@tonic-gate * RSM Auto-Halt Restart Returns to Incorrect RIP 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_93) 6017c478bd9Sstevel@tonic-gate /* 6027c478bd9Sstevel@tonic-gate * fix is in trap.c 6037c478bd9Sstevel@tonic-gate */ 6047c478bd9Sstevel@tonic-gate opteron_erratum_93++; 6057c478bd9Sstevel@tonic-gate #else 6067c478bd9Sstevel@tonic-gate WARNING(cpu, 93); 6077c478bd9Sstevel@tonic-gate missing++; 6087c478bd9Sstevel@tonic-gate #endif 6097c478bd9Sstevel@tonic-gate } 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate /*LINTED*/ 6127c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 95) > 0) { 6137c478bd9Sstevel@tonic-gate /* 6147c478bd9Sstevel@tonic-gate * RET Instruction May Return to Incorrect EIP 6157c478bd9Sstevel@tonic-gate */ 6167c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_95) 6177c478bd9Sstevel@tonic-gate #if defined(_LP64) 6187c478bd9Sstevel@tonic-gate /* 6197c478bd9Sstevel@tonic-gate * Workaround this by ensuring that 32-bit user code and 6207c478bd9Sstevel@tonic-gate * 64-bit kernel code never occupy the same address 6217c478bd9Sstevel@tonic-gate * range mod 4G. 6227c478bd9Sstevel@tonic-gate */ 6237c478bd9Sstevel@tonic-gate if (_userlimit32 > 0xc0000000ul) 6247c478bd9Sstevel@tonic-gate *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 6257c478bd9Sstevel@tonic-gate 6267c478bd9Sstevel@tonic-gate /*LINTED*/ 6277c478bd9Sstevel@tonic-gate ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 6287c478bd9Sstevel@tonic-gate #endif /* _LP64 */ 6297c478bd9Sstevel@tonic-gate #else 6307c478bd9Sstevel@tonic-gate WARNING(cpu, 95); 6317c478bd9Sstevel@tonic-gate missing++; 6327c478bd9Sstevel@tonic-gate #endif /* OPTERON_ERRATUM_95 */ 6337c478bd9Sstevel@tonic-gate } 6347c478bd9Sstevel@tonic-gate 6357c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 100) > 0) { 6367c478bd9Sstevel@tonic-gate /* 6377c478bd9Sstevel@tonic-gate * Compatibility Mode Branches Transfer to Illegal Address 6387c478bd9Sstevel@tonic-gate */ 6397c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_100) 6407c478bd9Sstevel@tonic-gate /* 6417c478bd9Sstevel@tonic-gate * fix is in trap.c 6427c478bd9Sstevel@tonic-gate */ 6437c478bd9Sstevel@tonic-gate opteron_erratum_100++; 6447c478bd9Sstevel@tonic-gate #else 6457c478bd9Sstevel@tonic-gate WARNING(cpu, 100); 6467c478bd9Sstevel@tonic-gate missing++; 6477c478bd9Sstevel@tonic-gate #endif 6487c478bd9Sstevel@tonic-gate } 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate /*LINTED*/ 6517c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 108) > 0) { 6527c478bd9Sstevel@tonic-gate /* 6537c478bd9Sstevel@tonic-gate * CPUID Instruction May Return Incorrect Model Number In 6547c478bd9Sstevel@tonic-gate * Some Processors 6557c478bd9Sstevel@tonic-gate */ 6567c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_108) 6577c478bd9Sstevel@tonic-gate /* 6587c478bd9Sstevel@tonic-gate * (Our cpuid-handling code corrects the model number on 6597c478bd9Sstevel@tonic-gate * those processors) 6607c478bd9Sstevel@tonic-gate */ 6617c478bd9Sstevel@tonic-gate #else 6627c478bd9Sstevel@tonic-gate WARNING(cpu, 108); 6637c478bd9Sstevel@tonic-gate missing++; 6647c478bd9Sstevel@tonic-gate #endif 6657c478bd9Sstevel@tonic-gate } 6667c478bd9Sstevel@tonic-gate 6677c478bd9Sstevel@tonic-gate /*LINTED*/ 6687c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 109) > 0) { 6697c478bd9Sstevel@tonic-gate /* 6707c478bd9Sstevel@tonic-gate * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 6717c478bd9Sstevel@tonic-gate */ 6727c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109) 6737c478bd9Sstevel@tonic-gate 6747c478bd9Sstevel@tonic-gate /* workaround is to print a warning to upgrade BIOS */ 6750ac7d7d8Skucharsk if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 6767c478bd9Sstevel@tonic-gate opteron_erratum_109++; 6777c478bd9Sstevel@tonic-gate #else 6787c478bd9Sstevel@tonic-gate WARNING(cpu, 109); 6797c478bd9Sstevel@tonic-gate missing++; 6807c478bd9Sstevel@tonic-gate #endif 6817c478bd9Sstevel@tonic-gate } 6827c478bd9Sstevel@tonic-gate /*LINTED*/ 6837c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 121) > 0) { 6847c478bd9Sstevel@tonic-gate /* 6857c478bd9Sstevel@tonic-gate * Sequential Execution Across Non_Canonical Boundary Caused 6867c478bd9Sstevel@tonic-gate * Processor Hang 6877c478bd9Sstevel@tonic-gate */ 6887c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_121) 6897c478bd9Sstevel@tonic-gate static int lma; 6907c478bd9Sstevel@tonic-gate 6917c478bd9Sstevel@tonic-gate if (opteron_erratum_121) 6927c478bd9Sstevel@tonic-gate opteron_erratum_121++; 6937c478bd9Sstevel@tonic-gate 6947c478bd9Sstevel@tonic-gate /* 6957c478bd9Sstevel@tonic-gate * Erratum 121 is only present in long (64 bit) mode. 6967c478bd9Sstevel@tonic-gate * Workaround is to include the page immediately before the 6977c478bd9Sstevel@tonic-gate * va hole to eliminate the possibility of system hangs due to 6987c478bd9Sstevel@tonic-gate * sequential execution across the va hole boundary. 6997c478bd9Sstevel@tonic-gate */ 7007c478bd9Sstevel@tonic-gate if (lma == 0) { 7017c478bd9Sstevel@tonic-gate /* 7027c478bd9Sstevel@tonic-gate * check LMA once: assume all cpus are in long mode 7037c478bd9Sstevel@tonic-gate * or not. 7047c478bd9Sstevel@tonic-gate */ 7057c478bd9Sstevel@tonic-gate lma = 1; 7067c478bd9Sstevel@tonic-gate 7070ac7d7d8Skucharsk if (rdmsr(MSR_AMD_EFER) & AMD_EFER_LMA) { 7087c478bd9Sstevel@tonic-gate if (hole_start) { 7097c478bd9Sstevel@tonic-gate hole_start -= PAGESIZE; 7107c478bd9Sstevel@tonic-gate } else { 7117c478bd9Sstevel@tonic-gate /* 7127c478bd9Sstevel@tonic-gate * hole_start not yet initialized by 7137c478bd9Sstevel@tonic-gate * mmu_init. Initialize hole_start 7147c478bd9Sstevel@tonic-gate * with value to be subtracted. 7157c478bd9Sstevel@tonic-gate */ 7167c478bd9Sstevel@tonic-gate hole_start = PAGESIZE; 7177c478bd9Sstevel@tonic-gate } 7187c478bd9Sstevel@tonic-gate opteron_erratum_121++; 7197c478bd9Sstevel@tonic-gate } 7207c478bd9Sstevel@tonic-gate } 7217c478bd9Sstevel@tonic-gate #else 7227c478bd9Sstevel@tonic-gate WARNING(cpu, 121); 7237c478bd9Sstevel@tonic-gate missing++; 7247c478bd9Sstevel@tonic-gate #endif 7257c478bd9Sstevel@tonic-gate } 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate /*LINTED*/ 7287c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 122) > 0) { 7297c478bd9Sstevel@tonic-gate /* 7307c478bd9Sstevel@tonic-gate * TLB Flush Filter May Cause Cohenrency Problem in 7317c478bd9Sstevel@tonic-gate * Multiprocessor Systems 7327c478bd9Sstevel@tonic-gate */ 7337c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_122) 7347c478bd9Sstevel@tonic-gate /* 7357c478bd9Sstevel@tonic-gate * Erratum 122 is only present in MP configurations (multi-core 7367c478bd9Sstevel@tonic-gate * or multi-processor). 7377c478bd9Sstevel@tonic-gate */ 7387c478bd9Sstevel@tonic-gate 7397c478bd9Sstevel@tonic-gate if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 7407c478bd9Sstevel@tonic-gate cpuid_get_ncpu_per_chip(cpu) > 1) { 7417c478bd9Sstevel@tonic-gate /* disable TLB Flush Filter */ 7420ac7d7d8Skucharsk wrmsr(MSR_AMD_HWCR, rdmsr(MSR_AMD_HWCR) | 7430ac7d7d8Skucharsk (uint64_t)(uintptr_t)AMD_HWCR_FFDIS); 7447c478bd9Sstevel@tonic-gate opteron_erratum_122++; 7457c478bd9Sstevel@tonic-gate } 7467c478bd9Sstevel@tonic-gate 7477c478bd9Sstevel@tonic-gate #else 7487c478bd9Sstevel@tonic-gate WARNING(cpu, 122); 7497c478bd9Sstevel@tonic-gate missing++; 7507c478bd9Sstevel@tonic-gate #endif 7517c478bd9Sstevel@tonic-gate } 752403c216aSkchow 753403c216aSkchow #if defined(OPTERON_ERRATUM_123) 7547c478bd9Sstevel@tonic-gate /*LINTED*/ 7557c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 123) > 0) { 7567c478bd9Sstevel@tonic-gate /* 7577c478bd9Sstevel@tonic-gate * Bypassed Reads May Cause Data Corruption of System Hang in 7587c478bd9Sstevel@tonic-gate * Dual Core Processors 7597c478bd9Sstevel@tonic-gate */ 7607c478bd9Sstevel@tonic-gate /* 7617c478bd9Sstevel@tonic-gate * Erratum 123 applies only to multi-core cpus. 7627c478bd9Sstevel@tonic-gate */ 7637c478bd9Sstevel@tonic-gate 7647c478bd9Sstevel@tonic-gate if (cpuid_get_ncpu_per_chip(cpu) > 1) { 7657c478bd9Sstevel@tonic-gate /* workaround is to print a warning to upgrade BIOS */ 7660ac7d7d8Skucharsk if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 7677c478bd9Sstevel@tonic-gate opteron_erratum_123++; 7687c478bd9Sstevel@tonic-gate } 7697c478bd9Sstevel@tonic-gate } 770403c216aSkchow #endif 7712201b277Skucharsk 7722201b277Skucharsk #if defined(OPTERON_ERRATUM_131) 7732201b277Skucharsk /*LINTED*/ 7742201b277Skucharsk if (cpuid_opteron_erratum(cpu, 131) > 0) { 7752201b277Skucharsk /* 7762201b277Skucharsk * Multiprocessor Systems with Four or More Cores May Deadlock 7772201b277Skucharsk * Waiting for a Probe Response 7782201b277Skucharsk */ 7792201b277Skucharsk /* 7802201b277Skucharsk * Erratum 131 applies to any system with four or more cores. 7812201b277Skucharsk */ 7822201b277Skucharsk if ((opteron_erratum_131 == 0) && ((lgrp_plat_node_cnt * 7832201b277Skucharsk cpuid_get_ncpu_per_chip(cpu)) >= 4)) { 7842201b277Skucharsk /* 7852201b277Skucharsk * Workaround is to print a warning to upgrade 7862201b277Skucharsk * the BIOS 7872201b277Skucharsk */ 7880ac7d7d8Skucharsk if (!(rdmsr(MSR_AMD_NB_CFG) & AMD_NB_CFG_SRQ_HEARTBEAT)) 7892201b277Skucharsk opteron_erratum_131++; 7902201b277Skucharsk } 791ef50d8c0Sesaxe } 7922201b277Skucharsk #endif 793ef50d8c0Sesaxe 794ef50d8c0Sesaxe #if defined(OPTERON_WORKAROUND_6336786) 795ef50d8c0Sesaxe /* 796ef50d8c0Sesaxe * This isn't really erratum, but for convenience the 797ef50d8c0Sesaxe * detection/workaround code lives here and in cpuid_opteron_erratum. 798ef50d8c0Sesaxe */ 799ef50d8c0Sesaxe if (cpuid_opteron_erratum(cpu, 6336786) > 0) { 800ef50d8c0Sesaxe int node; 801ef50d8c0Sesaxe uint8_t data; 802ef50d8c0Sesaxe 803ef50d8c0Sesaxe /* 804ef50d8c0Sesaxe * Disable C1-Clock ramping on multi-core/multi-processor 805ef50d8c0Sesaxe * K8 platforms to guard against TSC drift. 806ef50d8c0Sesaxe */ 807ef50d8c0Sesaxe if (opteron_workaround_6336786) { 808ef50d8c0Sesaxe opteron_workaround_6336786++; 809ef50d8c0Sesaxe } else if ((lgrp_plat_node_cnt * 810ef50d8c0Sesaxe cpuid_get_ncpu_per_chip(cpu) >= 2) || 811ef50d8c0Sesaxe opteron_workaround_6336786_UP) { 812ef50d8c0Sesaxe for (node = 0; node < lgrp_plat_node_cnt; node++) { 813ef50d8c0Sesaxe /* 814ef50d8c0Sesaxe * Clear PMM7[1:0] (function 3, offset 0x87) 815ef50d8c0Sesaxe * Northbridge device is the node id + 24. 816ef50d8c0Sesaxe */ 817ef50d8c0Sesaxe data = pci_getb_func(0, node + 24, 3, 0x87); 818ef50d8c0Sesaxe data &= 0xFC; 819ef50d8c0Sesaxe pci_putb_func(0, node + 24, 3, 0x87, data); 820ef50d8c0Sesaxe } 821ef50d8c0Sesaxe opteron_workaround_6336786++; 822ef50d8c0Sesaxe } 8232201b277Skucharsk } 824ef50d8c0Sesaxe #endif 8257c478bd9Sstevel@tonic-gate return (missing); 8267c478bd9Sstevel@tonic-gate } 8277c478bd9Sstevel@tonic-gate 8287c478bd9Sstevel@tonic-gate void 8297c478bd9Sstevel@tonic-gate workaround_errata_end() 8307c478bd9Sstevel@tonic-gate { 8317c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109) 8327c478bd9Sstevel@tonic-gate if (opteron_erratum_109) { 8332201b277Skucharsk cmn_err(CE_WARN, 8342201b277Skucharsk "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 8352201b277Skucharsk " processor\nerratum 109 was not detected; updating your" 8362201b277Skucharsk " system's BIOS to a version\ncontaining this" 8372201b277Skucharsk " microcode patch is HIGHLY recommended or erroneous" 8382201b277Skucharsk " system\noperation may occur.\n"); 8397c478bd9Sstevel@tonic-gate } 8402201b277Skucharsk #endif /* OPTERON_ERRATUM_109 */ 8417c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_123) 8427c478bd9Sstevel@tonic-gate if (opteron_erratum_123) { 8432201b277Skucharsk cmn_err(CE_WARN, 8442201b277Skucharsk "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 8452201b277Skucharsk " processor\nerratum 123 was not detected; updating your" 8462201b277Skucharsk " system's BIOS to a version\ncontaining this" 8472201b277Skucharsk " microcode patch is HIGHLY recommended or erroneous" 8482201b277Skucharsk " system\noperation may occur.\n"); 8497c478bd9Sstevel@tonic-gate } 8502201b277Skucharsk #endif /* OPTERON_ERRATUM_123 */ 8512201b277Skucharsk #if defined(OPTERON_ERRATUM_131) 8522201b277Skucharsk if (opteron_erratum_131) { 8532201b277Skucharsk cmn_err(CE_WARN, 8542201b277Skucharsk "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 8552201b277Skucharsk " processor\nerratum 131 was not detected; updating your" 8562201b277Skucharsk " system's BIOS to a version\ncontaining this" 8572201b277Skucharsk " microcode patch is HIGHLY recommended or erroneous" 8582201b277Skucharsk " system\noperation may occur.\n"); 8592201b277Skucharsk } 8602201b277Skucharsk #endif /* OPTERON_ERRATUM_131 */ 8617c478bd9Sstevel@tonic-gate } 8627c478bd9Sstevel@tonic-gate 8637c478bd9Sstevel@tonic-gate static ushort_t *mp_map_warm_reset_vector(); 8647c478bd9Sstevel@tonic-gate static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8677c478bd9Sstevel@tonic-gate void 8687c478bd9Sstevel@tonic-gate start_other_cpus(int cprboot) 8697c478bd9Sstevel@tonic-gate { 8707c478bd9Sstevel@tonic-gate unsigned who; 871d90554ebSdmick int cpuid = 0; 8727c478bd9Sstevel@tonic-gate int delays = 0; 8737c478bd9Sstevel@tonic-gate int started_cpu; 8747c478bd9Sstevel@tonic-gate ushort_t *warm_reset_vector = NULL; 8757c478bd9Sstevel@tonic-gate extern int procset; 8767c478bd9Sstevel@tonic-gate 8777c478bd9Sstevel@tonic-gate /* 8787c478bd9Sstevel@tonic-gate * Initialize our own cpu_info. 8797c478bd9Sstevel@tonic-gate */ 8807c478bd9Sstevel@tonic-gate init_cpu_info(CPU); 8817c478bd9Sstevel@tonic-gate 8827c478bd9Sstevel@tonic-gate /* 8837c478bd9Sstevel@tonic-gate * Initialize our syscall handlers 8847c478bd9Sstevel@tonic-gate */ 8857c478bd9Sstevel@tonic-gate init_cpu_syscall(CPU); 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate /* 8887c478bd9Sstevel@tonic-gate * if only 1 cpu or not using MP, skip the rest of this 8897c478bd9Sstevel@tonic-gate */ 8907c478bd9Sstevel@tonic-gate if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 8917c478bd9Sstevel@tonic-gate if (use_mp == 0) 8927c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?***** Not in MP mode\n"); 8937c478bd9Sstevel@tonic-gate goto done; 8947c478bd9Sstevel@tonic-gate } 8957c478bd9Sstevel@tonic-gate 8967c478bd9Sstevel@tonic-gate /* 8977c478bd9Sstevel@tonic-gate * perform such initialization as is needed 8987c478bd9Sstevel@tonic-gate * to be able to take CPUs on- and off-line. 8997c478bd9Sstevel@tonic-gate */ 9007c478bd9Sstevel@tonic-gate cpu_pause_init(); 9017c478bd9Sstevel@tonic-gate 9027c478bd9Sstevel@tonic-gate xc_init(); /* initialize processor crosscalls */ 9037c478bd9Sstevel@tonic-gate 9047c478bd9Sstevel@tonic-gate /* 9057c478bd9Sstevel@tonic-gate * Copy the real mode code at "real_mode_start" to the 9067c478bd9Sstevel@tonic-gate * page at rm_platter_va. 9077c478bd9Sstevel@tonic-gate */ 9087c478bd9Sstevel@tonic-gate warm_reset_vector = mp_map_warm_reset_vector(); 9097c478bd9Sstevel@tonic-gate if (warm_reset_vector == NULL) 9107c478bd9Sstevel@tonic-gate goto done; 9117c478bd9Sstevel@tonic-gate 9127c478bd9Sstevel@tonic-gate bcopy((caddr_t)real_mode_start, 9137c478bd9Sstevel@tonic-gate (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 9147c478bd9Sstevel@tonic-gate (size_t)real_mode_end - (size_t)real_mode_start); 9157c478bd9Sstevel@tonic-gate 9167c478bd9Sstevel@tonic-gate flushes_require_xcalls = 1; 9177c478bd9Sstevel@tonic-gate 9187c478bd9Sstevel@tonic-gate affinity_set(CPU_CURRENT); 9197c478bd9Sstevel@tonic-gate 9207c478bd9Sstevel@tonic-gate for (who = 0; who < NCPU; who++) { 9217c478bd9Sstevel@tonic-gate if (who == cpuid) 9227c478bd9Sstevel@tonic-gate continue; 9237c478bd9Sstevel@tonic-gate 9247c478bd9Sstevel@tonic-gate if ((mp_cpus & (1 << who)) == 0) 9257c478bd9Sstevel@tonic-gate continue; 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate mp_startup_init(who); 9287c478bd9Sstevel@tonic-gate started_cpu = 1; 9297c478bd9Sstevel@tonic-gate (*cpu_startf)(who, rm_platter_pa); 9307c478bd9Sstevel@tonic-gate 9317c478bd9Sstevel@tonic-gate while ((procset & (1 << who)) == 0) { 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate delay(1); 9347c478bd9Sstevel@tonic-gate if (++delays > (20 * hz)) { 9357c478bd9Sstevel@tonic-gate 9367c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 9377c478bd9Sstevel@tonic-gate "cpu%d failed to start", who); 9387c478bd9Sstevel@tonic-gate 9397c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 9407c478bd9Sstevel@tonic-gate cpu[who]->cpu_flags = 0; 941affbd3ccSkchow cpu_vm_data_destroy(cpu[who]); 9427c478bd9Sstevel@tonic-gate cpu_del_unit(who); 9437c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 9447c478bd9Sstevel@tonic-gate 9457c478bd9Sstevel@tonic-gate started_cpu = 0; 9467c478bd9Sstevel@tonic-gate break; 9477c478bd9Sstevel@tonic-gate } 9487c478bd9Sstevel@tonic-gate } 9497c478bd9Sstevel@tonic-gate if (!started_cpu) 9507c478bd9Sstevel@tonic-gate continue; 9517c478bd9Sstevel@tonic-gate if (tsc_gethrtime_enable) 9527c478bd9Sstevel@tonic-gate tsc_sync_master(who); 9537c478bd9Sstevel@tonic-gate 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate if (dtrace_cpu_init != NULL) { 9567c478bd9Sstevel@tonic-gate /* 9577c478bd9Sstevel@tonic-gate * DTrace CPU initialization expects cpu_lock 9587c478bd9Sstevel@tonic-gate * to be held. 9597c478bd9Sstevel@tonic-gate */ 9607c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 9617c478bd9Sstevel@tonic-gate (*dtrace_cpu_init)(who); 9627c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 9637c478bd9Sstevel@tonic-gate } 9647c478bd9Sstevel@tonic-gate } 9657c478bd9Sstevel@tonic-gate 9667c478bd9Sstevel@tonic-gate affinity_clear(); 9677c478bd9Sstevel@tonic-gate 9687c478bd9Sstevel@tonic-gate for (who = 0; who < NCPU; who++) { 9697c478bd9Sstevel@tonic-gate if (who == cpuid) 9707c478bd9Sstevel@tonic-gate continue; 9717c478bd9Sstevel@tonic-gate 9727c478bd9Sstevel@tonic-gate if (!(procset & (1 << who))) 9737c478bd9Sstevel@tonic-gate continue; 9747c478bd9Sstevel@tonic-gate 9757c478bd9Sstevel@tonic-gate while (!(cpu_ready_set & (1 << who))) 9767c478bd9Sstevel@tonic-gate delay(1); 9777c478bd9Sstevel@tonic-gate } 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate done: 9807c478bd9Sstevel@tonic-gate workaround_errata_end(); 9817c478bd9Sstevel@tonic-gate 9827c478bd9Sstevel@tonic-gate if (warm_reset_vector != NULL) 9837c478bd9Sstevel@tonic-gate mp_unmap_warm_reset_vector(warm_reset_vector); 9847c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 9857c478bd9Sstevel@tonic-gate HAT_UNLOAD); 9867c478bd9Sstevel@tonic-gate } 9877c478bd9Sstevel@tonic-gate 9887c478bd9Sstevel@tonic-gate /* 9897c478bd9Sstevel@tonic-gate * Dummy functions - no i86pc platforms support dynamic cpu allocation. 9907c478bd9Sstevel@tonic-gate */ 9917c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9927c478bd9Sstevel@tonic-gate int 9937c478bd9Sstevel@tonic-gate mp_cpu_configure(int cpuid) 9947c478bd9Sstevel@tonic-gate { 9957c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 9967c478bd9Sstevel@tonic-gate } 9977c478bd9Sstevel@tonic-gate 9987c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9997c478bd9Sstevel@tonic-gate int 10007c478bd9Sstevel@tonic-gate mp_cpu_unconfigure(int cpuid) 10017c478bd9Sstevel@tonic-gate { 10027c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 10037c478bd9Sstevel@tonic-gate } 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate /* 10067c478bd9Sstevel@tonic-gate * Startup function for 'other' CPUs (besides boot cpu). 10077c478bd9Sstevel@tonic-gate * Resumed from cpu_startup. 1008b4b46911Skchow * 1009b4b46911Skchow * WARNING: until CPU_READY is set, mp_startup and routines called by 1010b4b46911Skchow * mp_startup should not call routines (e.g. kmem_free) that could call 1011b4b46911Skchow * hat_unload which requires CPU_READY to be set. 10127c478bd9Sstevel@tonic-gate */ 10137c478bd9Sstevel@tonic-gate void 10147c478bd9Sstevel@tonic-gate mp_startup(void) 10157c478bd9Sstevel@tonic-gate { 10167c478bd9Sstevel@tonic-gate struct cpu *cp = CPU; 10177c478bd9Sstevel@tonic-gate extern int procset; 10187c478bd9Sstevel@tonic-gate uint_t new_x86_feature; 10197c478bd9Sstevel@tonic-gate 10207c478bd9Sstevel@tonic-gate new_x86_feature = cpuid_pass1(cp); 10217c478bd9Sstevel@tonic-gate 10227c478bd9Sstevel@tonic-gate /* 10237c478bd9Sstevel@tonic-gate * We need to Sync MTRR with cpu0's MTRR. We have to do 10247c478bd9Sstevel@tonic-gate * this with interrupts disabled. 10257c478bd9Sstevel@tonic-gate */ 10267c478bd9Sstevel@tonic-gate if (x86_feature & X86_MTRR) 10277c478bd9Sstevel@tonic-gate mtrr_sync(); 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate /* 10307c478bd9Sstevel@tonic-gate * Initialize this CPU's syscall handlers 10317c478bd9Sstevel@tonic-gate */ 10327c478bd9Sstevel@tonic-gate init_cpu_syscall(cp); 10337c478bd9Sstevel@tonic-gate 10347c478bd9Sstevel@tonic-gate /* 10357c478bd9Sstevel@tonic-gate * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 10367c478bd9Sstevel@tonic-gate * highest level at which a routine is permitted to block on 10377c478bd9Sstevel@tonic-gate * an adaptive mutex (allows for cpu poke interrupt in case 10387c478bd9Sstevel@tonic-gate * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 10397c478bd9Sstevel@tonic-gate * device interrupts that may end up in the hat layer issuing cross 10407c478bd9Sstevel@tonic-gate * calls before CPU_READY is set. 10417c478bd9Sstevel@tonic-gate */ 10427c478bd9Sstevel@tonic-gate (void) splx(ipltospl(LOCK_LEVEL)); 10437c478bd9Sstevel@tonic-gate 10447c478bd9Sstevel@tonic-gate /* 10457c478bd9Sstevel@tonic-gate * Do a sanity check to make sure this new CPU is a sane thing 10467c478bd9Sstevel@tonic-gate * to add to the collection of processors running this system. 10477c478bd9Sstevel@tonic-gate * 10487c478bd9Sstevel@tonic-gate * XXX Clearly this needs to get more sophisticated, if x86 10497c478bd9Sstevel@tonic-gate * systems start to get built out of heterogenous CPUs; as is 10507c478bd9Sstevel@tonic-gate * likely to happen once the number of processors in a configuration 10517c478bd9Sstevel@tonic-gate * gets large enough. 10527c478bd9Sstevel@tonic-gate */ 10537c478bd9Sstevel@tonic-gate if ((x86_feature & new_x86_feature) != x86_feature) { 10547c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?cpu%d: %b\n", 10557c478bd9Sstevel@tonic-gate cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 10567c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 10577c478bd9Sstevel@tonic-gate } 10587c478bd9Sstevel@tonic-gate 10597c478bd9Sstevel@tonic-gate /* 10607c478bd9Sstevel@tonic-gate * We could be more sophisticated here, and just mark the CPU 10617c478bd9Sstevel@tonic-gate * as "faulted" but at this point we'll opt for the easier 10627c478bd9Sstevel@tonic-gate * answer of dieing horribly. Provided the boot cpu is ok, 10637c478bd9Sstevel@tonic-gate * the system can be recovered by booting with use_mp set to zero. 10647c478bd9Sstevel@tonic-gate */ 10657c478bd9Sstevel@tonic-gate if (workaround_errata(cp) != 0) 10667c478bd9Sstevel@tonic-gate panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate cpuid_pass2(cp); 10697c478bd9Sstevel@tonic-gate cpuid_pass3(cp); 10707c478bd9Sstevel@tonic-gate (void) cpuid_pass4(cp); 10717c478bd9Sstevel@tonic-gate 10727c478bd9Sstevel@tonic-gate init_cpu_info(cp); 10737c478bd9Sstevel@tonic-gate 10747c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10757c478bd9Sstevel@tonic-gate procset |= 1 << cp->cpu_id; 10767c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10777c478bd9Sstevel@tonic-gate 10787c478bd9Sstevel@tonic-gate if (tsc_gethrtime_enable) 10797c478bd9Sstevel@tonic-gate tsc_sync_slave(); 10807c478bd9Sstevel@tonic-gate 10817c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10827c478bd9Sstevel@tonic-gate /* 10837c478bd9Sstevel@tonic-gate * It's unfortunate that chip_cpu_init() has to be called here. 10847c478bd9Sstevel@tonic-gate * It really belongs in cpu_add_unit(), but unfortunately it is 10857c478bd9Sstevel@tonic-gate * dependent on the cpuid probing, which must be done in the 10867c478bd9Sstevel@tonic-gate * context of the current CPU. Care must be taken on x86 to ensure 10877c478bd9Sstevel@tonic-gate * that mp_startup can safely block even though chip_cpu_init() and 10887c478bd9Sstevel@tonic-gate * cpu_add_active() have not yet been called. 10897c478bd9Sstevel@tonic-gate */ 10907c478bd9Sstevel@tonic-gate chip_cpu_init(cp); 10917c478bd9Sstevel@tonic-gate chip_cpu_startup(cp); 10927c478bd9Sstevel@tonic-gate 10937c478bd9Sstevel@tonic-gate cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 10947c478bd9Sstevel@tonic-gate cpu_add_active(cp); 10957c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10967c478bd9Sstevel@tonic-gate 1097b4b46911Skchow add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1098b4b46911Skchow 10997c478bd9Sstevel@tonic-gate (void) spl0(); /* enable interrupts */ 11007c478bd9Sstevel@tonic-gate 1101*7aec1d6eScindi /* 1102*7aec1d6eScindi * Set up the CPU module for this CPU. This can't be done before 1103*7aec1d6eScindi * this CPU is made CPU_READY, because we may (in heterogeneous systems) 1104*7aec1d6eScindi * need to go load another CPU module. The act of attempting to load 1105*7aec1d6eScindi * a module may trigger a cross-call, which will ASSERT unless this 1106*7aec1d6eScindi * cpu is CPU_READY. 1107*7aec1d6eScindi */ 1108*7aec1d6eScindi cmi_init(); 1109*7aec1d6eScindi 1110*7aec1d6eScindi if (x86_feature & X86_MCA) 1111*7aec1d6eScindi cmi_mca_init(); 1112*7aec1d6eScindi 11137c478bd9Sstevel@tonic-gate if (boothowto & RB_DEBUG) 11147c478bd9Sstevel@tonic-gate kdi_dvec_cpu_init(cp); 11157c478bd9Sstevel@tonic-gate 11167c478bd9Sstevel@tonic-gate /* 11177c478bd9Sstevel@tonic-gate * Setting the bit in cpu_ready_set must be the last operation in 11187c478bd9Sstevel@tonic-gate * processor initialization; the boot CPU will continue to boot once 11197c478bd9Sstevel@tonic-gate * it sees this bit set for all active CPUs. 11207c478bd9Sstevel@tonic-gate */ 11217c478bd9Sstevel@tonic-gate CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 11227c478bd9Sstevel@tonic-gate 11237c478bd9Sstevel@tonic-gate /* 11247c478bd9Sstevel@tonic-gate * Because mp_startup() gets fired off after init() starts, we 11257c478bd9Sstevel@tonic-gate * can't use the '?' trick to do 'boot -v' printing - so we 11267c478bd9Sstevel@tonic-gate * always direct the 'cpu .. online' messages to the log. 11277c478bd9Sstevel@tonic-gate */ 11287c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 11297c478bd9Sstevel@tonic-gate cp->cpu_id); 11307c478bd9Sstevel@tonic-gate 11317c478bd9Sstevel@tonic-gate /* 11327c478bd9Sstevel@tonic-gate * Now we are done with the startup thread, so free it up. 11337c478bd9Sstevel@tonic-gate */ 11347c478bd9Sstevel@tonic-gate thread_exit(); 11357c478bd9Sstevel@tonic-gate panic("mp_startup: cannot return"); 11367c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 11377c478bd9Sstevel@tonic-gate } 11387c478bd9Sstevel@tonic-gate 11397c478bd9Sstevel@tonic-gate 11407c478bd9Sstevel@tonic-gate /* 11417c478bd9Sstevel@tonic-gate * Start CPU on user request. 11427c478bd9Sstevel@tonic-gate */ 11437c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11447c478bd9Sstevel@tonic-gate int 11457c478bd9Sstevel@tonic-gate mp_cpu_start(struct cpu *cp) 11467c478bd9Sstevel@tonic-gate { 11477c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 11487c478bd9Sstevel@tonic-gate return (0); 11497c478bd9Sstevel@tonic-gate } 11507c478bd9Sstevel@tonic-gate 11517c478bd9Sstevel@tonic-gate /* 11527c478bd9Sstevel@tonic-gate * Stop CPU on user request. 11537c478bd9Sstevel@tonic-gate */ 11547c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11557c478bd9Sstevel@tonic-gate int 11567c478bd9Sstevel@tonic-gate mp_cpu_stop(struct cpu *cp) 11577c478bd9Sstevel@tonic-gate { 1158d90554ebSdmick extern int cbe_psm_timer_mode; 11597c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1160d90554ebSdmick 1161d90554ebSdmick /* 1162d90554ebSdmick * If TIMER_PERIODIC mode is used, CPU0 is the one running it; 1163d90554ebSdmick * can't stop it. (This is true only for machines with no TSC.) 1164d90554ebSdmick */ 1165d90554ebSdmick 1166d90554ebSdmick if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0)) 1167d90554ebSdmick return (1); 11687c478bd9Sstevel@tonic-gate 11697c478bd9Sstevel@tonic-gate return (0); 11707c478bd9Sstevel@tonic-gate } 11717c478bd9Sstevel@tonic-gate 11727c478bd9Sstevel@tonic-gate /* 11737c478bd9Sstevel@tonic-gate * Power on CPU. 11747c478bd9Sstevel@tonic-gate */ 11757c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11767c478bd9Sstevel@tonic-gate int 11777c478bd9Sstevel@tonic-gate mp_cpu_poweron(struct cpu *cp) 11787c478bd9Sstevel@tonic-gate { 11797c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 11807c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 11817c478bd9Sstevel@tonic-gate } 11827c478bd9Sstevel@tonic-gate 11837c478bd9Sstevel@tonic-gate /* 11847c478bd9Sstevel@tonic-gate * Power off CPU. 11857c478bd9Sstevel@tonic-gate */ 11867c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11877c478bd9Sstevel@tonic-gate int 11887c478bd9Sstevel@tonic-gate mp_cpu_poweroff(struct cpu *cp) 11897c478bd9Sstevel@tonic-gate { 11907c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 11917c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 11927c478bd9Sstevel@tonic-gate } 11937c478bd9Sstevel@tonic-gate 11947c478bd9Sstevel@tonic-gate 11957c478bd9Sstevel@tonic-gate /* 11967c478bd9Sstevel@tonic-gate * Take the specified CPU out of participation in interrupts. 11977c478bd9Sstevel@tonic-gate */ 11987c478bd9Sstevel@tonic-gate int 11997c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp) 12007c478bd9Sstevel@tonic-gate { 12017c478bd9Sstevel@tonic-gate if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 12027c478bd9Sstevel@tonic-gate return (EBUSY); 12037c478bd9Sstevel@tonic-gate 12047c478bd9Sstevel@tonic-gate cp->cpu_flags &= ~CPU_ENABLE; 12057c478bd9Sstevel@tonic-gate return (0); 12067c478bd9Sstevel@tonic-gate } 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate /* 12097c478bd9Sstevel@tonic-gate * Allow the specified CPU to participate in interrupts. 12107c478bd9Sstevel@tonic-gate */ 12117c478bd9Sstevel@tonic-gate void 12127c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp) 12137c478bd9Sstevel@tonic-gate { 12147c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 12157c478bd9Sstevel@tonic-gate cp->cpu_flags |= CPU_ENABLE; 12167c478bd9Sstevel@tonic-gate psm_enable_intr(cp->cpu_id); 12177c478bd9Sstevel@tonic-gate } 12187c478bd9Sstevel@tonic-gate 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate 12217c478bd9Sstevel@tonic-gate static ushort_t * 12227c478bd9Sstevel@tonic-gate mp_map_warm_reset_vector() 12237c478bd9Sstevel@tonic-gate { 12247c478bd9Sstevel@tonic-gate ushort_t *warm_reset_vector; 12257c478bd9Sstevel@tonic-gate 12267c478bd9Sstevel@tonic-gate if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 12277c478bd9Sstevel@tonic-gate sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 12287c478bd9Sstevel@tonic-gate return (NULL); 12297c478bd9Sstevel@tonic-gate 12307c478bd9Sstevel@tonic-gate /* 12317c478bd9Sstevel@tonic-gate * setup secondary cpu bios boot up vector 12327c478bd9Sstevel@tonic-gate */ 12337c478bd9Sstevel@tonic-gate *warm_reset_vector = (ushort_t)((caddr_t) 12347c478bd9Sstevel@tonic-gate ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 12357c478bd9Sstevel@tonic-gate + ((ulong_t)rm_platter_va & 0xf)); 12367c478bd9Sstevel@tonic-gate warm_reset_vector++; 12377c478bd9Sstevel@tonic-gate *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 12387c478bd9Sstevel@tonic-gate 12397c478bd9Sstevel@tonic-gate --warm_reset_vector; 12407c478bd9Sstevel@tonic-gate return (warm_reset_vector); 12417c478bd9Sstevel@tonic-gate } 12427c478bd9Sstevel@tonic-gate 12437c478bd9Sstevel@tonic-gate static void 12447c478bd9Sstevel@tonic-gate mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 12457c478bd9Sstevel@tonic-gate { 12467c478bd9Sstevel@tonic-gate psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 12477c478bd9Sstevel@tonic-gate } 12487c478bd9Sstevel@tonic-gate 12497c478bd9Sstevel@tonic-gate void 12507c478bd9Sstevel@tonic-gate mp_cpu_faulted_enter(struct cpu *cp) 1251*7aec1d6eScindi { 1252*7aec1d6eScindi cmi_faulted_enter(cp); 1253*7aec1d6eScindi } 12547c478bd9Sstevel@tonic-gate 12557c478bd9Sstevel@tonic-gate void 12567c478bd9Sstevel@tonic-gate mp_cpu_faulted_exit(struct cpu *cp) 1257*7aec1d6eScindi { 1258*7aec1d6eScindi cmi_faulted_exit(cp); 1259*7aec1d6eScindi } 12607c478bd9Sstevel@tonic-gate 12617c478bd9Sstevel@tonic-gate /* 12627c478bd9Sstevel@tonic-gate * The following two routines are used as context operators on threads belonging 12637c478bd9Sstevel@tonic-gate * to processes with a private LDT (see sysi86). Due to the rarity of such 12647c478bd9Sstevel@tonic-gate * processes, these routines are currently written for best code readability and 12657c478bd9Sstevel@tonic-gate * organization rather than speed. We could avoid checking x86_feature at every 12667c478bd9Sstevel@tonic-gate * context switch by installing different context ops, depending on the 12677c478bd9Sstevel@tonic-gate * x86_feature flags, at LDT creation time -- one for each combination of fast 12687c478bd9Sstevel@tonic-gate * syscall feature flags. 12697c478bd9Sstevel@tonic-gate */ 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12727c478bd9Sstevel@tonic-gate void 12737c478bd9Sstevel@tonic-gate cpu_fast_syscall_disable(void *arg) 12747c478bd9Sstevel@tonic-gate { 12757c478bd9Sstevel@tonic-gate if (x86_feature & X86_SEP) 12767c478bd9Sstevel@tonic-gate cpu_sep_disable(); 12777c478bd9Sstevel@tonic-gate if (x86_feature & X86_ASYSC) 12787c478bd9Sstevel@tonic-gate cpu_asysc_disable(); 12797c478bd9Sstevel@tonic-gate } 12807c478bd9Sstevel@tonic-gate 12817c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12827c478bd9Sstevel@tonic-gate void 12837c478bd9Sstevel@tonic-gate cpu_fast_syscall_enable(void *arg) 12847c478bd9Sstevel@tonic-gate { 12857c478bd9Sstevel@tonic-gate if (x86_feature & X86_SEP) 12867c478bd9Sstevel@tonic-gate cpu_sep_enable(); 12877c478bd9Sstevel@tonic-gate if (x86_feature & X86_ASYSC) 12887c478bd9Sstevel@tonic-gate cpu_asysc_enable(); 12897c478bd9Sstevel@tonic-gate } 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate static void 12927c478bd9Sstevel@tonic-gate cpu_sep_enable(void) 12937c478bd9Sstevel@tonic-gate { 12947c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_SEP); 12957c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 12967c478bd9Sstevel@tonic-gate 12970ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL); 12987c478bd9Sstevel@tonic-gate } 12997c478bd9Sstevel@tonic-gate 13007c478bd9Sstevel@tonic-gate static void 13017c478bd9Sstevel@tonic-gate cpu_sep_disable(void) 13027c478bd9Sstevel@tonic-gate { 13037c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_SEP); 13047c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 13057c478bd9Sstevel@tonic-gate 13067c478bd9Sstevel@tonic-gate /* 13077c478bd9Sstevel@tonic-gate * Setting the SYSENTER_CS_MSR register to 0 causes software executing 13087c478bd9Sstevel@tonic-gate * the sysenter or sysexit instruction to trigger a #gp fault. 13097c478bd9Sstevel@tonic-gate */ 13100ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_CS, 0ULL); 13117c478bd9Sstevel@tonic-gate } 13127c478bd9Sstevel@tonic-gate 13137c478bd9Sstevel@tonic-gate static void 13147c478bd9Sstevel@tonic-gate cpu_asysc_enable(void) 13157c478bd9Sstevel@tonic-gate { 13167c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_ASYSC); 13177c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 13187c478bd9Sstevel@tonic-gate 13190ac7d7d8Skucharsk wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) | 13200ac7d7d8Skucharsk (uint64_t)(uintptr_t)AMD_EFER_SCE); 13217c478bd9Sstevel@tonic-gate } 13227c478bd9Sstevel@tonic-gate 13237c478bd9Sstevel@tonic-gate static void 13247c478bd9Sstevel@tonic-gate cpu_asysc_disable(void) 13257c478bd9Sstevel@tonic-gate { 13267c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_ASYSC); 13277c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 13287c478bd9Sstevel@tonic-gate 13297c478bd9Sstevel@tonic-gate /* 13307c478bd9Sstevel@tonic-gate * Turn off the SCE (syscall enable) bit in the EFER register. Software 13317c478bd9Sstevel@tonic-gate * executing syscall or sysret with this bit off will incur a #ud trap. 13327c478bd9Sstevel@tonic-gate */ 13330ac7d7d8Skucharsk wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) & 13340ac7d7d8Skucharsk ~((uint64_t)(uintptr_t)AMD_EFER_SCE)); 13357c478bd9Sstevel@tonic-gate } 1336