17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*100b72f4Sandrei * Common Development and Distribution License (the "License"). 6*100b72f4Sandrei * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22b4b46911Skchow * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/types.h> 297c478bd9Sstevel@tonic-gate #include <sys/thread.h> 307c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 317c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 327c478bd9Sstevel@tonic-gate #include <sys/param.h> 337c478bd9Sstevel@tonic-gate #include <sys/proc.h> 347c478bd9Sstevel@tonic-gate #include <sys/disp.h> 357c478bd9Sstevel@tonic-gate #include <sys/mmu.h> 367c478bd9Sstevel@tonic-gate #include <sys/class.h> 377c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 387c478bd9Sstevel@tonic-gate #include <sys/debug.h> 397c478bd9Sstevel@tonic-gate #include <sys/asm_linkage.h> 407c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 417c478bd9Sstevel@tonic-gate #include <sys/systm.h> 427c478bd9Sstevel@tonic-gate #include <sys/var.h> 437c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 447c478bd9Sstevel@tonic-gate #include <vm/hat.h> 457c478bd9Sstevel@tonic-gate #include <sys/mmu.h> 467c478bd9Sstevel@tonic-gate #include <vm/as.h> 477c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 487c478bd9Sstevel@tonic-gate #include <sys/segments.h> 497c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 507c478bd9Sstevel@tonic-gate #include <sys/stack.h> 517c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h> 527c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 537c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 547c478bd9Sstevel@tonic-gate #include <sys/traptrace.h> 557c478bd9Sstevel@tonic-gate #include <sys/clock.h> 567c478bd9Sstevel@tonic-gate #include <sys/cpc_impl.h> 577c478bd9Sstevel@tonic-gate #include <sys/chip.h> 587c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 597c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 607c478bd9Sstevel@tonic-gate #include <sys/fp.h> 617c478bd9Sstevel@tonic-gate #include <sys/reboot.h> 627c478bd9Sstevel@tonic-gate #include <sys/kdi.h> 637c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h> 647c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 65ef50d8c0Sesaxe #include <sys/pci_cfgspace.h> 667aec1d6eScindi #include <sys/cpu_module.h> 677c478bd9Sstevel@tonic-gate 687c478bd9Sstevel@tonic-gate struct cpu cpus[1]; /* CPU data */ 697c478bd9Sstevel@tonic-gate struct cpu *cpu[NCPU] = {&cpus[0]}; /* pointers to all CPUs */ 707c478bd9Sstevel@tonic-gate cpu_core_t cpu_core[NCPU]; /* cpu_core structures */ 717c478bd9Sstevel@tonic-gate 727c478bd9Sstevel@tonic-gate /* 737c478bd9Sstevel@tonic-gate * Useful for disabling MP bring-up for an MP capable kernel 747c478bd9Sstevel@tonic-gate * (a kernel that was built with MP defined) 757c478bd9Sstevel@tonic-gate */ 767c478bd9Sstevel@tonic-gate int use_mp = 1; 777c478bd9Sstevel@tonic-gate 787c478bd9Sstevel@tonic-gate int mp_cpus = 0x1; /* to be set by platform specific module */ 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate /* 817c478bd9Sstevel@tonic-gate * This variable is used by the hat layer to decide whether or not 827c478bd9Sstevel@tonic-gate * critical sections are needed to prevent race conditions. For sun4m, 837c478bd9Sstevel@tonic-gate * this variable is set once enough MP initialization has been done in 847c478bd9Sstevel@tonic-gate * order to allow cross calls. 857c478bd9Sstevel@tonic-gate */ 867c478bd9Sstevel@tonic-gate int flushes_require_xcalls = 0; 877c478bd9Sstevel@tonic-gate ulong_t cpu_ready_set = 1; 887c478bd9Sstevel@tonic-gate 897c478bd9Sstevel@tonic-gate extern void real_mode_start(void); 907c478bd9Sstevel@tonic-gate extern void real_mode_end(void); 917c478bd9Sstevel@tonic-gate static void mp_startup(void); 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate static void cpu_sep_enable(void); 947c478bd9Sstevel@tonic-gate static void cpu_sep_disable(void); 957c478bd9Sstevel@tonic-gate static void cpu_asysc_enable(void); 967c478bd9Sstevel@tonic-gate static void cpu_asysc_disable(void); 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate extern int tsc_gethrtime_enable; 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate /* 1017c478bd9Sstevel@tonic-gate * Init CPU info - get CPU type info for processor_info system call. 1027c478bd9Sstevel@tonic-gate */ 1037c478bd9Sstevel@tonic-gate void 1047c478bd9Sstevel@tonic-gate init_cpu_info(struct cpu *cp) 1057c478bd9Sstevel@tonic-gate { 1067c478bd9Sstevel@tonic-gate processor_info_t *pi = &cp->cpu_type_info; 1077c478bd9Sstevel@tonic-gate char buf[CPU_IDSTRLEN]; 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate /* 1107c478bd9Sstevel@tonic-gate * Get clock-frequency property for the CPU. 1117c478bd9Sstevel@tonic-gate */ 1127c478bd9Sstevel@tonic-gate pi->pi_clock = cpu_freq; 1137c478bd9Sstevel@tonic-gate 1147c478bd9Sstevel@tonic-gate (void) strcpy(pi->pi_processor_type, "i386"); 1157c478bd9Sstevel@tonic-gate if (fpu_exists) 1167c478bd9Sstevel@tonic-gate (void) strcpy(pi->pi_fputypes, "i387 compatible"); 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate (void) cpuid_getidstr(cp, buf, sizeof (buf)); 1197c478bd9Sstevel@tonic-gate 1207c478bd9Sstevel@tonic-gate cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 1217c478bd9Sstevel@tonic-gate (void) strcpy(cp->cpu_idstr, buf); 1227c478bd9Sstevel@tonic-gate 1237c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); 1247c478bd9Sstevel@tonic-gate 1257c478bd9Sstevel@tonic-gate (void) cpuid_getbrandstr(cp, buf, sizeof (buf)); 1267c478bd9Sstevel@tonic-gate cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP); 1277c478bd9Sstevel@tonic-gate (void) strcpy(cp->cpu_brandstr, buf); 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); 1307c478bd9Sstevel@tonic-gate } 1317c478bd9Sstevel@tonic-gate 1327c478bd9Sstevel@tonic-gate /* 1337c478bd9Sstevel@tonic-gate * Configure syscall support on this CPU. 1347c478bd9Sstevel@tonic-gate */ 1357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 1367c478bd9Sstevel@tonic-gate static void 1377c478bd9Sstevel@tonic-gate init_cpu_syscall(struct cpu *cp) 1387c478bd9Sstevel@tonic-gate { 1397c478bd9Sstevel@tonic-gate kpreempt_disable(); 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate #if defined(__amd64) 1427c478bd9Sstevel@tonic-gate if (x86_feature & X86_ASYSC) { 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate #if !defined(__lint) 1457c478bd9Sstevel@tonic-gate /* 1467c478bd9Sstevel@tonic-gate * The syscall instruction imposes a certain ordering on 1477c478bd9Sstevel@tonic-gate * segment selectors, so we double-check that ordering 1487c478bd9Sstevel@tonic-gate * here. 1497c478bd9Sstevel@tonic-gate */ 1507c478bd9Sstevel@tonic-gate ASSERT(KDS_SEL == KCS_SEL + 8); 1517c478bd9Sstevel@tonic-gate ASSERT(UDS_SEL == U32CS_SEL + 8); 1527c478bd9Sstevel@tonic-gate ASSERT(UCS_SEL == U32CS_SEL + 16); 1537c478bd9Sstevel@tonic-gate #endif 1547c478bd9Sstevel@tonic-gate /* 1557c478bd9Sstevel@tonic-gate * Turn syscall/sysret extensions on. 1567c478bd9Sstevel@tonic-gate */ 1577c478bd9Sstevel@tonic-gate cpu_asysc_enable(); 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate /* 1607c478bd9Sstevel@tonic-gate * Program the magic registers .. 1617c478bd9Sstevel@tonic-gate */ 1620ac7d7d8Skucharsk wrmsr(MSR_AMD_STAR, ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 1630ac7d7d8Skucharsk 32); 1640ac7d7d8Skucharsk wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall); 1650ac7d7d8Skucharsk wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32); 1667c478bd9Sstevel@tonic-gate 1677c478bd9Sstevel@tonic-gate /* 1687c478bd9Sstevel@tonic-gate * This list of flags is masked off the incoming 1697c478bd9Sstevel@tonic-gate * %rfl when we enter the kernel. 1707c478bd9Sstevel@tonic-gate */ 1710ac7d7d8Skucharsk wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T)); 1727c478bd9Sstevel@tonic-gate } 1737c478bd9Sstevel@tonic-gate #endif 1747c478bd9Sstevel@tonic-gate 1757c478bd9Sstevel@tonic-gate /* 1767c478bd9Sstevel@tonic-gate * On 32-bit kernels, we use sysenter/sysexit because it's too 1777c478bd9Sstevel@tonic-gate * hard to use syscall/sysret, and it is more portable anyway. 1787c478bd9Sstevel@tonic-gate * 1797c478bd9Sstevel@tonic-gate * On 64-bit kernels on Nocona machines, the 32-bit syscall 1807c478bd9Sstevel@tonic-gate * variant isn't available to 32-bit applications, but sysenter is. 1817c478bd9Sstevel@tonic-gate */ 1827c478bd9Sstevel@tonic-gate if (x86_feature & X86_SEP) { 1837c478bd9Sstevel@tonic-gate 1847c478bd9Sstevel@tonic-gate #if !defined(__lint) 1857c478bd9Sstevel@tonic-gate /* 1867c478bd9Sstevel@tonic-gate * The sysenter instruction imposes a certain ordering on 1877c478bd9Sstevel@tonic-gate * segment selectors, so we double-check that ordering 1887c478bd9Sstevel@tonic-gate * here. See "sysenter" in Intel document 245471-012, "IA-32 1897c478bd9Sstevel@tonic-gate * Intel Architecture Software Developer's Manual Volume 2: 1907c478bd9Sstevel@tonic-gate * Instruction Set Reference" 1917c478bd9Sstevel@tonic-gate */ 1927c478bd9Sstevel@tonic-gate ASSERT(KDS_SEL == KCS_SEL + 8); 1937c478bd9Sstevel@tonic-gate 1947c478bd9Sstevel@tonic-gate ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3)); 1957c478bd9Sstevel@tonic-gate ASSERT32(UDS_SEL == UCS_SEL + 8); 1967c478bd9Sstevel@tonic-gate 1977c478bd9Sstevel@tonic-gate ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3)); 1987c478bd9Sstevel@tonic-gate ASSERT64(UDS_SEL == U32CS_SEL + 8); 1997c478bd9Sstevel@tonic-gate #endif 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate cpu_sep_enable(); 2027c478bd9Sstevel@tonic-gate 2037c478bd9Sstevel@tonic-gate /* 2047c478bd9Sstevel@tonic-gate * resume() sets this value to the base of the threads stack 2057c478bd9Sstevel@tonic-gate * via a context handler. 2067c478bd9Sstevel@tonic-gate */ 2070ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_ESP, 0ULL); 2080ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter); 2097c478bd9Sstevel@tonic-gate } 2107c478bd9Sstevel@tonic-gate 2117c478bd9Sstevel@tonic-gate kpreempt_enable(); 2127c478bd9Sstevel@tonic-gate } 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate /* 2157c478bd9Sstevel@tonic-gate * Multiprocessor initialization. 2167c478bd9Sstevel@tonic-gate * 2177c478bd9Sstevel@tonic-gate * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the 2187c478bd9Sstevel@tonic-gate * startup and idle threads for the specified CPU. 2197c478bd9Sstevel@tonic-gate */ 2207c478bd9Sstevel@tonic-gate static void 2217c478bd9Sstevel@tonic-gate mp_startup_init(int cpun) 2227c478bd9Sstevel@tonic-gate { 2237c478bd9Sstevel@tonic-gate #if defined(__amd64) 2247c478bd9Sstevel@tonic-gate extern void *long_mode_64(void); 2257c478bd9Sstevel@tonic-gate #endif /* __amd64 */ 2267c478bd9Sstevel@tonic-gate 2277c478bd9Sstevel@tonic-gate struct cpu *cp; 2287c478bd9Sstevel@tonic-gate struct tss *ntss; 2297c478bd9Sstevel@tonic-gate kthread_id_t tp; 2307c478bd9Sstevel@tonic-gate caddr_t sp; 2317c478bd9Sstevel@tonic-gate int size; 2327c478bd9Sstevel@tonic-gate proc_t *procp; 2337c478bd9Sstevel@tonic-gate extern void idle(); 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate struct cpu_tables *tablesp; 2367c478bd9Sstevel@tonic-gate rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 2377c478bd9Sstevel@tonic-gate 2387c478bd9Sstevel@tonic-gate #ifdef TRAPTRACE 2397c478bd9Sstevel@tonic-gate trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; 2407c478bd9Sstevel@tonic-gate #endif 2417c478bd9Sstevel@tonic-gate 2427c478bd9Sstevel@tonic-gate ASSERT(cpun < NCPU && cpu[cpun] == NULL); 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) { 2457c478bd9Sstevel@tonic-gate panic("mp_startup_init: cpu%d: " 2467c478bd9Sstevel@tonic-gate "no memory for cpu structure", cpun); 2477c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 2487c478bd9Sstevel@tonic-gate } 2497c478bd9Sstevel@tonic-gate procp = curthread->t_procp; 2507c478bd9Sstevel@tonic-gate 2517c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 2527c478bd9Sstevel@tonic-gate /* 2537c478bd9Sstevel@tonic-gate * Initialize the dispatcher first. 2547c478bd9Sstevel@tonic-gate */ 2557c478bd9Sstevel@tonic-gate disp_cpu_init(cp); 2567c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2577c478bd9Sstevel@tonic-gate 258affbd3ccSkchow cpu_vm_data_init(cp); 259affbd3ccSkchow 2607c478bd9Sstevel@tonic-gate /* 2617c478bd9Sstevel@tonic-gate * Allocate and initialize the startup thread for this CPU. 2627c478bd9Sstevel@tonic-gate * Interrupt and process switch stacks get allocated later 2637c478bd9Sstevel@tonic-gate * when the CPU starts running. 2647c478bd9Sstevel@tonic-gate */ 2657c478bd9Sstevel@tonic-gate tp = thread_create(NULL, 0, NULL, NULL, 0, procp, 2667c478bd9Sstevel@tonic-gate TS_STOPPED, maxclsyspri); 2677c478bd9Sstevel@tonic-gate 2687c478bd9Sstevel@tonic-gate /* 2697c478bd9Sstevel@tonic-gate * Set state to TS_ONPROC since this thread will start running 2707c478bd9Sstevel@tonic-gate * as soon as the CPU comes online. 2717c478bd9Sstevel@tonic-gate * 2727c478bd9Sstevel@tonic-gate * All the other fields of the thread structure are setup by 2737c478bd9Sstevel@tonic-gate * thread_create(). 2747c478bd9Sstevel@tonic-gate */ 2757c478bd9Sstevel@tonic-gate THREAD_ONPROC(tp, cp); 2767c478bd9Sstevel@tonic-gate tp->t_preempt = 1; 2777c478bd9Sstevel@tonic-gate tp->t_bound_cpu = cp; 2787c478bd9Sstevel@tonic-gate tp->t_affinitycnt = 1; 2797c478bd9Sstevel@tonic-gate tp->t_cpu = cp; 2807c478bd9Sstevel@tonic-gate tp->t_disp_queue = cp->cpu_disp; 2817c478bd9Sstevel@tonic-gate 2827c478bd9Sstevel@tonic-gate /* 2837c478bd9Sstevel@tonic-gate * Setup thread to start in mp_startup. 2847c478bd9Sstevel@tonic-gate */ 2857c478bd9Sstevel@tonic-gate sp = tp->t_stk; 2867c478bd9Sstevel@tonic-gate tp->t_pc = (uintptr_t)mp_startup; 2877c478bd9Sstevel@tonic-gate tp->t_sp = (uintptr_t)(sp - MINFRAME); 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate cp->cpu_id = cpun; 2907c478bd9Sstevel@tonic-gate cp->cpu_self = cp; 2917c478bd9Sstevel@tonic-gate cp->cpu_mask = 1 << cpun; 2927c478bd9Sstevel@tonic-gate cp->cpu_thread = tp; 2937c478bd9Sstevel@tonic-gate cp->cpu_lwp = NULL; 2947c478bd9Sstevel@tonic-gate cp->cpu_dispthread = tp; 2957c478bd9Sstevel@tonic-gate cp->cpu_dispatch_pri = DISP_PRIO(tp); 2967c478bd9Sstevel@tonic-gate 2977c478bd9Sstevel@tonic-gate /* 2987c478bd9Sstevel@tonic-gate * Now, initialize per-CPU idle thread for this CPU. 2997c478bd9Sstevel@tonic-gate */ 3007c478bd9Sstevel@tonic-gate tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); 3017c478bd9Sstevel@tonic-gate 3027c478bd9Sstevel@tonic-gate cp->cpu_idle_thread = tp; 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate tp->t_preempt = 1; 3057c478bd9Sstevel@tonic-gate tp->t_bound_cpu = cp; 3067c478bd9Sstevel@tonic-gate tp->t_affinitycnt = 1; 3077c478bd9Sstevel@tonic-gate tp->t_cpu = cp; 3087c478bd9Sstevel@tonic-gate tp->t_disp_queue = cp->cpu_disp; 3097c478bd9Sstevel@tonic-gate 310394b433dSesaxe /* 311394b433dSesaxe * Bootstrap the CPU for CMT aware scheduling 312394b433dSesaxe * The rest of the initialization will happen from 313394b433dSesaxe * mp_startup() 314394b433dSesaxe */ 315394b433dSesaxe chip_bootstrap_cpu(cp); 316394b433dSesaxe 3177c478bd9Sstevel@tonic-gate /* 3187c478bd9Sstevel@tonic-gate * Perform CPC intialization on the new CPU. 3197c478bd9Sstevel@tonic-gate */ 3207c478bd9Sstevel@tonic-gate kcpc_hw_init(cp); 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate /* 3237c478bd9Sstevel@tonic-gate * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 3247c478bd9Sstevel@tonic-gate * for each CPU. 3257c478bd9Sstevel@tonic-gate */ 3267c478bd9Sstevel@tonic-gate 3277c478bd9Sstevel@tonic-gate setup_vaddr_for_ppcopy(cp); 3287c478bd9Sstevel@tonic-gate 3297c478bd9Sstevel@tonic-gate /* 3307c478bd9Sstevel@tonic-gate * Allocate space for page directory, stack, tss, gdt and idt. 3317c478bd9Sstevel@tonic-gate * This assumes that kmem_alloc will return memory which is aligned 3327c478bd9Sstevel@tonic-gate * to the next higher power of 2 or a page(if size > MAXABIG) 3337c478bd9Sstevel@tonic-gate * If this assumption goes wrong at any time due to change in 3347c478bd9Sstevel@tonic-gate * kmem alloc, things may not work as the page directory has to be 3357c478bd9Sstevel@tonic-gate * page aligned 3367c478bd9Sstevel@tonic-gate */ 3377c478bd9Sstevel@tonic-gate if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL) 3387c478bd9Sstevel@tonic-gate panic("mp_startup_init: cpu%d cannot allocate tables", cpun); 3397c478bd9Sstevel@tonic-gate 3407c478bd9Sstevel@tonic-gate if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) { 3417c478bd9Sstevel@tonic-gate kmem_free(tablesp, sizeof (struct cpu_tables)); 3427c478bd9Sstevel@tonic-gate size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE; 3437c478bd9Sstevel@tonic-gate tablesp = kmem_zalloc(size, KM_NOSLEEP); 3447c478bd9Sstevel@tonic-gate tablesp = (struct cpu_tables *) 3457c478bd9Sstevel@tonic-gate (((uintptr_t)tablesp + MMU_STD_PAGESIZE) & 3467c478bd9Sstevel@tonic-gate MMU_STD_PAGEMASK); 3477c478bd9Sstevel@tonic-gate } 3487c478bd9Sstevel@tonic-gate 3497c478bd9Sstevel@tonic-gate ntss = cp->cpu_tss = &tablesp->ct_tss; 3507c478bd9Sstevel@tonic-gate cp->cpu_gdt = tablesp->ct_gdt; 3517c478bd9Sstevel@tonic-gate bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t))); 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate #if defined(__amd64) 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate /* 3567c478bd9Sstevel@tonic-gate * #DF (double fault). 3577c478bd9Sstevel@tonic-gate */ 3587c478bd9Sstevel@tonic-gate ntss->tss_ist1 = 3597c478bd9Sstevel@tonic-gate (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 3607c478bd9Sstevel@tonic-gate 3617c478bd9Sstevel@tonic-gate #elif defined(__i386) 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp = 3647c478bd9Sstevel@tonic-gate (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)]; 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL; 3677c478bd9Sstevel@tonic-gate 3687c478bd9Sstevel@tonic-gate ntss->tss_eip = (uint32_t)mp_startup; 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate ntss->tss_cs = KCS_SEL; 3717c478bd9Sstevel@tonic-gate ntss->tss_fs = KFS_SEL; 3727c478bd9Sstevel@tonic-gate ntss->tss_gs = KGS_SEL; 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate /* 3757c478bd9Sstevel@tonic-gate * setup kernel %gs. 3767c478bd9Sstevel@tonic-gate */ 3777c478bd9Sstevel@tonic-gate set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA, 3787c478bd9Sstevel@tonic-gate SEL_KPL, 0, 1); 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate #endif /* __i386 */ 3817c478bd9Sstevel@tonic-gate 3827c478bd9Sstevel@tonic-gate /* 3837c478bd9Sstevel@tonic-gate * Set I/O bit map offset equal to size of TSS segment limit 3847c478bd9Sstevel@tonic-gate * for no I/O permission map. This will cause all user I/O 3857c478bd9Sstevel@tonic-gate * instructions to generate #gp fault. 3867c478bd9Sstevel@tonic-gate */ 3877c478bd9Sstevel@tonic-gate ntss->tss_bitmapbase = sizeof (*ntss); 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate /* 3907c478bd9Sstevel@tonic-gate * setup kernel tss. 3917c478bd9Sstevel@tonic-gate */ 3927c478bd9Sstevel@tonic-gate set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, 3937c478bd9Sstevel@tonic-gate sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL); 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate /* 3967c478bd9Sstevel@tonic-gate * If we have more than one node, each cpu gets a copy of IDT 3977c478bd9Sstevel@tonic-gate * local to its node. If this is a Pentium box, we use cpu 0's 3987c478bd9Sstevel@tonic-gate * IDT. cpu 0's IDT has been made read-only to workaround the 3997c478bd9Sstevel@tonic-gate * cmpxchgl register bug 4007c478bd9Sstevel@tonic-gate */ 4017c478bd9Sstevel@tonic-gate cp->cpu_idt = CPU->cpu_idt; 4027c478bd9Sstevel@tonic-gate if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) { 4037c478bd9Sstevel@tonic-gate cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP); 4047c478bd9Sstevel@tonic-gate bcopy(idt0, cp->cpu_idt, sizeof (idt0)); 4057c478bd9Sstevel@tonic-gate } 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate /* 4087c478bd9Sstevel@tonic-gate * Get interrupt priority data from cpu 0 4097c478bd9Sstevel@tonic-gate */ 4107c478bd9Sstevel@tonic-gate cp->cpu_pri_data = CPU->cpu_pri_data; 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate hat_cpu_online(cp); 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate /* Should remove all entries for the current process/thread here */ 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate /* 4177c478bd9Sstevel@tonic-gate * Fill up the real mode platter to make it easy for real mode code to 4187c478bd9Sstevel@tonic-gate * kick it off. This area should really be one passed by boot to kernel 4197c478bd9Sstevel@tonic-gate * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 4207c478bd9Sstevel@tonic-gate * have identical physical and virtual address in paged mode. 4217c478bd9Sstevel@tonic-gate */ 4227c478bd9Sstevel@tonic-gate real_mode_platter->rm_idt_base = cp->cpu_idt; 4237c478bd9Sstevel@tonic-gate real_mode_platter->rm_idt_lim = sizeof (idt0) - 1; 4247c478bd9Sstevel@tonic-gate real_mode_platter->rm_gdt_base = cp->cpu_gdt; 4257c478bd9Sstevel@tonic-gate real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1; 4267c478bd9Sstevel@tonic-gate real_mode_platter->rm_pdbr = getcr3(); 4277c478bd9Sstevel@tonic-gate real_mode_platter->rm_cpu = cpun; 4287c478bd9Sstevel@tonic-gate real_mode_platter->rm_x86feature = x86_feature; 4297c478bd9Sstevel@tonic-gate real_mode_platter->rm_cr4 = cr4_value; 4307c478bd9Sstevel@tonic-gate 4317c478bd9Sstevel@tonic-gate #if defined(__amd64) 4327c478bd9Sstevel@tonic-gate if (getcr3() > 0xffffffffUL) 4337c478bd9Sstevel@tonic-gate panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 4347c478bd9Sstevel@tonic-gate "located above 4G in physical memory (@ 0x%llx).", 4357c478bd9Sstevel@tonic-gate (unsigned long long)getcr3()); 4367c478bd9Sstevel@tonic-gate 4377c478bd9Sstevel@tonic-gate /* 4387c478bd9Sstevel@tonic-gate * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 4397c478bd9Sstevel@tonic-gate * by code in real_mode_start(): 4407c478bd9Sstevel@tonic-gate * 4417c478bd9Sstevel@tonic-gate * GDT[0]: NULL selector 4427c478bd9Sstevel@tonic-gate * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 4437c478bd9Sstevel@tonic-gate * 4447c478bd9Sstevel@tonic-gate * Clear the IDT as interrupts will be off and a limit of 0 will cause 4457c478bd9Sstevel@tonic-gate * the CPU to triple fault and reset on an NMI, seemingly as reasonable 4467c478bd9Sstevel@tonic-gate * a course of action as any other, though it may cause the entire 4477c478bd9Sstevel@tonic-gate * platform to reset in some cases... 4487c478bd9Sstevel@tonic-gate */ 4497c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt[0] = 0ULL; 4507c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 4517c478bd9Sstevel@tonic-gate 4527c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt_lim = (ushort_t) 4537c478bd9Sstevel@tonic-gate (sizeof (real_mode_platter->rm_temp_gdt) - 1); 4547c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 4557c478bd9Sstevel@tonic-gate (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt); 4567c478bd9Sstevel@tonic-gate 4577c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_idt_lim = 0; 4587c478bd9Sstevel@tonic-gate real_mode_platter->rm_temp_idt_base = 0; 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate /* 4617c478bd9Sstevel@tonic-gate * Since the CPU needs to jump to protected mode using an identity 4627c478bd9Sstevel@tonic-gate * mapped address, we need to calculate it here. 4637c478bd9Sstevel@tonic-gate */ 4647c478bd9Sstevel@tonic-gate real_mode_platter->rm_longmode64_addr = rm_platter_pa + 4657c478bd9Sstevel@tonic-gate ((uint32_t)long_mode_64 - (uint32_t)real_mode_start); 4667c478bd9Sstevel@tonic-gate #endif /* __amd64 */ 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate #ifdef TRAPTRACE 4697c478bd9Sstevel@tonic-gate /* 4707c478bd9Sstevel@tonic-gate * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this 4717c478bd9Sstevel@tonic-gate * CPU. 4727c478bd9Sstevel@tonic-gate */ 4737c478bd9Sstevel@tonic-gate ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); 4747c478bd9Sstevel@tonic-gate ttc->ttc_next = ttc->ttc_first; 4757c478bd9Sstevel@tonic-gate ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; 4767c478bd9Sstevel@tonic-gate #endif 4777c478bd9Sstevel@tonic-gate 4787c478bd9Sstevel@tonic-gate /* 4797c478bd9Sstevel@tonic-gate * Record that we have another CPU. 4807c478bd9Sstevel@tonic-gate */ 4817c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 4827c478bd9Sstevel@tonic-gate /* 4837c478bd9Sstevel@tonic-gate * Initialize the interrupt threads for this CPU 4847c478bd9Sstevel@tonic-gate */ 485*100b72f4Sandrei cpu_intr_alloc(cp, NINTR_THREADS); 4867c478bd9Sstevel@tonic-gate /* 4877c478bd9Sstevel@tonic-gate * Add CPU to list of available CPUs. It'll be on the active list 4887c478bd9Sstevel@tonic-gate * after mp_startup(). 4897c478bd9Sstevel@tonic-gate */ 4907c478bd9Sstevel@tonic-gate cpu_add_unit(cp); 4917c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 4927c478bd9Sstevel@tonic-gate } 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate /* 4957c478bd9Sstevel@tonic-gate * Apply workarounds for known errata, and warn about those that are absent. 4967c478bd9Sstevel@tonic-gate * 4977c478bd9Sstevel@tonic-gate * System vendors occasionally create configurations which contain different 4987c478bd9Sstevel@tonic-gate * revisions of the CPUs that are almost but not exactly the same. At the 4997c478bd9Sstevel@tonic-gate * time of writing, this meant that their clock rates were the same, their 5007c478bd9Sstevel@tonic-gate * feature sets were the same, but the required workaround were -not- 5017c478bd9Sstevel@tonic-gate * necessarily the same. So, this routine is invoked on -every- CPU soon 5027c478bd9Sstevel@tonic-gate * after starting to make sure that the resulting system contains the most 5037c478bd9Sstevel@tonic-gate * pessimal set of workarounds needed to cope with *any* of the CPUs in the 5047c478bd9Sstevel@tonic-gate * system. 5057c478bd9Sstevel@tonic-gate * 506ef50d8c0Sesaxe * workaround_errata is invoked early in mlsetup() for CPU 0, and in 507ef50d8c0Sesaxe * mp_startup() for all slave CPUs. Slaves process workaround_errata prior 508ef50d8c0Sesaxe * to acknowledging their readiness to the master, so this routine will 509ef50d8c0Sesaxe * never be executed by multiple CPUs in parallel, thus making updates to 510ef50d8c0Sesaxe * global data safe. 511ef50d8c0Sesaxe * 5122201b277Skucharsk * These workarounds are based on Rev 3.57 of the Revision Guide for 5132201b277Skucharsk * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005. 5147c478bd9Sstevel@tonic-gate */ 5157c478bd9Sstevel@tonic-gate 5167c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_91) 5177c478bd9Sstevel@tonic-gate int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ 5187c478bd9Sstevel@tonic-gate #endif 5197c478bd9Sstevel@tonic-gate 5207c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_93) 5217c478bd9Sstevel@tonic-gate int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ 5227c478bd9Sstevel@tonic-gate #endif 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_100) 5257c478bd9Sstevel@tonic-gate int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ 5267c478bd9Sstevel@tonic-gate #endif 5277c478bd9Sstevel@tonic-gate 5287c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109) 5297c478bd9Sstevel@tonic-gate int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ 5307c478bd9Sstevel@tonic-gate #endif 5317c478bd9Sstevel@tonic-gate 5327c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_121) 5337c478bd9Sstevel@tonic-gate int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ 5347c478bd9Sstevel@tonic-gate #endif 5357c478bd9Sstevel@tonic-gate 5367c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_122) 5377c478bd9Sstevel@tonic-gate int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ 5387c478bd9Sstevel@tonic-gate #endif 5397c478bd9Sstevel@tonic-gate 5407c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_123) 5417c478bd9Sstevel@tonic-gate int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ 5427c478bd9Sstevel@tonic-gate #endif 5437c478bd9Sstevel@tonic-gate 5442201b277Skucharsk #if defined(OPTERON_ERRATUM_131) 5452201b277Skucharsk int opteron_erratum_131; /* if non-zero -> at least one cpu has it */ 5462201b277Skucharsk #endif 5477c478bd9Sstevel@tonic-gate 548ef50d8c0Sesaxe #if defined(OPTERON_WORKAROUND_6336786) 549ef50d8c0Sesaxe int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */ 550ef50d8c0Sesaxe int opteron_workaround_6336786_UP = 0; /* Not needed for UP */ 551ef50d8c0Sesaxe #endif 552ef50d8c0Sesaxe 5537c478bd9Sstevel@tonic-gate #define WARNING(cpu, n) \ 5547c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d", \ 5557c478bd9Sstevel@tonic-gate (cpu)->cpu_id, (n)) 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate uint_t 5587c478bd9Sstevel@tonic-gate workaround_errata(struct cpu *cpu) 5597c478bd9Sstevel@tonic-gate { 5607c478bd9Sstevel@tonic-gate uint_t missing = 0; 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate ASSERT(cpu == CPU); 5637c478bd9Sstevel@tonic-gate 5647c478bd9Sstevel@tonic-gate /*LINTED*/ 5657c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 88) > 0) { 5667c478bd9Sstevel@tonic-gate /* 5677c478bd9Sstevel@tonic-gate * SWAPGS May Fail To Read Correct GS Base 5687c478bd9Sstevel@tonic-gate */ 5697c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_88) 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * The workaround is an mfence in the relevant assembler code 5727c478bd9Sstevel@tonic-gate */ 5737c478bd9Sstevel@tonic-gate #else 5747c478bd9Sstevel@tonic-gate WARNING(cpu, 88); 5757c478bd9Sstevel@tonic-gate missing++; 5767c478bd9Sstevel@tonic-gate #endif 5777c478bd9Sstevel@tonic-gate } 5787c478bd9Sstevel@tonic-gate 5797c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 91) > 0) { 5807c478bd9Sstevel@tonic-gate /* 5817c478bd9Sstevel@tonic-gate * Software Prefetches May Report A Page Fault 5827c478bd9Sstevel@tonic-gate */ 5837c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_91) 5847c478bd9Sstevel@tonic-gate /* 5857c478bd9Sstevel@tonic-gate * fix is in trap.c 5867c478bd9Sstevel@tonic-gate */ 5877c478bd9Sstevel@tonic-gate opteron_erratum_91++; 5887c478bd9Sstevel@tonic-gate #else 5897c478bd9Sstevel@tonic-gate WARNING(cpu, 91); 5907c478bd9Sstevel@tonic-gate missing++; 5917c478bd9Sstevel@tonic-gate #endif 5927c478bd9Sstevel@tonic-gate } 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 93) > 0) { 5957c478bd9Sstevel@tonic-gate /* 5967c478bd9Sstevel@tonic-gate * RSM Auto-Halt Restart Returns to Incorrect RIP 5977c478bd9Sstevel@tonic-gate */ 5987c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_93) 5997c478bd9Sstevel@tonic-gate /* 6007c478bd9Sstevel@tonic-gate * fix is in trap.c 6017c478bd9Sstevel@tonic-gate */ 6027c478bd9Sstevel@tonic-gate opteron_erratum_93++; 6037c478bd9Sstevel@tonic-gate #else 6047c478bd9Sstevel@tonic-gate WARNING(cpu, 93); 6057c478bd9Sstevel@tonic-gate missing++; 6067c478bd9Sstevel@tonic-gate #endif 6077c478bd9Sstevel@tonic-gate } 6087c478bd9Sstevel@tonic-gate 6097c478bd9Sstevel@tonic-gate /*LINTED*/ 6107c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 95) > 0) { 6117c478bd9Sstevel@tonic-gate /* 6127c478bd9Sstevel@tonic-gate * RET Instruction May Return to Incorrect EIP 6137c478bd9Sstevel@tonic-gate */ 6147c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_95) 6157c478bd9Sstevel@tonic-gate #if defined(_LP64) 6167c478bd9Sstevel@tonic-gate /* 6177c478bd9Sstevel@tonic-gate * Workaround this by ensuring that 32-bit user code and 6187c478bd9Sstevel@tonic-gate * 64-bit kernel code never occupy the same address 6197c478bd9Sstevel@tonic-gate * range mod 4G. 6207c478bd9Sstevel@tonic-gate */ 6217c478bd9Sstevel@tonic-gate if (_userlimit32 > 0xc0000000ul) 6227c478bd9Sstevel@tonic-gate *(uintptr_t *)&_userlimit32 = 0xc0000000ul; 6237c478bd9Sstevel@tonic-gate 6247c478bd9Sstevel@tonic-gate /*LINTED*/ 6257c478bd9Sstevel@tonic-gate ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); 6267c478bd9Sstevel@tonic-gate #endif /* _LP64 */ 6277c478bd9Sstevel@tonic-gate #else 6287c478bd9Sstevel@tonic-gate WARNING(cpu, 95); 6297c478bd9Sstevel@tonic-gate missing++; 6307c478bd9Sstevel@tonic-gate #endif /* OPTERON_ERRATUM_95 */ 6317c478bd9Sstevel@tonic-gate } 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 100) > 0) { 6347c478bd9Sstevel@tonic-gate /* 6357c478bd9Sstevel@tonic-gate * Compatibility Mode Branches Transfer to Illegal Address 6367c478bd9Sstevel@tonic-gate */ 6377c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_100) 6387c478bd9Sstevel@tonic-gate /* 6397c478bd9Sstevel@tonic-gate * fix is in trap.c 6407c478bd9Sstevel@tonic-gate */ 6417c478bd9Sstevel@tonic-gate opteron_erratum_100++; 6427c478bd9Sstevel@tonic-gate #else 6437c478bd9Sstevel@tonic-gate WARNING(cpu, 100); 6447c478bd9Sstevel@tonic-gate missing++; 6457c478bd9Sstevel@tonic-gate #endif 6467c478bd9Sstevel@tonic-gate } 6477c478bd9Sstevel@tonic-gate 6487c478bd9Sstevel@tonic-gate /*LINTED*/ 6497c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 108) > 0) { 6507c478bd9Sstevel@tonic-gate /* 6517c478bd9Sstevel@tonic-gate * CPUID Instruction May Return Incorrect Model Number In 6527c478bd9Sstevel@tonic-gate * Some Processors 6537c478bd9Sstevel@tonic-gate */ 6547c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_108) 6557c478bd9Sstevel@tonic-gate /* 6567c478bd9Sstevel@tonic-gate * (Our cpuid-handling code corrects the model number on 6577c478bd9Sstevel@tonic-gate * those processors) 6587c478bd9Sstevel@tonic-gate */ 6597c478bd9Sstevel@tonic-gate #else 6607c478bd9Sstevel@tonic-gate WARNING(cpu, 108); 6617c478bd9Sstevel@tonic-gate missing++; 6627c478bd9Sstevel@tonic-gate #endif 6637c478bd9Sstevel@tonic-gate } 6647c478bd9Sstevel@tonic-gate 6657c478bd9Sstevel@tonic-gate /*LINTED*/ 6667c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 109) > 0) { 6677c478bd9Sstevel@tonic-gate /* 6687c478bd9Sstevel@tonic-gate * Certain Reverse REP MOVS May Produce Unpredictable Behaviour 6697c478bd9Sstevel@tonic-gate */ 6707c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109) 6717c478bd9Sstevel@tonic-gate 6727c478bd9Sstevel@tonic-gate /* workaround is to print a warning to upgrade BIOS */ 6730ac7d7d8Skucharsk if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 6747c478bd9Sstevel@tonic-gate opteron_erratum_109++; 6757c478bd9Sstevel@tonic-gate #else 6767c478bd9Sstevel@tonic-gate WARNING(cpu, 109); 6777c478bd9Sstevel@tonic-gate missing++; 6787c478bd9Sstevel@tonic-gate #endif 6797c478bd9Sstevel@tonic-gate } 6807c478bd9Sstevel@tonic-gate /*LINTED*/ 6817c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 121) > 0) { 6827c478bd9Sstevel@tonic-gate /* 6837c478bd9Sstevel@tonic-gate * Sequential Execution Across Non_Canonical Boundary Caused 6847c478bd9Sstevel@tonic-gate * Processor Hang 6857c478bd9Sstevel@tonic-gate */ 6867c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_121) 6877c478bd9Sstevel@tonic-gate static int lma; 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate if (opteron_erratum_121) 6907c478bd9Sstevel@tonic-gate opteron_erratum_121++; 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate /* 6937c478bd9Sstevel@tonic-gate * Erratum 121 is only present in long (64 bit) mode. 6947c478bd9Sstevel@tonic-gate * Workaround is to include the page immediately before the 6957c478bd9Sstevel@tonic-gate * va hole to eliminate the possibility of system hangs due to 6967c478bd9Sstevel@tonic-gate * sequential execution across the va hole boundary. 6977c478bd9Sstevel@tonic-gate */ 6987c478bd9Sstevel@tonic-gate if (lma == 0) { 6997c478bd9Sstevel@tonic-gate /* 7007c478bd9Sstevel@tonic-gate * check LMA once: assume all cpus are in long mode 7017c478bd9Sstevel@tonic-gate * or not. 7027c478bd9Sstevel@tonic-gate */ 7037c478bd9Sstevel@tonic-gate lma = 1; 7047c478bd9Sstevel@tonic-gate 7050ac7d7d8Skucharsk if (rdmsr(MSR_AMD_EFER) & AMD_EFER_LMA) { 7067c478bd9Sstevel@tonic-gate if (hole_start) { 7077c478bd9Sstevel@tonic-gate hole_start -= PAGESIZE; 7087c478bd9Sstevel@tonic-gate } else { 7097c478bd9Sstevel@tonic-gate /* 7107c478bd9Sstevel@tonic-gate * hole_start not yet initialized by 7117c478bd9Sstevel@tonic-gate * mmu_init. Initialize hole_start 7127c478bd9Sstevel@tonic-gate * with value to be subtracted. 7137c478bd9Sstevel@tonic-gate */ 7147c478bd9Sstevel@tonic-gate hole_start = PAGESIZE; 7157c478bd9Sstevel@tonic-gate } 7167c478bd9Sstevel@tonic-gate opteron_erratum_121++; 7177c478bd9Sstevel@tonic-gate } 7187c478bd9Sstevel@tonic-gate } 7197c478bd9Sstevel@tonic-gate #else 7207c478bd9Sstevel@tonic-gate WARNING(cpu, 121); 7217c478bd9Sstevel@tonic-gate missing++; 7227c478bd9Sstevel@tonic-gate #endif 7237c478bd9Sstevel@tonic-gate } 7247c478bd9Sstevel@tonic-gate 7257c478bd9Sstevel@tonic-gate /*LINTED*/ 7267c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 122) > 0) { 7277c478bd9Sstevel@tonic-gate /* 7287c478bd9Sstevel@tonic-gate * TLB Flush Filter May Cause Cohenrency Problem in 7297c478bd9Sstevel@tonic-gate * Multiprocessor Systems 7307c478bd9Sstevel@tonic-gate */ 7317c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_122) 7327c478bd9Sstevel@tonic-gate /* 7337c478bd9Sstevel@tonic-gate * Erratum 122 is only present in MP configurations (multi-core 7347c478bd9Sstevel@tonic-gate * or multi-processor). 7357c478bd9Sstevel@tonic-gate */ 7367c478bd9Sstevel@tonic-gate 7377c478bd9Sstevel@tonic-gate if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 || 7387c478bd9Sstevel@tonic-gate cpuid_get_ncpu_per_chip(cpu) > 1) { 7397c478bd9Sstevel@tonic-gate /* disable TLB Flush Filter */ 7400ac7d7d8Skucharsk wrmsr(MSR_AMD_HWCR, rdmsr(MSR_AMD_HWCR) | 7410ac7d7d8Skucharsk (uint64_t)(uintptr_t)AMD_HWCR_FFDIS); 7427c478bd9Sstevel@tonic-gate opteron_erratum_122++; 7437c478bd9Sstevel@tonic-gate } 7447c478bd9Sstevel@tonic-gate 7457c478bd9Sstevel@tonic-gate #else 7467c478bd9Sstevel@tonic-gate WARNING(cpu, 122); 7477c478bd9Sstevel@tonic-gate missing++; 7487c478bd9Sstevel@tonic-gate #endif 7497c478bd9Sstevel@tonic-gate } 750403c216aSkchow 751403c216aSkchow #if defined(OPTERON_ERRATUM_123) 7527c478bd9Sstevel@tonic-gate /*LINTED*/ 7537c478bd9Sstevel@tonic-gate if (cpuid_opteron_erratum(cpu, 123) > 0) { 7547c478bd9Sstevel@tonic-gate /* 7557c478bd9Sstevel@tonic-gate * Bypassed Reads May Cause Data Corruption of System Hang in 7567c478bd9Sstevel@tonic-gate * Dual Core Processors 7577c478bd9Sstevel@tonic-gate */ 7587c478bd9Sstevel@tonic-gate /* 7597c478bd9Sstevel@tonic-gate * Erratum 123 applies only to multi-core cpus. 7607c478bd9Sstevel@tonic-gate */ 7617c478bd9Sstevel@tonic-gate 7627c478bd9Sstevel@tonic-gate if (cpuid_get_ncpu_per_chip(cpu) > 1) { 7637c478bd9Sstevel@tonic-gate /* workaround is to print a warning to upgrade BIOS */ 7640ac7d7d8Skucharsk if (rdmsr(MSR_AMD_PATCHLEVEL) == 0) 7657c478bd9Sstevel@tonic-gate opteron_erratum_123++; 7667c478bd9Sstevel@tonic-gate } 7677c478bd9Sstevel@tonic-gate } 768403c216aSkchow #endif 7692201b277Skucharsk 7702201b277Skucharsk #if defined(OPTERON_ERRATUM_131) 7712201b277Skucharsk /*LINTED*/ 7722201b277Skucharsk if (cpuid_opteron_erratum(cpu, 131) > 0) { 7732201b277Skucharsk /* 7742201b277Skucharsk * Multiprocessor Systems with Four or More Cores May Deadlock 7752201b277Skucharsk * Waiting for a Probe Response 7762201b277Skucharsk */ 7772201b277Skucharsk /* 7782201b277Skucharsk * Erratum 131 applies to any system with four or more cores. 7792201b277Skucharsk */ 7802201b277Skucharsk if ((opteron_erratum_131 == 0) && ((lgrp_plat_node_cnt * 7812201b277Skucharsk cpuid_get_ncpu_per_chip(cpu)) >= 4)) { 7822201b277Skucharsk /* 7832201b277Skucharsk * Workaround is to print a warning to upgrade 7842201b277Skucharsk * the BIOS 7852201b277Skucharsk */ 7860ac7d7d8Skucharsk if (!(rdmsr(MSR_AMD_NB_CFG) & AMD_NB_CFG_SRQ_HEARTBEAT)) 7872201b277Skucharsk opteron_erratum_131++; 7882201b277Skucharsk } 789ef50d8c0Sesaxe } 7902201b277Skucharsk #endif 791ef50d8c0Sesaxe 792ef50d8c0Sesaxe #if defined(OPTERON_WORKAROUND_6336786) 793ef50d8c0Sesaxe /* 794ef50d8c0Sesaxe * This isn't really erratum, but for convenience the 795ef50d8c0Sesaxe * detection/workaround code lives here and in cpuid_opteron_erratum. 796ef50d8c0Sesaxe */ 797ef50d8c0Sesaxe if (cpuid_opteron_erratum(cpu, 6336786) > 0) { 798ef50d8c0Sesaxe int node; 799ef50d8c0Sesaxe uint8_t data; 800ef50d8c0Sesaxe 801ef50d8c0Sesaxe /* 802ef50d8c0Sesaxe * Disable C1-Clock ramping on multi-core/multi-processor 803ef50d8c0Sesaxe * K8 platforms to guard against TSC drift. 804ef50d8c0Sesaxe */ 805ef50d8c0Sesaxe if (opteron_workaround_6336786) { 806ef50d8c0Sesaxe opteron_workaround_6336786++; 807ef50d8c0Sesaxe } else if ((lgrp_plat_node_cnt * 808ef50d8c0Sesaxe cpuid_get_ncpu_per_chip(cpu) >= 2) || 809ef50d8c0Sesaxe opteron_workaround_6336786_UP) { 810ef50d8c0Sesaxe for (node = 0; node < lgrp_plat_node_cnt; node++) { 811ef50d8c0Sesaxe /* 812ef50d8c0Sesaxe * Clear PMM7[1:0] (function 3, offset 0x87) 813ef50d8c0Sesaxe * Northbridge device is the node id + 24. 814ef50d8c0Sesaxe */ 815ef50d8c0Sesaxe data = pci_getb_func(0, node + 24, 3, 0x87); 816ef50d8c0Sesaxe data &= 0xFC; 817ef50d8c0Sesaxe pci_putb_func(0, node + 24, 3, 0x87, data); 818ef50d8c0Sesaxe } 819ef50d8c0Sesaxe opteron_workaround_6336786++; 820ef50d8c0Sesaxe } 8212201b277Skucharsk } 822ef50d8c0Sesaxe #endif 8237c478bd9Sstevel@tonic-gate return (missing); 8247c478bd9Sstevel@tonic-gate } 8257c478bd9Sstevel@tonic-gate 8267c478bd9Sstevel@tonic-gate void 8277c478bd9Sstevel@tonic-gate workaround_errata_end() 8287c478bd9Sstevel@tonic-gate { 8297c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109) 8307c478bd9Sstevel@tonic-gate if (opteron_erratum_109) { 8312201b277Skucharsk cmn_err(CE_WARN, 8322201b277Skucharsk "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 8332201b277Skucharsk " processor\nerratum 109 was not detected; updating your" 8342201b277Skucharsk " system's BIOS to a version\ncontaining this" 8352201b277Skucharsk " microcode patch is HIGHLY recommended or erroneous" 8362201b277Skucharsk " system\noperation may occur.\n"); 8377c478bd9Sstevel@tonic-gate } 8382201b277Skucharsk #endif /* OPTERON_ERRATUM_109 */ 8397c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_123) 8407c478bd9Sstevel@tonic-gate if (opteron_erratum_123) { 8412201b277Skucharsk cmn_err(CE_WARN, 8422201b277Skucharsk "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 8432201b277Skucharsk " processor\nerratum 123 was not detected; updating your" 8442201b277Skucharsk " system's BIOS to a version\ncontaining this" 8452201b277Skucharsk " microcode patch is HIGHLY recommended or erroneous" 8462201b277Skucharsk " system\noperation may occur.\n"); 8477c478bd9Sstevel@tonic-gate } 8482201b277Skucharsk #endif /* OPTERON_ERRATUM_123 */ 8492201b277Skucharsk #if defined(OPTERON_ERRATUM_131) 8502201b277Skucharsk if (opteron_erratum_131) { 8512201b277Skucharsk cmn_err(CE_WARN, 8522201b277Skucharsk "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" 8532201b277Skucharsk " processor\nerratum 131 was not detected; updating your" 8542201b277Skucharsk " system's BIOS to a version\ncontaining this" 8552201b277Skucharsk " microcode patch is HIGHLY recommended or erroneous" 8562201b277Skucharsk " system\noperation may occur.\n"); 8572201b277Skucharsk } 8582201b277Skucharsk #endif /* OPTERON_ERRATUM_131 */ 8597c478bd9Sstevel@tonic-gate } 8607c478bd9Sstevel@tonic-gate 8617c478bd9Sstevel@tonic-gate static ushort_t *mp_map_warm_reset_vector(); 8627c478bd9Sstevel@tonic-gate static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector); 8637c478bd9Sstevel@tonic-gate 8647c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8657c478bd9Sstevel@tonic-gate void 8667c478bd9Sstevel@tonic-gate start_other_cpus(int cprboot) 8677c478bd9Sstevel@tonic-gate { 8687c478bd9Sstevel@tonic-gate unsigned who; 869d90554ebSdmick int cpuid = 0; 8707c478bd9Sstevel@tonic-gate int delays = 0; 8717c478bd9Sstevel@tonic-gate int started_cpu; 8727c478bd9Sstevel@tonic-gate ushort_t *warm_reset_vector = NULL; 8737c478bd9Sstevel@tonic-gate extern int procset; 8747c478bd9Sstevel@tonic-gate 8757c478bd9Sstevel@tonic-gate /* 8767c478bd9Sstevel@tonic-gate * Initialize our own cpu_info. 8777c478bd9Sstevel@tonic-gate */ 8787c478bd9Sstevel@tonic-gate init_cpu_info(CPU); 8797c478bd9Sstevel@tonic-gate 8807c478bd9Sstevel@tonic-gate /* 8817c478bd9Sstevel@tonic-gate * Initialize our syscall handlers 8827c478bd9Sstevel@tonic-gate */ 8837c478bd9Sstevel@tonic-gate init_cpu_syscall(CPU); 8847c478bd9Sstevel@tonic-gate 8857c478bd9Sstevel@tonic-gate /* 8867c478bd9Sstevel@tonic-gate * if only 1 cpu or not using MP, skip the rest of this 8877c478bd9Sstevel@tonic-gate */ 8887c478bd9Sstevel@tonic-gate if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) { 8897c478bd9Sstevel@tonic-gate if (use_mp == 0) 8907c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?***** Not in MP mode\n"); 8917c478bd9Sstevel@tonic-gate goto done; 8927c478bd9Sstevel@tonic-gate } 8937c478bd9Sstevel@tonic-gate 8947c478bd9Sstevel@tonic-gate /* 8957c478bd9Sstevel@tonic-gate * perform such initialization as is needed 8967c478bd9Sstevel@tonic-gate * to be able to take CPUs on- and off-line. 8977c478bd9Sstevel@tonic-gate */ 8987c478bd9Sstevel@tonic-gate cpu_pause_init(); 8997c478bd9Sstevel@tonic-gate 9007c478bd9Sstevel@tonic-gate xc_init(); /* initialize processor crosscalls */ 9017c478bd9Sstevel@tonic-gate 9027c478bd9Sstevel@tonic-gate /* 9037c478bd9Sstevel@tonic-gate * Copy the real mode code at "real_mode_start" to the 9047c478bd9Sstevel@tonic-gate * page at rm_platter_va. 9057c478bd9Sstevel@tonic-gate */ 9067c478bd9Sstevel@tonic-gate warm_reset_vector = mp_map_warm_reset_vector(); 9077c478bd9Sstevel@tonic-gate if (warm_reset_vector == NULL) 9087c478bd9Sstevel@tonic-gate goto done; 9097c478bd9Sstevel@tonic-gate 9107c478bd9Sstevel@tonic-gate bcopy((caddr_t)real_mode_start, 9117c478bd9Sstevel@tonic-gate (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code, 9127c478bd9Sstevel@tonic-gate (size_t)real_mode_end - (size_t)real_mode_start); 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate flushes_require_xcalls = 1; 9157c478bd9Sstevel@tonic-gate 9167c478bd9Sstevel@tonic-gate affinity_set(CPU_CURRENT); 9177c478bd9Sstevel@tonic-gate 9187c478bd9Sstevel@tonic-gate for (who = 0; who < NCPU; who++) { 9197c478bd9Sstevel@tonic-gate if (who == cpuid) 9207c478bd9Sstevel@tonic-gate continue; 9217c478bd9Sstevel@tonic-gate 9227c478bd9Sstevel@tonic-gate if ((mp_cpus & (1 << who)) == 0) 9237c478bd9Sstevel@tonic-gate continue; 9247c478bd9Sstevel@tonic-gate 9257c478bd9Sstevel@tonic-gate mp_startup_init(who); 9267c478bd9Sstevel@tonic-gate started_cpu = 1; 9277c478bd9Sstevel@tonic-gate (*cpu_startf)(who, rm_platter_pa); 9287c478bd9Sstevel@tonic-gate 9297c478bd9Sstevel@tonic-gate while ((procset & (1 << who)) == 0) { 9307c478bd9Sstevel@tonic-gate 9317c478bd9Sstevel@tonic-gate delay(1); 9327c478bd9Sstevel@tonic-gate if (++delays > (20 * hz)) { 9337c478bd9Sstevel@tonic-gate 9347c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, 9357c478bd9Sstevel@tonic-gate "cpu%d failed to start", who); 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 9387c478bd9Sstevel@tonic-gate cpu[who]->cpu_flags = 0; 939affbd3ccSkchow cpu_vm_data_destroy(cpu[who]); 9407c478bd9Sstevel@tonic-gate cpu_del_unit(who); 9417c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 9427c478bd9Sstevel@tonic-gate 9437c478bd9Sstevel@tonic-gate started_cpu = 0; 9447c478bd9Sstevel@tonic-gate break; 9457c478bd9Sstevel@tonic-gate } 9467c478bd9Sstevel@tonic-gate } 9477c478bd9Sstevel@tonic-gate if (!started_cpu) 9487c478bd9Sstevel@tonic-gate continue; 9497c478bd9Sstevel@tonic-gate if (tsc_gethrtime_enable) 9507c478bd9Sstevel@tonic-gate tsc_sync_master(who); 9517c478bd9Sstevel@tonic-gate 9527c478bd9Sstevel@tonic-gate 9537c478bd9Sstevel@tonic-gate if (dtrace_cpu_init != NULL) { 9547c478bd9Sstevel@tonic-gate /* 9557c478bd9Sstevel@tonic-gate * DTrace CPU initialization expects cpu_lock 9567c478bd9Sstevel@tonic-gate * to be held. 9577c478bd9Sstevel@tonic-gate */ 9587c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 9597c478bd9Sstevel@tonic-gate (*dtrace_cpu_init)(who); 9607c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 9617c478bd9Sstevel@tonic-gate } 9627c478bd9Sstevel@tonic-gate } 9637c478bd9Sstevel@tonic-gate 9647c478bd9Sstevel@tonic-gate affinity_clear(); 9657c478bd9Sstevel@tonic-gate 9667c478bd9Sstevel@tonic-gate for (who = 0; who < NCPU; who++) { 9677c478bd9Sstevel@tonic-gate if (who == cpuid) 9687c478bd9Sstevel@tonic-gate continue; 9697c478bd9Sstevel@tonic-gate 9707c478bd9Sstevel@tonic-gate if (!(procset & (1 << who))) 9717c478bd9Sstevel@tonic-gate continue; 9727c478bd9Sstevel@tonic-gate 9737c478bd9Sstevel@tonic-gate while (!(cpu_ready_set & (1 << who))) 9747c478bd9Sstevel@tonic-gate delay(1); 9757c478bd9Sstevel@tonic-gate } 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate done: 9787c478bd9Sstevel@tonic-gate workaround_errata_end(); 9797c478bd9Sstevel@tonic-gate 9807c478bd9Sstevel@tonic-gate if (warm_reset_vector != NULL) 9817c478bd9Sstevel@tonic-gate mp_unmap_warm_reset_vector(warm_reset_vector); 9827c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 9837c478bd9Sstevel@tonic-gate HAT_UNLOAD); 9847c478bd9Sstevel@tonic-gate } 9857c478bd9Sstevel@tonic-gate 9867c478bd9Sstevel@tonic-gate /* 9877c478bd9Sstevel@tonic-gate * Dummy functions - no i86pc platforms support dynamic cpu allocation. 9887c478bd9Sstevel@tonic-gate */ 9897c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9907c478bd9Sstevel@tonic-gate int 9917c478bd9Sstevel@tonic-gate mp_cpu_configure(int cpuid) 9927c478bd9Sstevel@tonic-gate { 9937c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 9947c478bd9Sstevel@tonic-gate } 9957c478bd9Sstevel@tonic-gate 9967c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9977c478bd9Sstevel@tonic-gate int 9987c478bd9Sstevel@tonic-gate mp_cpu_unconfigure(int cpuid) 9997c478bd9Sstevel@tonic-gate { 10007c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 10017c478bd9Sstevel@tonic-gate } 10027c478bd9Sstevel@tonic-gate 10037c478bd9Sstevel@tonic-gate /* 10047c478bd9Sstevel@tonic-gate * Startup function for 'other' CPUs (besides boot cpu). 10057c478bd9Sstevel@tonic-gate * Resumed from cpu_startup. 1006b4b46911Skchow * 1007b4b46911Skchow * WARNING: until CPU_READY is set, mp_startup and routines called by 1008b4b46911Skchow * mp_startup should not call routines (e.g. kmem_free) that could call 1009b4b46911Skchow * hat_unload which requires CPU_READY to be set. 10107c478bd9Sstevel@tonic-gate */ 10117c478bd9Sstevel@tonic-gate void 10127c478bd9Sstevel@tonic-gate mp_startup(void) 10137c478bd9Sstevel@tonic-gate { 10147c478bd9Sstevel@tonic-gate struct cpu *cp = CPU; 10157c478bd9Sstevel@tonic-gate extern int procset; 10167c478bd9Sstevel@tonic-gate uint_t new_x86_feature; 10177c478bd9Sstevel@tonic-gate 10187c478bd9Sstevel@tonic-gate new_x86_feature = cpuid_pass1(cp); 10197c478bd9Sstevel@tonic-gate 10207c478bd9Sstevel@tonic-gate /* 10217c478bd9Sstevel@tonic-gate * We need to Sync MTRR with cpu0's MTRR. We have to do 10227c478bd9Sstevel@tonic-gate * this with interrupts disabled. 10237c478bd9Sstevel@tonic-gate */ 10247c478bd9Sstevel@tonic-gate if (x86_feature & X86_MTRR) 10257c478bd9Sstevel@tonic-gate mtrr_sync(); 10267c478bd9Sstevel@tonic-gate 10277c478bd9Sstevel@tonic-gate /* 10287c478bd9Sstevel@tonic-gate * Initialize this CPU's syscall handlers 10297c478bd9Sstevel@tonic-gate */ 10307c478bd9Sstevel@tonic-gate init_cpu_syscall(cp); 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate /* 10337c478bd9Sstevel@tonic-gate * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the 10347c478bd9Sstevel@tonic-gate * highest level at which a routine is permitted to block on 10357c478bd9Sstevel@tonic-gate * an adaptive mutex (allows for cpu poke interrupt in case 10367c478bd9Sstevel@tonic-gate * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks 10377c478bd9Sstevel@tonic-gate * device interrupts that may end up in the hat layer issuing cross 10387c478bd9Sstevel@tonic-gate * calls before CPU_READY is set. 10397c478bd9Sstevel@tonic-gate */ 10407c478bd9Sstevel@tonic-gate (void) splx(ipltospl(LOCK_LEVEL)); 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate /* 10437c478bd9Sstevel@tonic-gate * Do a sanity check to make sure this new CPU is a sane thing 10447c478bd9Sstevel@tonic-gate * to add to the collection of processors running this system. 10457c478bd9Sstevel@tonic-gate * 10467c478bd9Sstevel@tonic-gate * XXX Clearly this needs to get more sophisticated, if x86 10477c478bd9Sstevel@tonic-gate * systems start to get built out of heterogenous CPUs; as is 10487c478bd9Sstevel@tonic-gate * likely to happen once the number of processors in a configuration 10497c478bd9Sstevel@tonic-gate * gets large enough. 10507c478bd9Sstevel@tonic-gate */ 10517c478bd9Sstevel@tonic-gate if ((x86_feature & new_x86_feature) != x86_feature) { 10527c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "?cpu%d: %b\n", 10537c478bd9Sstevel@tonic-gate cp->cpu_id, new_x86_feature, FMT_X86_FEATURE); 10547c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); 10557c478bd9Sstevel@tonic-gate } 10567c478bd9Sstevel@tonic-gate 10577c478bd9Sstevel@tonic-gate /* 10587c478bd9Sstevel@tonic-gate * We could be more sophisticated here, and just mark the CPU 10597c478bd9Sstevel@tonic-gate * as "faulted" but at this point we'll opt for the easier 10607c478bd9Sstevel@tonic-gate * answer of dieing horribly. Provided the boot cpu is ok, 10617c478bd9Sstevel@tonic-gate * the system can be recovered by booting with use_mp set to zero. 10627c478bd9Sstevel@tonic-gate */ 10637c478bd9Sstevel@tonic-gate if (workaround_errata(cp) != 0) 10647c478bd9Sstevel@tonic-gate panic("critical workaround(s) missing for cpu%d", cp->cpu_id); 10657c478bd9Sstevel@tonic-gate 10667c478bd9Sstevel@tonic-gate cpuid_pass2(cp); 10677c478bd9Sstevel@tonic-gate cpuid_pass3(cp); 10687c478bd9Sstevel@tonic-gate (void) cpuid_pass4(cp); 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate init_cpu_info(cp); 10717c478bd9Sstevel@tonic-gate 10727c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10737c478bd9Sstevel@tonic-gate procset |= 1 << cp->cpu_id; 10747c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10757c478bd9Sstevel@tonic-gate 10767c478bd9Sstevel@tonic-gate if (tsc_gethrtime_enable) 10777c478bd9Sstevel@tonic-gate tsc_sync_slave(); 10787c478bd9Sstevel@tonic-gate 10797c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10807c478bd9Sstevel@tonic-gate /* 10817c478bd9Sstevel@tonic-gate * It's unfortunate that chip_cpu_init() has to be called here. 10827c478bd9Sstevel@tonic-gate * It really belongs in cpu_add_unit(), but unfortunately it is 10837c478bd9Sstevel@tonic-gate * dependent on the cpuid probing, which must be done in the 10847c478bd9Sstevel@tonic-gate * context of the current CPU. Care must be taken on x86 to ensure 10857c478bd9Sstevel@tonic-gate * that mp_startup can safely block even though chip_cpu_init() and 10867c478bd9Sstevel@tonic-gate * cpu_add_active() have not yet been called. 10877c478bd9Sstevel@tonic-gate */ 10887c478bd9Sstevel@tonic-gate chip_cpu_init(cp); 10897c478bd9Sstevel@tonic-gate chip_cpu_startup(cp); 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS; 10927c478bd9Sstevel@tonic-gate cpu_add_active(cp); 10937c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10947c478bd9Sstevel@tonic-gate 1095b4b46911Skchow add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi); 1096b4b46911Skchow 10977c478bd9Sstevel@tonic-gate (void) spl0(); /* enable interrupts */ 10987c478bd9Sstevel@tonic-gate 10997aec1d6eScindi /* 11007aec1d6eScindi * Set up the CPU module for this CPU. This can't be done before 11017aec1d6eScindi * this CPU is made CPU_READY, because we may (in heterogeneous systems) 11027aec1d6eScindi * need to go load another CPU module. The act of attempting to load 11037aec1d6eScindi * a module may trigger a cross-call, which will ASSERT unless this 11047aec1d6eScindi * cpu is CPU_READY. 11057aec1d6eScindi */ 11067aec1d6eScindi cmi_init(); 11077aec1d6eScindi 11087aec1d6eScindi if (x86_feature & X86_MCA) 11097aec1d6eScindi cmi_mca_init(); 11107aec1d6eScindi 11117c478bd9Sstevel@tonic-gate if (boothowto & RB_DEBUG) 11127c478bd9Sstevel@tonic-gate kdi_dvec_cpu_init(cp); 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate /* 11157c478bd9Sstevel@tonic-gate * Setting the bit in cpu_ready_set must be the last operation in 11167c478bd9Sstevel@tonic-gate * processor initialization; the boot CPU will continue to boot once 11177c478bd9Sstevel@tonic-gate * it sees this bit set for all active CPUs. 11187c478bd9Sstevel@tonic-gate */ 11197c478bd9Sstevel@tonic-gate CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); 11207c478bd9Sstevel@tonic-gate 11217c478bd9Sstevel@tonic-gate /* 11227c478bd9Sstevel@tonic-gate * Because mp_startup() gets fired off after init() starts, we 11237c478bd9Sstevel@tonic-gate * can't use the '?' trick to do 'boot -v' printing - so we 11247c478bd9Sstevel@tonic-gate * always direct the 'cpu .. online' messages to the log. 11257c478bd9Sstevel@tonic-gate */ 11267c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "!cpu%d initialization complete - online\n", 11277c478bd9Sstevel@tonic-gate cp->cpu_id); 11287c478bd9Sstevel@tonic-gate 11297c478bd9Sstevel@tonic-gate /* 11307c478bd9Sstevel@tonic-gate * Now we are done with the startup thread, so free it up. 11317c478bd9Sstevel@tonic-gate */ 11327c478bd9Sstevel@tonic-gate thread_exit(); 11337c478bd9Sstevel@tonic-gate panic("mp_startup: cannot return"); 11347c478bd9Sstevel@tonic-gate /*NOTREACHED*/ 11357c478bd9Sstevel@tonic-gate } 11367c478bd9Sstevel@tonic-gate 11377c478bd9Sstevel@tonic-gate 11387c478bd9Sstevel@tonic-gate /* 11397c478bd9Sstevel@tonic-gate * Start CPU on user request. 11407c478bd9Sstevel@tonic-gate */ 11417c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11427c478bd9Sstevel@tonic-gate int 11437c478bd9Sstevel@tonic-gate mp_cpu_start(struct cpu *cp) 11447c478bd9Sstevel@tonic-gate { 11457c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 11467c478bd9Sstevel@tonic-gate return (0); 11477c478bd9Sstevel@tonic-gate } 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate /* 11507c478bd9Sstevel@tonic-gate * Stop CPU on user request. 11517c478bd9Sstevel@tonic-gate */ 11527c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11537c478bd9Sstevel@tonic-gate int 11547c478bd9Sstevel@tonic-gate mp_cpu_stop(struct cpu *cp) 11557c478bd9Sstevel@tonic-gate { 1156d90554ebSdmick extern int cbe_psm_timer_mode; 11577c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 1158d90554ebSdmick 1159d90554ebSdmick /* 1160d90554ebSdmick * If TIMER_PERIODIC mode is used, CPU0 is the one running it; 1161d90554ebSdmick * can't stop it. (This is true only for machines with no TSC.) 1162d90554ebSdmick */ 1163d90554ebSdmick 1164d90554ebSdmick if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0)) 1165d90554ebSdmick return (1); 11667c478bd9Sstevel@tonic-gate 11677c478bd9Sstevel@tonic-gate return (0); 11687c478bd9Sstevel@tonic-gate } 11697c478bd9Sstevel@tonic-gate 11707c478bd9Sstevel@tonic-gate /* 11717c478bd9Sstevel@tonic-gate * Power on CPU. 11727c478bd9Sstevel@tonic-gate */ 11737c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11747c478bd9Sstevel@tonic-gate int 11757c478bd9Sstevel@tonic-gate mp_cpu_poweron(struct cpu *cp) 11767c478bd9Sstevel@tonic-gate { 11777c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 11787c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 11797c478bd9Sstevel@tonic-gate } 11807c478bd9Sstevel@tonic-gate 11817c478bd9Sstevel@tonic-gate /* 11827c478bd9Sstevel@tonic-gate * Power off CPU. 11837c478bd9Sstevel@tonic-gate */ 11847c478bd9Sstevel@tonic-gate /* ARGSUSED */ 11857c478bd9Sstevel@tonic-gate int 11867c478bd9Sstevel@tonic-gate mp_cpu_poweroff(struct cpu *cp) 11877c478bd9Sstevel@tonic-gate { 11887c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 11897c478bd9Sstevel@tonic-gate return (ENOTSUP); /* not supported */ 11907c478bd9Sstevel@tonic-gate } 11917c478bd9Sstevel@tonic-gate 11927c478bd9Sstevel@tonic-gate 11937c478bd9Sstevel@tonic-gate /* 11947c478bd9Sstevel@tonic-gate * Take the specified CPU out of participation in interrupts. 11957c478bd9Sstevel@tonic-gate */ 11967c478bd9Sstevel@tonic-gate int 11977c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp) 11987c478bd9Sstevel@tonic-gate { 11997c478bd9Sstevel@tonic-gate if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) 12007c478bd9Sstevel@tonic-gate return (EBUSY); 12017c478bd9Sstevel@tonic-gate 12027c478bd9Sstevel@tonic-gate cp->cpu_flags &= ~CPU_ENABLE; 12037c478bd9Sstevel@tonic-gate return (0); 12047c478bd9Sstevel@tonic-gate } 12057c478bd9Sstevel@tonic-gate 12067c478bd9Sstevel@tonic-gate /* 12077c478bd9Sstevel@tonic-gate * Allow the specified CPU to participate in interrupts. 12087c478bd9Sstevel@tonic-gate */ 12097c478bd9Sstevel@tonic-gate void 12107c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp) 12117c478bd9Sstevel@tonic-gate { 12127c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 12137c478bd9Sstevel@tonic-gate cp->cpu_flags |= CPU_ENABLE; 12147c478bd9Sstevel@tonic-gate psm_enable_intr(cp->cpu_id); 12157c478bd9Sstevel@tonic-gate } 12167c478bd9Sstevel@tonic-gate 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate 12197c478bd9Sstevel@tonic-gate static ushort_t * 12207c478bd9Sstevel@tonic-gate mp_map_warm_reset_vector() 12217c478bd9Sstevel@tonic-gate { 12227c478bd9Sstevel@tonic-gate ushort_t *warm_reset_vector; 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 12257c478bd9Sstevel@tonic-gate sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 12267c478bd9Sstevel@tonic-gate return (NULL); 12277c478bd9Sstevel@tonic-gate 12287c478bd9Sstevel@tonic-gate /* 12297c478bd9Sstevel@tonic-gate * setup secondary cpu bios boot up vector 12307c478bd9Sstevel@tonic-gate */ 12317c478bd9Sstevel@tonic-gate *warm_reset_vector = (ushort_t)((caddr_t) 12327c478bd9Sstevel@tonic-gate ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 12337c478bd9Sstevel@tonic-gate + ((ulong_t)rm_platter_va & 0xf)); 12347c478bd9Sstevel@tonic-gate warm_reset_vector++; 12357c478bd9Sstevel@tonic-gate *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 12367c478bd9Sstevel@tonic-gate 12377c478bd9Sstevel@tonic-gate --warm_reset_vector; 12387c478bd9Sstevel@tonic-gate return (warm_reset_vector); 12397c478bd9Sstevel@tonic-gate } 12407c478bd9Sstevel@tonic-gate 12417c478bd9Sstevel@tonic-gate static void 12427c478bd9Sstevel@tonic-gate mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector) 12437c478bd9Sstevel@tonic-gate { 12447c478bd9Sstevel@tonic-gate psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 12457c478bd9Sstevel@tonic-gate } 12467c478bd9Sstevel@tonic-gate 12477c478bd9Sstevel@tonic-gate void 12487c478bd9Sstevel@tonic-gate mp_cpu_faulted_enter(struct cpu *cp) 12497aec1d6eScindi { 12507aec1d6eScindi cmi_faulted_enter(cp); 12517aec1d6eScindi } 12527c478bd9Sstevel@tonic-gate 12537c478bd9Sstevel@tonic-gate void 12547c478bd9Sstevel@tonic-gate mp_cpu_faulted_exit(struct cpu *cp) 12557aec1d6eScindi { 12567aec1d6eScindi cmi_faulted_exit(cp); 12577aec1d6eScindi } 12587c478bd9Sstevel@tonic-gate 12597c478bd9Sstevel@tonic-gate /* 12607c478bd9Sstevel@tonic-gate * The following two routines are used as context operators on threads belonging 12617c478bd9Sstevel@tonic-gate * to processes with a private LDT (see sysi86). Due to the rarity of such 12627c478bd9Sstevel@tonic-gate * processes, these routines are currently written for best code readability and 12637c478bd9Sstevel@tonic-gate * organization rather than speed. We could avoid checking x86_feature at every 12647c478bd9Sstevel@tonic-gate * context switch by installing different context ops, depending on the 12657c478bd9Sstevel@tonic-gate * x86_feature flags, at LDT creation time -- one for each combination of fast 12667c478bd9Sstevel@tonic-gate * syscall feature flags. 12677c478bd9Sstevel@tonic-gate */ 12687c478bd9Sstevel@tonic-gate 12697c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12707c478bd9Sstevel@tonic-gate void 12717c478bd9Sstevel@tonic-gate cpu_fast_syscall_disable(void *arg) 12727c478bd9Sstevel@tonic-gate { 12737c478bd9Sstevel@tonic-gate if (x86_feature & X86_SEP) 12747c478bd9Sstevel@tonic-gate cpu_sep_disable(); 12757c478bd9Sstevel@tonic-gate if (x86_feature & X86_ASYSC) 12767c478bd9Sstevel@tonic-gate cpu_asysc_disable(); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate 12797c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12807c478bd9Sstevel@tonic-gate void 12817c478bd9Sstevel@tonic-gate cpu_fast_syscall_enable(void *arg) 12827c478bd9Sstevel@tonic-gate { 12837c478bd9Sstevel@tonic-gate if (x86_feature & X86_SEP) 12847c478bd9Sstevel@tonic-gate cpu_sep_enable(); 12857c478bd9Sstevel@tonic-gate if (x86_feature & X86_ASYSC) 12867c478bd9Sstevel@tonic-gate cpu_asysc_enable(); 12877c478bd9Sstevel@tonic-gate } 12887c478bd9Sstevel@tonic-gate 12897c478bd9Sstevel@tonic-gate static void 12907c478bd9Sstevel@tonic-gate cpu_sep_enable(void) 12917c478bd9Sstevel@tonic-gate { 12927c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_SEP); 12937c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 12947c478bd9Sstevel@tonic-gate 12950ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL); 12967c478bd9Sstevel@tonic-gate } 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate static void 12997c478bd9Sstevel@tonic-gate cpu_sep_disable(void) 13007c478bd9Sstevel@tonic-gate { 13017c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_SEP); 13027c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 13037c478bd9Sstevel@tonic-gate 13047c478bd9Sstevel@tonic-gate /* 13057c478bd9Sstevel@tonic-gate * Setting the SYSENTER_CS_MSR register to 0 causes software executing 13067c478bd9Sstevel@tonic-gate * the sysenter or sysexit instruction to trigger a #gp fault. 13077c478bd9Sstevel@tonic-gate */ 13080ac7d7d8Skucharsk wrmsr(MSR_INTC_SEP_CS, 0ULL); 13097c478bd9Sstevel@tonic-gate } 13107c478bd9Sstevel@tonic-gate 13117c478bd9Sstevel@tonic-gate static void 13127c478bd9Sstevel@tonic-gate cpu_asysc_enable(void) 13137c478bd9Sstevel@tonic-gate { 13147c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_ASYSC); 13157c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 13167c478bd9Sstevel@tonic-gate 13170ac7d7d8Skucharsk wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) | 13180ac7d7d8Skucharsk (uint64_t)(uintptr_t)AMD_EFER_SCE); 13197c478bd9Sstevel@tonic-gate } 13207c478bd9Sstevel@tonic-gate 13217c478bd9Sstevel@tonic-gate static void 13227c478bd9Sstevel@tonic-gate cpu_asysc_disable(void) 13237c478bd9Sstevel@tonic-gate { 13247c478bd9Sstevel@tonic-gate ASSERT(x86_feature & X86_ASYSC); 13257c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); 13267c478bd9Sstevel@tonic-gate 13277c478bd9Sstevel@tonic-gate /* 13287c478bd9Sstevel@tonic-gate * Turn off the SCE (syscall enable) bit in the EFER register. Software 13297c478bd9Sstevel@tonic-gate * executing syscall or sysret with this bit off will incur a #ud trap. 13307c478bd9Sstevel@tonic-gate */ 13310ac7d7d8Skucharsk wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) & 13320ac7d7d8Skucharsk ~((uint64_t)(uintptr_t)AMD_EFER_SCE)); 13337c478bd9Sstevel@tonic-gate } 1334