xref: /illumos-gate/usr/src/uts/i86pc/os/mp_startup.c (revision 100b72f4)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*100b72f4Sandrei  * Common Development and Distribution License (the "License").
6*100b72f4Sandrei  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22b4b46911Skchow  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate #include <sys/types.h>
297c478bd9Sstevel@tonic-gate #include <sys/thread.h>
307c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
317c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
327c478bd9Sstevel@tonic-gate #include <sys/param.h>
337c478bd9Sstevel@tonic-gate #include <sys/proc.h>
347c478bd9Sstevel@tonic-gate #include <sys/disp.h>
357c478bd9Sstevel@tonic-gate #include <sys/mmu.h>
367c478bd9Sstevel@tonic-gate #include <sys/class.h>
377c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
387c478bd9Sstevel@tonic-gate #include <sys/debug.h>
397c478bd9Sstevel@tonic-gate #include <sys/asm_linkage.h>
407c478bd9Sstevel@tonic-gate #include <sys/x_call.h>
417c478bd9Sstevel@tonic-gate #include <sys/systm.h>
427c478bd9Sstevel@tonic-gate #include <sys/var.h>
437c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
447c478bd9Sstevel@tonic-gate #include <vm/hat.h>
457c478bd9Sstevel@tonic-gate #include <sys/mmu.h>
467c478bd9Sstevel@tonic-gate #include <vm/as.h>
477c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
487c478bd9Sstevel@tonic-gate #include <sys/segments.h>
497c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
507c478bd9Sstevel@tonic-gate #include <sys/stack.h>
517c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h>
527c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h>
537c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
547c478bd9Sstevel@tonic-gate #include <sys/traptrace.h>
557c478bd9Sstevel@tonic-gate #include <sys/clock.h>
567c478bd9Sstevel@tonic-gate #include <sys/cpc_impl.h>
577c478bd9Sstevel@tonic-gate #include <sys/chip.h>
587c478bd9Sstevel@tonic-gate #include <sys/dtrace.h>
597c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
607c478bd9Sstevel@tonic-gate #include <sys/fp.h>
617c478bd9Sstevel@tonic-gate #include <sys/reboot.h>
627c478bd9Sstevel@tonic-gate #include <sys/kdi.h>
637c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
647c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
65ef50d8c0Sesaxe #include <sys/pci_cfgspace.h>
667aec1d6eScindi #include <sys/cpu_module.h>
677c478bd9Sstevel@tonic-gate 
687c478bd9Sstevel@tonic-gate struct cpu	cpus[1];			/* CPU data */
697c478bd9Sstevel@tonic-gate struct cpu	*cpu[NCPU] = {&cpus[0]};	/* pointers to all CPUs */
707c478bd9Sstevel@tonic-gate cpu_core_t	cpu_core[NCPU];			/* cpu_core structures */
717c478bd9Sstevel@tonic-gate 
727c478bd9Sstevel@tonic-gate /*
737c478bd9Sstevel@tonic-gate  * Useful for disabling MP bring-up for an MP capable kernel
747c478bd9Sstevel@tonic-gate  * (a kernel that was built with MP defined)
757c478bd9Sstevel@tonic-gate  */
767c478bd9Sstevel@tonic-gate int use_mp = 1;
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate int mp_cpus = 0x1;	/* to be set by platform specific module	*/
797c478bd9Sstevel@tonic-gate 
807c478bd9Sstevel@tonic-gate /*
817c478bd9Sstevel@tonic-gate  * This variable is used by the hat layer to decide whether or not
827c478bd9Sstevel@tonic-gate  * critical sections are needed to prevent race conditions.  For sun4m,
837c478bd9Sstevel@tonic-gate  * this variable is set once enough MP initialization has been done in
847c478bd9Sstevel@tonic-gate  * order to allow cross calls.
857c478bd9Sstevel@tonic-gate  */
867c478bd9Sstevel@tonic-gate int flushes_require_xcalls = 0;
877c478bd9Sstevel@tonic-gate ulong_t	cpu_ready_set = 1;
887c478bd9Sstevel@tonic-gate 
897c478bd9Sstevel@tonic-gate extern	void	real_mode_start(void);
907c478bd9Sstevel@tonic-gate extern	void	real_mode_end(void);
917c478bd9Sstevel@tonic-gate static 	void	mp_startup(void);
927c478bd9Sstevel@tonic-gate 
937c478bd9Sstevel@tonic-gate static void cpu_sep_enable(void);
947c478bd9Sstevel@tonic-gate static void cpu_sep_disable(void);
957c478bd9Sstevel@tonic-gate static void cpu_asysc_enable(void);
967c478bd9Sstevel@tonic-gate static void cpu_asysc_disable(void);
977c478bd9Sstevel@tonic-gate 
987c478bd9Sstevel@tonic-gate extern int tsc_gethrtime_enable;
997c478bd9Sstevel@tonic-gate 
1007c478bd9Sstevel@tonic-gate /*
1017c478bd9Sstevel@tonic-gate  * Init CPU info - get CPU type info for processor_info system call.
1027c478bd9Sstevel@tonic-gate  */
1037c478bd9Sstevel@tonic-gate void
1047c478bd9Sstevel@tonic-gate init_cpu_info(struct cpu *cp)
1057c478bd9Sstevel@tonic-gate {
1067c478bd9Sstevel@tonic-gate 	processor_info_t *pi = &cp->cpu_type_info;
1077c478bd9Sstevel@tonic-gate 	char buf[CPU_IDSTRLEN];
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 	/*
1107c478bd9Sstevel@tonic-gate 	 * Get clock-frequency property for the CPU.
1117c478bd9Sstevel@tonic-gate 	 */
1127c478bd9Sstevel@tonic-gate 	pi->pi_clock = cpu_freq;
1137c478bd9Sstevel@tonic-gate 
1147c478bd9Sstevel@tonic-gate 	(void) strcpy(pi->pi_processor_type, "i386");
1157c478bd9Sstevel@tonic-gate 	if (fpu_exists)
1167c478bd9Sstevel@tonic-gate 		(void) strcpy(pi->pi_fputypes, "i387 compatible");
1177c478bd9Sstevel@tonic-gate 
1187c478bd9Sstevel@tonic-gate 	(void) cpuid_getidstr(cp, buf, sizeof (buf));
1197c478bd9Sstevel@tonic-gate 
1207c478bd9Sstevel@tonic-gate 	cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP);
1217c478bd9Sstevel@tonic-gate 	(void) strcpy(cp->cpu_idstr, buf);
1227c478bd9Sstevel@tonic-gate 
1237c478bd9Sstevel@tonic-gate 	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1247c478bd9Sstevel@tonic-gate 
1257c478bd9Sstevel@tonic-gate 	(void) cpuid_getbrandstr(cp, buf, sizeof (buf));
1267c478bd9Sstevel@tonic-gate 	cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP);
1277c478bd9Sstevel@tonic-gate 	(void) strcpy(cp->cpu_brandstr, buf);
1287c478bd9Sstevel@tonic-gate 
1297c478bd9Sstevel@tonic-gate 	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1307c478bd9Sstevel@tonic-gate }
1317c478bd9Sstevel@tonic-gate 
1327c478bd9Sstevel@tonic-gate /*
1337c478bd9Sstevel@tonic-gate  * Configure syscall support on this CPU.
1347c478bd9Sstevel@tonic-gate  */
1357c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1367c478bd9Sstevel@tonic-gate static void
1377c478bd9Sstevel@tonic-gate init_cpu_syscall(struct cpu *cp)
1387c478bd9Sstevel@tonic-gate {
1397c478bd9Sstevel@tonic-gate 	kpreempt_disable();
1407c478bd9Sstevel@tonic-gate 
1417c478bd9Sstevel@tonic-gate #if defined(__amd64)
1427c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_ASYSC) {
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate #if !defined(__lint)
1457c478bd9Sstevel@tonic-gate 		/*
1467c478bd9Sstevel@tonic-gate 		 * The syscall instruction imposes a certain ordering on
1477c478bd9Sstevel@tonic-gate 		 * segment selectors, so we double-check that ordering
1487c478bd9Sstevel@tonic-gate 		 * here.
1497c478bd9Sstevel@tonic-gate 		 */
1507c478bd9Sstevel@tonic-gate 		ASSERT(KDS_SEL == KCS_SEL + 8);
1517c478bd9Sstevel@tonic-gate 		ASSERT(UDS_SEL == U32CS_SEL + 8);
1527c478bd9Sstevel@tonic-gate 		ASSERT(UCS_SEL == U32CS_SEL + 16);
1537c478bd9Sstevel@tonic-gate #endif
1547c478bd9Sstevel@tonic-gate 		/*
1557c478bd9Sstevel@tonic-gate 		 * Turn syscall/sysret extensions on.
1567c478bd9Sstevel@tonic-gate 		 */
1577c478bd9Sstevel@tonic-gate 		cpu_asysc_enable();
1587c478bd9Sstevel@tonic-gate 
1597c478bd9Sstevel@tonic-gate 		/*
1607c478bd9Sstevel@tonic-gate 		 * Program the magic registers ..
1617c478bd9Sstevel@tonic-gate 		 */
1620ac7d7d8Skucharsk 		wrmsr(MSR_AMD_STAR, ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) <<
1630ac7d7d8Skucharsk 		    32);
1640ac7d7d8Skucharsk 		wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall);
1650ac7d7d8Skucharsk 		wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32);
1667c478bd9Sstevel@tonic-gate 
1677c478bd9Sstevel@tonic-gate 		/*
1687c478bd9Sstevel@tonic-gate 		 * This list of flags is masked off the incoming
1697c478bd9Sstevel@tonic-gate 		 * %rfl when we enter the kernel.
1707c478bd9Sstevel@tonic-gate 		 */
1710ac7d7d8Skucharsk 		wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T));
1727c478bd9Sstevel@tonic-gate 	}
1737c478bd9Sstevel@tonic-gate #endif
1747c478bd9Sstevel@tonic-gate 
1757c478bd9Sstevel@tonic-gate 	/*
1767c478bd9Sstevel@tonic-gate 	 * On 32-bit kernels, we use sysenter/sysexit because it's too
1777c478bd9Sstevel@tonic-gate 	 * hard to use syscall/sysret, and it is more portable anyway.
1787c478bd9Sstevel@tonic-gate 	 *
1797c478bd9Sstevel@tonic-gate 	 * On 64-bit kernels on Nocona machines, the 32-bit syscall
1807c478bd9Sstevel@tonic-gate 	 * variant isn't available to 32-bit applications, but sysenter is.
1817c478bd9Sstevel@tonic-gate 	 */
1827c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_SEP) {
1837c478bd9Sstevel@tonic-gate 
1847c478bd9Sstevel@tonic-gate #if !defined(__lint)
1857c478bd9Sstevel@tonic-gate 		/*
1867c478bd9Sstevel@tonic-gate 		 * The sysenter instruction imposes a certain ordering on
1877c478bd9Sstevel@tonic-gate 		 * segment selectors, so we double-check that ordering
1887c478bd9Sstevel@tonic-gate 		 * here. See "sysenter" in Intel document 245471-012, "IA-32
1897c478bd9Sstevel@tonic-gate 		 * Intel Architecture Software Developer's Manual Volume 2:
1907c478bd9Sstevel@tonic-gate 		 * Instruction Set Reference"
1917c478bd9Sstevel@tonic-gate 		 */
1927c478bd9Sstevel@tonic-gate 		ASSERT(KDS_SEL == KCS_SEL + 8);
1937c478bd9Sstevel@tonic-gate 
1947c478bd9Sstevel@tonic-gate 		ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3));
1957c478bd9Sstevel@tonic-gate 		ASSERT32(UDS_SEL == UCS_SEL + 8);
1967c478bd9Sstevel@tonic-gate 
1977c478bd9Sstevel@tonic-gate 		ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3));
1987c478bd9Sstevel@tonic-gate 		ASSERT64(UDS_SEL == U32CS_SEL + 8);
1997c478bd9Sstevel@tonic-gate #endif
2007c478bd9Sstevel@tonic-gate 
2017c478bd9Sstevel@tonic-gate 		cpu_sep_enable();
2027c478bd9Sstevel@tonic-gate 
2037c478bd9Sstevel@tonic-gate 		/*
2047c478bd9Sstevel@tonic-gate 		 * resume() sets this value to the base of the threads stack
2057c478bd9Sstevel@tonic-gate 		 * via a context handler.
2067c478bd9Sstevel@tonic-gate 		 */
2070ac7d7d8Skucharsk 		wrmsr(MSR_INTC_SEP_ESP, 0ULL);
2080ac7d7d8Skucharsk 		wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter);
2097c478bd9Sstevel@tonic-gate 	}
2107c478bd9Sstevel@tonic-gate 
2117c478bd9Sstevel@tonic-gate 	kpreempt_enable();
2127c478bd9Sstevel@tonic-gate }
2137c478bd9Sstevel@tonic-gate 
2147c478bd9Sstevel@tonic-gate /*
2157c478bd9Sstevel@tonic-gate  * Multiprocessor initialization.
2167c478bd9Sstevel@tonic-gate  *
2177c478bd9Sstevel@tonic-gate  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
2187c478bd9Sstevel@tonic-gate  * startup and idle threads for the specified CPU.
2197c478bd9Sstevel@tonic-gate  */
2207c478bd9Sstevel@tonic-gate static void
2217c478bd9Sstevel@tonic-gate mp_startup_init(int cpun)
2227c478bd9Sstevel@tonic-gate {
2237c478bd9Sstevel@tonic-gate #if defined(__amd64)
2247c478bd9Sstevel@tonic-gate extern void *long_mode_64(void);
2257c478bd9Sstevel@tonic-gate #endif	/* __amd64 */
2267c478bd9Sstevel@tonic-gate 
2277c478bd9Sstevel@tonic-gate 	struct cpu *cp;
2287c478bd9Sstevel@tonic-gate 	struct tss *ntss;
2297c478bd9Sstevel@tonic-gate 	kthread_id_t tp;
2307c478bd9Sstevel@tonic-gate 	caddr_t	sp;
2317c478bd9Sstevel@tonic-gate 	int size;
2327c478bd9Sstevel@tonic-gate 	proc_t *procp;
2337c478bd9Sstevel@tonic-gate 	extern void idle();
2347c478bd9Sstevel@tonic-gate 
2357c478bd9Sstevel@tonic-gate 	struct cpu_tables *tablesp;
2367c478bd9Sstevel@tonic-gate 	rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va;
2377c478bd9Sstevel@tonic-gate 
2387c478bd9Sstevel@tonic-gate #ifdef TRAPTRACE
2397c478bd9Sstevel@tonic-gate 	trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
2407c478bd9Sstevel@tonic-gate #endif
2417c478bd9Sstevel@tonic-gate 
2427c478bd9Sstevel@tonic-gate 	ASSERT(cpun < NCPU && cpu[cpun] == NULL);
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	if ((cp = kmem_zalloc(sizeof (*cp), KM_NOSLEEP)) == NULL) {
2457c478bd9Sstevel@tonic-gate 		panic("mp_startup_init: cpu%d: "
2467c478bd9Sstevel@tonic-gate 		    "no memory for cpu structure", cpun);
2477c478bd9Sstevel@tonic-gate 		/*NOTREACHED*/
2487c478bd9Sstevel@tonic-gate 	}
2497c478bd9Sstevel@tonic-gate 	procp = curthread->t_procp;
2507c478bd9Sstevel@tonic-gate 
2517c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
2527c478bd9Sstevel@tonic-gate 	/*
2537c478bd9Sstevel@tonic-gate 	 * Initialize the dispatcher first.
2547c478bd9Sstevel@tonic-gate 	 */
2557c478bd9Sstevel@tonic-gate 	disp_cpu_init(cp);
2567c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
2577c478bd9Sstevel@tonic-gate 
258affbd3ccSkchow 	cpu_vm_data_init(cp);
259affbd3ccSkchow 
2607c478bd9Sstevel@tonic-gate 	/*
2617c478bd9Sstevel@tonic-gate 	 * Allocate and initialize the startup thread for this CPU.
2627c478bd9Sstevel@tonic-gate 	 * Interrupt and process switch stacks get allocated later
2637c478bd9Sstevel@tonic-gate 	 * when the CPU starts running.
2647c478bd9Sstevel@tonic-gate 	 */
2657c478bd9Sstevel@tonic-gate 	tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
2667c478bd9Sstevel@tonic-gate 	    TS_STOPPED, maxclsyspri);
2677c478bd9Sstevel@tonic-gate 
2687c478bd9Sstevel@tonic-gate 	/*
2697c478bd9Sstevel@tonic-gate 	 * Set state to TS_ONPROC since this thread will start running
2707c478bd9Sstevel@tonic-gate 	 * as soon as the CPU comes online.
2717c478bd9Sstevel@tonic-gate 	 *
2727c478bd9Sstevel@tonic-gate 	 * All the other fields of the thread structure are setup by
2737c478bd9Sstevel@tonic-gate 	 * thread_create().
2747c478bd9Sstevel@tonic-gate 	 */
2757c478bd9Sstevel@tonic-gate 	THREAD_ONPROC(tp, cp);
2767c478bd9Sstevel@tonic-gate 	tp->t_preempt = 1;
2777c478bd9Sstevel@tonic-gate 	tp->t_bound_cpu = cp;
2787c478bd9Sstevel@tonic-gate 	tp->t_affinitycnt = 1;
2797c478bd9Sstevel@tonic-gate 	tp->t_cpu = cp;
2807c478bd9Sstevel@tonic-gate 	tp->t_disp_queue = cp->cpu_disp;
2817c478bd9Sstevel@tonic-gate 
2827c478bd9Sstevel@tonic-gate 	/*
2837c478bd9Sstevel@tonic-gate 	 * Setup thread to start in mp_startup.
2847c478bd9Sstevel@tonic-gate 	 */
2857c478bd9Sstevel@tonic-gate 	sp = tp->t_stk;
2867c478bd9Sstevel@tonic-gate 	tp->t_pc = (uintptr_t)mp_startup;
2877c478bd9Sstevel@tonic-gate 	tp->t_sp = (uintptr_t)(sp - MINFRAME);
2887c478bd9Sstevel@tonic-gate 
2897c478bd9Sstevel@tonic-gate 	cp->cpu_id = cpun;
2907c478bd9Sstevel@tonic-gate 	cp->cpu_self = cp;
2917c478bd9Sstevel@tonic-gate 	cp->cpu_mask = 1 << cpun;
2927c478bd9Sstevel@tonic-gate 	cp->cpu_thread = tp;
2937c478bd9Sstevel@tonic-gate 	cp->cpu_lwp = NULL;
2947c478bd9Sstevel@tonic-gate 	cp->cpu_dispthread = tp;
2957c478bd9Sstevel@tonic-gate 	cp->cpu_dispatch_pri = DISP_PRIO(tp);
2967c478bd9Sstevel@tonic-gate 
2977c478bd9Sstevel@tonic-gate 	/*
2987c478bd9Sstevel@tonic-gate 	 * Now, initialize per-CPU idle thread for this CPU.
2997c478bd9Sstevel@tonic-gate 	 */
3007c478bd9Sstevel@tonic-gate 	tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
3017c478bd9Sstevel@tonic-gate 
3027c478bd9Sstevel@tonic-gate 	cp->cpu_idle_thread = tp;
3037c478bd9Sstevel@tonic-gate 
3047c478bd9Sstevel@tonic-gate 	tp->t_preempt = 1;
3057c478bd9Sstevel@tonic-gate 	tp->t_bound_cpu = cp;
3067c478bd9Sstevel@tonic-gate 	tp->t_affinitycnt = 1;
3077c478bd9Sstevel@tonic-gate 	tp->t_cpu = cp;
3087c478bd9Sstevel@tonic-gate 	tp->t_disp_queue = cp->cpu_disp;
3097c478bd9Sstevel@tonic-gate 
310394b433dSesaxe 	/*
311394b433dSesaxe 	 * Bootstrap the CPU for CMT aware scheduling
312394b433dSesaxe 	 * The rest of the initialization will happen from
313394b433dSesaxe 	 * mp_startup()
314394b433dSesaxe 	 */
315394b433dSesaxe 	chip_bootstrap_cpu(cp);
316394b433dSesaxe 
3177c478bd9Sstevel@tonic-gate 	/*
3187c478bd9Sstevel@tonic-gate 	 * Perform CPC intialization on the new CPU.
3197c478bd9Sstevel@tonic-gate 	 */
3207c478bd9Sstevel@tonic-gate 	kcpc_hw_init(cp);
3217c478bd9Sstevel@tonic-gate 
3227c478bd9Sstevel@tonic-gate 	/*
3237c478bd9Sstevel@tonic-gate 	 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
3247c478bd9Sstevel@tonic-gate 	 * for each CPU.
3257c478bd9Sstevel@tonic-gate 	 */
3267c478bd9Sstevel@tonic-gate 
3277c478bd9Sstevel@tonic-gate 	setup_vaddr_for_ppcopy(cp);
3287c478bd9Sstevel@tonic-gate 
3297c478bd9Sstevel@tonic-gate 	/*
3307c478bd9Sstevel@tonic-gate 	 * Allocate space for page directory, stack, tss, gdt and idt.
3317c478bd9Sstevel@tonic-gate 	 * This assumes that kmem_alloc will return memory which is aligned
3327c478bd9Sstevel@tonic-gate 	 * to the next higher power of 2 or a page(if size > MAXABIG)
3337c478bd9Sstevel@tonic-gate 	 * If this assumption goes wrong at any time due to change in
3347c478bd9Sstevel@tonic-gate 	 * kmem alloc, things may not work as the page directory has to be
3357c478bd9Sstevel@tonic-gate 	 * page aligned
3367c478bd9Sstevel@tonic-gate 	 */
3377c478bd9Sstevel@tonic-gate 	if ((tablesp = kmem_zalloc(sizeof (*tablesp), KM_NOSLEEP)) == NULL)
3387c478bd9Sstevel@tonic-gate 		panic("mp_startup_init: cpu%d cannot allocate tables", cpun);
3397c478bd9Sstevel@tonic-gate 
3407c478bd9Sstevel@tonic-gate 	if ((uintptr_t)tablesp & ~MMU_STD_PAGEMASK) {
3417c478bd9Sstevel@tonic-gate 		kmem_free(tablesp, sizeof (struct cpu_tables));
3427c478bd9Sstevel@tonic-gate 		size = sizeof (struct cpu_tables) + MMU_STD_PAGESIZE;
3437c478bd9Sstevel@tonic-gate 		tablesp = kmem_zalloc(size, KM_NOSLEEP);
3447c478bd9Sstevel@tonic-gate 		tablesp = (struct cpu_tables *)
3457c478bd9Sstevel@tonic-gate 		    (((uintptr_t)tablesp + MMU_STD_PAGESIZE) &
3467c478bd9Sstevel@tonic-gate 		    MMU_STD_PAGEMASK);
3477c478bd9Sstevel@tonic-gate 	}
3487c478bd9Sstevel@tonic-gate 
3497c478bd9Sstevel@tonic-gate 	ntss = cp->cpu_tss = &tablesp->ct_tss;
3507c478bd9Sstevel@tonic-gate 	cp->cpu_gdt = tablesp->ct_gdt;
3517c478bd9Sstevel@tonic-gate 	bcopy(CPU->cpu_gdt, cp->cpu_gdt, NGDT * (sizeof (user_desc_t)));
3527c478bd9Sstevel@tonic-gate 
3537c478bd9Sstevel@tonic-gate #if defined(__amd64)
3547c478bd9Sstevel@tonic-gate 
3557c478bd9Sstevel@tonic-gate 	/*
3567c478bd9Sstevel@tonic-gate 	 * #DF (double fault).
3577c478bd9Sstevel@tonic-gate 	 */
3587c478bd9Sstevel@tonic-gate 	ntss->tss_ist1 =
3597c478bd9Sstevel@tonic-gate 	    (uint64_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)];
3607c478bd9Sstevel@tonic-gate 
3617c478bd9Sstevel@tonic-gate #elif defined(__i386)
3627c478bd9Sstevel@tonic-gate 
3637c478bd9Sstevel@tonic-gate 	ntss->tss_esp0 = ntss->tss_esp1 = ntss->tss_esp2 = ntss->tss_esp =
3647c478bd9Sstevel@tonic-gate 	    (uint32_t)&tablesp->ct_stack[sizeof (tablesp->ct_stack)];
3657c478bd9Sstevel@tonic-gate 
3667c478bd9Sstevel@tonic-gate 	ntss->tss_ss0 = ntss->tss_ss1 = ntss->tss_ss2 = ntss->tss_ss = KDS_SEL;
3677c478bd9Sstevel@tonic-gate 
3687c478bd9Sstevel@tonic-gate 	ntss->tss_eip = (uint32_t)mp_startup;
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate 	ntss->tss_cs = KCS_SEL;
3717c478bd9Sstevel@tonic-gate 	ntss->tss_fs = KFS_SEL;
3727c478bd9Sstevel@tonic-gate 	ntss->tss_gs = KGS_SEL;
3737c478bd9Sstevel@tonic-gate 
3747c478bd9Sstevel@tonic-gate 	/*
3757c478bd9Sstevel@tonic-gate 	 * setup kernel %gs.
3767c478bd9Sstevel@tonic-gate 	 */
3777c478bd9Sstevel@tonic-gate 	set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,
3787c478bd9Sstevel@tonic-gate 	    SEL_KPL, 0, 1);
3797c478bd9Sstevel@tonic-gate 
3807c478bd9Sstevel@tonic-gate #endif	/* __i386 */
3817c478bd9Sstevel@tonic-gate 
3827c478bd9Sstevel@tonic-gate 	/*
3837c478bd9Sstevel@tonic-gate 	 * Set I/O bit map offset equal to size of TSS segment limit
3847c478bd9Sstevel@tonic-gate 	 * for no I/O permission map. This will cause all user I/O
3857c478bd9Sstevel@tonic-gate 	 * instructions to generate #gp fault.
3867c478bd9Sstevel@tonic-gate 	 */
3877c478bd9Sstevel@tonic-gate 	ntss->tss_bitmapbase = sizeof (*ntss);
3887c478bd9Sstevel@tonic-gate 
3897c478bd9Sstevel@tonic-gate 	/*
3907c478bd9Sstevel@tonic-gate 	 * setup kernel tss.
3917c478bd9Sstevel@tonic-gate 	 */
3927c478bd9Sstevel@tonic-gate 	set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss,
3937c478bd9Sstevel@tonic-gate 	    sizeof (*cp->cpu_tss) -1, SDT_SYSTSS, SEL_KPL);
3947c478bd9Sstevel@tonic-gate 
3957c478bd9Sstevel@tonic-gate 	/*
3967c478bd9Sstevel@tonic-gate 	 * If we have more than one node, each cpu gets a copy of IDT
3977c478bd9Sstevel@tonic-gate 	 * local to its node. If this is a Pentium box, we use cpu 0's
3987c478bd9Sstevel@tonic-gate 	 * IDT. cpu 0's IDT has been made read-only to workaround the
3997c478bd9Sstevel@tonic-gate 	 * cmpxchgl register bug
4007c478bd9Sstevel@tonic-gate 	 */
4017c478bd9Sstevel@tonic-gate 	cp->cpu_idt = CPU->cpu_idt;
4027c478bd9Sstevel@tonic-gate 	if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) {
4037c478bd9Sstevel@tonic-gate 		cp->cpu_idt = kmem_alloc(sizeof (idt0), KM_SLEEP);
4047c478bd9Sstevel@tonic-gate 		bcopy(idt0, cp->cpu_idt, sizeof (idt0));
4057c478bd9Sstevel@tonic-gate 	}
4067c478bd9Sstevel@tonic-gate 
4077c478bd9Sstevel@tonic-gate 	/*
4087c478bd9Sstevel@tonic-gate 	 * Get interrupt priority data from cpu 0
4097c478bd9Sstevel@tonic-gate 	 */
4107c478bd9Sstevel@tonic-gate 	cp->cpu_pri_data = CPU->cpu_pri_data;
4117c478bd9Sstevel@tonic-gate 
4127c478bd9Sstevel@tonic-gate 	hat_cpu_online(cp);
4137c478bd9Sstevel@tonic-gate 
4147c478bd9Sstevel@tonic-gate 	/* Should remove all entries for the current process/thread here */
4157c478bd9Sstevel@tonic-gate 
4167c478bd9Sstevel@tonic-gate 	/*
4177c478bd9Sstevel@tonic-gate 	 * Fill up the real mode platter to make it easy for real mode code to
4187c478bd9Sstevel@tonic-gate 	 * kick it off. This area should really be one passed by boot to kernel
4197c478bd9Sstevel@tonic-gate 	 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also
4207c478bd9Sstevel@tonic-gate 	 * have identical physical and virtual address in paged mode.
4217c478bd9Sstevel@tonic-gate 	 */
4227c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_idt_base = cp->cpu_idt;
4237c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_idt_lim = sizeof (idt0) - 1;
4247c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_gdt_base = cp->cpu_gdt;
4257c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_gdt_lim = sizeof (gdt0) -1;
4267c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_pdbr = getcr3();
4277c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_cpu = cpun;
4287c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_x86feature = x86_feature;
4297c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_cr4 = cr4_value;
4307c478bd9Sstevel@tonic-gate 
4317c478bd9Sstevel@tonic-gate #if defined(__amd64)
4327c478bd9Sstevel@tonic-gate 	if (getcr3() > 0xffffffffUL)
4337c478bd9Sstevel@tonic-gate 		panic("Cannot initialize CPUs; kernel's 64-bit page tables\n"
4347c478bd9Sstevel@tonic-gate 			"located above 4G in physical memory (@ 0x%llx).",
4357c478bd9Sstevel@tonic-gate 			(unsigned long long)getcr3());
4367c478bd9Sstevel@tonic-gate 
4377c478bd9Sstevel@tonic-gate 	/*
4387c478bd9Sstevel@tonic-gate 	 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY
4397c478bd9Sstevel@tonic-gate 	 * by code in real_mode_start():
4407c478bd9Sstevel@tonic-gate 	 *
4417c478bd9Sstevel@tonic-gate 	 * GDT[0]:  NULL selector
4427c478bd9Sstevel@tonic-gate 	 * GDT[1]:  64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1
4437c478bd9Sstevel@tonic-gate 	 *
4447c478bd9Sstevel@tonic-gate 	 * Clear the IDT as interrupts will be off and a limit of 0 will cause
4457c478bd9Sstevel@tonic-gate 	 * the CPU to triple fault and reset on an NMI, seemingly as reasonable
4467c478bd9Sstevel@tonic-gate 	 * a course of action as any other, though it may cause the entire
4477c478bd9Sstevel@tonic-gate 	 * platform to reset in some cases...
4487c478bd9Sstevel@tonic-gate 	 */
4497c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_temp_gdt[0] = 0ULL;
4507c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL;
4517c478bd9Sstevel@tonic-gate 
4527c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_temp_gdt_lim = (ushort_t)
4537c478bd9Sstevel@tonic-gate 	    (sizeof (real_mode_platter->rm_temp_gdt) - 1);
4547c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_temp_gdt_base = rm_platter_pa +
4557c478bd9Sstevel@tonic-gate 	    (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt);
4567c478bd9Sstevel@tonic-gate 
4577c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_temp_idt_lim = 0;
4587c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_temp_idt_base = 0;
4597c478bd9Sstevel@tonic-gate 
4607c478bd9Sstevel@tonic-gate 	/*
4617c478bd9Sstevel@tonic-gate 	 * Since the CPU needs to jump to protected mode using an identity
4627c478bd9Sstevel@tonic-gate 	 * mapped address, we need to calculate it here.
4637c478bd9Sstevel@tonic-gate 	 */
4647c478bd9Sstevel@tonic-gate 	real_mode_platter->rm_longmode64_addr = rm_platter_pa +
4657c478bd9Sstevel@tonic-gate 	    ((uint32_t)long_mode_64 - (uint32_t)real_mode_start);
4667c478bd9Sstevel@tonic-gate #endif	/* __amd64 */
4677c478bd9Sstevel@tonic-gate 
4687c478bd9Sstevel@tonic-gate #ifdef TRAPTRACE
4697c478bd9Sstevel@tonic-gate 	/*
4707c478bd9Sstevel@tonic-gate 	 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers for this
4717c478bd9Sstevel@tonic-gate 	 * CPU.
4727c478bd9Sstevel@tonic-gate 	 */
4737c478bd9Sstevel@tonic-gate 	ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
4747c478bd9Sstevel@tonic-gate 	ttc->ttc_next = ttc->ttc_first;
4757c478bd9Sstevel@tonic-gate 	ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
4767c478bd9Sstevel@tonic-gate #endif
4777c478bd9Sstevel@tonic-gate 
4787c478bd9Sstevel@tonic-gate 	/*
4797c478bd9Sstevel@tonic-gate 	 * Record that we have another CPU.
4807c478bd9Sstevel@tonic-gate 	 */
4817c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
4827c478bd9Sstevel@tonic-gate 	/*
4837c478bd9Sstevel@tonic-gate 	 * Initialize the interrupt threads for this CPU
4847c478bd9Sstevel@tonic-gate 	 */
485*100b72f4Sandrei 	cpu_intr_alloc(cp, NINTR_THREADS);
4867c478bd9Sstevel@tonic-gate 	/*
4877c478bd9Sstevel@tonic-gate 	 * Add CPU to list of available CPUs.  It'll be on the active list
4887c478bd9Sstevel@tonic-gate 	 * after mp_startup().
4897c478bd9Sstevel@tonic-gate 	 */
4907c478bd9Sstevel@tonic-gate 	cpu_add_unit(cp);
4917c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
4927c478bd9Sstevel@tonic-gate }
4937c478bd9Sstevel@tonic-gate 
4947c478bd9Sstevel@tonic-gate /*
4957c478bd9Sstevel@tonic-gate  * Apply workarounds for known errata, and warn about those that are absent.
4967c478bd9Sstevel@tonic-gate  *
4977c478bd9Sstevel@tonic-gate  * System vendors occasionally create configurations which contain different
4987c478bd9Sstevel@tonic-gate  * revisions of the CPUs that are almost but not exactly the same.  At the
4997c478bd9Sstevel@tonic-gate  * time of writing, this meant that their clock rates were the same, their
5007c478bd9Sstevel@tonic-gate  * feature sets were the same, but the required workaround were -not-
5017c478bd9Sstevel@tonic-gate  * necessarily the same.  So, this routine is invoked on -every- CPU soon
5027c478bd9Sstevel@tonic-gate  * after starting to make sure that the resulting system contains the most
5037c478bd9Sstevel@tonic-gate  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
5047c478bd9Sstevel@tonic-gate  * system.
5057c478bd9Sstevel@tonic-gate  *
506ef50d8c0Sesaxe  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
507ef50d8c0Sesaxe  * mp_startup() for all slave CPUs. Slaves process workaround_errata prior
508ef50d8c0Sesaxe  * to acknowledging their readiness to the master, so this routine will
509ef50d8c0Sesaxe  * never be executed by multiple CPUs in parallel, thus making updates to
510ef50d8c0Sesaxe  * global data safe.
511ef50d8c0Sesaxe  *
5122201b277Skucharsk  * These workarounds are based on Rev 3.57 of the Revision Guide for
5132201b277Skucharsk  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
5147c478bd9Sstevel@tonic-gate  */
5157c478bd9Sstevel@tonic-gate 
5167c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_91)
5177c478bd9Sstevel@tonic-gate int opteron_erratum_91;		/* if non-zero -> at least one cpu has it */
5187c478bd9Sstevel@tonic-gate #endif
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_93)
5217c478bd9Sstevel@tonic-gate int opteron_erratum_93;		/* if non-zero -> at least one cpu has it */
5227c478bd9Sstevel@tonic-gate #endif
5237c478bd9Sstevel@tonic-gate 
5247c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_100)
5257c478bd9Sstevel@tonic-gate int opteron_erratum_100;	/* if non-zero -> at least one cpu has it */
5267c478bd9Sstevel@tonic-gate #endif
5277c478bd9Sstevel@tonic-gate 
5287c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109)
5297c478bd9Sstevel@tonic-gate int opteron_erratum_109;	/* if non-zero -> at least one cpu has it */
5307c478bd9Sstevel@tonic-gate #endif
5317c478bd9Sstevel@tonic-gate 
5327c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_121)
5337c478bd9Sstevel@tonic-gate int opteron_erratum_121;	/* if non-zero -> at least one cpu has it */
5347c478bd9Sstevel@tonic-gate #endif
5357c478bd9Sstevel@tonic-gate 
5367c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_122)
5377c478bd9Sstevel@tonic-gate int opteron_erratum_122;	/* if non-zero -> at least one cpu has it */
5387c478bd9Sstevel@tonic-gate #endif
5397c478bd9Sstevel@tonic-gate 
5407c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_123)
5417c478bd9Sstevel@tonic-gate int opteron_erratum_123;	/* if non-zero -> at least one cpu has it */
5427c478bd9Sstevel@tonic-gate #endif
5437c478bd9Sstevel@tonic-gate 
5442201b277Skucharsk #if defined(OPTERON_ERRATUM_131)
5452201b277Skucharsk int opteron_erratum_131;	/* if non-zero -> at least one cpu has it */
5462201b277Skucharsk #endif
5477c478bd9Sstevel@tonic-gate 
548ef50d8c0Sesaxe #if defined(OPTERON_WORKAROUND_6336786)
549ef50d8c0Sesaxe int opteron_workaround_6336786;	/* non-zero -> WA relevant and applied */
550ef50d8c0Sesaxe int opteron_workaround_6336786_UP = 0;	/* Not needed for UP */
551ef50d8c0Sesaxe #endif
552ef50d8c0Sesaxe 
5537c478bd9Sstevel@tonic-gate #define	WARNING(cpu, n)						\
5547c478bd9Sstevel@tonic-gate 	cmn_err(CE_WARN, "cpu%d: no workaround for erratum %d",	\
5557c478bd9Sstevel@tonic-gate 	    (cpu)->cpu_id, (n))
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate uint_t
5587c478bd9Sstevel@tonic-gate workaround_errata(struct cpu *cpu)
5597c478bd9Sstevel@tonic-gate {
5607c478bd9Sstevel@tonic-gate 	uint_t missing = 0;
5617c478bd9Sstevel@tonic-gate 
5627c478bd9Sstevel@tonic-gate 	ASSERT(cpu == CPU);
5637c478bd9Sstevel@tonic-gate 
5647c478bd9Sstevel@tonic-gate 	/*LINTED*/
5657c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 88) > 0) {
5667c478bd9Sstevel@tonic-gate 		/*
5677c478bd9Sstevel@tonic-gate 		 * SWAPGS May Fail To Read Correct GS Base
5687c478bd9Sstevel@tonic-gate 		 */
5697c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_88)
5707c478bd9Sstevel@tonic-gate 		/*
5717c478bd9Sstevel@tonic-gate 		 * The workaround is an mfence in the relevant assembler code
5727c478bd9Sstevel@tonic-gate 		 */
5737c478bd9Sstevel@tonic-gate #else
5747c478bd9Sstevel@tonic-gate 		WARNING(cpu, 88);
5757c478bd9Sstevel@tonic-gate 		missing++;
5767c478bd9Sstevel@tonic-gate #endif
5777c478bd9Sstevel@tonic-gate 	}
5787c478bd9Sstevel@tonic-gate 
5797c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 91) > 0) {
5807c478bd9Sstevel@tonic-gate 		/*
5817c478bd9Sstevel@tonic-gate 		 * Software Prefetches May Report A Page Fault
5827c478bd9Sstevel@tonic-gate 		 */
5837c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_91)
5847c478bd9Sstevel@tonic-gate 		/*
5857c478bd9Sstevel@tonic-gate 		 * fix is in trap.c
5867c478bd9Sstevel@tonic-gate 		 */
5877c478bd9Sstevel@tonic-gate 		opteron_erratum_91++;
5887c478bd9Sstevel@tonic-gate #else
5897c478bd9Sstevel@tonic-gate 		WARNING(cpu, 91);
5907c478bd9Sstevel@tonic-gate 		missing++;
5917c478bd9Sstevel@tonic-gate #endif
5927c478bd9Sstevel@tonic-gate 	}
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 93) > 0) {
5957c478bd9Sstevel@tonic-gate 		/*
5967c478bd9Sstevel@tonic-gate 		 * RSM Auto-Halt Restart Returns to Incorrect RIP
5977c478bd9Sstevel@tonic-gate 		 */
5987c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_93)
5997c478bd9Sstevel@tonic-gate 		/*
6007c478bd9Sstevel@tonic-gate 		 * fix is in trap.c
6017c478bd9Sstevel@tonic-gate 		 */
6027c478bd9Sstevel@tonic-gate 		opteron_erratum_93++;
6037c478bd9Sstevel@tonic-gate #else
6047c478bd9Sstevel@tonic-gate 		WARNING(cpu, 93);
6057c478bd9Sstevel@tonic-gate 		missing++;
6067c478bd9Sstevel@tonic-gate #endif
6077c478bd9Sstevel@tonic-gate 	}
6087c478bd9Sstevel@tonic-gate 
6097c478bd9Sstevel@tonic-gate 	/*LINTED*/
6107c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 95) > 0) {
6117c478bd9Sstevel@tonic-gate 		/*
6127c478bd9Sstevel@tonic-gate 		 * RET Instruction May Return to Incorrect EIP
6137c478bd9Sstevel@tonic-gate 		 */
6147c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_95)
6157c478bd9Sstevel@tonic-gate #if defined(_LP64)
6167c478bd9Sstevel@tonic-gate 		/*
6177c478bd9Sstevel@tonic-gate 		 * Workaround this by ensuring that 32-bit user code and
6187c478bd9Sstevel@tonic-gate 		 * 64-bit kernel code never occupy the same address
6197c478bd9Sstevel@tonic-gate 		 * range mod 4G.
6207c478bd9Sstevel@tonic-gate 		 */
6217c478bd9Sstevel@tonic-gate 		if (_userlimit32 > 0xc0000000ul)
6227c478bd9Sstevel@tonic-gate 			*(uintptr_t *)&_userlimit32 = 0xc0000000ul;
6237c478bd9Sstevel@tonic-gate 
6247c478bd9Sstevel@tonic-gate 		/*LINTED*/
6257c478bd9Sstevel@tonic-gate 		ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
6267c478bd9Sstevel@tonic-gate #endif	/* _LP64 */
6277c478bd9Sstevel@tonic-gate #else
6287c478bd9Sstevel@tonic-gate 		WARNING(cpu, 95);
6297c478bd9Sstevel@tonic-gate 		missing++;
6307c478bd9Sstevel@tonic-gate #endif	/* OPTERON_ERRATUM_95 */
6317c478bd9Sstevel@tonic-gate 	}
6327c478bd9Sstevel@tonic-gate 
6337c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 100) > 0) {
6347c478bd9Sstevel@tonic-gate 		/*
6357c478bd9Sstevel@tonic-gate 		 * Compatibility Mode Branches Transfer to Illegal Address
6367c478bd9Sstevel@tonic-gate 		 */
6377c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_100)
6387c478bd9Sstevel@tonic-gate 		/*
6397c478bd9Sstevel@tonic-gate 		 * fix is in trap.c
6407c478bd9Sstevel@tonic-gate 		 */
6417c478bd9Sstevel@tonic-gate 		opteron_erratum_100++;
6427c478bd9Sstevel@tonic-gate #else
6437c478bd9Sstevel@tonic-gate 		WARNING(cpu, 100);
6447c478bd9Sstevel@tonic-gate 		missing++;
6457c478bd9Sstevel@tonic-gate #endif
6467c478bd9Sstevel@tonic-gate 	}
6477c478bd9Sstevel@tonic-gate 
6487c478bd9Sstevel@tonic-gate 	/*LINTED*/
6497c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 108) > 0) {
6507c478bd9Sstevel@tonic-gate 		/*
6517c478bd9Sstevel@tonic-gate 		 * CPUID Instruction May Return Incorrect Model Number In
6527c478bd9Sstevel@tonic-gate 		 * Some Processors
6537c478bd9Sstevel@tonic-gate 		 */
6547c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_108)
6557c478bd9Sstevel@tonic-gate 		/*
6567c478bd9Sstevel@tonic-gate 		 * (Our cpuid-handling code corrects the model number on
6577c478bd9Sstevel@tonic-gate 		 * those processors)
6587c478bd9Sstevel@tonic-gate 		 */
6597c478bd9Sstevel@tonic-gate #else
6607c478bd9Sstevel@tonic-gate 		WARNING(cpu, 108);
6617c478bd9Sstevel@tonic-gate 		missing++;
6627c478bd9Sstevel@tonic-gate #endif
6637c478bd9Sstevel@tonic-gate 	}
6647c478bd9Sstevel@tonic-gate 
6657c478bd9Sstevel@tonic-gate 	/*LINTED*/
6667c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 109) > 0) {
6677c478bd9Sstevel@tonic-gate 		/*
6687c478bd9Sstevel@tonic-gate 		 * Certain Reverse REP MOVS May Produce Unpredictable Behaviour
6697c478bd9Sstevel@tonic-gate 		 */
6707c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109)
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate 		/* workaround is to print a warning to upgrade BIOS */
6730ac7d7d8Skucharsk 		if (rdmsr(MSR_AMD_PATCHLEVEL) == 0)
6747c478bd9Sstevel@tonic-gate 			opteron_erratum_109++;
6757c478bd9Sstevel@tonic-gate #else
6767c478bd9Sstevel@tonic-gate 		WARNING(cpu, 109);
6777c478bd9Sstevel@tonic-gate 		missing++;
6787c478bd9Sstevel@tonic-gate #endif
6797c478bd9Sstevel@tonic-gate 	}
6807c478bd9Sstevel@tonic-gate 	/*LINTED*/
6817c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 121) > 0) {
6827c478bd9Sstevel@tonic-gate 		/*
6837c478bd9Sstevel@tonic-gate 		 * Sequential Execution Across Non_Canonical Boundary Caused
6847c478bd9Sstevel@tonic-gate 		 * Processor Hang
6857c478bd9Sstevel@tonic-gate 		 */
6867c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_121)
6877c478bd9Sstevel@tonic-gate 		static int	lma;
6887c478bd9Sstevel@tonic-gate 
6897c478bd9Sstevel@tonic-gate 		if (opteron_erratum_121)
6907c478bd9Sstevel@tonic-gate 			opteron_erratum_121++;
6917c478bd9Sstevel@tonic-gate 
6927c478bd9Sstevel@tonic-gate 		/*
6937c478bd9Sstevel@tonic-gate 		 * Erratum 121 is only present in long (64 bit) mode.
6947c478bd9Sstevel@tonic-gate 		 * Workaround is to include the page immediately before the
6957c478bd9Sstevel@tonic-gate 		 * va hole to eliminate the possibility of system hangs due to
6967c478bd9Sstevel@tonic-gate 		 * sequential execution across the va hole boundary.
6977c478bd9Sstevel@tonic-gate 		 */
6987c478bd9Sstevel@tonic-gate 		if (lma == 0) {
6997c478bd9Sstevel@tonic-gate 			/*
7007c478bd9Sstevel@tonic-gate 			 * check LMA once: assume all cpus are in long mode
7017c478bd9Sstevel@tonic-gate 			 * or not.
7027c478bd9Sstevel@tonic-gate 			 */
7037c478bd9Sstevel@tonic-gate 			lma = 1;
7047c478bd9Sstevel@tonic-gate 
7050ac7d7d8Skucharsk 			if (rdmsr(MSR_AMD_EFER) & AMD_EFER_LMA) {
7067c478bd9Sstevel@tonic-gate 				if (hole_start) {
7077c478bd9Sstevel@tonic-gate 					hole_start -= PAGESIZE;
7087c478bd9Sstevel@tonic-gate 				} else {
7097c478bd9Sstevel@tonic-gate 					/*
7107c478bd9Sstevel@tonic-gate 					 * hole_start not yet initialized by
7117c478bd9Sstevel@tonic-gate 					 * mmu_init. Initialize hole_start
7127c478bd9Sstevel@tonic-gate 					 * with value to be subtracted.
7137c478bd9Sstevel@tonic-gate 					 */
7147c478bd9Sstevel@tonic-gate 					hole_start = PAGESIZE;
7157c478bd9Sstevel@tonic-gate 				}
7167c478bd9Sstevel@tonic-gate 				opteron_erratum_121++;
7177c478bd9Sstevel@tonic-gate 			}
7187c478bd9Sstevel@tonic-gate 		}
7197c478bd9Sstevel@tonic-gate #else
7207c478bd9Sstevel@tonic-gate 		WARNING(cpu, 121);
7217c478bd9Sstevel@tonic-gate 		missing++;
7227c478bd9Sstevel@tonic-gate #endif
7237c478bd9Sstevel@tonic-gate 	}
7247c478bd9Sstevel@tonic-gate 
7257c478bd9Sstevel@tonic-gate 	/*LINTED*/
7267c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 122) > 0) {
7277c478bd9Sstevel@tonic-gate 		/*
7287c478bd9Sstevel@tonic-gate 		 * TLB Flush Filter May Cause Cohenrency Problem in
7297c478bd9Sstevel@tonic-gate 		 * Multiprocessor Systems
7307c478bd9Sstevel@tonic-gate 		 */
7317c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_122)
7327c478bd9Sstevel@tonic-gate 		/*
7337c478bd9Sstevel@tonic-gate 		 * Erratum 122 is only present in MP configurations (multi-core
7347c478bd9Sstevel@tonic-gate 		 * or multi-processor).
7357c478bd9Sstevel@tonic-gate 		 */
7367c478bd9Sstevel@tonic-gate 
7377c478bd9Sstevel@tonic-gate 		if (opteron_erratum_122 || lgrp_plat_node_cnt > 1 ||
7387c478bd9Sstevel@tonic-gate 		    cpuid_get_ncpu_per_chip(cpu) > 1) {
7397c478bd9Sstevel@tonic-gate 			/* disable TLB Flush Filter */
7400ac7d7d8Skucharsk 			wrmsr(MSR_AMD_HWCR, rdmsr(MSR_AMD_HWCR) |
7410ac7d7d8Skucharsk 			    (uint64_t)(uintptr_t)AMD_HWCR_FFDIS);
7427c478bd9Sstevel@tonic-gate 			opteron_erratum_122++;
7437c478bd9Sstevel@tonic-gate 		}
7447c478bd9Sstevel@tonic-gate 
7457c478bd9Sstevel@tonic-gate #else
7467c478bd9Sstevel@tonic-gate 		WARNING(cpu, 122);
7477c478bd9Sstevel@tonic-gate 		missing++;
7487c478bd9Sstevel@tonic-gate #endif
7497c478bd9Sstevel@tonic-gate 	}
750403c216aSkchow 
751403c216aSkchow #if defined(OPTERON_ERRATUM_123)
7527c478bd9Sstevel@tonic-gate 	/*LINTED*/
7537c478bd9Sstevel@tonic-gate 	if (cpuid_opteron_erratum(cpu, 123) > 0) {
7547c478bd9Sstevel@tonic-gate 		/*
7557c478bd9Sstevel@tonic-gate 		 * Bypassed Reads May Cause Data Corruption of System Hang in
7567c478bd9Sstevel@tonic-gate 		 * Dual Core Processors
7577c478bd9Sstevel@tonic-gate 		 */
7587c478bd9Sstevel@tonic-gate 		/*
7597c478bd9Sstevel@tonic-gate 		 * Erratum 123 applies only to multi-core cpus.
7607c478bd9Sstevel@tonic-gate 		 */
7617c478bd9Sstevel@tonic-gate 
7627c478bd9Sstevel@tonic-gate 		if (cpuid_get_ncpu_per_chip(cpu) > 1) {
7637c478bd9Sstevel@tonic-gate 			/* workaround is to print a warning to upgrade BIOS */
7640ac7d7d8Skucharsk 			if (rdmsr(MSR_AMD_PATCHLEVEL) == 0)
7657c478bd9Sstevel@tonic-gate 				opteron_erratum_123++;
7667c478bd9Sstevel@tonic-gate 		}
7677c478bd9Sstevel@tonic-gate 	}
768403c216aSkchow #endif
7692201b277Skucharsk 
7702201b277Skucharsk #if defined(OPTERON_ERRATUM_131)
7712201b277Skucharsk 	/*LINTED*/
7722201b277Skucharsk 	if (cpuid_opteron_erratum(cpu, 131) > 0) {
7732201b277Skucharsk 		/*
7742201b277Skucharsk 		 * Multiprocessor Systems with Four or More Cores May Deadlock
7752201b277Skucharsk 		 * Waiting for a Probe Response
7762201b277Skucharsk 		 */
7772201b277Skucharsk 		/*
7782201b277Skucharsk 		 * Erratum 131 applies to any system with four or more cores.
7792201b277Skucharsk 		 */
7802201b277Skucharsk 		if ((opteron_erratum_131 == 0) && ((lgrp_plat_node_cnt *
7812201b277Skucharsk 		    cpuid_get_ncpu_per_chip(cpu)) >= 4)) {
7822201b277Skucharsk 			/*
7832201b277Skucharsk 			 * Workaround is to print a warning to upgrade
7842201b277Skucharsk 			 * the BIOS
7852201b277Skucharsk 			 */
7860ac7d7d8Skucharsk 			if (!(rdmsr(MSR_AMD_NB_CFG) & AMD_NB_CFG_SRQ_HEARTBEAT))
7872201b277Skucharsk 				opteron_erratum_131++;
7882201b277Skucharsk 		}
789ef50d8c0Sesaxe 	}
7902201b277Skucharsk #endif
791ef50d8c0Sesaxe 
792ef50d8c0Sesaxe #if defined(OPTERON_WORKAROUND_6336786)
793ef50d8c0Sesaxe 	/*
794ef50d8c0Sesaxe 	 * This isn't really erratum, but for convenience the
795ef50d8c0Sesaxe 	 * detection/workaround code lives here and in cpuid_opteron_erratum.
796ef50d8c0Sesaxe 	 */
797ef50d8c0Sesaxe 	if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
798ef50d8c0Sesaxe 		int	node;
799ef50d8c0Sesaxe 		uint8_t data;
800ef50d8c0Sesaxe 
801ef50d8c0Sesaxe 		/*
802ef50d8c0Sesaxe 		 * Disable C1-Clock ramping on multi-core/multi-processor
803ef50d8c0Sesaxe 		 * K8 platforms to guard against TSC drift.
804ef50d8c0Sesaxe 		 */
805ef50d8c0Sesaxe 		if (opteron_workaround_6336786) {
806ef50d8c0Sesaxe 			opteron_workaround_6336786++;
807ef50d8c0Sesaxe 		} else if ((lgrp_plat_node_cnt *
808ef50d8c0Sesaxe 		    cpuid_get_ncpu_per_chip(cpu) >= 2) ||
809ef50d8c0Sesaxe 		    opteron_workaround_6336786_UP) {
810ef50d8c0Sesaxe 			for (node = 0; node < lgrp_plat_node_cnt; node++) {
811ef50d8c0Sesaxe 				/*
812ef50d8c0Sesaxe 				 * Clear PMM7[1:0] (function 3, offset 0x87)
813ef50d8c0Sesaxe 				 * Northbridge device is the node id + 24.
814ef50d8c0Sesaxe 				 */
815ef50d8c0Sesaxe 				data = pci_getb_func(0, node + 24, 3, 0x87);
816ef50d8c0Sesaxe 				data &= 0xFC;
817ef50d8c0Sesaxe 				pci_putb_func(0, node + 24, 3, 0x87, data);
818ef50d8c0Sesaxe 			}
819ef50d8c0Sesaxe 			opteron_workaround_6336786++;
820ef50d8c0Sesaxe 		}
8212201b277Skucharsk 	}
822ef50d8c0Sesaxe #endif
8237c478bd9Sstevel@tonic-gate 	return (missing);
8247c478bd9Sstevel@tonic-gate }
8257c478bd9Sstevel@tonic-gate 
8267c478bd9Sstevel@tonic-gate void
8277c478bd9Sstevel@tonic-gate workaround_errata_end()
8287c478bd9Sstevel@tonic-gate {
8297c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_109)
8307c478bd9Sstevel@tonic-gate 	if (opteron_erratum_109) {
8312201b277Skucharsk 		cmn_err(CE_WARN,
8322201b277Skucharsk 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
8332201b277Skucharsk 		    " processor\nerratum 109 was not detected; updating your"
8342201b277Skucharsk 		    " system's BIOS to a version\ncontaining this"
8352201b277Skucharsk 		    " microcode patch is HIGHLY recommended or erroneous"
8362201b277Skucharsk 		    " system\noperation may occur.\n");
8377c478bd9Sstevel@tonic-gate 	}
8382201b277Skucharsk #endif	/* OPTERON_ERRATUM_109 */
8397c478bd9Sstevel@tonic-gate #if defined(OPTERON_ERRATUM_123)
8407c478bd9Sstevel@tonic-gate 	if (opteron_erratum_123) {
8412201b277Skucharsk 		cmn_err(CE_WARN,
8422201b277Skucharsk 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
8432201b277Skucharsk 		    " processor\nerratum 123 was not detected; updating your"
8442201b277Skucharsk 		    " system's BIOS to a version\ncontaining this"
8452201b277Skucharsk 		    " microcode patch is HIGHLY recommended or erroneous"
8462201b277Skucharsk 		    " system\noperation may occur.\n");
8477c478bd9Sstevel@tonic-gate 	}
8482201b277Skucharsk #endif	/* OPTERON_ERRATUM_123 */
8492201b277Skucharsk #if defined(OPTERON_ERRATUM_131)
8502201b277Skucharsk 	if (opteron_erratum_131) {
8512201b277Skucharsk 		cmn_err(CE_WARN,
8522201b277Skucharsk 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
8532201b277Skucharsk 		    " processor\nerratum 131 was not detected; updating your"
8542201b277Skucharsk 		    " system's BIOS to a version\ncontaining this"
8552201b277Skucharsk 		    " microcode patch is HIGHLY recommended or erroneous"
8562201b277Skucharsk 		    " system\noperation may occur.\n");
8572201b277Skucharsk 	}
8582201b277Skucharsk #endif	/* OPTERON_ERRATUM_131 */
8597c478bd9Sstevel@tonic-gate }
8607c478bd9Sstevel@tonic-gate 
8617c478bd9Sstevel@tonic-gate static ushort_t *mp_map_warm_reset_vector();
8627c478bd9Sstevel@tonic-gate static void mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector);
8637c478bd9Sstevel@tonic-gate 
8647c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8657c478bd9Sstevel@tonic-gate void
8667c478bd9Sstevel@tonic-gate start_other_cpus(int cprboot)
8677c478bd9Sstevel@tonic-gate {
8687c478bd9Sstevel@tonic-gate 	unsigned who;
869d90554ebSdmick 	int cpuid = 0;
8707c478bd9Sstevel@tonic-gate 	int delays = 0;
8717c478bd9Sstevel@tonic-gate 	int started_cpu;
8727c478bd9Sstevel@tonic-gate 	ushort_t *warm_reset_vector = NULL;
8737c478bd9Sstevel@tonic-gate 	extern int procset;
8747c478bd9Sstevel@tonic-gate 
8757c478bd9Sstevel@tonic-gate 	/*
8767c478bd9Sstevel@tonic-gate 	 * Initialize our own cpu_info.
8777c478bd9Sstevel@tonic-gate 	 */
8787c478bd9Sstevel@tonic-gate 	init_cpu_info(CPU);
8797c478bd9Sstevel@tonic-gate 
8807c478bd9Sstevel@tonic-gate 	/*
8817c478bd9Sstevel@tonic-gate 	 * Initialize our syscall handlers
8827c478bd9Sstevel@tonic-gate 	 */
8837c478bd9Sstevel@tonic-gate 	init_cpu_syscall(CPU);
8847c478bd9Sstevel@tonic-gate 
8857c478bd9Sstevel@tonic-gate 	/*
8867c478bd9Sstevel@tonic-gate 	 * if only 1 cpu or not using MP, skip the rest of this
8877c478bd9Sstevel@tonic-gate 	 */
8887c478bd9Sstevel@tonic-gate 	if (!(mp_cpus & ~(1 << cpuid)) || use_mp == 0) {
8897c478bd9Sstevel@tonic-gate 		if (use_mp == 0)
8907c478bd9Sstevel@tonic-gate 			cmn_err(CE_CONT, "?***** Not in MP mode\n");
8917c478bd9Sstevel@tonic-gate 		goto done;
8927c478bd9Sstevel@tonic-gate 	}
8937c478bd9Sstevel@tonic-gate 
8947c478bd9Sstevel@tonic-gate 	/*
8957c478bd9Sstevel@tonic-gate 	 * perform such initialization as is needed
8967c478bd9Sstevel@tonic-gate 	 * to be able to take CPUs on- and off-line.
8977c478bd9Sstevel@tonic-gate 	 */
8987c478bd9Sstevel@tonic-gate 	cpu_pause_init();
8997c478bd9Sstevel@tonic-gate 
9007c478bd9Sstevel@tonic-gate 	xc_init();		/* initialize processor crosscalls */
9017c478bd9Sstevel@tonic-gate 
9027c478bd9Sstevel@tonic-gate 	/*
9037c478bd9Sstevel@tonic-gate 	 * Copy the real mode code at "real_mode_start" to the
9047c478bd9Sstevel@tonic-gate 	 * page at rm_platter_va.
9057c478bd9Sstevel@tonic-gate 	 */
9067c478bd9Sstevel@tonic-gate 	warm_reset_vector = mp_map_warm_reset_vector();
9077c478bd9Sstevel@tonic-gate 	if (warm_reset_vector == NULL)
9087c478bd9Sstevel@tonic-gate 		goto done;
9097c478bd9Sstevel@tonic-gate 
9107c478bd9Sstevel@tonic-gate 	bcopy((caddr_t)real_mode_start,
9117c478bd9Sstevel@tonic-gate 	    (caddr_t)((rm_platter_t *)rm_platter_va)->rm_code,
9127c478bd9Sstevel@tonic-gate 	    (size_t)real_mode_end - (size_t)real_mode_start);
9137c478bd9Sstevel@tonic-gate 
9147c478bd9Sstevel@tonic-gate 	flushes_require_xcalls = 1;
9157c478bd9Sstevel@tonic-gate 
9167c478bd9Sstevel@tonic-gate 	affinity_set(CPU_CURRENT);
9177c478bd9Sstevel@tonic-gate 
9187c478bd9Sstevel@tonic-gate 	for (who = 0; who < NCPU; who++) {
9197c478bd9Sstevel@tonic-gate 		if (who == cpuid)
9207c478bd9Sstevel@tonic-gate 			continue;
9217c478bd9Sstevel@tonic-gate 
9227c478bd9Sstevel@tonic-gate 		if ((mp_cpus & (1 << who)) == 0)
9237c478bd9Sstevel@tonic-gate 			continue;
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate 		mp_startup_init(who);
9267c478bd9Sstevel@tonic-gate 		started_cpu = 1;
9277c478bd9Sstevel@tonic-gate 		(*cpu_startf)(who, rm_platter_pa);
9287c478bd9Sstevel@tonic-gate 
9297c478bd9Sstevel@tonic-gate 		while ((procset & (1 << who)) == 0) {
9307c478bd9Sstevel@tonic-gate 
9317c478bd9Sstevel@tonic-gate 			delay(1);
9327c478bd9Sstevel@tonic-gate 			if (++delays > (20 * hz)) {
9337c478bd9Sstevel@tonic-gate 
9347c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
9357c478bd9Sstevel@tonic-gate 				    "cpu%d failed to start", who);
9367c478bd9Sstevel@tonic-gate 
9377c478bd9Sstevel@tonic-gate 				mutex_enter(&cpu_lock);
9387c478bd9Sstevel@tonic-gate 				cpu[who]->cpu_flags = 0;
939affbd3ccSkchow 				cpu_vm_data_destroy(cpu[who]);
9407c478bd9Sstevel@tonic-gate 				cpu_del_unit(who);
9417c478bd9Sstevel@tonic-gate 				mutex_exit(&cpu_lock);
9427c478bd9Sstevel@tonic-gate 
9437c478bd9Sstevel@tonic-gate 				started_cpu = 0;
9447c478bd9Sstevel@tonic-gate 				break;
9457c478bd9Sstevel@tonic-gate 			}
9467c478bd9Sstevel@tonic-gate 		}
9477c478bd9Sstevel@tonic-gate 		if (!started_cpu)
9487c478bd9Sstevel@tonic-gate 			continue;
9497c478bd9Sstevel@tonic-gate 		if (tsc_gethrtime_enable)
9507c478bd9Sstevel@tonic-gate 			tsc_sync_master(who);
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 
9537c478bd9Sstevel@tonic-gate 		if (dtrace_cpu_init != NULL) {
9547c478bd9Sstevel@tonic-gate 			/*
9557c478bd9Sstevel@tonic-gate 			 * DTrace CPU initialization expects cpu_lock
9567c478bd9Sstevel@tonic-gate 			 * to be held.
9577c478bd9Sstevel@tonic-gate 			 */
9587c478bd9Sstevel@tonic-gate 			mutex_enter(&cpu_lock);
9597c478bd9Sstevel@tonic-gate 			(*dtrace_cpu_init)(who);
9607c478bd9Sstevel@tonic-gate 			mutex_exit(&cpu_lock);
9617c478bd9Sstevel@tonic-gate 		}
9627c478bd9Sstevel@tonic-gate 	}
9637c478bd9Sstevel@tonic-gate 
9647c478bd9Sstevel@tonic-gate 	affinity_clear();
9657c478bd9Sstevel@tonic-gate 
9667c478bd9Sstevel@tonic-gate 	for (who = 0; who < NCPU; who++) {
9677c478bd9Sstevel@tonic-gate 		if (who == cpuid)
9687c478bd9Sstevel@tonic-gate 			continue;
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 		if (!(procset & (1 << who)))
9717c478bd9Sstevel@tonic-gate 			continue;
9727c478bd9Sstevel@tonic-gate 
9737c478bd9Sstevel@tonic-gate 		while (!(cpu_ready_set & (1 << who)))
9747c478bd9Sstevel@tonic-gate 			delay(1);
9757c478bd9Sstevel@tonic-gate 	}
9767c478bd9Sstevel@tonic-gate 
9777c478bd9Sstevel@tonic-gate done:
9787c478bd9Sstevel@tonic-gate 	workaround_errata_end();
9797c478bd9Sstevel@tonic-gate 
9807c478bd9Sstevel@tonic-gate 	if (warm_reset_vector != NULL)
9817c478bd9Sstevel@tonic-gate 		mp_unmap_warm_reset_vector(warm_reset_vector);
9827c478bd9Sstevel@tonic-gate 	hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE,
9837c478bd9Sstevel@tonic-gate 	    HAT_UNLOAD);
9847c478bd9Sstevel@tonic-gate }
9857c478bd9Sstevel@tonic-gate 
9867c478bd9Sstevel@tonic-gate /*
9877c478bd9Sstevel@tonic-gate  * Dummy functions - no i86pc platforms support dynamic cpu allocation.
9887c478bd9Sstevel@tonic-gate  */
9897c478bd9Sstevel@tonic-gate /*ARGSUSED*/
9907c478bd9Sstevel@tonic-gate int
9917c478bd9Sstevel@tonic-gate mp_cpu_configure(int cpuid)
9927c478bd9Sstevel@tonic-gate {
9937c478bd9Sstevel@tonic-gate 	return (ENOTSUP);		/* not supported */
9947c478bd9Sstevel@tonic-gate }
9957c478bd9Sstevel@tonic-gate 
9967c478bd9Sstevel@tonic-gate /*ARGSUSED*/
9977c478bd9Sstevel@tonic-gate int
9987c478bd9Sstevel@tonic-gate mp_cpu_unconfigure(int cpuid)
9997c478bd9Sstevel@tonic-gate {
10007c478bd9Sstevel@tonic-gate 	return (ENOTSUP);		/* not supported */
10017c478bd9Sstevel@tonic-gate }
10027c478bd9Sstevel@tonic-gate 
10037c478bd9Sstevel@tonic-gate /*
10047c478bd9Sstevel@tonic-gate  * Startup function for 'other' CPUs (besides boot cpu).
10057c478bd9Sstevel@tonic-gate  * Resumed from cpu_startup.
1006b4b46911Skchow  *
1007b4b46911Skchow  * WARNING: until CPU_READY is set, mp_startup and routines called by
1008b4b46911Skchow  * mp_startup should not call routines (e.g. kmem_free) that could call
1009b4b46911Skchow  * hat_unload which requires CPU_READY to be set.
10107c478bd9Sstevel@tonic-gate  */
10117c478bd9Sstevel@tonic-gate void
10127c478bd9Sstevel@tonic-gate mp_startup(void)
10137c478bd9Sstevel@tonic-gate {
10147c478bd9Sstevel@tonic-gate 	struct cpu *cp = CPU;
10157c478bd9Sstevel@tonic-gate 	extern int procset;
10167c478bd9Sstevel@tonic-gate 	uint_t new_x86_feature;
10177c478bd9Sstevel@tonic-gate 
10187c478bd9Sstevel@tonic-gate 	new_x86_feature = cpuid_pass1(cp);
10197c478bd9Sstevel@tonic-gate 
10207c478bd9Sstevel@tonic-gate 	/*
10217c478bd9Sstevel@tonic-gate 	 * We need to Sync MTRR with cpu0's MTRR. We have to do
10227c478bd9Sstevel@tonic-gate 	 * this with interrupts disabled.
10237c478bd9Sstevel@tonic-gate 	 */
10247c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_MTRR)
10257c478bd9Sstevel@tonic-gate 		mtrr_sync();
10267c478bd9Sstevel@tonic-gate 
10277c478bd9Sstevel@tonic-gate 	/*
10287c478bd9Sstevel@tonic-gate 	 * Initialize this CPU's syscall handlers
10297c478bd9Sstevel@tonic-gate 	 */
10307c478bd9Sstevel@tonic-gate 	init_cpu_syscall(cp);
10317c478bd9Sstevel@tonic-gate 
10327c478bd9Sstevel@tonic-gate 	/*
10337c478bd9Sstevel@tonic-gate 	 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
10347c478bd9Sstevel@tonic-gate 	 * highest level at which a routine is permitted to block on
10357c478bd9Sstevel@tonic-gate 	 * an adaptive mutex (allows for cpu poke interrupt in case
10367c478bd9Sstevel@tonic-gate 	 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
10377c478bd9Sstevel@tonic-gate 	 * device interrupts that may end up in the hat layer issuing cross
10387c478bd9Sstevel@tonic-gate 	 * calls before CPU_READY is set.
10397c478bd9Sstevel@tonic-gate 	 */
10407c478bd9Sstevel@tonic-gate 	(void) splx(ipltospl(LOCK_LEVEL));
10417c478bd9Sstevel@tonic-gate 
10427c478bd9Sstevel@tonic-gate 	/*
10437c478bd9Sstevel@tonic-gate 	 * Do a sanity check to make sure this new CPU is a sane thing
10447c478bd9Sstevel@tonic-gate 	 * to add to the collection of processors running this system.
10457c478bd9Sstevel@tonic-gate 	 *
10467c478bd9Sstevel@tonic-gate 	 * XXX	Clearly this needs to get more sophisticated, if x86
10477c478bd9Sstevel@tonic-gate 	 * systems start to get built out of heterogenous CPUs; as is
10487c478bd9Sstevel@tonic-gate 	 * likely to happen once the number of processors in a configuration
10497c478bd9Sstevel@tonic-gate 	 * gets large enough.
10507c478bd9Sstevel@tonic-gate 	 */
10517c478bd9Sstevel@tonic-gate 	if ((x86_feature & new_x86_feature) != x86_feature) {
10527c478bd9Sstevel@tonic-gate 		cmn_err(CE_CONT, "?cpu%d: %b\n",
10537c478bd9Sstevel@tonic-gate 		    cp->cpu_id, new_x86_feature, FMT_X86_FEATURE);
10547c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
10557c478bd9Sstevel@tonic-gate 	}
10567c478bd9Sstevel@tonic-gate 
10577c478bd9Sstevel@tonic-gate 	/*
10587c478bd9Sstevel@tonic-gate 	 * We could be more sophisticated here, and just mark the CPU
10597c478bd9Sstevel@tonic-gate 	 * as "faulted" but at this point we'll opt for the easier
10607c478bd9Sstevel@tonic-gate 	 * answer of dieing horribly.  Provided the boot cpu is ok,
10617c478bd9Sstevel@tonic-gate 	 * the system can be recovered by booting with use_mp set to zero.
10627c478bd9Sstevel@tonic-gate 	 */
10637c478bd9Sstevel@tonic-gate 	if (workaround_errata(cp) != 0)
10647c478bd9Sstevel@tonic-gate 		panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
10657c478bd9Sstevel@tonic-gate 
10667c478bd9Sstevel@tonic-gate 	cpuid_pass2(cp);
10677c478bd9Sstevel@tonic-gate 	cpuid_pass3(cp);
10687c478bd9Sstevel@tonic-gate 	(void) cpuid_pass4(cp);
10697c478bd9Sstevel@tonic-gate 
10707c478bd9Sstevel@tonic-gate 	init_cpu_info(cp);
10717c478bd9Sstevel@tonic-gate 
10727c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
10737c478bd9Sstevel@tonic-gate 	procset |= 1 << cp->cpu_id;
10747c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
10757c478bd9Sstevel@tonic-gate 
10767c478bd9Sstevel@tonic-gate 	if (tsc_gethrtime_enable)
10777c478bd9Sstevel@tonic-gate 		tsc_sync_slave();
10787c478bd9Sstevel@tonic-gate 
10797c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
10807c478bd9Sstevel@tonic-gate 	/*
10817c478bd9Sstevel@tonic-gate 	 * It's unfortunate that chip_cpu_init() has to be called here.
10827c478bd9Sstevel@tonic-gate 	 * It really belongs in cpu_add_unit(), but unfortunately it is
10837c478bd9Sstevel@tonic-gate 	 * dependent on the cpuid probing, which must be done in the
10847c478bd9Sstevel@tonic-gate 	 * context of the current CPU. Care must be taken on x86 to ensure
10857c478bd9Sstevel@tonic-gate 	 * that mp_startup can safely block even though chip_cpu_init() and
10867c478bd9Sstevel@tonic-gate 	 * cpu_add_active() have not yet been called.
10877c478bd9Sstevel@tonic-gate 	 */
10887c478bd9Sstevel@tonic-gate 	chip_cpu_init(cp);
10897c478bd9Sstevel@tonic-gate 	chip_cpu_startup(cp);
10907c478bd9Sstevel@tonic-gate 
10917c478bd9Sstevel@tonic-gate 	cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_ENABLE | CPU_EXISTS;
10927c478bd9Sstevel@tonic-gate 	cpu_add_active(cp);
10937c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
10947c478bd9Sstevel@tonic-gate 
1095b4b46911Skchow 	add_cpunode2devtree(cp->cpu_id, cp->cpu_m.mcpu_cpi);
1096b4b46911Skchow 
10977c478bd9Sstevel@tonic-gate 	(void) spl0();				/* enable interrupts */
10987c478bd9Sstevel@tonic-gate 
10997aec1d6eScindi 	/*
11007aec1d6eScindi 	 * Set up the CPU module for this CPU.  This can't be done before
11017aec1d6eScindi 	 * this CPU is made CPU_READY, because we may (in heterogeneous systems)
11027aec1d6eScindi 	 * need to go load another CPU module.  The act of attempting to load
11037aec1d6eScindi 	 * a module may trigger a cross-call, which will ASSERT unless this
11047aec1d6eScindi 	 * cpu is CPU_READY.
11057aec1d6eScindi 	 */
11067aec1d6eScindi 	cmi_init();
11077aec1d6eScindi 
11087aec1d6eScindi 	if (x86_feature & X86_MCA)
11097aec1d6eScindi 		cmi_mca_init();
11107aec1d6eScindi 
11117c478bd9Sstevel@tonic-gate 	if (boothowto & RB_DEBUG)
11127c478bd9Sstevel@tonic-gate 		kdi_dvec_cpu_init(cp);
11137c478bd9Sstevel@tonic-gate 
11147c478bd9Sstevel@tonic-gate 	/*
11157c478bd9Sstevel@tonic-gate 	 * Setting the bit in cpu_ready_set must be the last operation in
11167c478bd9Sstevel@tonic-gate 	 * processor initialization; the boot CPU will continue to boot once
11177c478bd9Sstevel@tonic-gate 	 * it sees this bit set for all active CPUs.
11187c478bd9Sstevel@tonic-gate 	 */
11197c478bd9Sstevel@tonic-gate 	CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
11207c478bd9Sstevel@tonic-gate 
11217c478bd9Sstevel@tonic-gate 	/*
11227c478bd9Sstevel@tonic-gate 	 * Because mp_startup() gets fired off after init() starts, we
11237c478bd9Sstevel@tonic-gate 	 * can't use the '?' trick to do 'boot -v' printing - so we
11247c478bd9Sstevel@tonic-gate 	 * always direct the 'cpu .. online' messages to the log.
11257c478bd9Sstevel@tonic-gate 	 */
11267c478bd9Sstevel@tonic-gate 	cmn_err(CE_CONT, "!cpu%d initialization complete - online\n",
11277c478bd9Sstevel@tonic-gate 	    cp->cpu_id);
11287c478bd9Sstevel@tonic-gate 
11297c478bd9Sstevel@tonic-gate 	/*
11307c478bd9Sstevel@tonic-gate 	 * Now we are done with the startup thread, so free it up.
11317c478bd9Sstevel@tonic-gate 	 */
11327c478bd9Sstevel@tonic-gate 	thread_exit();
11337c478bd9Sstevel@tonic-gate 	panic("mp_startup: cannot return");
11347c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
11357c478bd9Sstevel@tonic-gate }
11367c478bd9Sstevel@tonic-gate 
11377c478bd9Sstevel@tonic-gate 
11387c478bd9Sstevel@tonic-gate /*
11397c478bd9Sstevel@tonic-gate  * Start CPU on user request.
11407c478bd9Sstevel@tonic-gate  */
11417c478bd9Sstevel@tonic-gate /* ARGSUSED */
11427c478bd9Sstevel@tonic-gate int
11437c478bd9Sstevel@tonic-gate mp_cpu_start(struct cpu *cp)
11447c478bd9Sstevel@tonic-gate {
11457c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
11467c478bd9Sstevel@tonic-gate 	return (0);
11477c478bd9Sstevel@tonic-gate }
11487c478bd9Sstevel@tonic-gate 
11497c478bd9Sstevel@tonic-gate /*
11507c478bd9Sstevel@tonic-gate  * Stop CPU on user request.
11517c478bd9Sstevel@tonic-gate  */
11527c478bd9Sstevel@tonic-gate /* ARGSUSED */
11537c478bd9Sstevel@tonic-gate int
11547c478bd9Sstevel@tonic-gate mp_cpu_stop(struct cpu *cp)
11557c478bd9Sstevel@tonic-gate {
1156d90554ebSdmick 	extern int cbe_psm_timer_mode;
11577c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
1158d90554ebSdmick 
1159d90554ebSdmick 	/*
1160d90554ebSdmick 	 * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1161d90554ebSdmick 	 * can't stop it.  (This is true only for machines with no TSC.)
1162d90554ebSdmick 	 */
1163d90554ebSdmick 
1164d90554ebSdmick 	if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1165d90554ebSdmick 		return (1);
11667c478bd9Sstevel@tonic-gate 
11677c478bd9Sstevel@tonic-gate 	return (0);
11687c478bd9Sstevel@tonic-gate }
11697c478bd9Sstevel@tonic-gate 
11707c478bd9Sstevel@tonic-gate /*
11717c478bd9Sstevel@tonic-gate  * Power on CPU.
11727c478bd9Sstevel@tonic-gate  */
11737c478bd9Sstevel@tonic-gate /* ARGSUSED */
11747c478bd9Sstevel@tonic-gate int
11757c478bd9Sstevel@tonic-gate mp_cpu_poweron(struct cpu *cp)
11767c478bd9Sstevel@tonic-gate {
11777c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
11787c478bd9Sstevel@tonic-gate 	return (ENOTSUP);		/* not supported */
11797c478bd9Sstevel@tonic-gate }
11807c478bd9Sstevel@tonic-gate 
11817c478bd9Sstevel@tonic-gate /*
11827c478bd9Sstevel@tonic-gate  * Power off CPU.
11837c478bd9Sstevel@tonic-gate  */
11847c478bd9Sstevel@tonic-gate /* ARGSUSED */
11857c478bd9Sstevel@tonic-gate int
11867c478bd9Sstevel@tonic-gate mp_cpu_poweroff(struct cpu *cp)
11877c478bd9Sstevel@tonic-gate {
11887c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
11897c478bd9Sstevel@tonic-gate 	return (ENOTSUP);		/* not supported */
11907c478bd9Sstevel@tonic-gate }
11917c478bd9Sstevel@tonic-gate 
11927c478bd9Sstevel@tonic-gate 
11937c478bd9Sstevel@tonic-gate /*
11947c478bd9Sstevel@tonic-gate  * Take the specified CPU out of participation in interrupts.
11957c478bd9Sstevel@tonic-gate  */
11967c478bd9Sstevel@tonic-gate int
11977c478bd9Sstevel@tonic-gate cpu_disable_intr(struct cpu *cp)
11987c478bd9Sstevel@tonic-gate {
11997c478bd9Sstevel@tonic-gate 	if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
12007c478bd9Sstevel@tonic-gate 		return (EBUSY);
12017c478bd9Sstevel@tonic-gate 
12027c478bd9Sstevel@tonic-gate 	cp->cpu_flags &= ~CPU_ENABLE;
12037c478bd9Sstevel@tonic-gate 	return (0);
12047c478bd9Sstevel@tonic-gate }
12057c478bd9Sstevel@tonic-gate 
12067c478bd9Sstevel@tonic-gate /*
12077c478bd9Sstevel@tonic-gate  * Allow the specified CPU to participate in interrupts.
12087c478bd9Sstevel@tonic-gate  */
12097c478bd9Sstevel@tonic-gate void
12107c478bd9Sstevel@tonic-gate cpu_enable_intr(struct cpu *cp)
12117c478bd9Sstevel@tonic-gate {
12127c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
12137c478bd9Sstevel@tonic-gate 	cp->cpu_flags |= CPU_ENABLE;
12147c478bd9Sstevel@tonic-gate 	psm_enable_intr(cp->cpu_id);
12157c478bd9Sstevel@tonic-gate }
12167c478bd9Sstevel@tonic-gate 
12177c478bd9Sstevel@tonic-gate 
12187c478bd9Sstevel@tonic-gate 
12197c478bd9Sstevel@tonic-gate static ushort_t *
12207c478bd9Sstevel@tonic-gate mp_map_warm_reset_vector()
12217c478bd9Sstevel@tonic-gate {
12227c478bd9Sstevel@tonic-gate 	ushort_t *warm_reset_vector;
12237c478bd9Sstevel@tonic-gate 
12247c478bd9Sstevel@tonic-gate 	if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR,
12257c478bd9Sstevel@tonic-gate 	    sizeof (ushort_t *), PROT_READ|PROT_WRITE)))
12267c478bd9Sstevel@tonic-gate 		return (NULL);
12277c478bd9Sstevel@tonic-gate 
12287c478bd9Sstevel@tonic-gate 	/*
12297c478bd9Sstevel@tonic-gate 	 * setup secondary cpu bios boot up vector
12307c478bd9Sstevel@tonic-gate 	 */
12317c478bd9Sstevel@tonic-gate 	*warm_reset_vector = (ushort_t)((caddr_t)
12327c478bd9Sstevel@tonic-gate 		((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va
12337c478bd9Sstevel@tonic-gate 		+ ((ulong_t)rm_platter_va & 0xf));
12347c478bd9Sstevel@tonic-gate 	warm_reset_vector++;
12357c478bd9Sstevel@tonic-gate 	*warm_reset_vector = (ushort_t)(rm_platter_pa >> 4);
12367c478bd9Sstevel@tonic-gate 
12377c478bd9Sstevel@tonic-gate 	--warm_reset_vector;
12387c478bd9Sstevel@tonic-gate 	return (warm_reset_vector);
12397c478bd9Sstevel@tonic-gate }
12407c478bd9Sstevel@tonic-gate 
12417c478bd9Sstevel@tonic-gate static void
12427c478bd9Sstevel@tonic-gate mp_unmap_warm_reset_vector(ushort_t *warm_reset_vector)
12437c478bd9Sstevel@tonic-gate {
12447c478bd9Sstevel@tonic-gate 	psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *));
12457c478bd9Sstevel@tonic-gate }
12467c478bd9Sstevel@tonic-gate 
12477c478bd9Sstevel@tonic-gate void
12487c478bd9Sstevel@tonic-gate mp_cpu_faulted_enter(struct cpu *cp)
12497aec1d6eScindi {
12507aec1d6eScindi 	cmi_faulted_enter(cp);
12517aec1d6eScindi }
12527c478bd9Sstevel@tonic-gate 
12537c478bd9Sstevel@tonic-gate void
12547c478bd9Sstevel@tonic-gate mp_cpu_faulted_exit(struct cpu *cp)
12557aec1d6eScindi {
12567aec1d6eScindi 	cmi_faulted_exit(cp);
12577aec1d6eScindi }
12587c478bd9Sstevel@tonic-gate 
12597c478bd9Sstevel@tonic-gate /*
12607c478bd9Sstevel@tonic-gate  * The following two routines are used as context operators on threads belonging
12617c478bd9Sstevel@tonic-gate  * to processes with a private LDT (see sysi86).  Due to the rarity of such
12627c478bd9Sstevel@tonic-gate  * processes, these routines are currently written for best code readability and
12637c478bd9Sstevel@tonic-gate  * organization rather than speed.  We could avoid checking x86_feature at every
12647c478bd9Sstevel@tonic-gate  * context switch by installing different context ops, depending on the
12657c478bd9Sstevel@tonic-gate  * x86_feature flags, at LDT creation time -- one for each combination of fast
12667c478bd9Sstevel@tonic-gate  * syscall feature flags.
12677c478bd9Sstevel@tonic-gate  */
12687c478bd9Sstevel@tonic-gate 
12697c478bd9Sstevel@tonic-gate /*ARGSUSED*/
12707c478bd9Sstevel@tonic-gate void
12717c478bd9Sstevel@tonic-gate cpu_fast_syscall_disable(void *arg)
12727c478bd9Sstevel@tonic-gate {
12737c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_SEP)
12747c478bd9Sstevel@tonic-gate 		cpu_sep_disable();
12757c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_ASYSC)
12767c478bd9Sstevel@tonic-gate 		cpu_asysc_disable();
12777c478bd9Sstevel@tonic-gate }
12787c478bd9Sstevel@tonic-gate 
12797c478bd9Sstevel@tonic-gate /*ARGSUSED*/
12807c478bd9Sstevel@tonic-gate void
12817c478bd9Sstevel@tonic-gate cpu_fast_syscall_enable(void *arg)
12827c478bd9Sstevel@tonic-gate {
12837c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_SEP)
12847c478bd9Sstevel@tonic-gate 		cpu_sep_enable();
12857c478bd9Sstevel@tonic-gate 	if (x86_feature & X86_ASYSC)
12867c478bd9Sstevel@tonic-gate 		cpu_asysc_enable();
12877c478bd9Sstevel@tonic-gate }
12887c478bd9Sstevel@tonic-gate 
12897c478bd9Sstevel@tonic-gate static void
12907c478bd9Sstevel@tonic-gate cpu_sep_enable(void)
12917c478bd9Sstevel@tonic-gate {
12927c478bd9Sstevel@tonic-gate 	ASSERT(x86_feature & X86_SEP);
12937c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
12947c478bd9Sstevel@tonic-gate 
12950ac7d7d8Skucharsk 	wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
12967c478bd9Sstevel@tonic-gate }
12977c478bd9Sstevel@tonic-gate 
12987c478bd9Sstevel@tonic-gate static void
12997c478bd9Sstevel@tonic-gate cpu_sep_disable(void)
13007c478bd9Sstevel@tonic-gate {
13017c478bd9Sstevel@tonic-gate 	ASSERT(x86_feature & X86_SEP);
13027c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
13037c478bd9Sstevel@tonic-gate 
13047c478bd9Sstevel@tonic-gate 	/*
13057c478bd9Sstevel@tonic-gate 	 * Setting the SYSENTER_CS_MSR register to 0 causes software executing
13067c478bd9Sstevel@tonic-gate 	 * the sysenter or sysexit instruction to trigger a #gp fault.
13077c478bd9Sstevel@tonic-gate 	 */
13080ac7d7d8Skucharsk 	wrmsr(MSR_INTC_SEP_CS, 0ULL);
13097c478bd9Sstevel@tonic-gate }
13107c478bd9Sstevel@tonic-gate 
13117c478bd9Sstevel@tonic-gate static void
13127c478bd9Sstevel@tonic-gate cpu_asysc_enable(void)
13137c478bd9Sstevel@tonic-gate {
13147c478bd9Sstevel@tonic-gate 	ASSERT(x86_feature & X86_ASYSC);
13157c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
13167c478bd9Sstevel@tonic-gate 
13170ac7d7d8Skucharsk 	wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
13180ac7d7d8Skucharsk 	    (uint64_t)(uintptr_t)AMD_EFER_SCE);
13197c478bd9Sstevel@tonic-gate }
13207c478bd9Sstevel@tonic-gate 
13217c478bd9Sstevel@tonic-gate static void
13227c478bd9Sstevel@tonic-gate cpu_asysc_disable(void)
13237c478bd9Sstevel@tonic-gate {
13247c478bd9Sstevel@tonic-gate 	ASSERT(x86_feature & X86_ASYSC);
13257c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
13267c478bd9Sstevel@tonic-gate 
13277c478bd9Sstevel@tonic-gate 	/*
13287c478bd9Sstevel@tonic-gate 	 * Turn off the SCE (syscall enable) bit in the EFER register. Software
13297c478bd9Sstevel@tonic-gate 	 * executing syscall or sysret with this bit off will incur a #ud trap.
13307c478bd9Sstevel@tonic-gate 	 */
13310ac7d7d8Skucharsk 	wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
13320ac7d7d8Skucharsk 	    ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
13337c478bd9Sstevel@tonic-gate }
1334