xref: /illumos-gate/usr/src/uts/i86pc/os/mp_startup.c (revision 8515d723)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5100b72f4Sandrei  * Common Development and Distribution License (the "License").
6100b72f4Sandrei  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21ae115bc7Smrj 
227c478bd9Sstevel@tonic-gate /*
237417cfdeSKuriakose Kuruvilla  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
247c478bd9Sstevel@tonic-gate  */
25a3114836SGerry Liu /*
26a3114836SGerry Liu  * Copyright (c) 2010, Intel Corporation.
27a3114836SGerry Liu  * All rights reserved.
28a3114836SGerry Liu  */
29ebb8ac07SRobert Mustacchi /*
30e25cb0e7SJohn Levon  * Copyright 2020 Joyent, Inc.
31850ad55aSHans Rosenfeld  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
3239df76fbSAndy Fiddaman  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
33ebb8ac07SRobert Mustacchi  */
347c478bd9Sstevel@tonic-gate 
357c478bd9Sstevel@tonic-gate #include <sys/types.h>
367c478bd9Sstevel@tonic-gate #include <sys/thread.h>
377c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
38a3114836SGerry Liu #include <sys/cpu.h>
397c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
407c478bd9Sstevel@tonic-gate #include <sys/param.h>
417c478bd9Sstevel@tonic-gate #include <sys/proc.h>
427c478bd9Sstevel@tonic-gate #include <sys/disp.h>
437c478bd9Sstevel@tonic-gate #include <sys/class.h>
447c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
457c478bd9Sstevel@tonic-gate #include <sys/debug.h>
46a3114836SGerry Liu #include <sys/note.h>
477c478bd9Sstevel@tonic-gate #include <sys/asm_linkage.h>
487c478bd9Sstevel@tonic-gate #include <sys/x_call.h>
497c478bd9Sstevel@tonic-gate #include <sys/systm.h>
507c478bd9Sstevel@tonic-gate #include <sys/var.h>
517c478bd9Sstevel@tonic-gate #include <sys/vtrace.h>
527c478bd9Sstevel@tonic-gate #include <vm/hat.h>
537c478bd9Sstevel@tonic-gate #include <vm/as.h>
547c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
55ae115bc7Smrj #include <vm/seg_kp.h>
567c478bd9Sstevel@tonic-gate #include <sys/segments.h>
577c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
587c478bd9Sstevel@tonic-gate #include <sys/stack.h>
597c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h>
607c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h>
617c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
627c478bd9Sstevel@tonic-gate #include <sys/traptrace.h>
637c478bd9Sstevel@tonic-gate #include <sys/clock.h>
647c478bd9Sstevel@tonic-gate #include <sys/cpc_impl.h>
65fb2f18f8Sesaxe #include <sys/pg.h>
66fb2f18f8Sesaxe #include <sys/cmt.h>
677c478bd9Sstevel@tonic-gate #include <sys/dtrace.h>
687c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>
697c478bd9Sstevel@tonic-gate #include <sys/fp.h>
707c478bd9Sstevel@tonic-gate #include <sys/reboot.h>
71ae115bc7Smrj #include <sys/kdi_machimpl.h>
727c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
73a3114836SGerry Liu #include <vm/vm_dep.h>
747c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
75ef50d8c0Sesaxe #include <sys/pci_cfgspace.h>
76ae115bc7Smrj #include <sys/mach_mmu.h>
77ae115bc7Smrj #include <sys/sysmacros.h>
78843e1988Sjohnlev #if defined(__xpv)
79843e1988Sjohnlev #include <sys/hypervisor.h>
80761dea5eSPatrick Mooney #else
81761dea5eSPatrick Mooney #include <sys/hma.h>
82843e1988Sjohnlev #endif
837aec1d6eScindi #include <sys/cpu_module.h>
84850ad55aSHans Rosenfeld #include <sys/ontrap.h>
857c478bd9Sstevel@tonic-gate 
8674ecdb51SJohn Levon struct cpu	cpus[1] __aligned(MMU_PAGESIZE);
8774ecdb51SJohn Levon struct cpu	*cpu[NCPU] = {&cpus[0]};
8874ecdb51SJohn Levon struct cpu	*cpu_free_list;
8974ecdb51SJohn Levon cpu_core_t	cpu_core[NCPU];
907c478bd9Sstevel@tonic-gate 
91a3114836SGerry Liu #define	cpu_next_free	cpu_prev
92a3114836SGerry Liu 
937c478bd9Sstevel@tonic-gate /*
94ae115bc7Smrj  * Useful for disabling MP bring-up on a MP capable system.
957c478bd9Sstevel@tonic-gate  */
967c478bd9Sstevel@tonic-gate int use_mp = 1;
977c478bd9Sstevel@tonic-gate 
9841791439Sandrei /*
99ae115bc7Smrj  * to be set by a PSM to indicate what cpus
100ae115bc7Smrj  * are sitting around on the system.
10141791439Sandrei  */
102ae115bc7Smrj cpuset_t mp_cpus;
1037c478bd9Sstevel@tonic-gate 
1047c478bd9Sstevel@tonic-gate /*
1057c478bd9Sstevel@tonic-gate  * This variable is used by the hat layer to decide whether or not
1067c478bd9Sstevel@tonic-gate  * critical sections are needed to prevent race conditions.  For sun4m,
1077c478bd9Sstevel@tonic-gate  * this variable is set once enough MP initialization has been done in
1087c478bd9Sstevel@tonic-gate  * order to allow cross calls.
1097c478bd9Sstevel@tonic-gate  */
110ae115bc7Smrj int flushes_require_xcalls;
111a563a037Sbholler 
112a563a037Sbholler cpuset_t cpu_ready_set;		/* initialized in startup() */
1137c478bd9Sstevel@tonic-gate 
114a3114836SGerry Liu static void mp_startup_boot(void);
115a3114836SGerry Liu static void mp_startup_hotplug(void);
1167c478bd9Sstevel@tonic-gate 
1177c478bd9Sstevel@tonic-gate static void cpu_sep_enable(void);
1187c478bd9Sstevel@tonic-gate static void cpu_sep_disable(void);
1197c478bd9Sstevel@tonic-gate static void cpu_asysc_enable(void);
1207c478bd9Sstevel@tonic-gate static void cpu_asysc_disable(void);
1217c478bd9Sstevel@tonic-gate 
1227c478bd9Sstevel@tonic-gate /*
1237c478bd9Sstevel@tonic-gate  * Init CPU info - get CPU type info for processor_info system call.
1247c478bd9Sstevel@tonic-gate  */
1257c478bd9Sstevel@tonic-gate void
init_cpu_info(struct cpu * cp)1267c478bd9Sstevel@tonic-gate init_cpu_info(struct cpu *cp)
1277c478bd9Sstevel@tonic-gate {
1287c478bd9Sstevel@tonic-gate 	processor_info_t *pi = &cp->cpu_type_info;
1297c478bd9Sstevel@tonic-gate 
1307c478bd9Sstevel@tonic-gate 	/*
1317c478bd9Sstevel@tonic-gate 	 * Get clock-frequency property for the CPU.
1327c478bd9Sstevel@tonic-gate 	 */
1337c478bd9Sstevel@tonic-gate 	pi->pi_clock = cpu_freq;
1347c478bd9Sstevel@tonic-gate 
1355cff7825Smh 	/*
1365cff7825Smh 	 * Current frequency in Hz.
1375cff7825Smh 	 */
138cf74e62bSmh 	cp->cpu_curr_clock = cpu_freq_hz;
1395cff7825Smh 
14037d22dc0SAnup Pemmaiah 	/*
14137d22dc0SAnup Pemmaiah 	 * Supported frequencies.
14237d22dc0SAnup Pemmaiah 	 */
14337d22dc0SAnup Pemmaiah 	if (cp->cpu_supp_freqs == NULL) {
14437d22dc0SAnup Pemmaiah 		cpu_set_supp_freqs(cp, NULL);
14537d22dc0SAnup Pemmaiah 	}
14637d22dc0SAnup Pemmaiah 
1477c478bd9Sstevel@tonic-gate 	(void) strcpy(pi->pi_processor_type, "i386");
1487c478bd9Sstevel@tonic-gate 	if (fpu_exists)
1497c478bd9Sstevel@tonic-gate 		(void) strcpy(pi->pi_fputypes, "i387 compatible");
1507c478bd9Sstevel@tonic-gate 
151a3114836SGerry Liu 	cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
152a3114836SGerry Liu 	cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
1537c478bd9Sstevel@tonic-gate 
154a3114836SGerry Liu 	/*
155a3114836SGerry Liu 	 * If called for the BSP, cp is equal to current CPU.
156a3114836SGerry Liu 	 * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
157a3114836SGerry Liu 	 * of current CPU as default values for cpu_idstr and cpu_brandstr.
158a3114836SGerry Liu 	 * They will be corrected in mp_startup_common() after cpuid_pass1()
159a3114836SGerry Liu 	 * has been invoked on target CPU.
160a3114836SGerry Liu 	 */
161a3114836SGerry Liu 	(void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
162a3114836SGerry Liu 	(void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
1637c478bd9Sstevel@tonic-gate }
1647c478bd9Sstevel@tonic-gate 
1657c478bd9Sstevel@tonic-gate /*
1667c478bd9Sstevel@tonic-gate  * Configure syscall support on this CPU.
1677c478bd9Sstevel@tonic-gate  */
1687c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1692df1fe9cSrandyf void
init_cpu_syscall(struct cpu * cp)1707c478bd9Sstevel@tonic-gate init_cpu_syscall(struct cpu *cp)
1717c478bd9Sstevel@tonic-gate {
1727c478bd9Sstevel@tonic-gate 	kpreempt_disable();
1737c478bd9Sstevel@tonic-gate 
1747417cfdeSKuriakose Kuruvilla 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
1757417cfdeSKuriakose Kuruvilla 	    is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
1763ce2fcdcSRobert Mustacchi 		uint64_t flags;
1777c478bd9Sstevel@tonic-gate 
17874ecdb51SJohn Levon #if !defined(__xpv)
1797c478bd9Sstevel@tonic-gate 		/*
1807c478bd9Sstevel@tonic-gate 		 * The syscall instruction imposes a certain ordering on
1817c478bd9Sstevel@tonic-gate 		 * segment selectors, so we double-check that ordering
1827c478bd9Sstevel@tonic-gate 		 * here.
1837c478bd9Sstevel@tonic-gate 		 */
18474ecdb51SJohn Levon 		CTASSERT(KDS_SEL == KCS_SEL + 8);
18574ecdb51SJohn Levon 		CTASSERT(UDS_SEL == U32CS_SEL + 8);
18674ecdb51SJohn Levon 		CTASSERT(UCS_SEL == U32CS_SEL + 16);
1877c478bd9Sstevel@tonic-gate #endif
18874ecdb51SJohn Levon 
1897c478bd9Sstevel@tonic-gate 		/*
1907c478bd9Sstevel@tonic-gate 		 * Turn syscall/sysret extensions on.
1917c478bd9Sstevel@tonic-gate 		 */
1927c478bd9Sstevel@tonic-gate 		cpu_asysc_enable();
1937c478bd9Sstevel@tonic-gate 
1947c478bd9Sstevel@tonic-gate 		/*
1957c478bd9Sstevel@tonic-gate 		 * Program the magic registers ..
1967c478bd9Sstevel@tonic-gate 		 */
197ae115bc7Smrj 		wrmsr(MSR_AMD_STAR,
198ae115bc7Smrj 		    ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
19974ecdb51SJohn Levon 		if (kpti_enable == 1) {
20074ecdb51SJohn Levon 			wrmsr(MSR_AMD_LSTAR,
20174ecdb51SJohn Levon 			    (uint64_t)(uintptr_t)tr_sys_syscall);
20274ecdb51SJohn Levon 			wrmsr(MSR_AMD_CSTAR,
20374ecdb51SJohn Levon 			    (uint64_t)(uintptr_t)tr_sys_syscall32);
20474ecdb51SJohn Levon 		} else {
20574ecdb51SJohn Levon 			wrmsr(MSR_AMD_LSTAR,
20674ecdb51SJohn Levon 			    (uint64_t)(uintptr_t)sys_syscall);
20774ecdb51SJohn Levon 			wrmsr(MSR_AMD_CSTAR,
20874ecdb51SJohn Levon 			    (uint64_t)(uintptr_t)sys_syscall32);
20974ecdb51SJohn Levon 		}
2107c478bd9Sstevel@tonic-gate 
2117c478bd9Sstevel@tonic-gate 		/*
2127c478bd9Sstevel@tonic-gate 		 * This list of flags is masked off the incoming
2137c478bd9Sstevel@tonic-gate 		 * %rfl when we enter the kernel.
2147c478bd9Sstevel@tonic-gate 		 */
2153ce2fcdcSRobert Mustacchi 		flags = PS_IE | PS_T;
2163ce2fcdcSRobert Mustacchi 		if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
2173ce2fcdcSRobert Mustacchi 			flags |= PS_ACHK;
2183ce2fcdcSRobert Mustacchi 		wrmsr(MSR_AMD_SFMASK, flags);
2197c478bd9Sstevel@tonic-gate 	}
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate 	/*
2227c478bd9Sstevel@tonic-gate 	 * On 64-bit kernels on Nocona machines, the 32-bit syscall
2237c478bd9Sstevel@tonic-gate 	 * variant isn't available to 32-bit applications, but sysenter is.
2247c478bd9Sstevel@tonic-gate 	 */
2257417cfdeSKuriakose Kuruvilla 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2267417cfdeSKuriakose Kuruvilla 	    is_x86_feature(x86_featureset, X86FSET_SEP)) {
2277c478bd9Sstevel@tonic-gate 
22874ecdb51SJohn Levon #if !defined(__xpv)
2297c478bd9Sstevel@tonic-gate 		/*
2307c478bd9Sstevel@tonic-gate 		 * The sysenter instruction imposes a certain ordering on
2317c478bd9Sstevel@tonic-gate 		 * segment selectors, so we double-check that ordering
2327c478bd9Sstevel@tonic-gate 		 * here. See "sysenter" in Intel document 245471-012, "IA-32
2337c478bd9Sstevel@tonic-gate 		 * Intel Architecture Software Developer's Manual Volume 2:
2347c478bd9Sstevel@tonic-gate 		 * Instruction Set Reference"
2357c478bd9Sstevel@tonic-gate 		 */
23674ecdb51SJohn Levon 		CTASSERT(KDS_SEL == KCS_SEL + 8);
2377c478bd9Sstevel@tonic-gate 
23874ecdb51SJohn Levon 		CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3));
23974ecdb51SJohn Levon 		CTASSERT(UDS_SEL == U32CS_SEL + 8);
2407c478bd9Sstevel@tonic-gate #endif
2417c478bd9Sstevel@tonic-gate 
2427c478bd9Sstevel@tonic-gate 		cpu_sep_enable();
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 		/*
2457c478bd9Sstevel@tonic-gate 		 * resume() sets this value to the base of the threads stack
2467c478bd9Sstevel@tonic-gate 		 * via a context handler.
2477c478bd9Sstevel@tonic-gate 		 */
248