17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
5100b72fandrei * Common Development and Distribution License (the "License").
6100b72fandrei * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
21ae115bcmrj
227c478bdstevel@tonic-gate/*
237417cfdKuriakose Kuruvilla * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
247c478bdstevel@tonic-gate */
25a311483Gerry Liu/*
26a311483Gerry Liu * Copyright (c) 2010, Intel Corporation.
27a311483Gerry Liu * All rights reserved.
28a311483Gerry Liu */
29ebb8ac0Robert Mustacchi/*
30bf9b145Robert Mustacchi * Copyright 2019 Joyent, Inc.
31850ad55Hans Rosenfeld * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
3239df76fAndy Fiddaman * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
33ebb8ac0Robert Mustacchi */
347c478bdstevel@tonic-gate
357c478bdstevel@tonic-gate#include <sys/types.h>
367c478bdstevel@tonic-gate#include <sys/thread.h>
377c478bdstevel@tonic-gate#include <sys/cpuvar.h>
38a311483Gerry Liu#include <sys/cpu.h>
397c478bdstevel@tonic-gate#include <sys/t_lock.h>
407c478bdstevel@tonic-gate#include <sys/param.h>
417c478bdstevel@tonic-gate#include <sys/proc.h>
427c478bdstevel@tonic-gate#include <sys/disp.h>
437c478bdstevel@tonic-gate#include <sys/class.h>
447c478bdstevel@tonic-gate#include <sys/cmn_err.h>
457c478bdstevel@tonic-gate#include <sys/debug.h>
46a311483Gerry Liu#include <sys/note.h>
477c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
487c478bdstevel@tonic-gate#include <sys/x_call.h>
497c478bdstevel@tonic-gate#include <sys/systm.h>
507c478bdstevel@tonic-gate#include <sys/var.h>
517c478bdstevel@tonic-gate#include <sys/vtrace.h>
527c478bdstevel@tonic-gate#include <vm/hat.h>
537c478bdstevel@tonic-gate#include <vm/as.h>
547c478bdstevel@tonic-gate#include <vm/seg_kmem.h>
55ae115bcmrj#include <vm/seg_kp.h>
567c478bdstevel@tonic-gate#include <sys/segments.h>
577c478bdstevel@tonic-gate#include <sys/kmem.h>
587c478bdstevel@tonic-gate#include <sys/stack.h>
597c478bdstevel@tonic-gate#include <sys/smp_impldefs.h>
607c478bdstevel@tonic-gate#include <sys/x86_archext.h>
617c478bdstevel@tonic-gate#include <sys/machsystm.h>
627c478bdstevel@tonic-gate#include <sys/traptrace.h>
637c478bdstevel@tonic-gate#include <sys/clock.h>
647c478bdstevel@tonic-gate#include <sys/cpc_impl.h>
65fb2f18fesaxe#include <sys/pg.h>
66fb2f18fesaxe#include <sys/cmt.h>
677c478bdstevel@tonic-gate#include <sys/dtrace.h>
687c478bdstevel@tonic-gate#include <sys/archsystm.h>
697c478bdstevel@tonic-gate#include <sys/fp.h>
707c478bdstevel@tonic-gate#include <sys/reboot.h>
71ae115bcmrj#include <sys/kdi_machimpl.h>
727c478bdstevel@tonic-gate#include <vm/hat_i86.h>
73a311483Gerry Liu#include <vm/vm_dep.h>
747c478bdstevel@tonic-gate#include <sys/memnode.h>
75ef50d8cesaxe#include <sys/pci_cfgspace.h>
76ae115bcmrj#include <sys/mach_mmu.h>
77ae115bcmrj#include <sys/sysmacros.h>
78843e198johnlev#if defined(__xpv)
79843e198johnlev#include <sys/hypervisor.h>
80843e198johnlev#endif
817aec1d6cindi#include <sys/cpu_module.h>
82850ad55Hans Rosenfeld#include <sys/ontrap.h>
837c478bdstevel@tonic-gate
8474ecdb5John Levonstruct cpu	cpus[1] __aligned(MMU_PAGESIZE);
8574ecdb5John Levonstruct cpu	*cpu[NCPU] = {&cpus[0]};
8674ecdb5John Levonstruct cpu	*cpu_free_list;
8774ecdb5John Levoncpu_core_t	cpu_core[NCPU];
887c478bdstevel@tonic-gate
89a311483Gerry Liu#define	cpu_next_free	cpu_prev
90a311483Gerry Liu
917c478bdstevel@tonic-gate/*
92ae115bcmrj * Useful for disabling MP bring-up on a MP capable system.
937c478bdstevel@tonic-gate */
947c478bdstevel@tonic-gateint use_mp = 1;
957c478bdstevel@tonic-gate
964179143andrei/*
97ae115bcmrj * to be set by a PSM to indicate what cpus
98ae115bcmrj * are sitting around on the system.
994179143andrei */
100ae115bcmrjcpuset_t mp_cpus;
1017c478bdstevel@tonic-gate
1027c478bdstevel@tonic-gate/*
1037c478bdstevel@tonic-gate * This variable is used by the hat layer to decide whether or not
1047c478bdstevel@tonic-gate * critical sections are needed to prevent race conditions.  For sun4m,
1057c478bdstevel@tonic-gate * this variable is set once enough MP initialization has been done in
1067c478bdstevel@tonic-gate * order to allow cross calls.
1077c478bdstevel@tonic-gate */
108ae115bcmrjint flushes_require_xcalls;
109a563a03bholler
110a563a03bhollercpuset_t cpu_ready_set;		/* initialized in startup() */
1117c478bdstevel@tonic-gate
112a311483Gerry Liustatic void mp_startup_boot(void);
113a311483Gerry Liustatic void mp_startup_hotplug(void);
1147c478bdstevel@tonic-gate
1157c478bdstevel@tonic-gatestatic void cpu_sep_enable(void);
1167c478bdstevel@tonic-gatestatic void cpu_sep_disable(void);
1177c478bdstevel@tonic-gatestatic void cpu_asysc_enable(void);
1187c478bdstevel@tonic-gatestatic void cpu_asysc_disable(void);
1197c478bdstevel@tonic-gate
1207c478bdstevel@tonic-gate/*
1217c478bdstevel@tonic-gate * Init CPU info - get CPU type info for processor_info system call.
1227c478bdstevel@tonic-gate */
1237c478bdstevel@tonic-gatevoid
1247c478bdstevel@tonic-gateinit_cpu_info(struct cpu *cp)
1257c478bdstevel@tonic-gate{
1267c478bdstevel@tonic-gate	processor_info_t *pi = &cp->cpu_type_info;
1277c478bdstevel@tonic-gate
1287c478bdstevel@tonic-gate	/*
1297c478bdstevel@tonic-gate	 * Get clock-frequency property for the CPU.
1307c478bdstevel@tonic-gate	 */
1317c478bdstevel@tonic-gate	pi->pi_clock = cpu_freq;
1327c478bdstevel@tonic-gate
1335cff782mh	/*
1345cff782mh	 * Current frequency in Hz.
1355cff782mh	 */
136cf74e62mh	cp->cpu_curr_clock = cpu_freq_hz;
1375cff782mh
13837d22dcAnup Pemmaiah	/*
13937d22dcAnup Pemmaiah	 * Supported frequencies.
14037d22dcAnup Pemmaiah	 */
14137d22dcAnup Pemmaiah	if (cp->cpu_supp_freqs == NULL) {
14237d22dcAnup Pemmaiah		cpu_set_supp_freqs(cp, NULL);
14337d22dcAnup Pemmaiah	}
14437d22dcAnup Pemmaiah
1457c478bdstevel@tonic-gate	(void) strcpy(pi->pi_processor_type, "i386");
1467c478bdstevel@tonic-gate	if (fpu_exists)
1477c478bdstevel@tonic-gate		(void) strcpy(pi->pi_fputypes, "i387 compatible");
1487c478bdstevel@tonic-gate
149a311483Gerry Liu	cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
150a311483Gerry Liu	cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
1517c478bdstevel@tonic-gate
152a311483Gerry Liu	/*
153a311483Gerry Liu	 * If called for the BSP, cp is equal to current CPU.
154a311483Gerry Liu	 * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
155a311483Gerry Liu	 * of current CPU as default values for cpu_idstr and cpu_brandstr.
156a311483Gerry Liu	 * They will be corrected in mp_startup_common() after cpuid_pass1()
157a311483Gerry Liu	 * has been invoked on target CPU.
158a311483Gerry Liu	 */
159a311483Gerry Liu	(void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
160a311483Gerry Liu	(void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
1617c478bdstevel@tonic-gate}
1627c478bdstevel@tonic-gate
1637c478bdstevel@tonic-gate/*
1647c478bdstevel@tonic-gate * Configure syscall support on this CPU.
1657c478bdstevel@tonic-gate */
1667c478bdstevel@tonic-gate/*ARGSUSED*/
1672df1fe9randyfvoid
1687c478bdstevel@tonic-gateinit_cpu_syscall(struct cpu *cp)
1697c478bdstevel@tonic-gate{
1707c478bdstevel@tonic-gate	kpreempt_disable();
1717c478bdstevel@tonic-gate
1727417cfdKuriakose Kuruvilla	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
1737417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
1743ce2fcdRobert Mustacchi		uint64_t flags;
1757c478bdstevel@tonic-gate
17674ecdb5John Levon#if !defined(__xpv)
1777c478bdstevel@tonic-gate		/*
1787c478bdstevel@tonic-gate		 * The syscall instruction imposes a certain ordering on
1797c478bdstevel@tonic-gate		 * segment selectors, so we double-check that ordering
1807c478bdstevel@tonic-gate		 * here.
1817c478bdstevel@tonic-gate		 */
18274ecdb5John Levon		CTASSERT(KDS_SEL == KCS_SEL + 8);
18374ecdb5John Levon		CTASSERT(UDS_SEL == U32CS_SEL + 8);
18474ecdb5John Levon		CTASSERT(UCS_SEL == U32CS_SEL + 16);
1857c478bdstevel@tonic-gate#endif
18674ecdb5John Levon
1877c478bdstevel@tonic-gate		/*
1887c478bdstevel@tonic-gate		 * Turn syscall/sysret extensions on.
1897c478bdstevel@tonic-gate		 */
1907c478bdstevel@tonic-gate		cpu_asysc_enable();
1917c478bdstevel@tonic-gate
1927c478bdstevel@tonic-gate		/*
1937c478bdstevel@tonic-gate		 * Program the magic registers ..
1947c478bdstevel@tonic-gate		 */
195ae115bcmrj		wrmsr(MSR_AMD_STAR,
196ae115bcmrj		    ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
19774ecdb5John Levon		if (kpti_enable == 1) {
19874ecdb5John Levon			wrmsr(MSR_AMD_LSTAR,
19974ecdb5John Levon			    (uint64_t)(uintptr_t)tr_sys_syscall);
20074ecdb5John Levon			wrmsr(MSR_AMD_CSTAR,
20174ecdb5John Levon			    (uint64_t)(uintptr_t)tr_sys_syscall32);
20274ecdb5John Levon		} else {
20374ecdb5John Levon			wrmsr(MSR_AMD_LSTAR,
20474ecdb5John Levon			    (uint64_t)(uintptr_t)sys_syscall);
20574ecdb5John Levon			wrmsr(MSR_AMD_CSTAR,
20674ecdb5John Levon			    (uint64_t)(uintptr_t)sys_syscall32);
20774ecdb5John Levon		}
2087c478bdstevel@tonic-gate
2097c478bdstevel@tonic-gate		/*
2107c478bdstevel@tonic-gate		 * This list of flags is masked off the incoming
2117c478bdstevel@tonic-gate		 * %rfl when we enter the kernel.
2127c478bdstevel@tonic-gate		 */
2133ce2fcdRobert Mustacchi		flags = PS_IE | PS_T;
2143ce2fcdRobert Mustacchi		if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
2153ce2fcdRobert Mustacchi			flags |= PS_ACHK;
2163ce2fcdRobert Mustacchi		wrmsr(MSR_AMD_SFMASK, flags);
2177c478bdstevel@tonic-gate	}
2187c478bdstevel@tonic-gate
2197c478bdstevel@tonic-gate	/*
2207c478bdstevel@tonic-gate	 * On 64-bit kernels on Nocona machines, the 32-bit syscall
2217c478bdstevel@tonic-gate	 * variant isn't available to 32-bit applications, but sysenter is.
2227c478bdstevel@tonic-gate	 */
2237417cfdKuriakose Kuruvilla	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2247417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_SEP)) {
2257c478bdstevel@tonic-gate
22674ecdb5John Levon#if !defined(__xpv)
2277c478bdstevel@tonic-gate		/*
2287c478bdstevel@tonic-gate		 * The sysenter instruction imposes a certain ordering on
2297c478bdstevel@tonic-gate		 * segment selectors, so we double-check that ordering
2307c478bdstevel@tonic-gate		 * here. See "sysenter" in Intel document 245471-012, "IA-32
2317c478bdstevel@tonic-gate		 * Intel Architecture Software Developer's Manual Volume 2:
2327c478bdstevel@tonic-gate		 * Instruction Set Reference"
2337c478bdstevel@tonic-gate		 */
23474ecdb5John Levon		CTASSERT(KDS_SEL == KCS_SEL + 8);
2357c478bdstevel@tonic-gate
23674ecdb5John Levon		CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3));
23774ecdb5John Levon		CTASSERT(UDS_SEL == U32CS_SEL + 8);
2387c478bdstevel@tonic-gate#endif
2397c478bdstevel@tonic-gate
2407c478bdstevel@tonic-gate		cpu_sep_enable();
2417c478bdstevel@tonic-gate
2427c478bdstevel@tonic-gate		/*
2437c478bdstevel@tonic-gate		 * resume() sets this value to the base of the threads stack
2447c478bdstevel@tonic-gate		 * via a context handler.
2457c478bdstevel@tonic-gate		 */
246ae115bcmrj		wrmsr(MSR_INTC_SEP_ESP, 0);
24774ecdb5John Levon
24874ecdb5John Levon		if (kpti_enable == 1) {
24974ecdb5John Levon			wrmsr(MSR_INTC_SEP_EIP,
25074ecdb5John Levon			    (uint64_t)(uintptr_t)tr_sys_sysenter);
25174ecdb5John Levon		} else {
25274ecdb5John Levon			wrmsr(MSR_INTC_SEP_EIP,
25374ecdb5John Levon			    (uint64_t)(uintptr_t)sys_sysenter);
25474ecdb5John Levon		}
2557c478bdstevel@tonic-gate	}
2567c478bdstevel@tonic-gate
2577c478bdstevel@tonic-gate	kpreempt_enable();
2587c478bdstevel@tonic-gate}
2597c478bdstevel@tonic-gate
2602428aadPatrick Mooney#if !defined(__xpv)
2612428aadPatrick Mooney/*
2622428aadPatrick Mooney * Configure per-cpu ID GDT
2632428aadPatrick Mooney */
2642428aadPatrick Mooneystatic void
2652428aadPatrick Mooneyinit_cpu_id_gdt(struct cpu *cp)
2662428aadPatrick Mooney{
2672428aadPatrick Mooney	/* Write cpu_id into limit field of GDT for usermode retrieval */
2682428aadPatrick Mooney#if defined(__amd64)
2692428aadPatrick Mooney	set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id,
2702428aadPatrick Mooney	    SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32);
2712428aadPatrick Mooney#elif defined(__i386)
2722428aadPatrick Mooney	set_usegd(&cp->cpu_gdt[GDT_CPUID], NULL, cp->cpu_id, SDT_MEMRODA,
2732428aadPatrick Mooney	    SEL_UPL, SDP_BYTES, SDP_OP32);
2742428aadPatrick Mooney#endif
2752428aadPatrick Mooney}
2762428aadPatrick Mooney#endif /* !defined(__xpv) */
2772428aadPatrick Mooney
2787c478bdstevel@tonic-gate/*
2797c478bdstevel@tonic-gate * Multiprocessor initialization.
2807c478bdstevel@tonic-gate *
2817c478bdstevel@tonic-gate * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
2827c478bdstevel@tonic-gate * startup and idle threads for the specified CPU.
283a311483Gerry Liu * Parameter boot is true for boot time operations and is false for CPU
284a311483Gerry Liu * DR operations.
2857c478bdstevel@tonic-gate */
286a311483Gerry Liustatic struct cpu *
287a311483Gerry Liump_cpu_configure_common(int cpun, boolean_t boot)
2887c478bdstevel@tonic-gate{
2897c478bdstevel@tonic-gate	struct cpu *cp;
2907c478bdstevel@tonic-gate	kthread_id_t tp;
2917c478bdstevel@tonic-gate	caddr_t	sp;
2927c478bdstevel@tonic-gate	proc_t *procp;
293843e198johnlev#if !defined(__xpv)
294