17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
5100b72fandrei * Common Development and Distribution License (the "License").
6100b72fandrei * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
21ae115bcmrj
227c478bdstevel@tonic-gate/*
237417cfdKuriakose Kuruvilla * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
247c478bdstevel@tonic-gate */
25a311483Gerry Liu/*
26a311483Gerry Liu * Copyright (c) 2010, Intel Corporation.
27a311483Gerry Liu * All rights reserved.
28a311483Gerry Liu */
29ebb8ac0Robert Mustacchi/*
30e25cb0eJohn Levon * Copyright 2020 Joyent, Inc.
31850ad55Hans Rosenfeld * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
3239df76fAndy Fiddaman * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
33ebb8ac0Robert Mustacchi */
347c478bdstevel@tonic-gate
357c478bdstevel@tonic-gate#include <sys/types.h>
367c478bdstevel@tonic-gate#include <sys/thread.h>
377c478bdstevel@tonic-gate#include <sys/cpuvar.h>
38a311483Gerry Liu#include <sys/cpu.h>
397c478bdstevel@tonic-gate#include <sys/t_lock.h>
407c478bdstevel@tonic-gate#include <sys/param.h>
417c478bdstevel@tonic-gate#include <sys/proc.h>
427c478bdstevel@tonic-gate#include <sys/disp.h>
437c478bdstevel@tonic-gate#include <sys/class.h>
447c478bdstevel@tonic-gate#include <sys/cmn_err.h>
457c478bdstevel@tonic-gate#include <sys/debug.h>
46a311483Gerry Liu#include <sys/note.h>
477c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
487c478bdstevel@tonic-gate#include <sys/x_call.h>
497c478bdstevel@tonic-gate#include <sys/systm.h>
507c478bdstevel@tonic-gate#include <sys/var.h>
517c478bdstevel@tonic-gate#include <sys/vtrace.h>
527c478bdstevel@tonic-gate#include <vm/hat.h>
537c478bdstevel@tonic-gate#include <vm/as.h>
547c478bdstevel@tonic-gate#include <vm/seg_kmem.h>
55ae115bcmrj#include <vm/seg_kp.h>
567c478bdstevel@tonic-gate#include <sys/segments.h>
577c478bdstevel@tonic-gate#include <sys/kmem.h>
587c478bdstevel@tonic-gate#include <sys/stack.h>
597c478bdstevel@tonic-gate#include <sys/smp_impldefs.h>
607c478bdstevel@tonic-gate#include <sys/x86_archext.h>
617c478bdstevel@tonic-gate#include <sys/machsystm.h>
627c478bdstevel@tonic-gate#include <sys/traptrace.h>
637c478bdstevel@tonic-gate#include <sys/clock.h>
647c478bdstevel@tonic-gate#include <sys/cpc_impl.h>
65fb2f18fesaxe#include <sys/pg.h>
66fb2f18fesaxe#include <sys/cmt.h>
677c478bdstevel@tonic-gate#include <sys/dtrace.h>
687c478bdstevel@tonic-gate#include <sys/archsystm.h>
697c478bdstevel@tonic-gate#include <sys/fp.h>
707c478bdstevel@tonic-gate#include <sys/reboot.h>
71ae115bcmrj#include <sys/kdi_machimpl.h>
727c478bdstevel@tonic-gate#include <vm/hat_i86.h>
73a311483Gerry Liu#include <vm/vm_dep.h>
747c478bdstevel@tonic-gate#include <sys/memnode.h>
75ef50d8cesaxe#include <sys/pci_cfgspace.h>
76ae115bcmrj#include <sys/mach_mmu.h>
77ae115bcmrj#include <sys/sysmacros.h>
78843e198johnlev#if defined(__xpv)
79843e198johnlev#include <sys/hypervisor.h>
80761dea5Patrick Mooney#else
81761dea5Patrick Mooney#include <sys/hma.h>
82843e198johnlev#endif
837aec1d6cindi#include <sys/cpu_module.h>
84850ad55Hans Rosenfeld#include <sys/ontrap.h>
857c478bdstevel@tonic-gate
8674ecdb5John Levonstruct cpu	cpus[1] __aligned(MMU_PAGESIZE);
8774ecdb5John Levonstruct cpu	*cpu[NCPU] = {&cpus[0]};
8874ecdb5John Levonstruct cpu	*cpu_free_list;
8974ecdb5John Levoncpu_core_t	cpu_core[NCPU];
907c478bdstevel@tonic-gate
91a311483Gerry Liu#define	cpu_next_free	cpu_prev
92a311483Gerry Liu
937c478bdstevel@tonic-gate/*
94ae115bcmrj * Useful for disabling MP bring-up on a MP capable system.
957c478bdstevel@tonic-gate */
967c478bdstevel@tonic-gateint use_mp = 1;
977c478bdstevel@tonic-gate
984179143andrei/*
99ae115bcmrj * to be set by a PSM to indicate what cpus
100ae115bcmrj * are sitting around on the system.
1014179143andrei */
102ae115bcmrjcpuset_t mp_cpus;
1037c478bdstevel@tonic-gate
1047c478bdstevel@tonic-gate/*
1057c478bdstevel@tonic-gate * This variable is used by the hat layer to decide whether or not
1067c478bdstevel@tonic-gate * critical sections are needed to prevent race conditions.  For sun4m,
1077c478bdstevel@tonic-gate * this variable is set once enough MP initialization has been done in
1087c478bdstevel@tonic-gate * order to allow cross calls.
1097c478bdstevel@tonic-gate */
110ae115bcmrjint flushes_require_xcalls;
111a563a03bholler
112a563a03bhollercpuset_t cpu_ready_set;		/* initialized in startup() */
1137c478bdstevel@tonic-gate
114a311483Gerry Liustatic void mp_startup_boot(void);
115a311483Gerry Liustatic void mp_startup_hotplug(void);
1167c478bdstevel@tonic-gate
1177c478bdstevel@tonic-gatestatic void cpu_sep_enable(void);
1187c478bdstevel@tonic-gatestatic void cpu_sep_disable(void);
1197c478bdstevel@tonic-gatestatic void cpu_asysc_enable(void);
1207c478bdstevel@tonic-gatestatic void cpu_asysc_disable(void);
1217c478bdstevel@tonic-gate
1227c478bdstevel@tonic-gate/*
1237c478bdstevel@tonic-gate * Init CPU info - get CPU type info for processor_info system call.
1247c478bdstevel@tonic-gate */
1257c478bdstevel@tonic-gatevoid
1267c478bdstevel@tonic-gateinit_cpu_info(struct cpu *cp)
1277c478bdstevel@tonic-gate{
1287c478bdstevel@tonic-gate	processor_info_t *pi = &cp->cpu_type_info;
1297c478bdstevel@tonic-gate
1307c478bdstevel@tonic-gate	/*
1317c478bdstevel@tonic-gate	 * Get clock-frequency property for the CPU.
1327c478bdstevel@tonic-gate	 */
1337c478bdstevel@tonic-gate	pi->pi_clock = cpu_freq;
1347c478bdstevel@tonic-gate
1355cff782mh	/*
1365cff782mh	 * Current frequency in Hz.
1375cff782mh	 */
138cf74e62mh	cp->cpu_curr_clock = cpu_freq_hz;
1395cff782mh
14037d22dcAnup Pemmaiah	/*
14137d22dcAnup Pemmaiah	 * Supported frequencies.
14237d22dcAnup Pemmaiah	 */
14337d22dcAnup Pemmaiah	if (cp->cpu_supp_freqs == NULL) {
14437d22dcAnup Pemmaiah		cpu_set_supp_freqs(cp, NULL);
14537d22dcAnup Pemmaiah	}
14637d22dcAnup Pemmaiah
1477c478bdstevel@tonic-gate	(void) strcpy(pi->pi_processor_type, "i386");
1487c478bdstevel@tonic-gate	if (fpu_exists)
1497c478bdstevel@tonic-gate		(void) strcpy(pi->pi_fputypes, "i387 compatible");
1507c478bdstevel@tonic-gate
151a311483Gerry Liu	cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
152a311483Gerry Liu	cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
1537c478bdstevel@tonic-gate
154a311483Gerry Liu	/*
155a311483Gerry Liu	 * If called for the BSP, cp is equal to current CPU.
156a311483Gerry Liu	 * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
157a311483Gerry Liu	 * of current CPU as default values for cpu_idstr and cpu_brandstr.
158a311483Gerry Liu	 * They will be corrected in mp_startup_common() after cpuid_pass1()
159a311483Gerry Liu	 * has been invoked on target CPU.
160a311483Gerry Liu	 */
161a311483Gerry Liu	(void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
162a311483Gerry Liu	(void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
1637c478bdstevel@tonic-gate}
1647c478bdstevel@tonic-gate
1657c478bdstevel@tonic-gate/*
1667c478bdstevel@tonic-gate * Configure syscall support on this CPU.
1677c478bdstevel@tonic-gate */
1687c478bdstevel@tonic-gate/*ARGSUSED*/
1692df1fe9randyfvoid
1707c478bdstevel@tonic-gateinit_cpu_syscall(struct cpu *cp)
1717c478bdstevel@tonic-gate{
1727c478bdstevel@tonic-gate	kpreempt_disable();
1737c478bdstevel@tonic-gate
1747417cfdKuriakose Kuruvilla	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
1757417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
1763ce2fcdRobert Mustacchi		uint64_t flags;
1777c478bdstevel@tonic-gate
17874ecdb5John Levon#if !defined(__xpv)
1797c478bdstevel@tonic-gate		/*
1807c478bdstevel@tonic-gate		 * The syscall instruction imposes a certain ordering on
1817c478bdstevel@tonic-gate		 * segment selectors, so we double-check that ordering
1827c478bdstevel@tonic-gate		 * here.
1837c478bdstevel@tonic-gate		 */
18474ecdb5John Levon		CTASSERT(KDS_SEL == KCS_SEL + 8);
18574ecdb5John Levon		CTASSERT(UDS_SEL == U32CS_SEL + 8);
18674ecdb5John Levon		CTASSERT(UCS_SEL == U32CS_SEL + 16);
1877c478bdstevel@tonic-gate#endif
18874ecdb5John Levon
1897c478bdstevel@tonic-gate		/*
1907c478bdstevel@tonic-gate		 * Turn syscall/sysret extensions on.
1917c478bdstevel@tonic-gate		 */
1927c478bdstevel@tonic-gate		cpu_asysc_enable();
1937c478bdstevel@tonic-gate
1947c478bdstevel@tonic-gate		/*
1957c478bdstevel@tonic-gate		 * Program the magic registers ..
1967c478bdstevel@tonic-gate		 */
197ae115bcmrj		wrmsr(MSR_AMD_STAR,
198ae115bcmrj		    ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
19974ecdb5John Levon		if (kpti_enable == 1) {
20074ecdb5John Levon			wrmsr(MSR_AMD_LSTAR,
20174ecdb5John Levon			    (uint64_t)(uintptr_t)tr_sys_syscall);
20274ecdb5John Levon			wrmsr(MSR_AMD_CSTAR,
20374ecdb5John Levon			    (uint64_t)(uintptr_t)tr_sys_syscall32);
20474ecdb5John Levon		} else {
20574ecdb5John Levon			wrmsr(MSR_AMD_LSTAR,
20674ecdb5John Levon			    (uint64_t)(uintptr_t)sys_syscall);
20774ecdb5John Levon			wrmsr(MSR_AMD_CSTAR,
20874ecdb5John Levon			    (uint64_t)(uintptr_t)sys_syscall32);
20974ecdb5John Levon		}
2107c478bdstevel@tonic-gate
2117c478bdstevel@tonic-gate		/*
2127c478bdstevel@tonic-gate		 * This list of flags is masked off the incoming
2137c478bdstevel@tonic-gate		 * %rfl when we enter the kernel.
2147c478bdstevel@tonic-gate		 */
2153ce2fcdRobert Mustacchi		flags = PS_IE | PS_T;
2163ce2fcdRobert Mustacchi		if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
2173ce2fcdRobert Mustacchi			flags |= PS_ACHK;
2183ce2fcdRobert Mustacchi		wrmsr(MSR_AMD_SFMASK, flags);
2197c478bdstevel@tonic-gate	}
2207c478bdstevel@tonic-gate
2217c478bdstevel@tonic-gate	/*
2227c478bdstevel@tonic-gate	 * On 64-bit kernels on Nocona machines, the 32-bit syscall
2237c478bdstevel@tonic-gate	 * variant isn't available to 32-bit applications, but sysenter is.
2247c478bdstevel@tonic-gate	 */
2257417cfdKuriakose Kuruvilla	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2267417cfdKuriakose Kuruvilla	    is_x86_feature(x86_featureset, X86FSET_SEP)) {
2277c478bdstevel@tonic-gate
22874ecdb5John Levon#if !defined(__xpv)
2297c478bdstevel@tonic-gate		/*
2307c478bdstevel@tonic-gate		 * The sysenter instruction imposes a certain ordering on
2317c478bdstevel@tonic-gate		 * segment selectors, so we double-check that ordering
2327c478bdstevel@tonic-gate		 * here. See "sysenter" in Intel document 245471-012, "IA-32
2337c478bdstevel@tonic-gate		 * Intel Architecture Software Developer's Manual Volume 2:
2347c478bdstevel@tonic-gate		 * Instruction Set Reference"
2357c478bdstevel@tonic-gate		 */
23674ecdb5John Levon		CTASSERT(KDS_SEL == KCS_SEL + 8);
2377c478bdstevel@tonic-gate
23874ecdb5John Levon		CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3));
23974ecdb5John Levon		CTASSERT(UDS_SEL == U32CS_SEL + 8);
2407c478bdstevel@tonic-gate#endif
2417c478bdstevel@tonic-gate
2427c478bdstevel@tonic-gate		cpu_sep_enable();
2437c478bdstevel@tonic-gate
2447c478bdstevel@tonic-gate		/*
2457c478bdstevel@tonic-gate		 * resume() sets this value to the base of the threads stack
2467c478bdstevel@tonic-gate		 * via a context handler.
2477c478bdstevel@tonic-gate		 */
248ae115bcmrj		wrmsr(MSR_INTC_SEP_ESP, 0);
24974ecdb5John Levon
25074ecdb5John Levon		if (kpti_enable == 1) {
25174ecdb5John Levon			wrmsr(MSR_INTC_SEP_EIP,
25274ecdb5John Levon			    (uint64_t)(uintptr_t)tr_sys_sysenter);
25374ecdb5John Levon		} else {
25474ecdb5John Levon			wrmsr(MSR_INTC_SEP_EIP,
25574ecdb5John Levon			    (uint64_t)(uintptr_t)sys_sysenter);
25674ecdb5John Levon		}
2577c478bdstevel@tonic-gate	}
2587c478bdstevel@tonic-gate
2597c478bdstevel@tonic-gate	kpreempt_enable();
2607c478bdstevel@tonic-gate}
2617c478bdstevel@tonic-gate
2622428aadPatrick Mooney#if !defined(__xpv)
2632428aadPatrick Mooney/*
2642428aadPatrick Mooney * Configure per-cpu ID GDT
2652428aadPatrick Mooney */
2662428aadPatrick Mooneystatic void
2672428aadPatrick Mooneyinit_cpu_id_gdt(struct cpu *cp)
2682428aadPatrick Mooney{
2692428aadPatrick Mooney	/* Write cpu_id into limit field of GDT for usermode retrieval */
2702428aadPatrick Mooney#if defined(__amd64)
2712428aadPatrick Mooney	set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id,
2722428aadPatrick Mooney	    SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32);
2732428aadPatrick Mooney#elif defined(__i386)
2742428aadPatrick Mooney	set_usegd(&cp->cpu_gdt[GDT_CPUID], NULL, cp->cpu_id, SDT_MEMRODA,
2752428aadPatrick Mooney	    SEL_UPL, SDP_BYTES, SDP_OP32);
2762428aadPatrick Mooney#endif
2772428aadPatrick Mooney}
2782428aadPatrick Mooney#endif /* !defined(__xpv) */
2792428aadPatrick Mooney
2807c478bdstevel@tonic-gate/*
2817c478bdstevel@tonic-gate * Multiprocessor initialization.
2827c478bdstevel@tonic-gate *
2837c478bdstevel@tonic-gate * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
2847c478bdstevel@tonic-gate * startup and idle threads for the specified CPU.
285a311483Gerry Liu * Parameter boot is true for boot time operations and is false for CPU
286a311483Gerry Liu * DR operations.
2877c478bdstevel@tonic-gate */
288a311483Gerry Liustatic struct cpu *
289a311483Gerry Liump_cpu_configure_common(int cpun, boolean_t boot)
2907c478bdstevel@tonic-gate{
2917c478bdstevel@tonic-gate	struct cpu *cp;
2927c478bdstevel@tonic-gate	kthread_id_t tp;
2937c478bdstevel@tonic-gate	caddr_t	sp;
2947c478bdstevel@tonic-gate	proc_t *procp;
295