xref: /illumos-gate/usr/src/uts/i86pc/os/mp_startup.c (revision bbf21555)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 /*
26  * Copyright (c) 2010, Intel Corporation.
27  * All rights reserved.
28  */
29 /*
30  * Copyright 2020 Joyent, Inc.
31  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
32  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
33  */
34 
35 #include <sys/types.h>
36 #include <sys/thread.h>
37 #include <sys/cpuvar.h>
38 #include <sys/cpu.h>
39 #include <sys/t_lock.h>
40 #include <sys/param.h>
41 #include <sys/proc.h>
42 #include <sys/disp.h>
43 #include <sys/class.h>
44 #include <sys/cmn_err.h>
45 #include <sys/debug.h>
46 #include <sys/note.h>
47 #include <sys/asm_linkage.h>
48 #include <sys/x_call.h>
49 #include <sys/systm.h>
50 #include <sys/var.h>
51 #include <sys/vtrace.h>
52 #include <vm/hat.h>
53 #include <vm/as.h>
54 #include <vm/seg_kmem.h>
55 #include <vm/seg_kp.h>
56 #include <sys/segments.h>
57 #include <sys/kmem.h>
58 #include <sys/stack.h>
59 #include <sys/smp_impldefs.h>
60 #include <sys/x86_archext.h>
61 #include <sys/machsystm.h>
62 #include <sys/traptrace.h>
63 #include <sys/clock.h>
64 #include <sys/cpc_impl.h>
65 #include <sys/pg.h>
66 #include <sys/cmt.h>
67 #include <sys/dtrace.h>
68 #include <sys/archsystm.h>
69 #include <sys/fp.h>
70 #include <sys/reboot.h>
71 #include <sys/kdi_machimpl.h>
72 #include <vm/hat_i86.h>
73 #include <vm/vm_dep.h>
74 #include <sys/memnode.h>
75 #include <sys/pci_cfgspace.h>
76 #include <sys/mach_mmu.h>
77 #include <sys/sysmacros.h>
78 #if defined(__xpv)
79 #include <sys/hypervisor.h>
80 #else
81 #include <sys/hma.h>
82 #endif
83 #include <sys/cpu_module.h>
84 #include <sys/ontrap.h>
85 
86 struct cpu	cpus[1] __aligned(MMU_PAGESIZE);
87 struct cpu	*cpu[NCPU] = {&cpus[0]};
88 struct cpu	*cpu_free_list;
89 cpu_core_t	cpu_core[NCPU];
90 
91 #define	cpu_next_free	cpu_prev
92 
93 /*
94  * Useful for disabling MP bring-up on a MP capable system.
95  */
96 int use_mp = 1;
97 
98 /*
99  * to be set by a PSM to indicate what cpus
100  * are sitting around on the system.
101  */
102 cpuset_t mp_cpus;
103 
104 /*
105  * This variable is used by the hat layer to decide whether or not
106  * critical sections are needed to prevent race conditions.  For sun4m,
107  * this variable is set once enough MP initialization has been done in
108  * order to allow cross calls.
109  */
110 int flushes_require_xcalls;
111 
112 cpuset_t cpu_ready_set;		/* initialized in startup() */
113 
114 static void mp_startup_boot(void);
115 static void mp_startup_hotplug(void);
116 
117 static void cpu_sep_enable(void);
118 static void cpu_sep_disable(void);
119 static void cpu_asysc_enable(void);
120 static void cpu_asysc_disable(void);
121 
122 /*
123  * Init CPU info - get CPU type info for processor_info system call.
124  */
125 void
init_cpu_info(struct cpu * cp)126 init_cpu_info(struct cpu *cp)
127 {
128 	processor_info_t *pi = &cp->cpu_type_info;
129 
130 	/*
131 	 * Get clock-frequency property for the CPU.
132 	 */
133 	pi->pi_clock = cpu_freq;
134 
135 	/*
136 	 * Current frequency in Hz.
137 	 */
138 	cp->cpu_curr_clock = cpu_freq_hz;
139 
140 	/*
141 	 * Supported frequencies.
142 	 */
143 	if (cp->cpu_supp_freqs == NULL) {
144 		cpu_set_supp_freqs(cp, NULL);
145 	}
146 
147 	(void) strcpy(pi->pi_processor_type, "i386");
148 	if (fpu_exists)
149 		(void) strcpy(pi->pi_fputypes, "i387 compatible");
150 
151 	cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
152 	cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP);
153 
154 	/*
155 	 * If called for the BSP, cp is equal to current CPU.
156 	 * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info
157 	 * of current CPU as default values for cpu_idstr and cpu_brandstr.
158 	 * They will be corrected in mp_startup_common() after cpuid_pass1()
159 	 * has been invoked on target CPU.
160 	 */
161 	(void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN);
162 	(void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN);
163 }
164 
165 /*
166  * Configure syscall support on this CPU.
167  */
168 /*ARGSUSED*/
169 void
init_cpu_syscall(struct cpu * cp)170 init_cpu_syscall(struct cpu *cp)
171 {
172 	kpreempt_disable();
173 
174 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
175 	    is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
176 		uint64_t flags;
177 
178 #if !defined(__xpv)
179 		/*
180 		 * The syscall instruction imposes a certain ordering on
181 		 * segment selectors, so we double-check that ordering
182 		 * here.
183 		 */
184 		CTASSERT(KDS_SEL == KCS_SEL + 8);
185 		CTASSERT(UDS_SEL == U32CS_SEL + 8);
186 		CTASSERT(UCS_SEL == U32CS_SEL + 16);
187 #endif
188 
189 		/*
190 		 * Turn syscall/sysret extensions on.
191 		 */
192 		cpu_asysc_enable();
193 
194 		/*
195 		 * Program the magic registers ..
196 		 */
197 		wrmsr(MSR_AMD_STAR,
198 		    ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);
199 		if (kpti_enable == 1) {
200 			wrmsr(MSR_AMD_LSTAR,
201 			    (uint64_t)(uintptr_t)tr_sys_syscall);
202 			wrmsr(MSR_AMD_CSTAR,
203 			    (uint64_t)(uintptr_t)tr_sys_syscall32);
204 		} else {
205 			wrmsr(MSR_AMD_LSTAR,
206 			    (uint64_t)(uintptr_t)sys_syscall);
207 			wrmsr(MSR_AMD_CSTAR,
208 			    (uint64_t)(uintptr_t)sys_syscall32);
209 		}
210 
211 		/*
212 		 * This list of flags is masked off the incoming
213 		 * %rfl when we enter the kernel.
214 		 */
215 		flags = PS_IE | PS_T;
216 		if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE)
217 			flags |= PS_ACHK;
218 		wrmsr(MSR_AMD_SFMASK, flags);
219 	}
220 
221 	/*
222 	 * On 64-bit kernels on Nocona machines, the 32-bit syscall
223 	 * variant isn't available to 32-bit applications, but sysenter is.
224 	 */
225 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
226 	    is_x86_feature(x86_featureset, X86FSET_SEP)) {
227 
228 #if !defined(__xpv)
229 		/*
230 		 * The sysenter instruction imposes a certain ordering on
231 		 * segment selectors, so we double-check that ordering
232 		 * here. See "sysenter" in Intel document 245471-012, "IA-32
233 		 * Intel Architecture Software Developer's Manual Volume 2:
234 		 * Instruction Set Reference"
235 		 */
236 		CTASSERT(KDS_SEL == KCS_SEL + 8);
237 
238 		CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3));
239 		CTASSERT(UDS_SEL == U32CS_SEL + 8);
240 #endif
241 
242 		cpu_sep_enable();
243 
244 		/*
245 		 * resume() sets this value to the base of the threads stack
246 		 * via a context handler.
247 		 */
248 		wrmsr(MSR_INTC_SEP_ESP, 0);
249 
250 		if (kpti_enable == 1) {
251 			wrmsr(MSR_INTC_SEP_EIP,
252 			    (uint64_t)(uintptr_t)tr_sys_sysenter);
253 		} else {
254 			wrmsr(MSR_INTC_SEP_EIP,
255 			    (uint64_t)(uintptr_t)sys_sysenter);
256 		}
257 	}
258 
259 	kpreempt_enable();
260 }
261 
262 #if !defined(__xpv)
263 /*
264  * Configure per-cpu ID GDT
265  */
266 static void
init_cpu_id_gdt(struct cpu * cp)267 init_cpu_id_gdt(struct cpu *cp)
268 {
269 	/* Write cpu_id into limit field of GDT for usermode retrieval */
270 	set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id,
271 	    SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32);
272 }
273 #endif /* !defined(__xpv) */
274 
275 /*
276  * Multiprocessor initialization.
277  *
278  * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the
279  * startup and idle threads for the specified CPU.
280  * Parameter boot is true for boot time operations and is false for CPU
281  * DR operations.
282  */
283 static struct cpu *
mp_cpu_configure_common(int cpun,boolean_t boot)284 mp_cpu_configure_common(int cpun, boolean_t boot)
285 {
286 	struct cpu *cp;
287 	kthread_id_t tp;
288 	caddr_t	sp;
289 	proc_t *procp;
290 #if !defined(__xpv)
291 	extern int idle_cpu_prefer_mwait;
292 	extern void cpu_idle_mwait();
293 #endif
294 	extern void idle();
295 	extern void cpu_idle();
296 
297 #ifdef TRAPTRACE
298 	trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];
299 #endif
300 
301 	ASSERT(MUTEX_HELD(&cpu_lock));
302 	ASSERT(cpun < NCPU && cpu[cpun] == NULL);
303 
304 	if (cpu_free_list == NULL) {
305 		cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);
306 	} else {
307 		cp = cpu_free_list;
308 		cpu_free_list = cp->cpu_next_free;
309 	}
310 
311 	cp->cpu_m.mcpu_istamp = cpun << 16;
312 
313 	/* Create per CPU specific threads in the process p0. */
314 	procp = &p0;
315 
316 	/*
317 	 * Initialize the dispatcher first.
318 	 */
319 	disp_cpu_init(cp);
320 
321 	cpu_vm_data_init(cp);
322 
323 	/*
324 	 * Allocate and initialize the startup thread for this CPU.
325 	 * Interrupt and process switch stacks get allocated later
326 	 * when the CPU starts running.
327 	 */
328 	tp = thread_create(NULL, 0, NULL, NULL, 0, procp,
329 	    TS_STOPPED, maxclsyspri);
330 
331 	/*
332 	 * Set state to TS_ONPROC since this thread will start running
333 	 * as soon as the CPU comes online.
334 	 *
335 	 * All the other fields of the thread structure are setup by
336 	 * thread_create().
337 	 */
338 	THREAD_ONPROC(tp, cp);
339 	tp->t_preempt = 1;
340 	tp->t_bound_cpu = cp;
341 	tp->t_affinitycnt = 1;
342 	tp->t_cpu = cp;
343 	tp->t_disp_queue = cp->cpu_disp;
344 
345 	/*
346 	 * Setup thread to start in mp_startup_common.
347 	 */
348 	sp = tp->t_stk;
349 	tp->t_sp = (uintptr_t)(sp - MINFRAME);
350 	tp->t_sp -= STACK_ENTRY_ALIGN;		/* fake a call */
351 	/*
352 	 * Setup thread start entry point for boot or hotplug.
353 	 */
354 	if (boot) {
355 		tp->t_pc = (uintptr_t)mp_startup_boot;
356 	} else {
357 		tp->t_pc = (uintptr_t)mp_startup_hotplug;
358 	}
359 
360 	cp->cpu_id = cpun;
361 	cp->cpu_self = cp;
362 	cp->cpu_thread = tp;
363 	cp->cpu_lwp = NULL;
364 	cp->cpu_dispthread = tp;
365 	cp->cpu_dispatch_pri = DISP_PRIO(tp);
366 
367 	/*
368 	 * cpu_base_spl must be set explicitly here to prevent any blocking
369 	 * operations in mp_startup_common from causing the spl of the cpu
370 	 * to drop to 0 (allowing device interrupts before we're ready) in
371 	 * resume().
372 	 * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.
373 	 * As an extra bit of security on DEBUG kernels, this is enforced with
374 	 * an assertion in mp_startup_common() -- before cpu_base_spl is set
375 	 * to its proper value.
376 	 */
377 	cp->cpu_base_spl = ipltospl(LOCK_LEVEL);
378 
379 	/*
380 	 * Now, initialize per-CPU idle thread for this CPU.
381 	 */
382 	tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);
383 
384 	cp->cpu_idle_thread = tp;
385 
386 	tp->t_preempt = 1;
387 	tp->t_bound_cpu = cp;
388 	tp->t_affinitycnt = 1;
389 	tp->t_cpu = cp;
390 	tp->t_disp_queue = cp->cpu_disp;
391 
392 	/*
393 	 * Bootstrap the CPU's PG data
394 	 */
395 	pg_cpu_bootstrap(cp);
396 
397 	/*
398 	 * Perform CPC initialization on the new CPU.
399 	 */
400 	kcpc_hw_init(cp);
401 
402 	/*
403 	 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2
404 	 * for each CPU.
405 	 */
406 	setup_vaddr_for_ppcopy(cp);
407 
408 	/*
409 	 * Allocate page for new GDT and initialize from current GDT.
410 	 */
411 #if !defined(__lint)
412 	ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);
413 #endif
414 	cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);
415 	bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));
416 
417 
418 	/*
419 	 * Allocate pages for the CPU LDT.
420 	 */
421 	cp->cpu_m.mcpu_ldt = kmem_zalloc(LDT_CPU_SIZE, KM_SLEEP);
422 	cp->cpu_m.mcpu_ldt_len = 0;
423 
424 	/*
425 	 * Allocate a per-CPU IDT and initialize the new IDT to the currently
426 	 * runing CPU.
427 	 */
428 #if !defined(__lint)
429 	ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);
430 #endif
431 	cp->cpu_idt = kmem_alloc(PAGESIZE, KM_SLEEP);
432 	bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);
433 
434 	/*
435 	 * alloc space for cpuid info
436 	 */
437 	cpuid_alloc_space(cp);
438 #if !defined(__xpv)
439 	if (is_x86_feature(x86_featureset, X86FSET_MWAIT) &&
440 	    idle_cpu_prefer_mwait) {
441 		cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(cp);
442 		cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait;
443 	} else
444 #endif
445 		cp->cpu_m.mcpu_idle_cpu = cpu_idle;
446 
447 	init_cpu_info(cp);
448 
449 #if !defined(__xpv)
450 	init_cpu_id_gdt(cp);
451 #endif
452 
453 	/*
454 	 * alloc space for ucode_info
455 	 */
456 	ucode_alloc_space(cp);
457 	xc_init_cpu(cp);
458 	hat_cpu_online(cp);
459 
460 #ifdef TRAPTRACE
461 	/*
462 	 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers
463 	 */
464 	ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);
465 	ttc->ttc_next = ttc->ttc_first;
466 	ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;
467 #endif
468 
469 	/*
470 	 * Record that we have another CPU.
471 	 */
472 	/*
473 	 * Initialize the interrupt threads for this CPU
474 	 */
475 	cpu_intr_alloc(cp, NINTR_THREADS);
476 
477 	cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF;
478 	cpu_set_state(cp);
479 
480 	/*
481 	 * Add CPU to list of available CPUs.  It'll be on the active list
482 	 * after mp_startup_common().
483 	 */
484 	cpu_add_unit(cp);
485 
486 	return (cp);
487 }
488 
489 /*
490  * Undo what was done in mp_cpu_configure_common
491  */
492 static void
mp_cpu_unconfigure_common(struct cpu * cp,int error)493 mp_cpu_unconfigure_common(struct cpu *cp, int error)
494 {
495 	ASSERT(MUTEX_HELD(&cpu_lock));
496 
497 	/*
498 	 * Remove the CPU from the list of available CPUs.
499 	 */
500 	cpu_del_unit(cp->cpu_id);
501 
502 	if (error == ETIMEDOUT) {
503 		/*
504 		 * The cpu was started, but never *seemed* to run any
505 		 * code in the kernel; it's probably off spinning in its
506 		 * own private world, though with potential references to
507 		 * our kmem-allocated IDTs and GDTs (for example).
508 		 *
509 		 * Worse still, it may actually wake up some time later,
510 		 * so rather than guess what it might or might not do, we
511 		 * leave the fundamental data structures intact.
512 		 */
513 		cp->cpu_flags = 0;
514 		return;
515 	}
516 
517 	/*
518 	 * At this point, the only threads bound to this CPU should
519 	 * special per-cpu threads: it's idle thread, it's pause threads,
520 	 * and it's interrupt threads.  Clean these up.
521 	 */
522 	cpu_destroy_bound_threads(cp);
523 	cp->cpu_idle_thread = NULL;
524 
525 	/*
526 	 * Free the interrupt stack.
527 	 */
528 	segkp_release(segkp,
529 	    cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));
530 	cp->cpu_intr_stack = NULL;
531 
532 #ifdef TRAPTRACE
533 	/*
534 	 * Discard the trap trace buffer
535 	 */
536 	{
537 		trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];
538 
539 		kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);
540 		ttc->ttc_first = (uintptr_t)NULL;
541 	}
542 #endif
543 
544 	hat_cpu_offline(cp);
545 
546 	ucode_free_space(cp);
547 
548 	/* Free CPU ID string and brand string. */
549 	if (cp->cpu_idstr) {
550 		kmem_free(cp->cpu_idstr, CPU_IDSTRLEN);
551 		cp->cpu_idstr = NULL;
552 	}
553 	if (cp->cpu_brandstr) {
554 		kmem_free(cp->cpu_brandstr, CPU_IDSTRLEN);
555 		cp->cpu_brandstr = NULL;
556 	}
557 
558 #if !defined(__xpv)
559 	if (cp->cpu_m.mcpu_mwait != NULL) {
560 		cpuid_mwait_free(cp);
561 		cp->cpu_m.mcpu_mwait = NULL;
562 	}
563 #endif
564 	cpuid_free_space(cp);
565 
566 	if (cp->cpu_idt != CPU->cpu_idt)
567 		kmem_free(cp->cpu_idt, PAGESIZE);
568 	cp->cpu_idt = NULL;
569 
570 	kmem_free(cp->cpu_m.mcpu_ldt, LDT_CPU_SIZE);
571 	cp->cpu_m.mcpu_ldt = NULL;
572 	cp->cpu_m.mcpu_ldt_len = 0;
573 
574 	kmem_free(cp->cpu_gdt, PAGESIZE);
575 	cp->cpu_gdt = NULL;
576 
577 	if (cp->cpu_supp_freqs != NULL) {
578 		size_t len = strlen(cp->cpu_supp_freqs) + 1;
579 		kmem_free(cp->cpu_supp_freqs, len);
580 		cp->cpu_supp_freqs = NULL;
581 	}
582 
583 	teardown_vaddr_for_ppcopy(cp);
584 
585 	kcpc_hw_fini(cp);
586 
587 	cp->cpu_dispthread = NULL;
588 	cp->cpu_thread = NULL;	/* discarded by cpu_destroy_bound_threads() */
589 
590 	cpu_vm_data_destroy(cp);
591 
592 	xc_fini_cpu(cp);
593 	disp_cpu_fini(cp);
594 
595 	ASSERT(cp != CPU0);
596 	bzero(cp, sizeof (*cp));
597 	cp->cpu_next_free = cpu_free_list;
598 	cpu_free_list = cp;
599 }
600 
601 /*
602  * Apply workarounds for known errata, and warn about those that are absent.
603  *
604  * System vendors occasionally create configurations which contain different
605  * revisions of the CPUs that are almost but not exactly the same.  At the
606  * time of writing, this meant that their clock rates were the same, their
607  * feature sets were the same, but the required workaround were -not-
608  * necessarily the same.  So, this routine is invoked on -every- CPU soon
609  * after starting to make sure that the resulting system contains the most
610  * pessimal set of workarounds needed to cope with *any* of the CPUs in the
611  * system.
612  *
613  * workaround_errata is invoked early in mlsetup() for CPU 0, and in
614  * mp_startup_common() for all slave CPUs. Slaves process workaround_errata
615  * prior to acknowledging their readiness to the master, so this routine will
616  * never be executed by multiple CPUs in parallel, thus making updates to
617  * global data safe.
618  *
619  * These workarounds are based on Rev 3.57 of the Revision Guide for
620  * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005.
621  */
622 
623 #if defined(OPTERON_ERRATUM_88)
624 int opteron_erratum_88;		/* if non-zero -> at least one cpu has it */
625 #endif
626 
627 #if defined(OPTERON_ERRATUM_91)
628 int opteron_erratum_91;		/* if non-zero -> at least one cpu has it */
629 #endif
630 
631 #if defined(OPTERON_ERRATUM_93)
632 int opteron_erratum_93;		/* if non-zero -> at least one cpu has it */
633 #endif
634 
635 #if defined(OPTERON_ERRATUM_95)
636 int opteron_erratum_95;		/* if non-zero -> at least one cpu has it */
637 #endif
638 
639 #if defined(OPTERON_ERRATUM_100)
640 int opteron_erratum_100;	/* if non-zero -> at least one cpu has it */
641 #endif
642 
643 #if defined(OPTERON_ERRATUM_108)
644 int opteron_erratum_108;	/* if non-zero -> at least one cpu has it */
645 #endif
646 
647 #if defined(OPTERON_ERRATUM_109)
648 int opteron_erratum_109;	/* if non-zero -> at least one cpu has it */
649 #endif
650 
651 #if defined(OPTERON_ERRATUM_121)
652 int opteron_erratum_121;	/* if non-zero -> at least one cpu has it */
653 #endif
654 
655 #if defined(OPTERON_ERRATUM_122)
656 int opteron_erratum_122;	/* if non-zero -> at least one cpu has it */
657 #endif
658 
659 #if defined(OPTERON_ERRATUM_123)
660 int opteron_erratum_123;	/* if non-zero -> at least one cpu has it */
661 #endif
662 
663 #if defined(OPTERON_ERRATUM_131)
664 int opteron_erratum_131;	/* if non-zero -> at least one cpu has it */
665 #endif
666 
667 #if defined(OPTERON_WORKAROUND_6336786)
668 int opteron_workaround_6336786;	/* non-zero -> WA relevant and applied */
669 int opteron_workaround_6336786_UP = 0;	/* Not needed for UP */
670 #endif
671 
672 #if defined(OPTERON_WORKAROUND_6323525)
673 int opteron_workaround_6323525;	/* if non-zero -> at least one cpu has it */
674 #endif
675 
676 #if defined(OPTERON_ERRATUM_298)
677 int opteron_erratum_298;
678 #endif
679 
680 #if defined(OPTERON_ERRATUM_721)
681 int opteron_erratum_721;
682 #endif
683 
684 static void
workaround_warning(cpu_t * cp,uint_t erratum)685 workaround_warning(cpu_t *cp, uint_t erratum)
686 {
687 	cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u",
688 	    cp->cpu_id, erratum);
689 }
690 
691 static void
workaround_applied(uint_t erratum)692 workaround_applied(uint_t erratum)
693 {
694 	if (erratum > 1000000)
695 		cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n",
696 		    erratum);
697 	else
698 		cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n",
699 		    erratum);
700 }
701 
702 static void
msr_warning(cpu_t * cp,const char * rw,uint_t msr,int error)703 msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error)
704 {
705 	cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d",
706 	    cp->cpu_id, rw, msr, error);
707 }
708 
709 /*
710  * Determine the number of nodes in a Hammer / Greyhound / Griffin family
711  * system.
712  */
713 static uint_t
opteron_get_nnodes(void)714 opteron_get_nnodes(void)
715 {
716 	static uint_t nnodes = 0;
717 
718 	if (nnodes == 0) {
719 #ifdef	DEBUG
720 		uint_t family;
721 
722 		/*
723 		 * This routine uses a PCI config space based mechanism
724 		 * for retrieving the number of nodes in the system.
725 		 * Device 24, function 0, offset 0x60 as used here is not
726 		 * AMD processor architectural, and may not work on processor
727 		 * families other than those listed below.
728 		 *
729 		 * Callers of this routine must ensure that we're running on
730 		 * a processor which supports this mechanism.
731 		 * The assertion below is meant to catch calls on unsupported
732 		 * processors.
733 		 */
734 		family = cpuid_getfamily(CPU);
735 		ASSERT(family == 0xf || family == 0x10 || family == 0x11);
736 #endif	/* DEBUG */
737 
738 		/*
739 		 * Obtain the number of nodes in the system from
740 		 * bits [6:4] of the Node ID register on node 0.
741 		 *
742 		 * The actual node count is NodeID[6:4] + 1
743 		 *
744 		 * The Node ID register is accessed via function 0,
745 		 * offset 0x60. Node 0 is device 24.
746 		 */
747 		nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1;
748 	}
749 	return (nnodes);
750 }
751 
752 uint_t
do_erratum_298(struct cpu * cpu)753 do_erratum_298(struct cpu *cpu)
754 {
755 	static int	osvwrc = -3;
756 	extern int	osvw_opteron_erratum(cpu_t *, uint_t);
757 
758 	/*
759 	 * L2 Eviction May Occur During Processor Operation To Set
760 	 * Accessed or Dirty Bit.
761 	 */
762 	if (osvwrc == -3) {
763 		osvwrc = osvw_opteron_erratum(cpu, 298);
764 	} else {
765 		/* osvw return codes should be consistent for all cpus */
766 		ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298));
767 	}
768 
769 	switch (osvwrc) {
770 	case 0:		/* erratum is not present: do nothing */
771 		break;
772 	case 1:		/* erratum is present: BIOS workaround applied */
773 		/*
774 		 * check if workaround is actually in place and issue warning
775 		 * if not.
776 		 */
777 		if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
778 		    ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) {
779 #if defined(OPTERON_ERRATUM_298)
780 			opteron_erratum_298++;
781 #else
782 			workaround_warning(cpu, 298);
783 			return (1);
784 #endif
785 		}
786 		break;
787 	case -1:	/* cannot determine via osvw: check cpuid */
788 		if ((cpuid_opteron_erratum(cpu, 298) > 0) &&
789 		    (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) ||
790 		    ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) {
791 #if defined(OPTERON_ERRATUM_298)
792 			opteron_erratum_298++;
793 #else
794 			workaround_warning(cpu, 298);
795 			return (1);
796 #endif
797 		}
798 		break;
799 	}
800 	return (0);
801 }
802 
803 uint_t
workaround_errata(struct cpu * cpu)804 workaround_errata(struct cpu *cpu)
805 {
806 	volatile uint_t missing = 0;
807 
808 	ASSERT(cpu == CPU);
809 
810 	/*LINTED*/
811 	if (cpuid_opteron_erratum(cpu, 88) > 0) {
812 		/*
813 		 * SWAPGS May Fail To Read Correct GS Base
814 		 */
815 #if defined(OPTERON_ERRATUM_88)
816 		/*
817 		 * The workaround is an mfence in the relevant assembler code
818 		 */
819 		opteron_erratum_88++;
820 #else
821 		workaround_warning(cpu, 88);
822 		missing++;
823 #endif
824 	}
825 
826 	if (cpuid_opteron_erratum(cpu, 91) > 0) {
827 		/*
828 		 * Software Prefetches May Report A Page Fault
829 		 */
830 #if defined(OPTERON_ERRATUM_91)
831 		/*
832 		 * fix is in trap.c
833 		 */
834 		opteron_erratum_91++;
835 #else
836 		workaround_warning(cpu, 91);
837 		missing++;
838 #endif
839 	}
840 
841 	if (cpuid_opteron_erratum(cpu, 93) > 0) {
842 		/*
843 		 * RSM Auto-Halt Restart Returns to Incorrect RIP
844 		 */
845 #if defined(OPTERON_ERRATUM_93)
846 		/*
847 		 * fix is in trap.c
848 		 */
849 		opteron_erratum_93++;
850 #else
851 		workaround_warning(cpu, 93);
852 		missing++;
853 #endif
854 	}
855 
856 	/*LINTED*/
857 	if (cpuid_opteron_erratum(cpu, 95) > 0) {
858 		/*
859 		 * RET Instruction May Return to Incorrect EIP
860 		 */
861 #if defined(OPTERON_ERRATUM_95)
862 #if defined(_LP64)
863 		/*
864 		 * Workaround this by ensuring that 32-bit user code and
865 		 * 64-bit kernel code never occupy the same address
866 		 * range mod 4G.
867 		 */
868 		if (_userlimit32 > 0xc0000000ul)
869 			*(uintptr_t *)&_userlimit32 = 0xc0000000ul;
870 
871 		/*LINTED*/
872 		ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u);
873 		opteron_erratum_95++;
874 #endif	/* _LP64 */
875 #else
876 		workaround_warning(cpu, 95);
877 		missing++;
878 #endif
879 	}
880 
881 	if (cpuid_opteron_erratum(cpu, 100) > 0) {
882 		/*
883 		 * Compatibility Mode Branches Transfer to Illegal Address
884 		 */
885 #if defined(OPTERON_ERRATUM_100)
886 		/*
887 		 * fix is in trap.c
888 		 */
889 		opteron_erratum_100++;
890 #else
891 		workaround_warning(cpu, 100);
892 		missing++;
893 #endif
894 	}
895 
896 	/*LINTED*/
897 	if (cpuid_opteron_erratum(cpu, 108) > 0) {
898 		/*
899 		 * CPUID Instruction May Return Incorrect Model Number In
900 		 * Some Processors
901 		 */
902 #if defined(OPTERON_ERRATUM_108)
903 		/*
904 		 * (Our cpuid-handling code corrects the model number on
905 		 * those processors)
906 		 */
907 #else
908 		workaround_warning(cpu, 108);
909 		missing++;
910 #endif
911 	}
912 
913 	/*LINTED*/
914 	if (cpuid_opteron_erratum(cpu, 109) > 0) do {
915 		/*
916 		 * Certain Reverse REP MOVS May Produce Unpredictable Behavior
917 		 */
918 #if defined(OPTERON_ERRATUM_109)
919 		/*
920 		 * The "workaround" is to print a warning to upgrade the BIOS
921 		 */
922 		uint64_t value;
923 		const uint_t msr = MSR_AMD_PATCHLEVEL;
924 		int err;
925 
926 		if ((err = checked_rdmsr(msr, &value)) != 0) {
927 			msr_warning(cpu, "rd", msr, err);
928 			workaround_warning(cpu, 109);
929 			missing++;
930 		}
931 		if (value == 0)
932 			opteron_erratum_109++;
933 #else
934 		workaround_warning(cpu, 109);
935 		missing++;
936 #endif
937 	/*CONSTANTCONDITION*/
938 	} while (0);
939 
940 	/*LINTED*/
941 	if (cpuid_opteron_erratum(cpu, 121) > 0) {
942 		/*
943 		 * Sequential Execution Across Non_Canonical Boundary Caused
944 		 * Processor Hang
945 		 */
946 #if defined(OPTERON_ERRATUM_121)
947 #if defined(_LP64)
948 		/*
949 		 * Erratum 121 is only present in long (64 bit) mode.
950 		 * Workaround is to include the page immediately before the
951 		 * va hole to eliminate the possibility of system hangs due to
952 		 * sequential execution across the va hole boundary.
953 		 */
954 		if (opteron_erratum_121)
955 			opteron_erratum_121++;
956 		else {
957 			if (hole_start) {
958 				hole_start -= PAGESIZE;
959 			} else {
960 				/*
961 				 * hole_start not yet initialized by
962 				 * mmu_init. Initialize hole_start
963 				 * with value to be subtracted.
964 				 */
965 				hole_start = PAGESIZE;
966 			}
967 			opteron_erratum_121++;
968 		}
969 #endif	/* _LP64 */
970 #else
971 		workaround_warning(cpu, 121);
972 		missing++;
973 #endif
974 	}
975 
976 	/*LINTED*/
977 	if (cpuid_opteron_erratum(cpu, 122) > 0) do {
978 		/*
979 		 * TLB Flush Filter May Cause Coherency Problem in
980 		 * Multiprocessor Systems
981 		 */
982 #if defined(OPTERON_ERRATUM_122)
983 		uint64_t value;
984 		const uint_t msr = MSR_AMD_HWCR;
985 		int error;
986 
987 		/*
988 		 * Erratum 122 is only present in MP configurations (multi-core
989 		 * or multi-processor).
990 		 */
991 #if defined(__xpv)
992 		if (!DOMAIN_IS_INITDOMAIN(xen_info))
993 			break;
994 		if (!opteron_erratum_122 && xpv_nr_phys_cpus() == 1)
995 			break;
996 #else
997 		if (!opteron_erratum_122 && opteron_get_nnodes() == 1 &&
998 		    cpuid_get_ncpu_per_chip(cpu) == 1)
999 			break;
1000 #endif
1001 		/* disable TLB Flush Filter */
1002 
1003 		if ((error = checked_rdmsr(msr, &value)) != 0) {
1004 			msr_warning(cpu, "rd", msr, error);
1005 			workaround_warning(cpu, 122);
1006 			missing++;
1007 		} else {
1008 			value |= (uint64_t)AMD_HWCR_FFDIS;
1009 			if ((error = checked_wrmsr(msr, value)) != 0) {
1010 				msr_warning(cpu, "wr", msr, error);
1011 				workaround_warning(cpu, 122);
1012 				missing++;
1013 			}
1014 		}
1015 		opteron_erratum_122++;
1016 #else
1017 		workaround_warning(cpu, 122);
1018 		missing++;
1019 #endif
1020 	/*CONSTANTCONDITION*/
1021 	} while (0);
1022 
1023 	/*LINTED*/
1024 	if (cpuid_opteron_erratum(cpu, 123) > 0) do {
1025 		/*
1026 		 * Bypassed Reads May Cause Data Corruption of System Hang in
1027 		 * Dual Core Processors
1028 		 */
1029 #if defined(OPTERON_ERRATUM_123)
1030 		uint64_t value;
1031 		const uint_t msr = MSR_AMD_PATCHLEVEL;
1032 		int err;
1033 
1034 		/*
1035 		 * Erratum 123 applies only to multi-core cpus.
1036 		 */
1037 		if (cpuid_get_ncpu_per_chip(cpu) < 2)
1038 			break;
1039 #if defined(__xpv)
1040 		if (!DOMAIN_IS_INITDOMAIN(xen_info))
1041 			break;
1042 #endif
1043 		/*
1044 		 * The "workaround" is to print a warning to upgrade the BIOS
1045 		 */
1046 		if ((err = checked_rdmsr(msr, &value)) != 0) {
1047 			msr_warning(cpu, "rd", msr, err);
1048 			workaround_warning(cpu, 123);
1049 			missing++;
1050 		}
1051 		if (value == 0)
1052 			opteron_erratum_123++;
1053 #else
1054 		workaround_warning(cpu, 123);
1055 		missing++;
1056 
1057 #endif
1058 	/*CONSTANTCONDITION*/
1059 	} while (0);
1060 
1061 	/*LINTED*/
1062 	if (cpuid_opteron_erratum(cpu, 131) > 0) do {
1063 		/*
1064 		 * Multiprocessor Systems with Four or More Cores May Deadlock
1065 		 * Waiting for a Probe Response
1066 		 */
1067 #if defined(OPTERON_ERRATUM_131)
1068 		uint64_t nbcfg;
1069 		const uint_t msr = MSR_AMD_NB_CFG;
1070 		const uint64_t wabits =
1071 		    AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR;
1072 		int error;
1073 
1074 		/*
1075 		 * Erratum 131 applies to any system with four or more cores.
1076 		 */
1077 		if (opteron_erratum_131)
1078 			break;
1079 #if defined(__xpv)
1080 		if (!DOMAIN_IS_INITDOMAIN(xen_info))
1081 			break;
1082 		if (xpv_nr_phys_cpus() < 4)
1083 			break;
1084 #else
1085 		if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4)
1086 			break;
1087 #endif
1088 		/*
1089 		 * Print a warning if neither of the workarounds for
1090 		 * erratum 131 is present.
1091 		 */
1092 		if ((error = checked_rdmsr(msr, &nbcfg)) != 0) {
1093 			msr_warning(cpu, "rd", msr, error);
1094 			workaround_warning(cpu, 131);
1095 			missing++;
1096 		} else if ((nbcfg & wabits) == 0) {
1097 			opteron_erratum_131++;
1098 		} else {
1099 			/* cannot have both workarounds set */
1100 			ASSERT((nbcfg & wabits) != wabits);
1101 		}
1102 #else
1103 		workaround_warning(cpu, 131);
1104 		missing++;
1105 #endif
1106 	/*CONSTANTCONDITION*/
1107 	} while (0);
1108 
1109 	/*
1110 	 * This isn't really an erratum, but for convenience the
1111 	 * detection/workaround code lives here and in cpuid_opteron_erratum.
1112 	 * Note, the technique only is valid on families before 12h and
1113 	 * certainly doesn't work when we're virtualized. This is checked for in
1114 	 * the erratum workaround.
1115 	 */
1116 	if (cpuid_opteron_erratum(cpu, 6336786) > 0) {
1117 #if defined(OPTERON_WORKAROUND_6336786)
1118 		/*
1119 		 * Disable C1-Clock ramping on multi-core/multi-processor
1120 		 * K8 platforms to guard against TSC drift.
1121 		 */
1122 		if (opteron_workaround_6336786) {
1123 			opteron_workaround_6336786++;
1124 #if defined(__xpv)
1125 		} else if ((DOMAIN_IS_INITDOMAIN(xen_info) &&
1126 		    xpv_nr_phys_cpus() > 1) ||
1127 		    opteron_workaround_6336786_UP) {
1128 			/*
1129 			 * XXPV	Hmm.  We can't walk the Northbridges on
1130 			 *	the hypervisor; so just complain and drive
1131 			 *	on.  This probably needs to be fixed in
1132 			 *	the hypervisor itself.
1133 			 */
1134 			opteron_workaround_6336786++;
1135 			workaround_warning(cpu, 6336786);
1136 #else	/* __xpv */
1137 		} else if ((opteron_get_nnodes() *
1138 		    cpuid_get_ncpu_per_chip(cpu) > 1) ||
1139 		    opteron_workaround_6336786_UP) {
1140 
1141 			uint_t	node, nnodes;
1142 			uint8_t data;
1143 
1144 			nnodes = opteron_get_nnodes();
1145 			for (node = 0; node < nnodes; node++) {
1146 				/*
1147 				 * Clear PMM7[1:0] (function 3, offset 0x87)
1148 				 * Northbridge device is the node id + 24.
1149 				 */
1150 				data = pci_getb_func(0, node + 24, 3, 0x87);
1151 				data &= 0xFC;
1152 				pci_putb_func(0, node + 24, 3, 0x87, data);
1153 			}
1154 			opteron_workaround_6336786++;
1155 #endif	/* __xpv */
1156 		}
1157 #else
1158 		workaround_warning(cpu, 6336786);
1159 		missing++;
1160 #endif
1161 	}
1162 
1163 	/*LINTED*/
1164 	/*
1165 	 * Mutex primitives don't work as expected. This is erratum #147 from
1166 	 * 'Revision Guide for AMD Athlon 64 and AMD Opteron Processors'
1167 	 * document 25759.
1168 	 */
1169 	if (cpuid_opteron_erratum(cpu, 6323525) > 0) {
1170 #if defined(OPTERON_WORKAROUND_6323525)
1171 		/*
1172 		 * This problem only occurs with 2 or more cores. If bit in
1173 		 * MSR_AMD_BU_CFG set, then not applicable. The workaround
1174 		 * is to patch the semaphone routines with the lfence
1175 		 * instruction to provide necessary load memory barrier with
1176 		 * possible subsequent read-modify-write ops.
1177 		 *
1178 		 * It is too early in boot to call the patch routine so
1179 		 * set erratum variable to be done in startup_end().
1180 		 */
1181 		if (opteron_workaround_6323525) {
1182 			opteron_workaround_6323525++;
1183 #if defined(__xpv)
1184 		} else if (is_x86_feature(x86_featureset, X86FSET_SSE2)) {
1185 			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1186 				/*
1187 				 * XXPV	Use dom0_msr here when extended
1188 				 *	operations are supported?
1189 				 */
1190 				if (xpv_nr_phys_cpus() > 1)
1191 					opteron_workaround_6323525++;
1192 			} else {
1193 				/*
1194 				 * We have no way to tell how many physical
1195 				 * cpus there are, or even if this processor
1196 				 * has the problem, so enable the workaround
1197 				 * unconditionally (at some performance cost).
1198 				 */
1199 				opteron_workaround_6323525++;
1200 			}
1201 #else	/* __xpv */
1202 		} else if (is_x86_feature(x86_featureset, X86FSET_SSE2) &&
1203 		    ((opteron_get_nnodes() *
1204 		    cpuid_get_ncpu_per_chip(cpu)) > 1)) {
1205 			if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0)
1206 				opteron_workaround_6323525++;
1207 #endif	/* __xpv */
1208 		}
1209 #else
1210 		workaround_warning(cpu, 6323525);
1211 		missing++;
1212 #endif
1213 	}
1214 
1215 	missing += do_erratum_298(cpu);
1216 
1217 	if (cpuid_opteron_erratum(cpu, 721) > 0) {
1218 #if defined(OPTERON_ERRATUM_721)
1219 		on_trap_data_t otd;
1220 
1221 		if (!on_trap(&otd, OT_DATA_ACCESS))
1222 			wrmsr(MSR_AMD_DE_CFG,
1223 			    rdmsr(MSR_AMD_DE_CFG) | AMD_DE_CFG_E721);
1224 		no_trap();
1225 
1226 		opteron_erratum_721++;
1227 #else
1228 		workaround_warning(cpu, 721);
1229 		missing++;
1230 #endif
1231 	}
1232 
1233 #ifdef __xpv
1234 	return (0);
1235 #else
1236 	return (missing);
1237 #endif
1238 }
1239 
1240 void
workaround_errata_end()1241 workaround_errata_end()
1242 {
1243 #if defined(OPTERON_ERRATUM_88)
1244 	if (opteron_erratum_88)
1245 		workaround_applied(88);
1246 #endif
1247 #if defined(OPTERON_ERRATUM_91)
1248 	if (opteron_erratum_91)
1249 		workaround_applied(91);
1250 #endif
1251 #if defined(OPTERON_ERRATUM_93)
1252 	if (opteron_erratum_93)
1253 		workaround_applied(93);
1254 #endif
1255 #if defined(OPTERON_ERRATUM_95)
1256 	if (opteron_erratum_95)
1257 		workaround_applied(95);
1258 #endif
1259 #if defined(OPTERON_ERRATUM_100)
1260 	if (opteron_erratum_100)
1261 		workaround_applied(100);
1262 #endif
1263 #if defined(OPTERON_ERRATUM_108)
1264 	if (opteron_erratum_108)
1265 		workaround_applied(108);
1266 #endif
1267 #if defined(OPTERON_ERRATUM_109)
1268 	if (opteron_erratum_109) {
1269 		cmn_err(CE_WARN,
1270 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1271 		    " processor\nerratum 109 was not detected; updating your"
1272 		    " system's BIOS to a version\ncontaining this"
1273 		    " microcode patch is HIGHLY recommended or erroneous"
1274 		    " system\noperation may occur.\n");
1275 	}
1276 #endif
1277 #if defined(OPTERON_ERRATUM_121)
1278 	if (opteron_erratum_121)
1279 		workaround_applied(121);
1280 #endif
1281 #if defined(OPTERON_ERRATUM_122)
1282 	if (opteron_erratum_122)
1283 		workaround_applied(122);
1284 #endif
1285 #if defined(OPTERON_ERRATUM_123)
1286 	if (opteron_erratum_123) {
1287 		cmn_err(CE_WARN,
1288 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1289 		    " processor\nerratum 123 was not detected; updating your"
1290 		    " system's BIOS to a version\ncontaining this"
1291 		    " microcode patch is HIGHLY recommended or erroneous"
1292 		    " system\noperation may occur.\n");
1293 	}
1294 #endif
1295 #if defined(OPTERON_ERRATUM_131)
1296 	if (opteron_erratum_131) {
1297 		cmn_err(CE_WARN,
1298 		    "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)"
1299 		    " processor\nerratum 131 was not detected; updating your"
1300 		    " system's BIOS to a version\ncontaining this"
1301 		    " microcode patch is HIGHLY recommended or erroneous"
1302 		    " system\noperation may occur.\n");
1303 	}
1304 #endif
1305 #if defined(OPTERON_WORKAROUND_6336786)
1306 	if (opteron_workaround_6336786)
1307 		workaround_applied(6336786);
1308 #endif
1309 #if defined(OPTERON_WORKAROUND_6323525)
1310 	if (opteron_workaround_6323525)
1311 		workaround_applied(6323525);
1312 #endif
1313 #if defined(OPTERON_ERRATUM_298)
1314 	if (opteron_erratum_298) {
1315 		cmn_err(CE_WARN,
1316 		    "BIOS microcode patch for AMD 64/Opteron(tm)"
1317 		    " processor\nerratum 298 was not detected; updating your"
1318 		    " system's BIOS to a version\ncontaining this"
1319 		    " microcode patch is HIGHLY recommended or erroneous"
1320 		    " system\noperation may occur.\n");
1321 	}
1322 #endif
1323 #if defined(OPTERON_ERRATUM_721)
1324 	if (opteron_erratum_721)
1325 		workaround_applied(721);
1326 #endif
1327 }
1328 
1329 /*
1330  * The procset_slave and procset_master are used to synchronize
1331  * between the control CPU and the target CPU when starting CPUs.
1332  */
1333 static cpuset_t procset_slave, procset_master;
1334 
1335 static void
mp_startup_wait(cpuset_t * sp,processorid_t cpuid)1336 mp_startup_wait(cpuset_t *sp, processorid_t cpuid)
1337 {
1338 	cpuset_t tempset;
1339 
1340 	for (tempset = *sp; !CPU_IN_SET(tempset, cpuid);
1341 	    tempset = *(volatile cpuset_t *)sp) {
1342 		SMT_PAUSE();
1343 	}
1344 	CPUSET_ATOMIC_DEL(*(cpuset_t *)sp, cpuid);
1345 }
1346 
1347 static void
mp_startup_signal(cpuset_t * sp,processorid_t cpuid)1348 mp_startup_signal(cpuset_t *sp, processorid_t cpuid)
1349 {
1350 	cpuset_t tempset;
1351 
1352 	CPUSET_ATOMIC_ADD(*(cpuset_t *)sp, cpuid);
1353 	for (tempset = *sp; CPU_IN_SET(tempset, cpuid);
1354 	    tempset = *(volatile cpuset_t *)sp) {
1355 		SMT_PAUSE();
1356 	}
1357 }
1358 
1359 int
mp_start_cpu_common(cpu_t * cp,boolean_t boot)1360 mp_start_cpu_common(cpu_t *cp, boolean_t boot)
1361 {
1362 	_NOTE(ARGUNUSED(boot));
1363 
1364 	void *ctx;
1365 	int delays;
1366 	int error = 0;
1367 	cpuset_t tempset;
1368 	processorid_t cpuid;
1369 #ifndef __xpv
1370 	extern void cpupm_init(cpu_t *);
1371 #endif
1372 
1373 	ASSERT(cp != NULL);
1374 	cpuid = cp->cpu_id;
1375 	ctx = mach_cpucontext_alloc(cp);
1376 	if (ctx == NULL) {
1377 		cmn_err(CE_WARN,
1378 		    "cpu%d: failed to allocate context", cp->cpu_id);
1379 		return (EAGAIN);
1380 	}
1381 	error = mach_cpu_start(cp, ctx);
1382 	if (error != 0) {
1383 		cmn_err(CE_WARN,
1384 		    "cpu%d: failed to start, error %d", cp->cpu_id, error);
1385 		mach_cpucontext_free(cp, ctx, error);
1386 		return (error);
1387 	}
1388 
1389 	for (delays = 0, tempset = procset_slave; !CPU_IN_SET(tempset, cpuid);
1390 	    delays++) {
1391 		if (delays == 500) {
1392 			/*
1393 			 * After five seconds, things are probably looking
1394 			 * a bit bleak - explain the hang.
1395 			 */
1396 			cmn_err(CE_NOTE, "cpu%d: started, "
1397 			    "but not running in the kernel yet", cpuid);
1398 		} else if (delays > 2000) {
1399 			/*
1400 			 * We waited at least 20 seconds, bail ..
1401 			 */
1402 			error = ETIMEDOUT;
1403 			cmn_err(CE_WARN, "cpu%d: timed out", cpuid);
1404 			mach_cpucontext_free(cp, ctx, error);
1405 			return (error);
1406 		}
1407 
1408 		/*
1409 		 * wait at least 10ms, then check again..
1410 		 */
1411 		delay(USEC_TO_TICK_ROUNDUP(10000));
1412 		tempset = *((volatile cpuset_t *)&procset_slave);
1413 	}
1414 	CPUSET_ATOMIC_DEL(procset_slave, cpuid);
1415 
1416 	mach_cpucontext_free(cp, ctx, 0);
1417 
1418 #ifndef __xpv
1419 	if (tsc_gethrtime_enable)
1420 		tsc_sync_master(cpuid);
1421 #endif
1422 
1423 	if (dtrace_cpu_init != NULL) {
1424 		(*dtrace_cpu_init)(cpuid);
1425 	}
1426 
1427 	/*
1428 	 * During CPU DR operations, the cpu_lock is held by current
1429 	 * (the control) thread. We can't release the cpu_lock here
1430 	 * because that will break the CPU DR logic.
1431 	 * On the other hand, CPUPM and processor group initialization
1432 	 * routines need to access the cpu_lock. So we invoke those
1433 	 * routines here on behalf of mp_startup_common().
1434 	 *
1435 	 * CPUPM and processor group initialization routines depend
1436 	 * on the cpuid probing results. Wait for mp_startup_common()
1437 	 * to signal that cpuid probing is done.
1438 	 */
1439 	mp_startup_wait(&procset_slave, cpuid);
1440 #ifndef __xpv
1441 	cpupm_init(cp);
1442 #endif
1443 	(void) pg_cpu_init(cp, B_FALSE);
1444 	cpu_set_state(cp);
1445 	mp_startup_signal(&procset_master, cpuid);
1446 
1447 	return (0);
1448 }
1449 
1450 /*
1451  * Start a single cpu, assuming that the kernel context is available
1452  * to successfully start another cpu.
1453  *
1454  * (For example, real mode code is mapped into the right place
1455  * in memory and is ready to be run.)
1456  */
1457 int
start_cpu(processorid_t who)1458 start_cpu(processorid_t who)
1459 {
1460 	cpu_t *cp;
1461 	int error = 0;
1462 	cpuset_t tempset;
1463 
1464 	ASSERT(who != 0);
1465 
1466 	/*
1467 	 * Check if there's at least a Mbyte of kmem available
1468 	 * before attempting to start the cpu.
1469 	 */
1470 	if (kmem_avail() < 1024 * 1024) {
1471 		/*
1472 		 * Kick off a reap in case that helps us with
1473 		 * later attempts ..
1474 		 */
1475 		kmem_reap();
1476 		return (ENOMEM);
1477 	}
1478 
1479 	/*
1480 	 * First configure cpu.
1481 	 */
1482 	cp = mp_cpu_configure_common(who, B_TRUE);
1483 	ASSERT(cp != NULL);
1484 
1485 	/*
1486 	 * Then start cpu.
1487 	 */
1488 	error = mp_start_cpu_common(cp, B_TRUE);
1489 	if (error != 0) {
1490 		mp_cpu_unconfigure_common(cp, error);
1491 		return (error);
1492 	}
1493 
1494 	mutex_exit(&cpu_lock);
1495 	tempset = cpu_ready_set;
1496 	while (!CPU_IN_SET(tempset, who)) {
1497 		drv_usecwait(1);
1498 		tempset = *((volatile cpuset_t *)&cpu_ready_set);
1499 	}
1500 	mutex_enter(&cpu_lock);
1501 
1502 	return (0);
1503 }
1504 
1505 void
start_other_cpus(int cprboot)1506 start_other_cpus(int cprboot)
1507 {
1508 	_NOTE(ARGUNUSED(cprboot));
1509 
1510 	uint_t who;
1511 	uint_t bootcpuid = 0;
1512 
1513 	/*
1514 	 * Initialize our own cpu_info.
1515 	 */
1516 	init_cpu_info(CPU);
1517 
1518 #if !defined(__xpv)
1519 	init_cpu_id_gdt(CPU);
1520 #endif
1521 
1522 	cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr);
1523 	cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr);
1524 
1525 	/*
1526 	 * KPTI initialisation happens very early in boot, before logging is
1527 	 * set up. Output a status message now as the boot CPU comes online.
1528 	 */
1529 	cmn_err(CE_CONT, "?KPTI %s (PCID %s, INVPCID %s)\n",
1530 	    kpti_enable ? "enabled" : "disabled",
1531 	    x86_use_pcid == 1 ? "in use" :
1532 	    (is_x86_feature(x86_featureset, X86FSET_PCID) ? "disabled" :
1533 	    "not supported"),
1534 	    x86_use_pcid == 1 && x86_use_invpcid == 1 ? "in use" :
1535 	    (is_x86_feature(x86_featureset, X86FSET_INVPCID) ? "disabled" :
1536 	    "not supported"));
1537 
1538 	/*
1539 	 * Initialize our syscall handlers
1540 	 */
1541 	init_cpu_syscall(CPU);
1542 
1543 	/*
1544 	 * Take the boot cpu out of the mp_cpus set because we know
1545 	 * it's already running.  Add it to the cpu_ready_set for
1546 	 * precisely the same reason.
1547 	 */
1548 	CPUSET_DEL(mp_cpus, bootcpuid);
1549 	CPUSET_ADD(cpu_ready_set, bootcpuid);
1550 
1551 	/*
1552 	 * skip the rest of this if
1553 	 * . only 1 cpu dectected and system isn't hotplug-capable
1554 	 * . not using MP
1555 	 */
1556 	if ((CPUSET_ISNULL(mp_cpus) && plat_dr_support_cpu() == 0) ||
1557 	    use_mp == 0) {
1558 		if (use_mp == 0)
1559 			cmn_err(CE_CONT, "?***** Not in MP mode\n");
1560 		goto done;
1561 	}
1562 
1563 	/*
1564 	 * perform such initialization as is needed
1565 	 * to be able to take CPUs on- and off-line.
1566 	 */
1567 	cpu_pause_init();
1568 
1569 	xc_init_cpu(CPU);		/* initialize processor crosscalls */
1570 
1571 	if (mach_cpucontext_init() != 0)
1572 		goto done;
1573 
1574 	flushes_require_xcalls = 1;
1575 
1576 	/*
1577 	 * We lock our affinity to the master CPU to ensure that all slave CPUs
1578 	 * do their TSC syncs with the same CPU.
1579 	 */
1580 	affinity_set(CPU_CURRENT);
1581 
1582 	for (who = 0; who < NCPU; who++) {
1583 		if (!CPU_IN_SET(mp_cpus, who))
1584 			continue;
1585 		ASSERT(who != bootcpuid);
1586 
1587 		mutex_enter(&cpu_lock);
1588 		if (start_cpu(who) != 0)
1589 			CPUSET_DEL(mp_cpus, who);
1590 		cpu_state_change_notify(who, CPU_SETUP);
1591 		mutex_exit(&cpu_lock);
1592 	}
1593 
1594 	/* Free the space allocated to hold the microcode file */
1595 	ucode_cleanup();
1596 
1597 	affinity_clear();
1598 
1599 	mach_cpucontext_fini();
1600 
1601 done:
1602 	if (get_hwenv() == HW_NATIVE)
1603 		workaround_errata_end();
1604 	cmi_post_mpstartup();
1605 
1606 #if !defined(__xpv)
1607 	/*
1608 	 * Once other CPUs have completed startup procedures, perform
1609 	 * initialization of hypervisor resources for HMA.
1610 	 */
1611 	hma_init();
1612 #endif
1613 
1614 	if (use_mp && ncpus != boot_max_ncpus) {
1615 		cmn_err(CE_NOTE,
1616 		    "System detected %d cpus, but "
1617 		    "only %d cpu(s) were enabled during boot.",
1618 		    boot_max_ncpus, ncpus);
1619 		cmn_err(CE_NOTE,
1620 		    "Use \"boot-ncpus\" parameter to enable more CPU(s). "
1621 		    "See eeprom(8).");
1622 	}
1623 }
1624 
1625 int
mp_cpu_configure(int cpuid)1626 mp_cpu_configure(int cpuid)
1627 {
1628 	cpu_t *cp;
1629 
1630 	if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1631 		return (ENOTSUP);
1632 	}
1633 
1634 	cp = cpu_get(cpuid);
1635 	if (cp != NULL) {
1636 		return (EALREADY);
1637 	}
1638 
1639 	/*
1640 	 * Check if there's at least a Mbyte of kmem available
1641 	 * before attempting to start the cpu.
1642 	 */
1643 	if (kmem_avail() < 1024 * 1024) {
1644 		/*
1645 		 * Kick off a reap in case that helps us with
1646 		 * later attempts ..
1647 		 */
1648 		kmem_reap();
1649 		return (ENOMEM);
1650 	}
1651 
1652 	cp = mp_cpu_configure_common(cpuid, B_FALSE);
1653 	ASSERT(cp != NULL && cpu_get(cpuid) == cp);
1654 
1655 	return (cp != NULL ? 0 : EAGAIN);
1656 }
1657 
1658 int
mp_cpu_unconfigure(int cpuid)1659 mp_cpu_unconfigure(int cpuid)
1660 {
1661 	cpu_t *cp;
1662 
1663 	if (use_mp == 0 || plat_dr_support_cpu() == 0) {
1664 		return (ENOTSUP);
1665 	} else if (cpuid < 0 || cpuid >= max_ncpus) {
1666 		return (EINVAL);
1667 	}
1668 
1669 	cp = cpu_get(cpuid);
1670 	if (cp == NULL) {
1671 		return (ENODEV);
1672 	}
1673 	mp_cpu_unconfigure_common(cp, 0);
1674 
1675 	return (0);
1676 }
1677 
1678 /*
1679  * Startup function for 'other' CPUs (besides boot cpu).
1680  * Called from real_mode_start.
1681  *
1682  * WARNING: until CPU_READY is set, mp_startup_common and routines called by
1683  * mp_startup_common should not call routines (e.g. kmem_free) that could call
1684  * hat_unload which requires CPU_READY to be set.
1685  */
1686 static void
mp_startup_common(boolean_t boot)1687 mp_startup_common(boolean_t boot)
1688 {
1689 	cpu_t *cp = CPU;
1690 	uchar_t new_x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];
1691 	extern void cpu_event_init_cpu(cpu_t *);
1692 
1693 	/*
1694 	 * We need to get TSC on this proc synced (i.e., any delta
1695 	 * from cpu0 accounted for) as soon as we can, because many
1696 	 * many things use gethrtime/pc_gethrestime, including
1697 	 * interrupts, cmn_err, etc.  Before we can do that, we want to
1698 	 * clear TSC if we're on a buggy Sandy/Ivy Bridge CPU, so do that
1699 	 * right away.
1700 	 */
1701 	bzero(new_x86_featureset, BT_SIZEOFMAP(NUM_X86_FEATURES));
1702 	cpuid_pass1(cp, new_x86_featureset);
1703 
1704 	if (boot && get_hwenv() == HW_NATIVE &&
1705 	    cpuid_getvendor(CPU) == X86_VENDOR_Intel &&
1706 	    cpuid_getfamily(CPU) == 6 &&
1707 	    (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) &&
1708 	    is_x86_feature(new_x86_featureset, X86FSET_TSC)) {
1709 		(void) wrmsr(REG_TSC, 0UL);
1710 	}
1711 
1712 	/* Let the control CPU continue into tsc_sync_master() */
1713 	mp_startup_signal(&procset_slave, cp->cpu_id);
1714 
1715 #ifndef __xpv
1716 	if (tsc_gethrtime_enable)
1717 		tsc_sync_slave();
1718 #endif
1719 
1720 	/*
1721 	 * Once this was done from assembly, but it's safer here; if
1722 	 * it blocks, we need to be able to swtch() to and from, and
1723 	 * since we get here by calling t_pc, we need to do that call
1724 	 * before swtch() overwrites it.
1725 	 */
1726 	(void) (*ap_mlsetup)();
1727 
1728 #ifndef __xpv
1729 	/*
1730 	 * Program this cpu's PAT
1731 	 */
1732 	pat_sync();
1733 #endif
1734 
1735 	/*
1736 	 * Set up TSC_AUX to contain the cpuid for this processor
1737 	 * for the rdtscp instruction.
1738 	 */
1739 	if (is_x86_feature(x86_featureset, X86FSET_TSCP))
1740 		(void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id);
1741 
1742 	/*
1743 	 * Initialize this CPU's syscall handlers
1744 	 */
1745 	init_cpu_syscall(cp);
1746 
1747 	/*
1748 	 * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the
1749 	 * highest level at which a routine is permitted to block on
1750 	 * an adaptive mutex (allows for cpu poke interrupt in case
1751 	 * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks
1752 	 * device interrupts that may end up in the hat layer issuing cross
1753 	 * calls before CPU_READY is set.
1754 	 */
1755 	splx(ipltospl(LOCK_LEVEL));
1756 	sti();
1757 
1758 	/*
1759 	 * There exists a small subset of systems which expose differing
1760 	 * MWAIT/MONITOR support between CPUs.  If MWAIT support is absent from
1761 	 * the boot CPU, but is found on a later CPU, the system continues to
1762 	 * operate as if no MWAIT support is available.
1763 	 *
1764 	 * The reverse case, where MWAIT is available on the boot CPU but not
1765 	 * on a subsequently initialized CPU, is not presently allowed and will
1766 	 * result in a panic.
1767 	 */
1768 	if (is_x86_feature(x86_featureset, X86FSET_MWAIT) !=
1769 	    is_x86_feature(new_x86_featureset, X86FSET_MWAIT)) {
1770 		if (!is_x86_feature(x86_featureset, X86FSET_MWAIT)) {
1771 			remove_x86_feature(new_x86_featureset, X86FSET_MWAIT);
1772 		} else {
1773 			panic("unsupported mixed cpu mwait support detected");
1774 		}
1775 	}
1776 
1777 	/*
1778 	 * We could be more sophisticated here, and just mark the CPU
1779 	 * as "faulted" but at this point we'll opt for the easier
1780 	 * answer of dying horribly.  Provided the boot cpu is ok,
1781 	 * the system can be recovered by booting with use_mp set to zero.
1782 	 */
1783 	if (workaround_errata(cp) != 0)
1784 		panic("critical workaround(s) missing for cpu%d", cp->cpu_id);
1785 
1786 	/*
1787 	 * We can touch cpu_flags here without acquiring the cpu_lock here
1788 	 * because the cpu_lock is held by the control CPU which is running
1789 	 * mp_start_cpu_common().
1790 	 * Need to clear CPU_QUIESCED flag before calling any function which
1791 	 * may cause thread context switching, such as kmem_alloc() etc.
1792 	 * The idle thread checks for CPU_QUIESCED flag and loops for ever if
1793 	 * it's set. So the startup thread may have no chance to switch back
1794 	 * again if it's switched away with CPU_QUIESCED set.
1795 	 */
1796 	cp->cpu_flags &= ~(CPU_POWEROFF | CPU_QUIESCED);
1797 
1798 	enable_pcid();
1799 
1800 	/*
1801 	 * Setup this processor for XSAVE.
1802 	 */
1803 	if (fp_save_mech == FP_XSAVE) {
1804 		xsave_setup_msr(cp);
1805 	}
1806 
1807 	cpuid_pass2(cp);
1808 	cpuid_pass3(cp);
1809 	cpuid_pass4(cp, NULL);
1810 
1811 	/*
1812 	 * Correct cpu_idstr and cpu_brandstr on target CPU after
1813 	 * cpuid_pass1() is done.
1814 	 */
1815 	(void) cpuid_getidstr(cp, cp->cpu_idstr, CPU_IDSTRLEN);
1816 	(void) cpuid_getbrandstr(cp, cp->cpu_brandstr, CPU_IDSTRLEN);
1817 
1818 	cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS;
1819 
1820 	post_startup_cpu_fixups();
1821 
1822 	cpu_event_init_cpu(cp);
1823 
1824 	/*
1825 	 * Enable preemption here so that contention for any locks acquired
1826 	 * later in mp_startup_common may be preempted if the thread owning
1827 	 * those locks is continuously executing on other CPUs (for example,
1828 	 * this CPU must be preemptible to allow other CPUs to pause it during
1829 	 * their startup phases).  It's safe to enable preemption here because
1830 	 * the CPU state is pretty-much fully constructed.
1831 	 */
1832 	curthread->t_preempt = 0;
1833 
1834 	/* The base spl should still be at LOCK LEVEL here */
1835 	ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL));
1836 	set_base_spl();		/* Restore the spl to its proper value */
1837 
1838 	pghw_physid_create(cp);
1839 	/*
1840 	 * Delegate initialization tasks, which need to access the cpu_lock,
1841 	 * to mp_start_cpu_common() because we can't acquire the cpu_lock here
1842 	 * during CPU DR operations.
1843 	 */
1844 	mp_startup_signal(&procset_slave, cp->cpu_id);
1845 	mp_startup_wait(&procset_master, cp->cpu_id);
1846 	pg_cmt_cpu_startup(cp);
1847 
1848 	if (boot) {
1849 		mutex_enter(&cpu_lock);
1850 		cp->cpu_flags &= ~CPU_OFFLINE;
1851 		cpu_enable_intr(cp);
1852 		cpu_add_active(cp);
1853 		mutex_exit(&cpu_lock);
1854 	}
1855 
1856 	/* Enable interrupts */
1857 	(void) spl0();
1858 
1859 	/*
1860 	 * Fill out cpu_ucode_info.  Update microcode if necessary. Note that
1861 	 * this is done after pass1 on the boot CPU, but it needs to be later on
1862 	 * for the other CPUs.
1863 	 */
1864 	ucode_check(cp);
1865 	cpuid_pass_ucode(cp, new_x86_featureset);
1866 
1867 	/*
1868 	 * Do a sanity check to make sure this new CPU is a sane thing
1869 	 * to add to the collection of processors running this system.
1870 	 *
1871 	 * XXX	Clearly this needs to get more sophisticated, if x86
1872 	 * systems start to get built out of heterogenous CPUs; as is
1873 	 * likely to happen once the number of processors in a configuration
1874 	 * gets large enough.
1875 	 */
1876 	if (compare_x86_featureset(x86_featureset, new_x86_featureset) ==
1877 	    B_FALSE) {
1878 		cmn_err(CE_CONT, "cpu%d: featureset\n", cp->cpu_id);
1879 		print_x86_featureset(new_x86_featureset);
1880 		cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id);
1881 	}
1882 
1883 #ifndef __xpv
1884 	{
1885 		/*
1886 		 * Set up the CPU module for this CPU.  This can't be done
1887 		 * before this CPU is made CPU_READY, because we may (in
1888 		 * heterogeneous systems) need to go load another CPU module.
1889 		 * The act of attempting to load a module may trigger a
1890 		 * cross-call, which will ASSERT unless this cpu is CPU_READY.
1891 		 */
1892 		cmi_hdl_t hdl;
1893 
1894 		if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
1895 		    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) {
1896 			if (is_x86_feature(x86_featureset, X86FSET_MCA))
1897 				cmi_mca_init(hdl);
1898 			cp->cpu_m.mcpu_cmi_hdl = hdl;
1899 		}
1900 	}
1901 #endif /* __xpv */
1902 
1903 	if (boothowto & RB_DEBUG)
1904 		kdi_cpu_init();
1905 
1906 	(void) mach_cpu_create_device_node(cp, NULL);
1907 
1908 	/*
1909 	 * Setting the bit in cpu_ready_set must be the last operation in
1910 	 * processor initialization; the boot CPU will continue to boot once
1911 	 * it sees this bit set for all active CPUs.
1912 	 */
1913 	CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id);
1914 
1915 	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);
1916 	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);
1917 	cmn_err(CE_CONT, "?cpu%d initialization complete - online\n",
1918 	    cp->cpu_id);
1919 
1920 	/*
1921 	 * Now we are done with the startup thread, so free it up.
1922 	 */
1923 	thread_exit();
1924 	/*NOTREACHED*/
1925 }
1926 
1927 /*
1928  * Startup function for 'other' CPUs at boot time (besides boot cpu).
1929  */
1930 static void
mp_startup_boot(void)1931 mp_startup_boot(void)
1932 {
1933 	mp_startup_common(B_TRUE);
1934 }
1935 
1936 /*
1937  * Startup function for hotplug CPUs at runtime.
1938  */
1939 void
mp_startup_hotplug(void)1940 mp_startup_hotplug(void)
1941 {
1942 	mp_startup_common(B_FALSE);
1943 }
1944 
1945 /*
1946  * Start CPU on user request.
1947  */
1948 /* ARGSUSED */
1949 int
mp_cpu_start(struct cpu * cp)1950 mp_cpu_start(struct cpu *cp)
1951 {
1952 	ASSERT(MUTEX_HELD(&cpu_lock));
1953 	return (0);
1954 }
1955 
1956 /*
1957  * Stop CPU on user request.
1958  */
1959 int
mp_cpu_stop(struct cpu * cp)1960 mp_cpu_stop(struct cpu *cp)
1961 {
1962 	extern int cbe_psm_timer_mode;
1963 	ASSERT(MUTEX_HELD(&cpu_lock));
1964 
1965 #ifdef __xpv
1966 	/*
1967 	 * We can't offline vcpu0.
1968 	 */
1969 	if (cp->cpu_id == 0)
1970 		return (EBUSY);
1971 #endif
1972 
1973 	/*
1974 	 * If TIMER_PERIODIC mode is used, CPU0 is the one running it;
1975 	 * can't stop it.  (This is true only for machines with no TSC.)
1976 	 */
1977 
1978 	if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0))
1979 		return (EBUSY);
1980 
1981 	return (0);
1982 }
1983 
1984 /*
1985  * Take the specified CPU out of participation in interrupts.
1986  *
1987  * Usually, we hold cpu_lock. But we cannot assert as such due to the
1988  * exception - i_cpr_save_context() - where we have mutual exclusion via a
1989  * separate mechanism.
1990  */
1991 int
cpu_disable_intr(struct cpu * cp)1992 cpu_disable_intr(struct cpu *cp)
1993 {
1994 	if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS)
1995 		return (EBUSY);
1996 
1997 	cp->cpu_flags &= ~CPU_ENABLE;
1998 	ncpus_intr_enabled--;
1999 	return (0);
2000 }
2001 
2002 /*
2003  * Allow the specified CPU to participate in interrupts.
2004  */
2005 void
cpu_enable_intr(struct cpu * cp)2006 cpu_enable_intr(struct cpu *cp)
2007 {
2008 	ASSERT(MUTEX_HELD(&cpu_lock));
2009 	cp->cpu_flags |= CPU_ENABLE;
2010 	ncpus_intr_enabled++;
2011 	psm_enable_intr(cp->cpu_id);
2012 }
2013 
2014 void
mp_cpu_faulted_enter(struct cpu * cp)2015 mp_cpu_faulted_enter(struct cpu *cp)
2016 {
2017 #ifdef __xpv
2018 	_NOTE(ARGUNUSED(cp));
2019 #else
2020 	cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
2021 
2022 	if (hdl != NULL) {
2023 		cmi_hdl_hold(hdl);
2024 	} else {
2025 		hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2026 		    cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2027 	}
2028 	if (hdl != NULL) {
2029 		cmi_faulted_enter(hdl);
2030 		cmi_hdl_rele(hdl);
2031 	}
2032 #endif
2033 }
2034 
2035 void
mp_cpu_faulted_exit(struct cpu * cp)2036 mp_cpu_faulted_exit(struct cpu *cp)
2037 {
2038 #ifdef __xpv
2039 	_NOTE(ARGUNUSED(cp));
2040 #else
2041 	cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl;
2042 
2043 	if (hdl != NULL) {
2044 		cmi_hdl_hold(hdl);
2045 	} else {
2046 		hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
2047 		    cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));
2048 	}
2049 	if (hdl != NULL) {
2050 		cmi_faulted_exit(hdl);
2051 		cmi_hdl_rele(hdl);
2052 	}
2053 #endif
2054 }
2055 
2056 /*
2057  * The following two routines are used as context operators on threads belonging
2058  * to processes with a private LDT (see sysi86).  Due to the rarity of such
2059  * processes, these routines are currently written for best code readability and
2060  * organization rather than speed.  We could avoid checking x86_featureset at
2061  * every context switch by installing different context ops, depending on
2062  * x86_featureset, at LDT creation time -- one for each combination of fast
2063  * syscall features.
2064  */
2065 
2066 void
cpu_fast_syscall_disable(void)2067 cpu_fast_syscall_disable(void)
2068 {
2069 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2070 	    is_x86_feature(x86_featureset, X86FSET_SEP))
2071 		cpu_sep_disable();
2072 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2073 	    is_x86_feature(x86_featureset, X86FSET_ASYSC))
2074 		cpu_asysc_disable();
2075 }
2076 
2077 void
cpu_fast_syscall_enable(void)2078 cpu_fast_syscall_enable(void)
2079 {
2080 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2081 	    is_x86_feature(x86_featureset, X86FSET_SEP))
2082 		cpu_sep_enable();
2083 	if (is_x86_feature(x86_featureset, X86FSET_MSR) &&
2084 	    is_x86_feature(x86_featureset, X86FSET_ASYSC))
2085 		cpu_asysc_enable();
2086 }
2087 
2088 static void
cpu_sep_enable(void)2089 cpu_sep_enable(void)
2090 {
2091 	ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2092 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2093 
2094 	wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL);
2095 
2096 	CPU->cpu_m.mcpu_fast_syscall_state |= FSS_SEP_ENABLED;
2097 }
2098 
2099 static void
cpu_sep_disable(void)2100 cpu_sep_disable(void)
2101 {
2102 	ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP));
2103 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2104 
2105 	/*
2106 	 * Setting the SYSENTER_CS_MSR register to 0 causes software executing
2107 	 * the sysenter or sysexit instruction to trigger a #gp fault.
2108 	 */
2109 	wrmsr(MSR_INTC_SEP_CS, 0);
2110 
2111 	CPU->cpu_m.mcpu_fast_syscall_state &= ~FSS_SEP_ENABLED;
2112 }
2113 
2114 static void
cpu_asysc_enable(void)2115 cpu_asysc_enable(void)
2116 {
2117 	ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2118 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2119 
2120 	wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) |
2121 	    (uint64_t)(uintptr_t)AMD_EFER_SCE);
2122 
2123 	CPU->cpu_m.mcpu_fast_syscall_state |= FSS_ASYSC_ENABLED;
2124 }
2125 
2126 static void
cpu_asysc_disable(void)2127 cpu_asysc_disable(void)
2128 {
2129 	ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC));
2130 	ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL);
2131 
2132 	/*
2133 	 * Turn off the SCE (syscall enable) bit in the EFER register. Software
2134 	 * executing syscall or sysret with this bit off will incur a #ud trap.
2135 	 */
2136 	wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) &
2137 	    ~((uint64_t)(uintptr_t)AMD_EFER_SCE));
2138 
2139 	CPU->cpu_m.mcpu_fast_syscall_state &= ~FSS_ASYSC_ENABLED;
2140 }
2141