xref: /illumos-gate/usr/src/uts/i86pc/io/pcplusmp/apic_common.c (revision 11ed32a0b3b424ec966d0330d0efaf049baaf8d2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 /*
26  * Copyright 2019, Joyent, Inc.
27  * Copyright (c) 2016, 2017 by Delphix. All rights reserved.
28  * Copyright 2019 Joshua M. Clulow <josh@sysmgr.org>
29  */
30 
31 /*
32  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
33  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
34  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
35  * PSMI 1.5 extensions are supported in Solaris Nevada.
36  * PSMI 1.6 extensions are supported in Solaris Nevada.
37  * PSMI 1.7 extensions are supported in Solaris Nevada.
38  */
39 #define	PSMI_1_7
40 
41 #include <sys/processor.h>
42 #include <sys/time.h>
43 #include <sys/psm.h>
44 #include <sys/smp_impldefs.h>
45 #include <sys/cram.h>
46 #include <sys/acpi/acpi.h>
47 #include <sys/acpica.h>
48 #include <sys/psm_common.h>
49 #include <sys/apic.h>
50 #include <sys/pit.h>
51 #include <sys/ddi.h>
52 #include <sys/sunddi.h>
53 #include <sys/ddi_impldefs.h>
54 #include <sys/pci.h>
55 #include <sys/promif.h>
56 #include <sys/x86_archext.h>
57 #include <sys/cpc_impl.h>
58 #include <sys/uadmin.h>
59 #include <sys/panic.h>
60 #include <sys/debug.h>
61 #include <sys/archsystm.h>
62 #include <sys/trap.h>
63 #include <sys/machsystm.h>
64 #include <sys/sysmacros.h>
65 #include <sys/cpuvar.h>
66 #include <sys/rm_platter.h>
67 #include <sys/privregs.h>
68 #include <sys/note.h>
69 #include <sys/pci_intr_lib.h>
70 #include <sys/spl.h>
71 #include <sys/clock.h>
72 #include <sys/dditypes.h>
73 #include <sys/sunddi.h>
74 #include <sys/x_call.h>
75 #include <sys/reboot.h>
76 #include <sys/hpet.h>
77 #include <sys/apic_common.h>
78 #include <sys/apic_timer.h>
79 
80 static void	apic_record_ioapic_rdt(void *intrmap_private,
81 		    ioapic_rdt_t *irdt);
82 static void	apic_record_msi(void *intrmap_private, msi_regs_t *mregs);
83 
84 /*
85  * Common routines between pcplusmp & apix (taken from apic.c).
86  */
87 
88 int	apic_clkinit(int);
89 hrtime_t apic_gethrtime(void);
90 void	apic_send_ipi(int, int);
91 void	apic_set_idlecpu(processorid_t);
92 void	apic_unset_idlecpu(processorid_t);
93 void	apic_shutdown(int, int);
94 void	apic_preshutdown(int, int);
95 processorid_t	apic_get_next_processorid(processorid_t);
96 
97 hrtime_t apic_gettime();
98 
99 enum apic_ioapic_method_type apix_mul_ioapic_method = APIC_MUL_IOAPIC_PCPLUSMP;
100 
101 /* Now the ones for Dynamic Interrupt distribution */
102 int	apic_enable_dynamic_migration = 0;
103 
104 /* maximum loop count when sending Start IPIs. */
105 int apic_sipi_max_loop_count = 0x1000;
106 
107 /*
108  * These variables are frequently accessed in apic_intr_enter(),
109  * apic_intr_exit and apic_setspl, so group them together
110  */
111 volatile uint32_t *apicadr =  NULL;	/* virtual addr of local APIC	*/
112 int apic_setspl_delay = 1;		/* apic_setspl - delay enable	*/
113 int apic_clkvect;
114 
115 /* vector at which error interrupts come in */
116 int apic_errvect;
117 int apic_enable_error_intr = 1;
118 int apic_error_display_delay = 100;
119 
120 /* vector at which performance counter overflow interrupts come in */
121 int apic_cpcovf_vect;
122 int apic_enable_cpcovf_intr = 1;
123 
124 /* vector at which CMCI interrupts come in */
125 int apic_cmci_vect;
126 extern void cmi_cmci_trap(void);
127 
128 lock_t apic_mode_switch_lock;
129 
130 int apic_pir_vect;
131 
132 /*
133  * Patchable global variables.
134  */
135 int	apic_forceload = 0;
136 
137 int	apic_coarse_hrtime = 1;		/* 0 - use accurate slow gethrtime() */
138 
139 int	apic_flat_model = 0;		/* 0 - clustered. 1 - flat */
140 int	apic_panic_on_nmi = 0;
141 int	apic_panic_on_apic_error = 0;
142 
143 int	apic_verbose = 0;	/* 0x1ff */
144 
145 #ifdef DEBUG
146 int	apic_debug = 0;
147 int	apic_restrict_vector = 0;
148 
149 int	apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE];
150 int	apic_debug_msgbufindex = 0;
151 
152 #endif /* DEBUG */
153 
154 uint_t apic_nticks = 0;
155 uint_t apic_skipped_redistribute = 0;
156 
157 uint_t last_count_read = 0;
158 lock_t	apic_gethrtime_lock;
159 volatile int	apic_hrtime_stamp = 0;
160 volatile hrtime_t apic_nsec_since_boot = 0;
161 
162 static	hrtime_t	apic_last_hrtime = 0;
163 int		apic_hrtime_error = 0;
164 int		apic_remote_hrterr = 0;
165 int		apic_num_nmis = 0;
166 int		apic_apic_error = 0;
167 int		apic_num_apic_errors = 0;
168 int		apic_num_cksum_errors = 0;
169 
170 int	apic_error = 0;
171 
172 static	int	apic_cmos_ssb_set = 0;
173 
174 /* use to make sure only one cpu handles the nmi */
175 lock_t	apic_nmi_lock;
176 /* use to make sure only one cpu handles the error interrupt */
177 lock_t	apic_error_lock;
178 
179 static	struct {
180 	uchar_t	cntl;
181 	uchar_t	data;
182 } aspen_bmc[] = {
183 	{ CC_SMS_WR_START,	0x18 },		/* NetFn/LUN */
184 	{ CC_SMS_WR_NEXT,	0x24 },		/* Cmd SET_WATCHDOG_TIMER */
185 	{ CC_SMS_WR_NEXT,	0x84 },		/* DataByte 1: SMS/OS no log */
186 	{ CC_SMS_WR_NEXT,	0x2 },		/* DataByte 2: Power Down */
187 	{ CC_SMS_WR_NEXT,	0x0 },		/* DataByte 3: no pre-timeout */
188 	{ CC_SMS_WR_NEXT,	0x0 },		/* DataByte 4: timer expir. */
189 	{ CC_SMS_WR_NEXT,	0xa },		/* DataByte 5: init countdown */
190 	{ CC_SMS_WR_END,	0x0 },		/* DataByte 6: init countdown */
191 
192 	{ CC_SMS_WR_START,	0x18 },		/* NetFn/LUN */
193 	{ CC_SMS_WR_END,	0x22 }		/* Cmd RESET_WATCHDOG_TIMER */
194 };
195 
196 static	struct {
197 	int	port;
198 	uchar_t	data;
199 } sitka_bmc[] = {
200 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_START },
201 	{ SMS_DATA_REGISTER,	0x18 },		/* NetFn/LUN */
202 	{ SMS_DATA_REGISTER,	0x24 },		/* Cmd SET_WATCHDOG_TIMER */
203 	{ SMS_DATA_REGISTER,	0x84 },		/* DataByte 1: SMS/OS no log */
204 	{ SMS_DATA_REGISTER,	0x2 },		/* DataByte 2: Power Down */
205 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 3: no pre-timeout */
206 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 4: timer expir. */
207 	{ SMS_DATA_REGISTER,	0xa },		/* DataByte 5: init countdown */
208 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_END },
209 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 6: init countdown */
210 
211 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_START },
212 	{ SMS_DATA_REGISTER,	0x18 },		/* NetFn/LUN */
213 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_END },
214 	{ SMS_DATA_REGISTER,	0x22 }		/* Cmd RESET_WATCHDOG_TIMER */
215 };
216 
217 /* Patchable global variables. */
218 int		apic_kmdb_on_nmi = 0;		/* 0 - no, 1 - yes enter kmdb */
219 uint32_t	apic_divide_reg_init = 0;	/* 0 - divide by 2 */
220 
221 /* default apic ops without interrupt remapping */
222 static apic_intrmap_ops_t apic_nointrmap_ops = {
223 	(int (*)(int))return_instr,
224 	(void (*)(int))return_instr,
225 	(void (*)(void **, dev_info_t *, uint16_t, int, uchar_t))return_instr,
226 	(void (*)(void *, void *, uint16_t, int))return_instr,
227 	(void (*)(void **))return_instr,
228 	apic_record_ioapic_rdt,
229 	apic_record_msi,
230 };
231 
232 apic_intrmap_ops_t *apic_vt_ops = &apic_nointrmap_ops;
233 apic_cpus_info_t	*apic_cpus = NULL;
234 cpuset_t	apic_cpumask;
235 uint_t		apic_picinit_called;
236 
237 /* Flag to indicate that we need to shut down all processors */
238 static uint_t	apic_shutdown_processors;
239 
240 /*
241  * Probe the ioapic method for apix module. Called in apic_probe_common()
242  */
243 int
244 apic_ioapic_method_probe()
245 {
246 	if (apix_enable == 0)
247 		return (PSM_SUCCESS);
248 
249 	/*
250 	 * Set IOAPIC EOI handling method. The priority from low to high is:
251 	 *	1. IOxAPIC: with EOI register
252 	 *	2. IOMMU interrupt mapping
253 	 *	3. Mask-Before-EOI method for systems without boot
254 	 *	interrupt routing, such as systems with only one IOAPIC;
255 	 *	NVIDIA CK8-04/MCP55 systems; systems with bridge solution
256 	 *	which disables the boot interrupt routing already.
257 	 *	4. Directed EOI
258 	 */
259 	if (apic_io_ver[0] >= 0x20)
260 		apix_mul_ioapic_method = APIC_MUL_IOAPIC_IOXAPIC;
261 	if ((apic_io_max == 1) || (apic_nvidia_io_max == apic_io_max))
262 		apix_mul_ioapic_method = APIC_MUL_IOAPIC_MASK;
263 	if (apic_directed_EOI_supported())
264 		apix_mul_ioapic_method = APIC_MUL_IOAPIC_DEOI;
265 
266 	/* fall back to pcplusmp */
267 	if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_PCPLUSMP) {
268 		/* make sure apix is after pcplusmp in /etc/mach */
269 		apix_enable = 0; /* go ahead with pcplusmp install next */
270 		return (PSM_FAILURE);
271 	}
272 
273 	return (PSM_SUCCESS);
274 }
275 
276 /*
277  * handler for APIC Error interrupt. Just print a warning and continue
278  */
279 int
280 apic_error_intr()
281 {
282 	uint_t	error0, error1, error;
283 	uint_t	i;
284 
285 	/*
286 	 * We need to write before read as per 7.4.17 of system prog manual.
287 	 * We do both and or the results to be safe
288 	 */
289 	error0 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
290 	apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
291 	error1 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
292 	error = error0 | error1;
293 
294 	/*
295 	 * Clear the APIC error status (do this on all cpus that enter here)
296 	 * (two writes are required due to the semantics of accessing the
297 	 * error status register.)
298 	 */
299 	apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
300 	apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
301 
302 	/*
303 	 * Prevent more than 1 CPU from handling error interrupt causing
304 	 * double printing (interleave of characters from multiple
305 	 * CPU's when using prom_printf)
306 	 */
307 	if (lock_try(&apic_error_lock) == 0)
308 		return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
309 	if (error) {
310 #if	DEBUG
311 		if (apic_debug)
312 			debug_enter("pcplusmp: APIC Error interrupt received");
313 #endif /* DEBUG */
314 		if (apic_panic_on_apic_error)
315 			cmn_err(CE_PANIC,
316 			    "APIC Error interrupt on CPU %d. Status = %x",
317 			    psm_get_cpu_id(), error);
318 		else {
319 			if ((error & ~APIC_CS_ERRORS) == 0) {
320 				/* cksum error only */
321 				apic_error |= APIC_ERR_APIC_ERROR;
322 				apic_apic_error |= error;
323 				apic_num_apic_errors++;
324 				apic_num_cksum_errors++;
325 			} else {
326 				/*
327 				 * prom_printf is the best shot we have of
328 				 * something which is problem free from
329 				 * high level/NMI type of interrupts
330 				 */
331 				prom_printf("APIC Error interrupt on CPU %d. "
332 				    "Status 0 = %x, Status 1 = %x\n",
333 				    psm_get_cpu_id(), error0, error1);
334 				apic_error |= APIC_ERR_APIC_ERROR;
335 				apic_apic_error |= error;
336 				apic_num_apic_errors++;
337 				for (i = 0; i < apic_error_display_delay; i++) {
338 					tenmicrosec();
339 				}
340 				/*
341 				 * provide more delay next time limited to
342 				 * roughly 1 clock tick time
343 				 */
344 				if (apic_error_display_delay < 500)
345 					apic_error_display_delay *= 2;
346 			}
347 		}
348 		lock_clear(&apic_error_lock);
349 		return (DDI_INTR_CLAIMED);
350 	} else {
351 		lock_clear(&apic_error_lock);
352 		return (DDI_INTR_UNCLAIMED);
353 	}
354 }
355 
356 /*
357  * Turn off the mask bit in the performance counter Local Vector Table entry.
358  */
359 void
360 apic_cpcovf_mask_clear(void)
361 {
362 	apic_reg_ops->apic_write(APIC_PCINT_VECT,
363 	    (apic_reg_ops->apic_read(APIC_PCINT_VECT) & ~APIC_LVT_MASK));
364 }
365 
366 /*ARGSUSED*/
367 static int
368 apic_cmci_enable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
369 {
370 	apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
371 	return (0);
372 }
373 
374 /*ARGSUSED*/
375 static int
376 apic_cmci_disable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
377 {
378 	apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect | AV_MASK);
379 	return (0);
380 }
381 
382 void
383 apic_cmci_setup(processorid_t cpuid, boolean_t enable)
384 {
385 	cpuset_t	cpu_set;
386 
387 	CPUSET_ONLY(cpu_set, cpuid);
388 
389 	if (enable) {
390 		xc_call(0, 0, 0, CPUSET2BV(cpu_set),
391 		    (xc_func_t)apic_cmci_enable);
392 	} else {
393 		xc_call(0, 0, 0, CPUSET2BV(cpu_set),
394 		    (xc_func_t)apic_cmci_disable);
395 	}
396 }
397 
398 static void
399 apic_disable_local_apic(void)
400 {
401 	apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
402 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK);
403 
404 	/* local intr reg 0 */
405 	apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK);
406 
407 	/* disable NMI */
408 	apic_reg_ops->apic_write(APIC_INT_VECT1, AV_MASK);
409 
410 	/* and error interrupt */
411 	apic_reg_ops->apic_write(APIC_ERR_VECT, AV_MASK);
412 
413 	/* and perf counter intr */
414 	apic_reg_ops->apic_write(APIC_PCINT_VECT, AV_MASK);
415 
416 	apic_reg_ops->apic_write(APIC_SPUR_INT_REG, APIC_SPUR_INTR);
417 }
418 
419 static void
420 apic_cpu_send_SIPI(processorid_t cpun, boolean_t start)
421 {
422 	int		loop_count;
423 	uint32_t	vector;
424 	uint_t		apicid;
425 	ulong_t		iflag;
426 
427 	apicid =  apic_cpus[cpun].aci_local_id;
428 
429 	/*
430 	 * Interrupts on current CPU will be disabled during the
431 	 * steps in order to avoid unwanted side effects from
432 	 * executing interrupt handlers on a problematic BIOS.
433 	 */
434 	iflag = intr_clear();
435 
436 	if (start) {
437 		outb(CMOS_ADDR, SSB);
438 		outb(CMOS_DATA, BIOS_SHUTDOWN);
439 	}
440 
441 	/*
442 	 * According to X2APIC specification in section '2.3.5.1' of
443 	 * Interrupt Command Register Semantics, the semantics of
444 	 * programming the Interrupt Command Register to dispatch an interrupt
445 	 * is simplified. A single MSR write to the 64-bit ICR is required
446 	 * for dispatching an interrupt. Specifically, with the 64-bit MSR
447 	 * interface to ICR, system software is not required to check the
448 	 * status of the delivery status bit prior to writing to the ICR
449 	 * to send an IPI. With the removal of the Delivery Status bit,
450 	 * system software no longer has a reason to read the ICR. It remains
451 	 * readable only to aid in debugging.
452 	 */
453 #ifdef	DEBUG
454 	APIC_AV_PENDING_SET();
455 #else
456 	if (apic_mode == LOCAL_APIC) {
457 		APIC_AV_PENDING_SET();
458 	}
459 #endif /* DEBUG */
460 
461 	/* for integrated - make sure there is one INIT IPI in buffer */
462 	/* for external - it will wake up the cpu */
463 	apic_reg_ops->apic_write_int_cmd(apicid, AV_ASSERT | AV_RESET);
464 
465 	/* If only 1 CPU is installed, PENDING bit will not go low */
466 	for (loop_count = apic_sipi_max_loop_count; loop_count; loop_count--) {
467 		if (apic_mode == LOCAL_APIC &&
468 		    apic_reg_ops->apic_read(APIC_INT_CMD1) & AV_PENDING)
469 			apic_ret();
470 		else
471 			break;
472 	}
473 
474 	apic_reg_ops->apic_write_int_cmd(apicid, AV_DEASSERT | AV_RESET);
475 	drv_usecwait(20000);		/* 20 milli sec */
476 
477 	if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) {
478 		/* integrated apic */
479 
480 		vector = (rm_platter_pa >> MMU_PAGESHIFT) &
481 		    (APIC_VECTOR_MASK | APIC_IPL_MASK);
482 
483 		/* to offset the INIT IPI queue up in the buffer */
484 		apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
485 		drv_usecwait(200);		/* 20 micro sec */
486 
487 		/*
488 		 * send the second SIPI (Startup IPI) as recommended by Intel
489 		 * software development manual.
490 		 */
491 		apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
492 		drv_usecwait(200);	/* 20 micro sec */
493 	}
494 
495 	intr_restore(iflag);
496 }
497 
498 /*ARGSUSED1*/
499 int
500 apic_cpu_start(processorid_t cpun, caddr_t arg)
501 {
502 	ASSERT(MUTEX_HELD(&cpu_lock));
503 
504 	if (!apic_cpu_in_range(cpun)) {
505 		return (EINVAL);
506 	}
507 
508 	/*
509 	 * Switch to apic_common_send_ipi for safety during starting other CPUs.
510 	 */
511 	if (apic_mode == LOCAL_X2APIC) {
512 		apic_switch_ipi_callback(B_TRUE);
513 	}
514 
515 	apic_cmos_ssb_set = 1;
516 	apic_cpu_send_SIPI(cpun, B_TRUE);
517 
518 	return (0);
519 }
520 
521 /*
522  * Put CPU into halted state with interrupts disabled.
523  */
524 /*ARGSUSED1*/
525 int
526 apic_cpu_stop(processorid_t cpun, caddr_t arg)
527 {
528 	int		rc;
529 	cpu_t		*cp;
530 	extern cpuset_t cpu_ready_set;
531 	extern void cpu_idle_intercept_cpu(cpu_t *cp);
532 
533 	ASSERT(MUTEX_HELD(&cpu_lock));
534 
535 	if (!apic_cpu_in_range(cpun)) {
536 		return (EINVAL);
537 	}
538 	if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
539 		return (ENOTSUP);
540 	}
541 
542 	cp = cpu_get(cpun);
543 	ASSERT(cp != NULL);
544 	ASSERT((cp->cpu_flags & CPU_OFFLINE) != 0);
545 	ASSERT((cp->cpu_flags & CPU_QUIESCED) != 0);
546 	ASSERT((cp->cpu_flags & CPU_ENABLE) == 0);
547 
548 	/* Clear CPU_READY flag to disable cross calls. */
549 	cp->cpu_flags &= ~CPU_READY;
550 	CPUSET_ATOMIC_DEL(cpu_ready_set, cpun);
551 	rc = xc_flush_cpu(cp);
552 	if (rc != 0) {
553 		CPUSET_ATOMIC_ADD(cpu_ready_set, cpun);
554 		cp->cpu_flags |= CPU_READY;
555 		return (rc);
556 	}
557 
558 	/* Intercept target CPU at a safe point before powering it off. */
559 	cpu_idle_intercept_cpu(cp);
560 
561 	apic_cpu_send_SIPI(cpun, B_FALSE);
562 	cp->cpu_flags &= ~CPU_RUNNING;
563 
564 	return (0);
565 }
566 
567 int
568 apic_cpu_ops(psm_cpu_request_t *reqp)
569 {
570 	if (reqp == NULL) {
571 		return (EINVAL);
572 	}
573 
574 	switch (reqp->pcr_cmd) {
575 	case PSM_CPU_ADD:
576 		return (apic_cpu_add(reqp));
577 
578 	case PSM_CPU_REMOVE:
579 		return (apic_cpu_remove(reqp));
580 
581 	case PSM_CPU_STOP:
582 		return (apic_cpu_stop(reqp->req.cpu_stop.cpuid,
583 		    reqp->req.cpu_stop.ctx));
584 
585 	default:
586 		return (ENOTSUP);
587 	}
588 }
589 
590 #ifdef	DEBUG
591 int	apic_break_on_cpu = 9;
592 int	apic_stretch_interrupts = 0;
593 int	apic_stretch_ISR = 1 << 3;	/* IPL of 3 matches nothing now */
594 #endif /* DEBUG */
595 
596 /*
597  * generates an interprocessor interrupt to another CPU. Any changes made to
598  * this routine must be accompanied by similar changes to
599  * apic_common_send_ipi().
600  */
601 void
602 apic_send_ipi(int cpun, int ipl)
603 {
604 	int vector;
605 	ulong_t flag;
606 
607 	vector = apic_resv_vector[ipl];
608 
609 	ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
610 
611 	flag = intr_clear();
612 
613 	APIC_AV_PENDING_SET();
614 
615 	apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
616 	    vector);
617 
618 	intr_restore(flag);
619 }
620 
621 void
622 apic_send_pir_ipi(processorid_t cpun)
623 {
624 	const int vector = apic_pir_vect;
625 	ulong_t flag;
626 
627 	ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
628 
629 	flag = intr_clear();
630 
631 	/* Self-IPI for inducing PIR makes no sense. */
632 	if ((cpun != psm_get_cpu_id())) {
633 		APIC_AV_PENDING_SET();
634 		apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
635 		    vector);
636 	}
637 
638 	intr_restore(flag);
639 }
640 
641 int
642 apic_get_pir_ipivect(void)
643 {
644 	return (apic_pir_vect);
645 }
646 
647 /*ARGSUSED*/
648 void
649 apic_set_idlecpu(processorid_t cpun)
650 {
651 }
652 
653 /*ARGSUSED*/
654 void
655 apic_unset_idlecpu(processorid_t cpun)
656 {
657 }
658 
659 
660 void
661 apic_ret()
662 {
663 }
664 
665 /*
666  * If apic_coarse_time == 1, then apic_gettime() is used instead of
667  * apic_gethrtime().  This is used for performance instead of accuracy.
668  */
669 
670 hrtime_t
671 apic_gettime()
672 {
673 	int old_hrtime_stamp;
674 	hrtime_t temp;
675 
676 	/*
677 	 * In one-shot mode, we do not keep time, so if anyone
678 	 * calls psm_gettime() directly, we vector over to
679 	 * gethrtime().
680 	 * one-shot mode MUST NOT be enabled if this psm is the source of
681 	 * hrtime.
682 	 */
683 
684 	if (apic_oneshot)
685 		return (gethrtime());
686 
687 
688 gettime_again:
689 	while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
690 		apic_ret();
691 
692 	temp = apic_nsec_since_boot;
693 
694 	if (apic_hrtime_stamp != old_hrtime_stamp) {	/* got an interrupt */
695 		goto gettime_again;
696 	}
697 	return (temp);
698 }
699 
700 /*
701  * Here we return the number of nanoseconds since booting.  Note every
702  * clock interrupt increments apic_nsec_since_boot by the appropriate
703  * amount.
704  */
705 hrtime_t
706 apic_gethrtime(void)
707 {
708 	int curr_timeval, countval, elapsed_ticks;
709 	int old_hrtime_stamp, status;
710 	hrtime_t temp;
711 	uint32_t cpun;
712 	ulong_t oflags;
713 
714 	/*
715 	 * In one-shot mode, we do not keep time, so if anyone
716 	 * calls psm_gethrtime() directly, we vector over to
717 	 * gethrtime().
718 	 * one-shot mode MUST NOT be enabled if this psm is the source of
719 	 * hrtime.
720 	 */
721 
722 	if (apic_oneshot)
723 		return (gethrtime());
724 
725 	oflags = intr_clear();	/* prevent migration */
726 
727 	cpun = apic_reg_ops->apic_read(APIC_LID_REG);
728 	if (apic_mode == LOCAL_APIC)
729 		cpun >>= APIC_ID_BIT_OFFSET;
730 
731 	lock_set(&apic_gethrtime_lock);
732 
733 gethrtime_again:
734 	while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
735 		apic_ret();
736 
737 	/*
738 	 * Check to see which CPU we are on.  Note the time is kept on
739 	 * the local APIC of CPU 0.  If on CPU 0, simply read the current
740 	 * counter.  If on another CPU, issue a remote read command to CPU 0.
741 	 */
742 	if (cpun == apic_cpus[0].aci_local_id) {
743 		countval = apic_reg_ops->apic_read(APIC_CURR_COUNT);
744 	} else {
745 #ifdef	DEBUG
746 		APIC_AV_PENDING_SET();
747 #else
748 		if (apic_mode == LOCAL_APIC)
749 			APIC_AV_PENDING_SET();
750 #endif /* DEBUG */
751 
752 		apic_reg_ops->apic_write_int_cmd(
753 		    apic_cpus[0].aci_local_id, APIC_CURR_ADD | AV_REMOTE);
754 
755 		while ((status = apic_reg_ops->apic_read(APIC_INT_CMD1))
756 		    & AV_READ_PENDING) {
757 			apic_ret();
758 		}
759 
760 		if (status & AV_REMOTE_STATUS)	/* 1 = valid */
761 			countval = apic_reg_ops->apic_read(APIC_REMOTE_READ);
762 		else {	/* 0 = invalid */
763 			apic_remote_hrterr++;
764 			/*
765 			 * return last hrtime right now, will need more
766 			 * testing if change to retry
767 			 */
768 			temp = apic_last_hrtime;
769 
770 			lock_clear(&apic_gethrtime_lock);
771 
772 			intr_restore(oflags);
773 
774 			return (temp);
775 		}
776 	}
777 	if (countval > last_count_read)
778 		countval = 0;
779 	else
780 		last_count_read = countval;
781 
782 	elapsed_ticks = apic_hertz_count - countval;
783 
784 	curr_timeval = APIC_TICKS_TO_NSECS(elapsed_ticks);
785 	temp = apic_nsec_since_boot + curr_timeval;
786 
787 	if (apic_hrtime_stamp != old_hrtime_stamp) {	/* got an interrupt */
788 		/* we might have clobbered last_count_read. Restore it */
789 		last_count_read = apic_hertz_count;
790 		goto gethrtime_again;
791 	}
792 
793 	if (temp < apic_last_hrtime) {
794 		/* return last hrtime if error occurs */
795 		apic_hrtime_error++;
796 		temp = apic_last_hrtime;
797 	}
798 	else
799 		apic_last_hrtime = temp;
800 
801 	lock_clear(&apic_gethrtime_lock);
802 	intr_restore(oflags);
803 
804 	return (temp);
805 }
806 
807 /* apic NMI handler */
808 /*ARGSUSED*/
809 void
810 apic_nmi_intr(caddr_t arg, struct regs *rp)
811 {
812 	nmi_action_t action = nmi_action;
813 
814 	if (apic_shutdown_processors) {
815 		apic_disable_local_apic();
816 		return;
817 	}
818 
819 	apic_error |= APIC_ERR_NMI;
820 
821 	if (!lock_try(&apic_nmi_lock))
822 		return;
823 	apic_num_nmis++;
824 
825 	/*
826 	 * "nmi_action" always over-rides the older way of doing this, unless we
827 	 * can't actually drop into kmdb when requested.
828 	 */
829 	if (action == NMI_ACTION_KMDB && !psm_debugger())
830 		action = NMI_ACTION_UNSET;
831 
832 	if (action == NMI_ACTION_UNSET) {
833 		if (apic_kmdb_on_nmi && psm_debugger())
834 			action = NMI_ACTION_KMDB;
835 		else if (apic_panic_on_nmi)
836 			action = NMI_ACTION_PANIC;
837 		else
838 			action = NMI_ACTION_IGNORE;
839 	}
840 
841 	switch (action) {
842 	case NMI_ACTION_IGNORE:
843 		/*
844 		 * prom_printf is the best shot we have of something which is
845 		 * problem free from high level/NMI type of interrupts
846 		 */
847 		prom_printf("NMI received\n");
848 		break;
849 
850 	case NMI_ACTION_PANIC:
851 		/* Keep panic from entering kmdb. */
852 		nopanicdebug = 1;
853 		panic("NMI received\n");
854 		break;
855 
856 	case NMI_ACTION_KMDB:
857 	default:
858 		debug_enter("NMI received: entering kmdb\n");
859 		break;
860 	}
861 
862 	lock_clear(&apic_nmi_lock);
863 }
864 
865 processorid_t
866 apic_get_next_processorid(processorid_t cpu_id)
867 {
868 
869 	int i;
870 
871 	if (cpu_id == -1)
872 		return ((processorid_t)0);
873 
874 	for (i = cpu_id + 1; i < NCPU; i++) {
875 		if (apic_cpu_in_range(i))
876 			return (i);
877 	}
878 
879 	return ((processorid_t)-1);
880 }
881 
882 int
883 apic_cpu_add(psm_cpu_request_t *reqp)
884 {
885 	int i, rv = 0;
886 	ulong_t iflag;
887 	boolean_t first = B_TRUE;
888 	uchar_t localver = 0;
889 	uint32_t localid, procid;
890 	processorid_t cpuid = (processorid_t)-1;
891 	mach_cpu_add_arg_t *ap;
892 
893 	ASSERT(reqp != NULL);
894 	reqp->req.cpu_add.cpuid = (processorid_t)-1;
895 
896 	/* Check whether CPU hotplug is supported. */
897 	if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
898 		return (ENOTSUP);
899 	}
900 
901 	ap = (mach_cpu_add_arg_t *)reqp->req.cpu_add.argp;
902 	switch (ap->type) {
903 	case MACH_CPU_ARG_LOCAL_APIC:
904 		localid = ap->arg.apic.apic_id;
905 		procid = ap->arg.apic.proc_id;
906 		if (localid >= 255 || procid > 255) {
907 			cmn_err(CE_WARN,
908 			    "!apic: apicid(%u) or procid(%u) is invalid.",
909 			    localid, procid);
910 			return (EINVAL);
911 		}
912 		break;
913 
914 	case MACH_CPU_ARG_LOCAL_X2APIC:
915 		localid = ap->arg.apic.apic_id;
916 		procid = ap->arg.apic.proc_id;
917 		if (localid >= UINT32_MAX) {
918 			cmn_err(CE_WARN,
919 			    "!apic: x2apicid(%u) is invalid.", localid);
920 			return (EINVAL);
921 		} else if (localid >= 255 && apic_mode == LOCAL_APIC) {
922 			cmn_err(CE_WARN, "!apic: system is in APIC mode, "
923 			    "can't support x2APIC processor.");
924 			return (ENOTSUP);
925 		}
926 		break;
927 
928 	default:
929 		cmn_err(CE_WARN,
930 		    "!apic: unknown argument type %d to apic_cpu_add().",
931 		    ap->type);
932 		return (EINVAL);
933 	}
934 
935 	/* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
936 	iflag = intr_clear();
937 	lock_set(&apic_ioapic_lock);
938 
939 	/* Check whether local APIC id already exists. */
940 	for (i = 0; i < apic_nproc; i++) {
941 		if (!CPU_IN_SET(apic_cpumask, i))
942 			continue;
943 		if (apic_cpus[i].aci_local_id == localid) {
944 			lock_clear(&apic_ioapic_lock);
945 			intr_restore(iflag);
946 			cmn_err(CE_WARN,
947 			    "!apic: local apic id %u already exists.",
948 			    localid);
949 			return (EEXIST);
950 		} else if (apic_cpus[i].aci_processor_id == procid) {
951 			lock_clear(&apic_ioapic_lock);
952 			intr_restore(iflag);
953 			cmn_err(CE_WARN,
954 			    "!apic: processor id %u already exists.",
955 			    (int)procid);
956 			return (EEXIST);
957 		}
958 
959 		/*
960 		 * There's no local APIC version number available in MADT table,
961 		 * so assume that all CPUs are homogeneous and use local APIC
962 		 * version number of the first existing CPU.
963 		 */
964 		if (first) {
965 			first = B_FALSE;
966 			localver = apic_cpus[i].aci_local_ver;
967 		}
968 	}
969 	ASSERT(first == B_FALSE);
970 
971 	/*
972 	 * Try to assign the same cpuid if APIC id exists in the dirty cache.
973 	 */
974 	for (i = 0; i < apic_max_nproc; i++) {
975 		if (CPU_IN_SET(apic_cpumask, i)) {
976 			ASSERT((apic_cpus[i].aci_status & APIC_CPU_FREE) == 0);
977 			continue;
978 		}
979 		ASSERT(apic_cpus[i].aci_status & APIC_CPU_FREE);
980 		if ((apic_cpus[i].aci_status & APIC_CPU_DIRTY) &&
981 		    apic_cpus[i].aci_local_id == localid &&
982 		    apic_cpus[i].aci_processor_id == procid) {
983 			cpuid = i;
984 			break;
985 		}
986 	}
987 
988 	/* Avoid the dirty cache and allocate fresh slot if possible. */
989 	if (cpuid == (processorid_t)-1) {
990 		for (i = 0; i < apic_max_nproc; i++) {
991 			if ((apic_cpus[i].aci_status & APIC_CPU_FREE) &&
992 			    (apic_cpus[i].aci_status & APIC_CPU_DIRTY) == 0) {
993 				cpuid = i;
994 				break;
995 			}
996 		}
997 	}
998 
999 	/* Try to find any free slot as last resort. */
1000 	if (cpuid == (processorid_t)-1) {
1001 		for (i = 0; i < apic_max_nproc; i++) {
1002 			if (apic_cpus[i].aci_status & APIC_CPU_FREE) {
1003 				cpuid = i;
1004 				break;
1005 			}
1006 		}
1007 	}
1008 
1009 	if (cpuid == (processorid_t)-1) {
1010 		lock_clear(&apic_ioapic_lock);
1011 		intr_restore(iflag);
1012 		cmn_err(CE_NOTE,
1013 		    "!apic: failed to allocate cpu id for processor %u.",
1014 		    procid);
1015 		rv = EAGAIN;
1016 	} else if (ACPI_FAILURE(acpica_map_cpu(cpuid, procid))) {
1017 		lock_clear(&apic_ioapic_lock);
1018 		intr_restore(iflag);
1019 		cmn_err(CE_NOTE,
1020 		    "!apic: failed to build mapping for processor %u.",
1021 		    procid);
1022 		rv = EBUSY;
1023 	} else {
1024 		ASSERT(cpuid >= 0 && cpuid < NCPU);
1025 		ASSERT(cpuid < apic_max_nproc && cpuid < max_ncpus);
1026 		bzero(&apic_cpus[cpuid], sizeof (apic_cpus[0]));
1027 		apic_cpus[cpuid].aci_processor_id = procid;
1028 		apic_cpus[cpuid].aci_local_id = localid;
1029 		apic_cpus[cpuid].aci_local_ver = localver;
1030 		CPUSET_ATOMIC_ADD(apic_cpumask, cpuid);
1031 		if (cpuid >= apic_nproc) {
1032 			apic_nproc = cpuid + 1;
1033 		}
1034 		lock_clear(&apic_ioapic_lock);
1035 		intr_restore(iflag);
1036 		reqp->req.cpu_add.cpuid = cpuid;
1037 	}
1038 
1039 	return (rv);
1040 }
1041 
1042 int
1043 apic_cpu_remove(psm_cpu_request_t *reqp)
1044 {
1045 	int i;
1046 	ulong_t iflag;
1047 	processorid_t cpuid;
1048 
1049 	/* Check whether CPU hotplug is supported. */
1050 	if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
1051 		return (ENOTSUP);
1052 	}
1053 
1054 	cpuid = reqp->req.cpu_remove.cpuid;
1055 
1056 	/* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
1057 	iflag = intr_clear();
1058 	lock_set(&apic_ioapic_lock);
1059 
1060 	if (!apic_cpu_in_range(cpuid)) {
1061 		lock_clear(&apic_ioapic_lock);
1062 		intr_restore(iflag);
1063 		cmn_err(CE_WARN,
1064 		    "!apic: cpuid %d doesn't exist in apic_cpus array.",
1065 		    cpuid);
1066 		return (ENODEV);
1067 	}
1068 	ASSERT((apic_cpus[cpuid].aci_status & APIC_CPU_FREE) == 0);
1069 
1070 	if (ACPI_FAILURE(acpica_unmap_cpu(cpuid))) {
1071 		lock_clear(&apic_ioapic_lock);
1072 		intr_restore(iflag);
1073 		return (ENOENT);
1074 	}
1075 
1076 	if (cpuid == apic_nproc - 1) {
1077 		/*
1078 		 * We are removing the highest numbered cpuid so we need to
1079 		 * find the next highest cpuid as the new value for apic_nproc.
1080 		 */
1081 		for (i = apic_nproc; i > 0; i--) {
1082 			if (CPU_IN_SET(apic_cpumask, i - 1)) {
1083 				apic_nproc = i;
1084 				break;
1085 			}
1086 		}
1087 		/* at least one CPU left */
1088 		ASSERT(i > 0);
1089 	}
1090 	CPUSET_ATOMIC_DEL(apic_cpumask, cpuid);
1091 	/* mark slot as free and keep it in the dirty cache */
1092 	apic_cpus[cpuid].aci_status = APIC_CPU_FREE | APIC_CPU_DIRTY;
1093 
1094 	lock_clear(&apic_ioapic_lock);
1095 	intr_restore(iflag);
1096 
1097 	return (0);
1098 }
1099 
1100 /*
1101  * Return the number of ticks the APIC decrements in SF nanoseconds.
1102  * The fixed-frequency PIT (aka 8254) is used for the measurement.
1103  */
1104 static uint64_t
1105 apic_calibrate_impl()
1106 {
1107 	uint8_t		pit_tick_lo;
1108 	uint16_t	pit_tick, target_pit_tick, pit_ticks_adj;
1109 	uint32_t	pit_ticks;
1110 	uint32_t	start_apic_tick, end_apic_tick, apic_ticks;
1111 	ulong_t		iflag;
1112 
1113 	apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1114 	apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
1115 
1116 	iflag = intr_clear();
1117 
1118 	/*
1119 	 * Put the PIT in mode 0, "Interrupt On Terminal Count":
1120 	 */
1121 	outb(PITCTL_PORT, PIT_C0 | PIT_LOADMODE | PIT_ENDSIGMODE);
1122 
1123 	/*
1124 	 * The PIT counts down and then the counter value wraps around.  Load
1125 	 * the maximum counter value:
1126 	 */
1127 	outb(PITCTR0_PORT, 0xFF);
1128 	outb(PITCTR0_PORT, 0xFF);
1129 
1130 	do {
1131 		pit_tick_lo = inb(PITCTR0_PORT);
1132 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1133 	} while (pit_tick < APIC_TIME_MIN ||
1134 	    pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX);
1135 
1136 	/*
1137 	 * Wait for the PIT to decrement by 5 ticks to ensure
1138 	 * we didn't start in the middle of a tick.
1139 	 * Compare with 0x10 for the wrap around case.
1140 	 */
1141 	target_pit_tick = pit_tick - 5;
1142 	do {
1143 		pit_tick_lo = inb(PITCTR0_PORT);
1144 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1145 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1146 
1147 	start_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
1148 
1149 	/*
1150 	 * Wait for the PIT to decrement by APIC_TIME_COUNT ticks
1151 	 */
1152 	target_pit_tick = pit_tick - APIC_TIME_COUNT;
1153 	do {
1154 		pit_tick_lo = inb(PITCTR0_PORT);
1155 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1156 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1157 
1158 	end_apic_tick = apic_reg_ops->apic_read(APIC_CURR_COUNT);
1159 
1160 	intr_restore(iflag);
1161 
1162 	apic_ticks = start_apic_tick - end_apic_tick;
1163 
1164 	/* The PIT might have decremented by more ticks than planned */
1165 	pit_ticks_adj = target_pit_tick - pit_tick;
1166 	/* total number of PIT ticks corresponding to apic_ticks */
1167 	pit_ticks = APIC_TIME_COUNT + pit_ticks_adj;
1168 
1169 	/*
1170 	 * Determine the number of nanoseconds per APIC clock tick
1171 	 * and then determine how many APIC ticks to interrupt at the
1172 	 * desired frequency
1173 	 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s
1174 	 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s
1175 	 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9)
1176 	 * apic_ticks_per_SFns =
1177 	 * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9)
1178 	 */
1179 	return ((SF * apic_ticks * PIT_HZ) / ((uint64_t)pit_ticks * NANOSEC));
1180 }
1181 
1182 /*
1183  * It was found empirically that 5 measurements seem sufficient to give a good
1184  * accuracy. Most spurious measurements are higher than the target value thus
1185  * we eliminate up to 2/5 spurious measurements.
1186  */
1187 #define	APIC_CALIBRATE_MEASUREMENTS		5
1188 
1189 #define	APIC_CALIBRATE_PERCENT_OFF_WARNING	10
1190 
1191 /*
1192  * Return the number of ticks the APIC decrements in SF nanoseconds.
1193  * Several measurements are taken to filter out outliers.
1194  */
1195 uint64_t
1196 apic_calibrate()
1197 {
1198 	uint64_t	measurements[APIC_CALIBRATE_MEASUREMENTS];
1199 	int		median_idx;
1200 	uint64_t	median;
1201 
1202 	/*
1203 	 * When running under a virtual machine, the emulated PIT and APIC
1204 	 * counters do not always return the right values and can roll over.
1205 	 * Those spurious measurements are relatively rare but could
1206 	 * significantly affect the calibration.
1207 	 * Therefore we take several measurements and then keep the median.
1208 	 * The median is preferred to the average here as we only want to
1209 	 * discard outliers.
1210 	 */
1211 	for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++)
1212 		measurements[i] = apic_calibrate_impl();
1213 
1214 	/*
1215 	 * sort results and retrieve median.
1216 	 */
1217 	for (int i = 0; i < APIC_CALIBRATE_MEASUREMENTS; i++) {
1218 		for (int j = i + 1; j < APIC_CALIBRATE_MEASUREMENTS; j++) {
1219 			if (measurements[j] < measurements[i]) {
1220 				uint64_t tmp = measurements[i];
1221 				measurements[i] = measurements[j];
1222 				measurements[j] = tmp;
1223 			}
1224 		}
1225 	}
1226 	median_idx = APIC_CALIBRATE_MEASUREMENTS / 2;
1227 	median = measurements[median_idx];
1228 
1229 #if (APIC_CALIBRATE_MEASUREMENTS >= 3)
1230 	/*
1231 	 * Check that measurements are consistent. Post a warning
1232 	 * if the three middle values are not close to each other.
1233 	 */
1234 	uint64_t delta_warn = median *
1235 	    APIC_CALIBRATE_PERCENT_OFF_WARNING / 100;
1236 	if ((median - measurements[median_idx - 1]) > delta_warn ||
1237 	    (measurements[median_idx + 1] - median) > delta_warn) {
1238 		cmn_err(CE_WARN, "apic_calibrate measurements lack "
1239 		    "precision: %llu, %llu, %llu.",
1240 		    (u_longlong_t)measurements[median_idx - 1],
1241 		    (u_longlong_t)median,
1242 		    (u_longlong_t)measurements[median_idx + 1]);
1243 	}
1244 #endif
1245 
1246 	return (median);
1247 }
1248 
1249 /*
1250  * Initialise the APIC timer on the local APIC of CPU 0 to the desired
1251  * frequency.  Note at this stage in the boot sequence, the boot processor
1252  * is the only active processor.
1253  * hertz value of 0 indicates a one-shot mode request.  In this case
1254  * the function returns the resolution (in nanoseconds) for the hardware
1255  * timer interrupt.  If one-shot mode capability is not available,
1256  * the return value will be 0. apic_enable_oneshot is a global switch
1257  * for disabling the functionality.
1258  * A non-zero positive value for hertz indicates a periodic mode request.
1259  * In this case the hardware will be programmed to generate clock interrupts
1260  * at hertz frequency and returns the resolution of interrupts in
1261  * nanosecond.
1262  */
1263 
1264 int
1265 apic_clkinit(int hertz)
1266 {
1267 	int		ret;
1268 
1269 	apic_int_busy_mark = (apic_int_busy_mark *
1270 	    apic_sample_factor_redistribution) / 100;
1271 	apic_int_free_mark = (apic_int_free_mark *
1272 	    apic_sample_factor_redistribution) / 100;
1273 	apic_diff_for_redistribution = (apic_diff_for_redistribution *
1274 	    apic_sample_factor_redistribution) / 100;
1275 
1276 	ret = apic_timer_init(hertz);
1277 	return (ret);
1278 
1279 }
1280 
1281 /*
1282  * apic_preshutdown:
1283  * Called early in shutdown whilst we can still access filesystems to do
1284  * things like loading modules which will be required to complete shutdown
1285  * after filesystems are all unmounted.
1286  */
1287 void
1288 apic_preshutdown(int cmd, int fcn)
1289 {
1290 	APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n",
1291 	    cmd, fcn, apic_poweroff_method, apic_enable_acpi));
1292 }
1293 
1294 void
1295 apic_shutdown(int cmd, int fcn)
1296 {
1297 	int restarts, attempts;
1298 	int i;
1299 	uchar_t	byte;
1300 	ulong_t iflag;
1301 
1302 	hpet_acpi_fini();
1303 
1304 	/* Send NMI to all CPUs except self to do per processor shutdown */
1305 	iflag = intr_clear();
1306 #ifdef	DEBUG
1307 	APIC_AV_PENDING_SET();
1308 #else
1309 	if (apic_mode == LOCAL_APIC)
1310 		APIC_AV_PENDING_SET();
1311 #endif /* DEBUG */
1312 	apic_shutdown_processors = 1;
1313 	apic_reg_ops->apic_write(APIC_INT_CMD1,
1314 	    AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF);
1315 
1316 	/* restore cmos shutdown byte before reboot */
1317 	if (apic_cmos_ssb_set) {
1318 		outb(CMOS_ADDR, SSB);
1319 		outb(CMOS_DATA, 0);
1320 	}
1321 
1322 	ioapic_disable_redirection();
1323 
1324 	/*	disable apic mode if imcr present	*/
1325 	if (apic_imcrp) {
1326 		outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
1327 		outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC);
1328 	}
1329 
1330 	apic_disable_local_apic();
1331 
1332 	intr_restore(iflag);
1333 
1334 	/* remainder of function is for shutdown cases only */
1335 	if (cmd != A_SHUTDOWN)
1336 		return;
1337 
1338 	/*
1339 	 * Switch system back into Legacy-Mode if using ACPI and
1340 	 * not powering-off.  Some BIOSes need to remain in ACPI-mode
1341 	 * for power-off to succeed (Dell Dimension 4600)
1342 	 * Do not disable ACPI while doing fastreboot
1343 	 */
1344 	if (apic_enable_acpi && fcn != AD_POWEROFF && fcn != AD_FASTREBOOT)
1345 		(void) AcpiDisable();
1346 
1347 	if (fcn == AD_FASTREBOOT) {
1348 		apic_reg_ops->apic_write(APIC_INT_CMD1,
1349 		    AV_ASSERT | AV_RESET | AV_SH_ALL_EXCSELF);
1350 	}
1351 
1352 	/* remainder of function is for shutdown+poweroff case only */
1353 	if (fcn != AD_POWEROFF)
1354 		return;
1355 
1356 	switch (apic_poweroff_method) {
1357 		case APIC_POWEROFF_VIA_RTC:
1358 
1359 			/* select the extended NVRAM bank in the RTC */
1360 			outb(CMOS_ADDR, RTC_REGA);
1361 			byte = inb(CMOS_DATA);
1362 			outb(CMOS_DATA, (byte | EXT_BANK));
1363 
1364 			outb(CMOS_ADDR, PFR_REG);
1365 
1366 			/* for Predator must toggle the PAB bit */
1367 			byte = inb(CMOS_DATA);
1368 
1369 			/*
1370 			 * clear power active bar, wakeup alarm and
1371 			 * kickstart
1372 			 */
1373 			byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG);
1374 			outb(CMOS_DATA, byte);
1375 
1376 			/* delay before next write */
1377 			drv_usecwait(1000);
1378 
1379 			/* for S40 the following would suffice */
1380 			byte = inb(CMOS_DATA);
1381 
1382 			/* power active bar control bit */
1383 			byte |= PAB_CBIT;
1384 			outb(CMOS_DATA, byte);
1385 
1386 			break;
1387 
1388 		case APIC_POWEROFF_VIA_ASPEN_BMC:
1389 			restarts = 0;
1390 restart_aspen_bmc:
1391 			if (++restarts == 3)
1392 				break;
1393 			attempts = 0;
1394 			do {
1395 				byte = inb(MISMIC_FLAG_REGISTER);
1396 				byte &= MISMIC_BUSY_MASK;
1397 				if (byte != 0) {
1398 					drv_usecwait(1000);
1399 					if (attempts >= 3)
1400 						goto restart_aspen_bmc;
1401 					++attempts;
1402 				}
1403 			} while (byte != 0);
1404 			outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS);
1405 			byte = inb(MISMIC_FLAG_REGISTER);
1406 			byte |= 0x1;
1407 			outb(MISMIC_FLAG_REGISTER, byte);
1408 			i = 0;
1409 			for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0]));
1410 			    i++) {
1411 				attempts = 0;
1412 				do {
1413 					byte = inb(MISMIC_FLAG_REGISTER);
1414 					byte &= MISMIC_BUSY_MASK;
1415 					if (byte != 0) {
1416 						drv_usecwait(1000);
1417 						if (attempts >= 3)
1418 							goto restart_aspen_bmc;
1419 						++attempts;
1420 					}
1421 				} while (byte != 0);
1422 				outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl);
1423 				outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data);
1424 				byte = inb(MISMIC_FLAG_REGISTER);
1425 				byte |= 0x1;
1426 				outb(MISMIC_FLAG_REGISTER, byte);
1427 			}
1428 			break;
1429 
1430 		case APIC_POWEROFF_VIA_SITKA_BMC:
1431 			restarts = 0;
1432 restart_sitka_bmc:
1433 			if (++restarts == 3)
1434 				break;
1435 			attempts = 0;
1436 			do {
1437 				byte = inb(SMS_STATUS_REGISTER);
1438 				byte &= SMS_STATE_MASK;
1439 				if ((byte == SMS_READ_STATE) ||
1440 				    (byte == SMS_WRITE_STATE)) {
1441 					drv_usecwait(1000);
1442 					if (attempts >= 3)
1443 						goto restart_sitka_bmc;
1444 					++attempts;
1445 				}
1446 			} while ((byte == SMS_READ_STATE) ||
1447 			    (byte == SMS_WRITE_STATE));
1448 			outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS);
1449 			i = 0;
1450 			for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0]));
1451 			    i++) {
1452 				attempts = 0;
1453 				do {
1454 					byte = inb(SMS_STATUS_REGISTER);
1455 					byte &= SMS_IBF_MASK;
1456 					if (byte != 0) {
1457 						drv_usecwait(1000);
1458 						if (attempts >= 3)
1459 							goto restart_sitka_bmc;
1460 						++attempts;
1461 					}
1462 				} while (byte != 0);
1463 				outb(sitka_bmc[i].port, sitka_bmc[i].data);
1464 			}
1465 			break;
1466 
1467 		case APIC_POWEROFF_NONE:
1468 
1469 			/* If no APIC direct method, we will try using ACPI */
1470 			if (apic_enable_acpi) {
1471 				if (acpi_poweroff() == 1)
1472 					return;
1473 			} else
1474 				return;
1475 
1476 			break;
1477 	}
1478 	/*
1479 	 * Wait a limited time here for power to go off.
1480 	 * If the power does not go off, then there was a
1481 	 * problem and we should continue to the halt which
1482 	 * prints a message for the user to press a key to
1483 	 * reboot.
1484 	 */
1485 	drv_usecwait(7000000); /* wait seven seconds */
1486 
1487 }
1488 
1489 cyclic_id_t apic_cyclic_id;
1490 
1491 /*
1492  * The following functions are in the platform specific file so that they
1493  * can be different functions depending on whether we are running on
1494  * bare metal or a hypervisor.
1495  */
1496 
1497 /*
1498  * map an apic for memory-mapped access
1499  */
1500 uint32_t *
1501 mapin_apic(uint32_t addr, size_t len, int flags)
1502 {
1503 	return ((void *)psm_map_phys(addr, len, flags));
1504 }
1505 
1506 uint32_t *
1507 mapin_ioapic(uint32_t addr, size_t len, int flags)
1508 {
1509 	return (mapin_apic(addr, len, flags));
1510 }
1511 
1512 /*
1513  * unmap an apic
1514  */
1515 void
1516 mapout_apic(caddr_t addr, size_t len)
1517 {
1518 	psm_unmap_phys(addr, len);
1519 }
1520 
1521 void
1522 mapout_ioapic(caddr_t addr, size_t len)
1523 {
1524 	mapout_apic(addr, len);
1525 }
1526 
1527 uint32_t
1528 ioapic_read(int ioapic_ix, uint32_t reg)
1529 {
1530 	volatile uint32_t *ioapic;
1531 
1532 	ioapic = apicioadr[ioapic_ix];
1533 	ioapic[APIC_IO_REG] = reg;
1534 	return (ioapic[APIC_IO_DATA]);
1535 }
1536 
1537 void
1538 ioapic_write(int ioapic_ix, uint32_t reg, uint32_t value)
1539 {
1540 	volatile uint32_t *ioapic;
1541 
1542 	ioapic = apicioadr[ioapic_ix];
1543 	ioapic[APIC_IO_REG] = reg;
1544 	ioapic[APIC_IO_DATA] = value;
1545 }
1546 
1547 void
1548 ioapic_write_eoi(int ioapic_ix, uint32_t value)
1549 {
1550 	volatile uint32_t *ioapic;
1551 
1552 	ioapic = apicioadr[ioapic_ix];
1553 	ioapic[APIC_IO_EOI] = value;
1554 }
1555 
1556 /*
1557  * Round-robin algorithm to find the next CPU with interrupts enabled.
1558  * It can't share the same static variable apic_next_bind_cpu with
1559  * apic_get_next_bind_cpu(), since that will cause all interrupts to be
1560  * bound to CPU1 at boot time.  During boot, only CPU0 is online with
1561  * interrupts enabled when apic_get_next_bind_cpu() and apic_find_cpu()
1562  * are called.  However, the pcplusmp driver assumes that there will be
1563  * boot_ncpus CPUs configured eventually so it tries to distribute all
1564  * interrupts among CPU0 - CPU[boot_ncpus - 1].  Thus to prevent all
1565  * interrupts being targetted at CPU1, we need to use a dedicated static
1566  * variable for find_next_cpu() instead of sharing apic_next_bind_cpu.
1567  */
1568 
1569 processorid_t
1570 apic_find_cpu(int flag)
1571 {
1572 	int i;
1573 	static processorid_t acid = 0;
1574 
1575 	/* Find the first CPU with the passed-in flag set */
1576 	for (i = 0; i < apic_nproc; i++) {
1577 		if (++acid >= apic_nproc) {
1578 			acid = 0;
1579 		}
1580 		if (apic_cpu_in_range(acid) &&
1581 		    (apic_cpus[acid].aci_status & flag)) {
1582 			break;
1583 		}
1584 	}
1585 
1586 	ASSERT((apic_cpus[acid].aci_status & flag) != 0);
1587 	return (acid);
1588 }
1589 
1590 void
1591 apic_intrmap_init(int apic_mode)
1592 {
1593 	int suppress_brdcst_eoi = 0;
1594 
1595 	/*
1596 	 * Intel Software Developer's Manual 3A, 10.12.7:
1597 	 *
1598 	 * Routing of device interrupts to local APIC units operating in
1599 	 * x2APIC mode requires use of the interrupt-remapping architecture
1600 	 * specified in the Intel Virtualization Technology for Directed
1601 	 * I/O, Revision 1.3.  Because of this, BIOS must enumerate support
1602 	 * for and software must enable this interrupt remapping with
1603 	 * Extended Interrupt Mode Enabled before it enabling x2APIC mode in
1604 	 * the local APIC units.
1605 	 *
1606 	 *
1607 	 * In other words, to use the APIC in x2APIC mode, we need interrupt
1608 	 * remapping.  Since we don't start up the IOMMU by default, we
1609 	 * won't be able to do any interrupt remapping and therefore have to
1610 	 * use the APIC in traditional 'local APIC' mode with memory mapped
1611 	 * I/O.
1612 	 */
1613 
1614 	if (psm_vt_ops != NULL) {
1615 		if (((apic_intrmap_ops_t *)psm_vt_ops)->
1616 		    apic_intrmap_init(apic_mode) == DDI_SUCCESS) {
1617 
1618 			apic_vt_ops = psm_vt_ops;
1619 
1620 			/*
1621 			 * We leverage the interrupt remapping engine to
1622 			 * suppress broadcast EOI; thus we must send the
1623 			 * directed EOI with the directed-EOI handler.
1624 			 */
1625 			if (apic_directed_EOI_supported() == 0) {
1626 				suppress_brdcst_eoi = 1;
1627 			}
1628 
1629 			apic_vt_ops->apic_intrmap_enable(suppress_brdcst_eoi);
1630 
1631 			if (apic_detect_x2apic()) {
1632 				apic_enable_x2apic();
1633 			}
1634 
1635 			if (apic_directed_EOI_supported() == 0) {
1636 				apic_set_directed_EOI_handler();
1637 			}
1638 		}
1639 	}
1640 }
1641 
1642 /*ARGSUSED*/
1643 static void
1644 apic_record_ioapic_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
1645 {
1646 	irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
1647 }
1648 
1649 /*ARGSUSED*/
1650 static void
1651 apic_record_msi(void *intrmap_private, msi_regs_t *mregs)
1652 {
1653 	mregs->mr_addr = MSI_ADDR_HDR |
1654 	    (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
1655 	    (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
1656 	    (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
1657 	mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
1658 	    mregs->mr_data;
1659 }
1660 
1661 /*
1662  * Functions from apic_introp.c
1663  *
1664  * Those functions are used by apic_intr_ops().
1665  */
1666 
1667 /*
1668  * MSI support flag:
1669  * reflects whether MSI is supported at APIC level
1670  * it can also be patched through /etc/system
1671  *
1672  *  0 = default value - don't know and need to call apic_check_msi_support()
1673  *      to find out then set it accordingly
1674  *  1 = supported
1675  * -1 = not supported
1676  */
1677 int	apic_support_msi = 0;
1678 
1679 /* Multiple vector support for MSI-X */
1680 int	apic_msix_enable = 1;
1681 
1682 /* Multiple vector support for MSI */
1683 int	apic_multi_msi_enable = 1;
1684 
1685 /*
1686  * Check whether the system supports MSI.
1687  *
1688  * MSI is required for PCI-E and for PCI versions later than 2.2, so if we find
1689  * a PCI-E bus or we find a PCI bus whose version we know is >= 2.2, then we
1690  * return PSM_SUCCESS to indicate this system supports MSI.
1691  *
1692  * (Currently the only way we check whether a given PCI bus supports >= 2.2 is
1693  * by detecting if we are running inside the KVM hypervisor, which guarantees
1694  * this version number.)
1695  */
1696 int
1697 apic_check_msi_support()
1698 {
1699 	dev_info_t *cdip;
1700 	char dev_type[16];
1701 	int dev_len;
1702 	int hwenv = get_hwenv();
1703 
1704 	DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support:\n"));
1705 
1706 	/*
1707 	 * check whether the first level children of root_node have
1708 	 * PCI-E or PCI capability.
1709 	 */
1710 	for (cdip = ddi_get_child(ddi_root_node()); cdip != NULL;
1711 	    cdip = ddi_get_next_sibling(cdip)) {
1712 
1713 		DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: cdip: 0x%p,"
1714 		    " driver: %s, binding: %s, nodename: %s\n", (void *)cdip,
1715 		    ddi_driver_name(cdip), ddi_binding_name(cdip),
1716 		    ddi_node_name(cdip)));
1717 		dev_len = sizeof (dev_type);
1718 		if (ddi_getlongprop_buf(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1719 		    "device_type", (caddr_t)dev_type, &dev_len)
1720 		    != DDI_PROP_SUCCESS)
1721 			continue;
1722 		if (strcmp(dev_type, "pciex") == 0)
1723 			return (PSM_SUCCESS);
1724 		if (strcmp(dev_type, "pci") == 0 &&
1725 		    (hwenv == HW_KVM || hwenv == HW_BHYVE))
1726 			return (PSM_SUCCESS);
1727 	}
1728 
1729 	/* MSI is not supported on this system */
1730 	DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: no 'pciex' "
1731 	    "device_type found\n"));
1732 	return (PSM_FAILURE);
1733 }
1734 
1735 /*
1736  * apic_pci_msi_unconfigure:
1737  *
1738  * This and next two interfaces are copied from pci_intr_lib.c
1739  * Do ensure that these two files stay in sync.
1740  * These needed to be copied over here to avoid a deadlock situation on
1741  * certain mp systems that use MSI interrupts.
1742  *
1743  * IMPORTANT regards next three interfaces:
1744  * i) are called only for MSI/X interrupts.
1745  * ii) called with interrupts disabled, and must not block
1746  */
1747 void
1748 apic_pci_msi_unconfigure(dev_info_t *rdip, int type, int inum)
1749 {
1750 	ushort_t		msi_ctrl;
1751 	int			cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1752 	ddi_acc_handle_t	handle = i_ddi_get_pci_config_handle(rdip);
1753 
1754 	ASSERT((handle != NULL) && (cap_ptr != 0));
1755 
1756 	if (type == DDI_INTR_TYPE_MSI) {
1757 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1758 		msi_ctrl &= (~PCI_MSI_MME_MASK);
1759 		pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1760 		pci_config_put32(handle, cap_ptr + PCI_MSI_ADDR_OFFSET, 0);
1761 
1762 		if (msi_ctrl &  PCI_MSI_64BIT_MASK) {
1763 			pci_config_put16(handle,
1764 			    cap_ptr + PCI_MSI_64BIT_DATA, 0);
1765 			pci_config_put32(handle,
1766 			    cap_ptr + PCI_MSI_ADDR_OFFSET + 4, 0);
1767 		} else {
1768 			pci_config_put16(handle,
1769 			    cap_ptr + PCI_MSI_32BIT_DATA, 0);
1770 		}
1771 
1772 	} else if (type == DDI_INTR_TYPE_MSIX) {
1773 		uintptr_t	off;
1774 		uint32_t	mask;
1775 		ddi_intr_msix_t	*msix_p = i_ddi_get_msix(rdip);
1776 
1777 		ASSERT(msix_p != NULL);
1778 
1779 		/* Offset into "inum"th entry in the MSI-X table & mask it */
1780 		off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1781 		    PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1782 
1783 		mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1784 
1785 		ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask | 1));
1786 
1787 		/* Offset into the "inum"th entry in the MSI-X table */
1788 		off = (uintptr_t)msix_p->msix_tbl_addr +
1789 		    (inum * PCI_MSIX_VECTOR_SIZE);
1790 
1791 		/* Reset the "data" and "addr" bits */
1792 		ddi_put32(msix_p->msix_tbl_hdl,
1793 		    (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), 0);
1794 		ddi_put64(msix_p->msix_tbl_hdl, (uint64_t *)off, 0);
1795 	}
1796 }
1797 
1798 /*
1799  * apic_pci_msi_disable_mode:
1800  */
1801 void
1802 apic_pci_msi_disable_mode(dev_info_t *rdip, int type)
1803 {
1804 	ushort_t		msi_ctrl;
1805 	int			cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1806 	ddi_acc_handle_t	handle = i_ddi_get_pci_config_handle(rdip);
1807 
1808 	ASSERT((handle != NULL) && (cap_ptr != 0));
1809 
1810 	if (type == DDI_INTR_TYPE_MSI) {
1811 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1812 		if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
1813 			return;
1814 
1815 		msi_ctrl &= ~PCI_MSI_ENABLE_BIT;	/* MSI disable */
1816 		pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1817 
1818 	} else if (type == DDI_INTR_TYPE_MSIX) {
1819 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1820 		if (msi_ctrl & PCI_MSIX_ENABLE_BIT) {
1821 			msi_ctrl &= ~PCI_MSIX_ENABLE_BIT;
1822 			pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
1823 			    msi_ctrl);
1824 		}
1825 	}
1826 }
1827 
1828 uint32_t
1829 apic_get_localapicid(uint32_t cpuid)
1830 {
1831 	ASSERT(cpuid < apic_nproc && apic_cpus != NULL);
1832 
1833 	return (apic_cpus[cpuid].aci_local_id);
1834 }
1835 
1836 uchar_t
1837 apic_get_ioapicid(uchar_t ioapicindex)
1838 {
1839 	ASSERT(ioapicindex < MAX_IO_APIC);
1840 
1841 	return (apic_io_id[ioapicindex]);
1842 }
1843