xref: /illumos-gate/usr/src/uts/sun4v/os/error.c (revision c4b03495)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/machsystm.h>
30 #include <sys/cpuvar.h>
31 #include <sys/async.h>
32 #include <sys/ontrap.h>
33 #include <sys/ddifm.h>
34 #include <sys/hypervisor_api.h>
35 #include <sys/errorq.h>
36 #include <sys/promif.h>
37 #include <sys/prom_plat.h>
38 #include <sys/x_call.h>
39 #include <sys/error.h>
40 #include <sys/fm/util.h>
41 #include <sys/ivintr.h>
42 
43 #define	MAX_CE_FLTS		10
44 #define	MAX_ASYNC_FLTS		6
45 
46 errorq_t *ue_queue;			/* queue of uncorrectable errors */
47 errorq_t *ce_queue;			/* queue of correctable errors */
48 
49 /*
50  * Being used by memory test driver.
51  * ce_verbose_memory - covers CEs in DIMMs
52  * ce_verbose_other - covers "others" (ecache, IO, etc.)
53  *
54  * If the value is 0, nothing is logged.
55  * If the value is 1, the error is logged to the log file, but not console.
56  * If the value is 2, the error is logged to the log file and console.
57  */
58 int	ce_verbose_memory = 1;
59 int	ce_verbose_other = 1;
60 
61 int	ce_show_data = 0;
62 int	ce_debug = 0;
63 int	ue_debug = 0;
64 int	reset_debug = 0;
65 
66 /*
67  * Tunables for controlling the handling of asynchronous faults (AFTs). Setting
68  * these to non-default values on a non-DEBUG kernel is NOT supported.
69  */
70 int	aft_verbose = 0;	/* log AFT messages > 1 to log only */
71 int	aft_panic = 0;		/* panic (not reboot) on fatal usermode AFLT */
72 int	aft_testfatal = 0;	/* force all AFTs to panic immediately */
73 
74 /*
75  * Used for vbsc hostshutdown (power-off buton)
76  */
77 int	err_shutdown_triggered = 0;	/* only once */
78 uint_t	err_shutdown_inum = 0;		/* used to pull the trigger */
79 
80 /*
81  * Defined in bus_func.c but initialised in error_init
82  */
83 extern kmutex_t bfd_lock;
84 
85 static uint32_t rq_overflow_count = 0;		/* counter for rq overflow */
86 
87 static void cpu_queue_one_event(errh_async_flt_t *);
88 static uint32_t count_entries_on_queue(uint64_t, uint64_t, uint32_t);
89 static void errh_page_retire(errh_async_flt_t *, uchar_t);
90 static int errh_error_protected(struct regs *, struct async_flt *, int *);
91 static void errh_rq_full(struct async_flt *);
92 static void ue_drain(void *, struct async_flt *, errorq_elem_t *);
93 static void ce_drain(void *, struct async_flt *, errorq_elem_t *);
94 
95 /*ARGSUSED*/
96 void
97 process_resumable_error(struct regs *rp, uint32_t head_offset,
98     uint32_t tail_offset)
99 {
100 	struct machcpu *mcpup;
101 	struct async_flt *aflt;
102 	errh_async_flt_t errh_flt;
103 	errh_er_t *head_va;
104 
105 	mcpup = &(CPU->cpu_m);
106 
107 	while (head_offset != tail_offset) {
108 		/* kernel buffer starts right after the resumable queue */
109 		head_va = (errh_er_t *)(mcpup->cpu_rq_va + head_offset +
110 		    CPU_RQ_SIZE);
111 		/* Copy the error report to local buffer */
112 		bzero(&errh_flt, sizeof (errh_async_flt_t));
113 		bcopy((char *)head_va, &(errh_flt.errh_er),
114 		    sizeof (errh_er_t));
115 
116 		/* Increment the queue head */
117 		head_offset += Q_ENTRY_SIZE;
118 		/* Wrap around */
119 		head_offset &= (CPU_RQ_SIZE - 1);
120 
121 		/* set error handle to zero so it can hold new error report */
122 		head_va->ehdl = 0;
123 
124 		switch (errh_flt.errh_er.desc) {
125 		case ERRH_DESC_UCOR_RE:
126 			break;
127 
128 		case ERRH_DESC_WARN_RE:
129 			/*
130 			 * Power-off requested, but handle it one time only.
131 			 */
132 			if (!err_shutdown_triggered) {
133 				setsoftint(err_shutdown_inum);
134 				++err_shutdown_triggered;
135 			}
136 			continue;
137 
138 		default:
139 			cmn_err(CE_WARN, "Error Descriptor 0x%llx "
140 			    " invalid in resumable error handler",
141 			    (long long) errh_flt.errh_er.desc);
142 			continue;
143 		}
144 
145 		aflt = (struct async_flt *)&(errh_flt.cmn_asyncflt);
146 		aflt->flt_id = gethrtime();
147 		aflt->flt_bus_id = getprocessorid();
148 		aflt->flt_class = CPU_FAULT;
149 		aflt->flt_prot = AFLT_PROT_NONE;
150 		aflt->flt_priv = (((errh_flt.errh_er.attr & ERRH_MODE_MASK)
151 		    >> ERRH_MODE_SHIFT) == ERRH_MODE_PRIV);
152 
153 		if (errh_flt.errh_er.attr & ERRH_ATTR_CPU)
154 			/* If it is an error on other cpu */
155 			aflt->flt_panic = 1;
156 		else
157 			aflt->flt_panic = 0;
158 
159 		/*
160 		 * Handle resumable queue full case.
161 		 */
162 		if (errh_flt.errh_er.attr & ERRH_ATTR_RQF) {
163 			(void) errh_rq_full(aflt);
164 		}
165 
166 		/*
167 		 * Queue the error on ce or ue queue depend on flt_panic.
168 		 * Even if flt_panic is set, the code still keep processing
169 		 * the rest element on rq until the panic starts.
170 		 */
171 		(void) cpu_queue_one_event(&errh_flt);
172 
173 		/*
174 		 * Panic here if aflt->flt_panic has been set.
175 		 * Enqueued errors will be logged as part of the panic flow.
176 		 */
177 		if (aflt->flt_panic) {
178 			fm_panic("Unrecoverable error on another CPU");
179 		}
180 	}
181 }
182 
183 void
184 process_nonresumable_error(struct regs *rp, uint64_t flags,
185     uint32_t head_offset, uint32_t tail_offset)
186 {
187 	struct machcpu *mcpup;
188 	struct async_flt *aflt;
189 	errh_async_flt_t errh_flt;
190 	errh_er_t *head_va;
191 	int trampolined = 0;
192 	int expected = DDI_FM_ERR_UNEXPECTED;
193 	uint64_t exec_mode;
194 	uint8_t u_spill_fill;
195 
196 	mcpup = &(CPU->cpu_m);
197 
198 	while (head_offset != tail_offset) {
199 		/* kernel buffer starts right after the nonresumable queue */
200 		head_va = (errh_er_t *)(mcpup->cpu_nrq_va + head_offset +
201 		    CPU_NRQ_SIZE);
202 
203 		/* Copy the error report to local buffer */
204 		bzero(&errh_flt, sizeof (errh_async_flt_t));
205 
206 		bcopy((char *)head_va, &(errh_flt.errh_er),
207 		    sizeof (errh_er_t));
208 
209 		/* Increment the queue head */
210 		head_offset += Q_ENTRY_SIZE;
211 		/* Wrap around */
212 		head_offset &= (CPU_NRQ_SIZE - 1);
213 
214 		/* set error handle to zero so it can hold new error report */
215 		head_va->ehdl = 0;
216 
217 		aflt = (struct async_flt *)&(errh_flt.cmn_asyncflt);
218 
219 		trampolined = 0;
220 
221 		if (errh_flt.errh_er.attr & ERRH_ATTR_PIO)
222 			aflt->flt_class = BUS_FAULT;
223 		else
224 			aflt->flt_class = CPU_FAULT;
225 
226 		aflt->flt_id = gethrtime();
227 		aflt->flt_bus_id = getprocessorid();
228 		aflt->flt_pc = (caddr_t)rp->r_pc;
229 		exec_mode = (errh_flt.errh_er.attr & ERRH_MODE_MASK)
230 		    >> ERRH_MODE_SHIFT;
231 		aflt->flt_priv = (exec_mode == ERRH_MODE_PRIV ||
232 		    exec_mode == ERRH_MODE_UNKNOWN);
233 		aflt->flt_prot = AFLT_PROT_NONE;
234 		aflt->flt_tl = (uchar_t)(flags & ERRH_TL_MASK);
235 		aflt->flt_panic = ((aflt->flt_tl != 0) ||
236 		    (aft_testfatal != 0));
237 
238 		/*
239 		 * For the first error packet on the queue, check if it
240 		 * happened in user fill/spill trap.
241 		 */
242 		if (flags & ERRH_U_SPILL_FILL) {
243 			u_spill_fill = 1;
244 			/* clear the user fill/spill flag in flags */
245 			flags = (uint64_t)aflt->flt_tl;
246 		} else
247 			u_spill_fill = 0;
248 
249 		switch (errh_flt.errh_er.desc) {
250 		case ERRH_DESC_PR_NRE:
251 			if (u_spill_fill) {
252 				aflt->flt_panic = 0;
253 				break;
254 			}
255 			/*
256 			 * Fall through, precise fault also need to check
257 			 * to see if it was protected.
258 			 */
259 			/*FALLTHRU*/
260 
261 		case ERRH_DESC_DEF_NRE:
262 			/*
263 			 * If the trap occurred in privileged mode at TL=0,
264 			 * we need to check to see if we were executing
265 			 * in kernel under on_trap() or t_lofault
266 			 * protection. If so, and if it was a PIO or MEM
267 			 * error, then modify the saved registers so that
268 			 * we return from the trap to the appropriate
269 			 * trampoline routine.
270 			 */
271 			if (aflt->flt_priv == 1 && aflt->flt_tl == 0 &&
272 			    ((errh_flt.errh_er.attr & ERRH_ATTR_PIO) ||
273 			    (errh_flt.errh_er.attr & ERRH_ATTR_MEM))) {
274 				trampolined =
275 				    errh_error_protected(rp, aflt, &expected);
276 			}
277 
278 			if (!aflt->flt_priv || aflt->flt_prot ==
279 			    AFLT_PROT_COPY) {
280 				aflt->flt_panic |= aft_panic;
281 			} else if (!trampolined &&
282 			    (aflt->flt_class != BUS_FAULT)) {
283 				aflt->flt_panic = 1;
284 			}
285 
286 			/*
287 			 * If PIO error, we need to query the bus nexus
288 			 * for fatal errors.
289 			 */
290 			if (aflt->flt_class == BUS_FAULT) {
291 				aflt->flt_addr = errh_flt.errh_er.ra;
292 				errh_cpu_run_bus_error_handlers(aflt,
293 				    expected);
294 			}
295 
296 			break;
297 
298 		default:
299 			cmn_err(CE_WARN, "Panic - Error Descriptor 0x%llx "
300 			    " invalid in non-resumable error handler",
301 			    (long long) errh_flt.errh_er.desc);
302 			aflt->flt_panic = 1;
303 			break;
304 		}
305 
306 		/*
307 		 * Queue the error report for further processing. If
308 		 * flt_panic is set, code still process other errors
309 		 * in the queue until the panic routine stops the
310 		 * kernel.
311 		 */
312 		(void) cpu_queue_one_event(&errh_flt);
313 
314 		/*
315 		 * Panic here if aflt->flt_panic has been set.
316 		 * Enqueued errors will be logged as part of the panic flow.
317 		 */
318 		if (aflt->flt_panic) {
319 			fm_panic("Unrecoverable hardware error");
320 		}
321 
322 		/*
323 		 * Call page_retire() to handle memory errors.
324 		 */
325 		if (errh_flt.errh_er.attr & ERRH_ATTR_MEM)
326 			errh_page_retire(&errh_flt, PR_UE);
327 
328 		/*
329 		 * If we queued an error and the it was in user mode, or
330 		 * protected by t_lofault, or user_spill_fill is set, we
331 		 * set AST flag so the queue will be drained before
332 		 * returning to user mode.
333 		 */
334 		if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY ||
335 		    u_spill_fill) {
336 			int pcb_flag = 0;
337 
338 			if (aflt->flt_class == CPU_FAULT)
339 				pcb_flag |= ASYNC_HWERR;
340 			else if (aflt->flt_class == BUS_FAULT)
341 				pcb_flag |= ASYNC_BERR;
342 
343 			ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag;
344 			aston(curthread);
345 		}
346 	}
347 }
348 
349 /*
350  * For PIO errors, this routine calls nexus driver's error
351  * callback routines. If the callback routine returns fatal, and
352  * we are in kernel or unknow mode without any error protection,
353  * we need to turn on the panic flag.
354  */
355 void
356 errh_cpu_run_bus_error_handlers(struct async_flt *aflt, int expected)
357 {
358 	int status;
359 	ddi_fm_error_t de;
360 
361 	bzero(&de, sizeof (ddi_fm_error_t));
362 
363 	de.fme_version = DDI_FME_VERSION;
364 	de.fme_ena = fm_ena_generate(aflt->flt_id, FM_ENA_FMT1);
365 	de.fme_flag = expected;
366 	de.fme_bus_specific = (void *)aflt->flt_addr;
367 	status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de);
368 
369 	/*
370 	 * If error is protected, it will jump to proper routine
371 	 * to handle the handle; if it is in user level, we just
372 	 * kill the user process; if the driver thinks the error is
373 	 * not fatal, we can drive on. If none of above are true,
374 	 * we panic
375 	 */
376 	if ((aflt->flt_prot == AFLT_PROT_NONE) && (aflt->flt_priv == 1) &&
377 	    (status == DDI_FM_FATAL))
378 		aflt->flt_panic = 1;
379 }
380 
381 /*
382  * This routine checks to see if we are under any error protection when
383  * the error happens. If we are under error protection, we unwind to
384  * the protection and indicate fault.
385  */
386 static int
387 errh_error_protected(struct regs *rp, struct async_flt *aflt, int *expected)
388 {
389 	int trampolined = 0;
390 	ddi_acc_hdl_t *hp;
391 
392 	if (curthread->t_ontrap != NULL) {
393 		on_trap_data_t *otp = curthread->t_ontrap;
394 
395 		if (otp->ot_prot & OT_DATA_EC) {
396 			aflt->flt_prot = AFLT_PROT_EC;
397 			otp->ot_trap |= OT_DATA_EC;
398 			rp->r_pc = otp->ot_trampoline;
399 			rp->r_npc = rp->r_pc +4;
400 			trampolined = 1;
401 		}
402 
403 		if (otp->ot_prot & OT_DATA_ACCESS) {
404 			aflt->flt_prot = AFLT_PROT_ACCESS;
405 			otp->ot_trap |= OT_DATA_ACCESS;
406 			rp->r_pc = otp->ot_trampoline;
407 			rp->r_npc = rp->r_pc + 4;
408 			trampolined = 1;
409 			/*
410 			 * for peek and caut_gets
411 			 * errors are expected
412 			 */
413 			hp = (ddi_acc_hdl_t *)otp->ot_handle;
414 			if (!hp)
415 				*expected = DDI_FM_ERR_PEEK;
416 			else if (hp->ah_acc.devacc_attr_access ==
417 			    DDI_CAUTIOUS_ACC)
418 				*expected = DDI_FM_ERR_EXPECTED;
419 		}
420 	} else if (curthread->t_lofault) {
421 		aflt->flt_prot = AFLT_PROT_COPY;
422 		rp->r_g1 = EFAULT;
423 		rp->r_pc = curthread->t_lofault;
424 		rp->r_npc = rp->r_pc + 4;
425 		trampolined = 1;
426 	}
427 
428 	return (trampolined);
429 }
430 
431 /*
432  * Queue one event.
433  */
434 static void
435 cpu_queue_one_event(errh_async_flt_t *errh_fltp)
436 {
437 	struct async_flt *aflt = (struct async_flt *)errh_fltp;
438 	errorq_t *eqp;
439 
440 	if (aflt->flt_panic)
441 		eqp = ue_queue;
442 	else
443 		eqp = ce_queue;
444 
445 	errorq_dispatch(eqp, errh_fltp, sizeof (errh_async_flt_t),
446 	    aflt->flt_panic);
447 }
448 
449 /*
450  * The cpu_async_log_err() function is called by the ce/ue_drain() function to
451  * handle logging for CPU events that are dequeued.  As such, it can be invoked
452  * from softint context, from AST processing in the trap() flow, or from the
453  * panic flow.  We decode the CPU-specific data, and log appropriate messages.
454  */
455 void
456 cpu_async_log_err(void *flt)
457 {
458 	errh_async_flt_t *errh_fltp = (errh_async_flt_t *)flt;
459 	errh_er_t *errh_erp = (errh_er_t *)&errh_fltp->errh_er;
460 
461 	switch (errh_erp->desc) {
462 	case ERRH_DESC_UCOR_RE:
463 		if (errh_erp->attr & ERRH_ATTR_MEM) {
464 			/*
465 			 * Turn on the PR_UE flag. The page will be
466 			 * scrubbed when it is freed.
467 			 */
468 			errh_page_retire(errh_fltp, PR_UE);
469 		}
470 
471 		break;
472 
473 	case ERRH_DESC_PR_NRE:
474 	case ERRH_DESC_DEF_NRE:
475 		if (errh_erp->attr & ERRH_ATTR_MEM) {
476 			/*
477 			 * For non-resumable memory error, retire
478 			 * the page here.
479 			 */
480 			errh_page_retire(errh_fltp, PR_UE);
481 
482 			/*
483 			 * If we are going to panic, scrub the page first
484 			 */
485 			if (errh_fltp->cmn_asyncflt.flt_panic)
486 				mem_scrub(errh_fltp->errh_er.ra,
487 				    errh_fltp->errh_er.sz);
488 		}
489 		break;
490 
491 	default:
492 		break;
493 	}
494 }
495 
496 /*
497  * Called from ce_drain().
498  */
499 void
500 cpu_ce_log_err(struct async_flt *aflt)
501 {
502 	switch (aflt->flt_class) {
503 	case CPU_FAULT:
504 		cpu_async_log_err(aflt);
505 		break;
506 
507 	case BUS_FAULT:
508 		cpu_async_log_err(aflt);
509 		break;
510 
511 	default:
512 		break;
513 	}
514 }
515 
516 /*
517  * Called from ue_drain().
518  */
519 void
520 cpu_ue_log_err(struct async_flt *aflt)
521 {
522 	switch (aflt->flt_class) {
523 	case CPU_FAULT:
524 		cpu_async_log_err(aflt);
525 		break;
526 
527 	case BUS_FAULT:
528 		cpu_async_log_err(aflt);
529 		break;
530 
531 	default:
532 		break;
533 	}
534 }
535 
536 /*
537  * Turn on flag on the error memory region.
538  */
539 static void
540 errh_page_retire(errh_async_flt_t *errh_fltp, uchar_t flag)
541 {
542 	uint64_t flt_real_addr_start = errh_fltp->errh_er.ra;
543 	uint64_t flt_real_addr_end = flt_real_addr_start +
544 	    errh_fltp->errh_er.sz - 1;
545 	int64_t current_addr;
546 
547 	if (errh_fltp->errh_er.sz == 0)
548 		return;
549 
550 	for (current_addr = flt_real_addr_start;
551 	    current_addr < flt_real_addr_end; current_addr += MMU_PAGESIZE) {
552 		(void) page_retire(current_addr, flag);
553 	}
554 }
555 
556 void
557 mem_scrub(uint64_t paddr, uint64_t len)
558 {
559 	uint64_t pa, length, scrubbed_len;
560 
561 	pa = paddr;
562 	length = len;
563 	scrubbed_len = 0;
564 
565 	while (length > 0) {
566 		if (hv_mem_scrub(pa, length, &scrubbed_len) != H_EOK)
567 			break;
568 
569 		pa += scrubbed_len;
570 		length -= scrubbed_len;
571 	}
572 }
573 
574 /*
575  * Call hypervisor to flush the memory region. The memory region
576  * must be within the same page frame.
577  */
578 void
579 mem_sync(caddr_t va, size_t len)
580 {
581 	uint64_t pa, length, flushed;
582 
583 	pa = va_to_pa((caddr_t)va);
584 
585 	if (pa == (uint64_t)-1)
586 		return;
587 
588 	ASSERT((pa >> MMU_PAGESHIFT) == ((pa + len - 1) >> MMU_PAGESHIFT));
589 
590 	length = len;
591 	flushed = 0;
592 
593 	while (length > 0) {
594 		if (hv_mem_sync(pa, length, &flushed) != H_EOK)
595 			break;
596 
597 		pa += flushed;
598 		length -= flushed;
599 	}
600 }
601 
602 /*
603  * If resumable queue is full, we need to check if any cpu is in
604  * error state. If not, we drive on. If yes, we need to panic. The
605  * hypervisor call hv_cpu_state() is being used for checking the
606  * cpu state.
607  */
608 static void
609 errh_rq_full(struct async_flt *afltp)
610 {
611 	processorid_t who;
612 	uint64_t cpu_state;
613 	uint64_t retval;
614 
615 	for (who = 0; who < NCPU; who++)
616 		if (CPU_IN_SET(cpu_ready_set, who)) {
617 			retval = hv_cpu_state(who, &cpu_state);
618 			if (retval != H_EOK || cpu_state == CPU_STATE_ERROR) {
619 				afltp->flt_panic = 1;
620 				break;
621 			}
622 		}
623 }
624 
625 /*
626  * Return processor specific async error structure
627  * size used.
628  */
629 int
630 cpu_aflt_size(void)
631 {
632 	return (sizeof (errh_async_flt_t));
633 }
634 
635 #define	SZ_TO_ETRS_SHIFT	6
636 
637 /*
638  * Message print out when resumable queue is overflown
639  */
640 /*ARGSUSED*/
641 void
642 rq_overflow(struct regs *rp, uint64_t head_offset,
643     uint64_t tail_offset)
644 {
645 	rq_overflow_count++;
646 }
647 
648 /*
649  * Handler to process a fatal error.  This routine can be called from a
650  * softint, called from trap()'s AST handling, or called from the panic flow.
651  */
652 /*ARGSUSED*/
653 static void
654 ue_drain(void *ignored, struct async_flt *aflt, errorq_elem_t *eqep)
655 {
656 	cpu_ue_log_err(aflt);
657 }
658 
659 /*
660  * Handler to process a correctable error.  This routine can be called from a
661  * softint.  We just call the CPU module's logging routine.
662  */
663 /*ARGSUSED*/
664 static void
665 ce_drain(void *ignored, struct async_flt *aflt, errorq_elem_t *eqep)
666 {
667 	cpu_ce_log_err(aflt);
668 }
669 
670 /*
671  * Handler to process vbsc hostshutdown (power-off button).
672  */
673 static int
674 err_shutdown_softintr()
675 {
676 	cmn_err(CE_WARN, "Power-off requested, system will now shutdown.");
677 	do_shutdown();
678 
679 	/*
680 	 * just in case do_shutdown() fails
681 	 */
682 	(void) timeout((void(*)(void *))power_down, NULL, 100 * hz);
683 	return (DDI_INTR_CLAIMED);
684 }
685 
686 /*
687  * Allocate error queue sizes based on max_ncpus.  max_ncpus is set just
688  * after ncpunode has been determined.  ncpus is set in start_other_cpus
689  * which is called after error_init() but may change dynamically.
690  */
691 void
692 error_init(void)
693 {
694 	char tmp_name[MAXSYSNAME];
695 	pnode_t node;
696 	size_t size = cpu_aflt_size();
697 
698 	/*
699 	 * Initialize the correctable and uncorrectable error queues.
700 	 */
701 	ue_queue = errorq_create("ue_queue", (errorq_func_t)ue_drain, NULL,
702 	    MAX_ASYNC_FLTS * (max_ncpus + 1), size, PIL_2, ERRORQ_VITAL);
703 
704 	ce_queue = errorq_create("ce_queue", (errorq_func_t)ce_drain, NULL,
705 	    MAX_CE_FLTS * (max_ncpus + 1), size, PIL_1, 0);
706 
707 	if (ue_queue == NULL || ce_queue == NULL)
708 		panic("failed to create required system error queue");
709 
710 	/*
711 	 * Setup interrupt handler for power-off button.
712 	 */
713 	err_shutdown_inum = add_softintr(PIL_9,
714 	    (softintrfunc)err_shutdown_softintr, NULL);
715 
716 	/*
717 	 * Initialize the busfunc list mutex.  This must be a PIL_15 spin lock
718 	 * because we will need to acquire it from cpu_async_error().
719 	 */
720 	mutex_init(&bfd_lock, NULL, MUTEX_SPIN, (void *)PIL_15);
721 
722 	node = prom_rootnode();
723 	if ((node == OBP_NONODE) || (node == OBP_BADNODE)) {
724 		cmn_err(CE_CONT, "error_init: node 0x%x\n", (uint_t)node);
725 		return;
726 	}
727 
728 	if (((size = prom_getproplen(node, "reset-reason")) != -1) &&
729 	    (size <= MAXSYSNAME) &&
730 	    (prom_getprop(node, "reset-reason", tmp_name) != -1)) {
731 		if (reset_debug) {
732 			cmn_err(CE_CONT, "System booting after %s\n", tmp_name);
733 		} else if (strncmp(tmp_name, "FATAL", 5) == 0) {
734 			cmn_err(CE_CONT,
735 			    "System booting after fatal error %s\n", tmp_name);
736 		}
737 	}
738 }
739 
740 /*
741  * Nonresumable queue is full, panic here
742  */
743 /*ARGSUSED*/
744 void
745 nrq_overflow(struct regs *rp)
746 {
747 	fm_panic("Nonresumable queue full");
748 }
749