xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 940f2f58)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/types.h>
26 #include <fmadm.h>
27 #include <errno.h>
28 #include <limits.h>
29 #include <strings.h>
30 #include <stdio.h>
31 #include <unistd.h>
32 #include <sys/wait.h>
33 #include <sys/stat.h>
34 #include <fcntl.h>
35 #include <fm/fmd_log.h>
36 #include <sys/fm/protocol.h>
37 #include <fm/libtopo.h>
38 #include <fm/fmd_adm.h>
39 #include <fm/fmd_msg.h>
40 #include <dlfcn.h>
41 #include <sys/systeminfo.h>
42 #include <sys/utsname.h>
43 #include <libintl.h>
44 #include <locale.h>
45 #include <sys/smbios.h>
46 #include <libdevinfo.h>
47 #include <stdlib.h>
48 #include <stddef.h>
49 
50 /*
51  * Fault records are added to catalog by calling add_fault_record_to_catalog()
52  * records are stored in order of importance to the system.
53  * If -g flag is set or not_suppressed is not set and the class fru, fault,
54  * type are the same then details are merged into an existing record, with uuid
55  * records are stored in time order.
56  * For each record information is extracted from nvlist and merged into linked
57  * list each is checked for identical records for which percentage certainty are
58  * added together.
59  * print_catalog() is called to print out catalog and release external resources
60  *
61  *                         /---------------\
62  *	status_rec_list -> |               | -|
63  *                         \---------------/
64  *                                \/
65  *                         /---------------\    /-------\    /-------\
66  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
67  *            \/           |               | |- |       | <- |       |
68  *      /-------------\    |               |    \-------/    \-------/
69  *      |             | -> |               |       \/           \/
70  *      \-------------/    |               |    /-------\    /-------\
71  *            \/           |               | -> | asru  | -> | asru  |
72  *            ---          |               |    |       | <- |       |
73  *                         |               |    \-------/    \-------/
74  *      status_asru_list   |  class        |
75  *            \/           |  resource     |    /-------\    /-------\
76  *      /-------------\    |  fru          | -> | list  | -> | list  |
77  *      |             | -> |  serial       |    |       | <- |       |
78  *      \-------------/    |               |    \-------/    \-------/
79  *            \/           \---------------/
80  *            ---               \/    /\
81  *                         /---------------\
82  *                         | status_record |
83  *                         \---------------/
84  *
85  * Fmadm faulty takes a number of options which affect the format of the
86  * output displayed. By default, the display reports the FRU and ASRU along
87  * with other information on per-case basis as in the example below.
88  *
89  * --------------- ------------------------------------  -------------- -------
90  * TIME            EVENT-ID                              MSG-ID         SEVERITY
91  * --------------- ------------------------------------  -------------- -------
92  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
93  *
94  * Fault class	: fault.memory.dimm_sb
95  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
96  *		    faulted but still in service
97  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
98  *		    faulty
99  *
100  * Description	: The number of errors associated with this memory module has
101  *		exceeded acceptable levels.  Refer to
102  *		http://illumos.org/msg/AMD-8000-2F for more information.
103  *
104  * Response	: Pages of memory associated with this memory module are being
105  *		removed from service as errors are reported.
106  *
107  * Impact	: Total system memory capacity will be reduced as pages are
108  *		retired.
109  *
110  * Action	: Schedule a repair procedure to replace the affected memory
111  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
112  *
113  * The -v flag is similar, but adds some additonal information such as the
114  * resource. The -s flag is also similar but just gives the top line summary.
115  * All these options (ie without the -f or -r flags) use the print_catalog()
116  * function to do the display.
117  *
118  * The -f flag changes the output so that it appears sorted on a per-fru basis.
119  * The output is somewhat cut down compared to the default output. If -f is
120  * used, then print_fru() is used to print the output.
121  *
122  * -----------------------------------------------------------------------------
123  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
124  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
125  *
126  * Description	: A problem was detected for a PCI device.
127  *		Refer to http://illumos.org/msg/PCI-8000-7J
128  *		for more information.
129  *
130  * Response	: One or more device instances may be disabled
131  *
132  * Impact	: Possible loss of services provided by the device instances
133  *		associated with this fault
134  *
135  * Action	: Schedule a repair procedure to replace the affected device.
136  *		Use fmdump -v -u <EVENT_ID> to identify the device or contact
137  *		Sun for support.
138  *
139  * The -r flag changes the output so that it appears sorted on a per-asru basis.
140  * The output is very much cut down compared to the default output, just giving
141  * the asru fmri and state. Here print_asru() is used to print the output.
142  *
143  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
144  *
145  * For all fmadm faulty options, the sequence of events is
146  *
147  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
148  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
149  * This will extract the data from the nvlist and call catalog_new_record() to
150  * save the data away in various linked lists in the catalogue.
151  *
152  * 2) Once this is done, the data can be supplemented by using
153  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
154  *
155  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
156  * appropriate to display the information from the catalogue sorted in the
157  * requested way.
158  *
159  */
160 
161 typedef struct name_list {
162 	struct name_list *next;
163 	struct name_list *prev;
164 	char *name;
165 	uint8_t pct;
166 	uint8_t max_pct;
167 	ushort_t count;
168 	int status;
169 	char *label;
170 } name_list_t;
171 
172 typedef struct ari_list {
173 	char *ari_uuid;
174 	struct ari_list *next;
175 } ari_list_t;
176 
177 typedef struct uurec {
178 	struct uurec *next;
179 	struct uurec *prev;
180 	char *uuid;
181 	ari_list_t *ari_uuid_list;
182 	name_list_t *asru;
183 	uint64_t sec;
184 	nvlist_t *event;
185 } uurec_t;
186 
187 typedef struct uurec_select {
188 	struct uurec_select *next;
189 	char *uuid;
190 } uurec_select_t;
191 
192 typedef struct host_id {
193 	char *chassis;
194 	char *server;
195 	char *platform;
196 	char *domain;
197 	char *product_sn;
198 } hostid_t;
199 
200 typedef struct host_id_list {
201 	hostid_t hostid;
202 	struct host_id_list *next;
203 } host_id_list_t;
204 
205 typedef struct status_record {
206 	hostid_t *host;
207 	int nrecs;
208 	uurec_t *uurec;
209 	char *severity;			/* in C locale */
210 	char *msgid;
211 	name_list_t *class;
212 	name_list_t *resource;
213 	name_list_t *asru;
214 	name_list_t *fru;
215 	name_list_t *serial;
216 	uint8_t not_suppressed;
217 	uint8_t injected;
218 } status_record_t;
219 
220 typedef struct sr_list {
221 	struct sr_list *next;
222 	struct sr_list *prev;
223 	struct status_record *status_record;
224 } sr_list_t;
225 
226 typedef struct resource_list {
227 	struct resource_list *next;
228 	struct resource_list *prev;
229 	sr_list_t *status_rec_list;
230 	char *resource;
231 	uint8_t not_suppressed;
232 	uint8_t injected;
233 	uint8_t max_pct;
234 } resource_list_t;
235 
236 sr_list_t *status_rec_list;
237 resource_list_t *status_fru_list;
238 resource_list_t *status_asru_list;
239 
240 static int max_display;
241 static int max_fault = 0;
242 static topo_hdl_t *topo_handle;
243 static host_id_list_t *host_list;
244 static int n_server;
245 static int opt_g;
246 static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */
247 
248 static char *
249 format_date(char *buf, size_t len, uint64_t sec)
250 {
251 	if (sec > LONG_MAX) {
252 		(void) fprintf(stderr,
253 		    "record time is too large for 32-bit utility\n");
254 		(void) snprintf(buf, len, "0x%llx", sec);
255 	} else {
256 		time_t tod = (time_t)sec;
257 		time_t now = time(NULL);
258 		if (tod > now+60 ||
259 		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
260 			(void) strftime(buf, len, "%b %d %Y    ",
261 			    localtime(&tod));
262 		} else {
263 			(void) strftime(buf, len, "%b %d %T", localtime(&tod));
264 		}
265 	}
266 
267 	return (buf);
268 }
269 
270 static hostid_t *
271 find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
272     char *product_sn)
273 {
274 	hostid_t *rt = NULL;
275 	host_id_list_t *hostp;
276 
277 	if (platform == NULL)
278 		platform = "-";
279 	if (server == NULL)
280 		server = "-";
281 	hostp = host_list;
282 	while (hostp) {
283 		if (hostp->hostid.platform &&
284 		    strcmp(hostp->hostid.platform, platform) == 0 &&
285 		    hostp->hostid.server &&
286 		    strcmp(hostp->hostid.server, server) == 0 &&
287 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
288 		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
289 		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
290 		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
291 		    (domain == NULL || hostp->hostid.domain == NULL ||
292 		    strcmp(domain, hostp->hostid.domain) == 0)) {
293 			rt = &hostp->hostid;
294 			break;
295 		}
296 		hostp = hostp->next;
297 	}
298 	if (rt == NULL) {
299 		hostp = malloc(sizeof (host_id_list_t));
300 		hostp->hostid.platform = strdup(platform);
301 		hostp->hostid.product_sn =
302 		    product_sn ? strdup(product_sn) : NULL;
303 		hostp->hostid.server = strdup(server);
304 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
305 		hostp->hostid.domain = domain ? strdup(domain) : NULL;
306 		hostp->next = host_list;
307 		host_list = hostp;
308 		rt = &hostp->hostid;
309 		n_server++;
310 	}
311 	return (rt);
312 }
313 
314 static hostid_t *
315 find_hostid(nvlist_t *nvl)
316 {
317 	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
318 	char *product_sn = NULL;
319 	nvlist_t *auth, *fmri;
320 	hostid_t *rt = NULL;
321 
322 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
323 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
324 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
325 		    &platform);
326 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
327 		    &product_sn);
328 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
329 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
330 		    &chassis);
331 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
332 		rt = find_hostid_in_list(platform, chassis, server,
333 		    domain, product_sn);
334 	}
335 	return (rt);
336 }
337 
338 static char *
339 get_nvl2str_topo(nvlist_t *nvl)
340 {
341 	char *name = NULL;
342 	char *tname;
343 	int err;
344 	char *scheme = NULL;
345 	char *mod_name = NULL;
346 	char buf[128];
347 
348 	if (topo_handle == NULL)
349 		topo_handle = topo_open(TOPO_VERSION, 0, &err);
350 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
351 		name = strdup(tname);
352 		topo_hdl_strfree(topo_handle, tname);
353 	} else {
354 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
355 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
356 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
357 		    mod_name) {
358 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
359 			    scheme, mod_name);
360 			name = strdup(buf);
361 		}
362 	}
363 	return (name);
364 }
365 
366 static int
367 set_priority(char *s)
368 {
369 	int rt = 0;
370 
371 	if (s) {
372 		if (strcmp(s, "Minor") == 0)
373 			rt = 1;
374 		else if (strcmp(s, "Major") == 0)
375 			rt = 10;
376 		else if (strcmp(s, "Critical") == 0)
377 			rt = 100;
378 	}
379 	return (rt);
380 }
381 
382 static int
383 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
384     uint8_t p2)
385 {
386 	int r1, r2;
387 	int rt;
388 
389 	r1 = set_priority(s1);
390 	r2 = set_priority(s2);
391 	rt = r1 - r2;
392 	if (rt == 0) {
393 		if (t1 > t2)
394 			rt = 1;
395 		else if (t1 < t2)
396 			rt = -1;
397 		else
398 			rt = p1 - p2;
399 	}
400 	return (rt);
401 }
402 
403 /*
404  * merge two lists into one, by comparing enties in new and moving into list if
405  * name is not there or free off memory for names which are already there
406  * add_pct indicates if pct is the sum or highest pct
407  */
408 static name_list_t *
409 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
410 {
411 	name_list_t *lp, *np, *sp, *rt = NULL;
412 	int max_pct;
413 
414 	rt = *list;
415 	np = new;
416 	while (np) {
417 		lp = *list;
418 		while (lp) {
419 			if (strcmp(lp->name, np->name) == 0)
420 				break;
421 			lp = lp->next;
422 			if (lp == *list)
423 				lp = NULL;
424 		}
425 		if (np->next == new)
426 			sp = NULL;
427 		else
428 			sp = np->next;
429 		if (lp) {
430 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
431 			if (add_pct) {
432 				lp->pct += np->pct;
433 				lp->count += np->count;
434 			} else if (np->pct > lp->pct) {
435 				lp->pct = np->pct;
436 			}
437 			max_pct = np->max_pct;
438 			if (np->label)
439 				free(np->label);
440 			free(np->name);
441 			free(np);
442 			np = NULL;
443 			if (max_pct > lp->max_pct) {
444 				lp->max_pct = max_pct;
445 				if (lp->max_pct > lp->prev->max_pct &&
446 				    lp != *list) {
447 					lp->prev->next = lp->next;
448 					lp->next->prev = lp->prev;
449 					np = lp;
450 				}
451 			}
452 		}
453 		if (np) {
454 			lp = *list;
455 			if (lp) {
456 				if (np->max_pct > lp->max_pct) {
457 					np->next = lp;
458 					np->prev = lp->prev;
459 					lp->prev->next = np;
460 					lp->prev = np;
461 					*list = np;
462 					rt = np;
463 				} else {
464 					lp = lp->next;
465 					while (lp != *list &&
466 					    np->max_pct < lp->max_pct) {
467 						lp = lp->next;
468 					}
469 					np->next = lp;
470 					np->prev = lp->prev;
471 					lp->prev->next = np;
472 					lp->prev = np;
473 				}
474 			} else {
475 				*list = np;
476 				np->next = np;
477 				np->prev = np;
478 				rt = np;
479 			}
480 		}
481 		np = sp;
482 	}
483 	return (rt);
484 }
485 
486 static name_list_t *
487 alloc_name_list(char *name, uint8_t pct)
488 {
489 	name_list_t *nlp;
490 
491 	nlp = malloc(sizeof (*nlp));
492 	nlp->name = strdup(name);
493 	nlp->pct = pct;
494 	nlp->max_pct = pct;
495 	nlp->count = 1;
496 	nlp->next = nlp;
497 	nlp->prev = nlp;
498 	nlp->status = 0;
499 	nlp->label = NULL;
500 	return (nlp);
501 }
502 
503 static status_record_t *
504 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
505     name_list_t *fru, name_list_t *asru, name_list_t *resource,
506     name_list_t *serial, boolean_t not_suppressed,
507     hostid_t *hostid, boolean_t injected)
508 {
509 	status_record_t *status_rec_p;
510 
511 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
512 	status_rec_p->nrecs = 1;
513 	status_rec_p->host = hostid;
514 	status_rec_p->uurec = uurec_p;
515 	uurec_p->next = NULL;
516 	uurec_p->prev = NULL;
517 	uurec_p->asru = asru;
518 	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
519 	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
520 		status_rec_p->severity = strdup("unknown");
521 	status_rec_p->class = class;
522 	status_rec_p->fru = fru;
523 	status_rec_p->asru = asru;
524 	status_rec_p->resource = resource;
525 	status_rec_p->serial = serial;
526 	status_rec_p->msgid = strdup(msgid);
527 	status_rec_p->not_suppressed = not_suppressed;
528 	status_rec_p->injected = injected;
529 	return (status_rec_p);
530 }
531 
532 /*
533  * add record to given list maintaining order higher priority first.
534  */
535 static void
536 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
537 {
538 	sr_list_t *tp, *np, *sp;
539 	int order;
540 	uint64_t sec;
541 
542 	np = malloc(sizeof (sr_list_t));
543 	np->status_record = status_rec_p;
544 	sec = status_rec_p->uurec->sec;
545 	if ((sp = *list_pp) == NULL) {
546 		*list_pp = np;
547 		np->next = np;
548 		np->prev = np;
549 	} else {
550 		/* insert new record in front of lower priority */
551 		tp = sp;
552 		order = cmp_priority(status_rec_p->severity,
553 		    sp->status_record->severity, sec,
554 		    tp->status_record->uurec->sec, 0, 0);
555 		if (order > 0) {
556 			*list_pp = np;
557 		} else {
558 			tp = sp->next;
559 			while (tp != sp &&
560 			    cmp_priority(status_rec_p->severity,
561 			    tp->status_record->severity, sec,
562 			    tp->status_record->uurec->sec, 0, 0)) {
563 				tp = tp->next;
564 			}
565 		}
566 		np->next = tp;
567 		np->prev = tp->prev;
568 		tp->prev->next = np;
569 		tp->prev = np;
570 	}
571 }
572 
573 static void
574 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
575     resource_list_t *np)
576 {
577 	int order;
578 	uint64_t sec;
579 	resource_list_t *sp, *tp;
580 	status_record_t *srp;
581 	char *severity = status_rec_p->severity;
582 
583 	add_rec_list(status_rec_p, &np->status_rec_list);
584 	if ((sp = *rp) == NULL) {
585 		np->next = np;
586 		np->prev = np;
587 		*rp = np;
588 	} else {
589 		/*
590 		 * insert new record in front of lower priority
591 		 */
592 		tp = sp->next;
593 		srp = sp->status_rec_list->status_record;
594 		sec = status_rec_p->uurec->sec;
595 		order = cmp_priority(severity, srp->severity, sec,
596 		    srp->uurec->sec, np->max_pct, sp->max_pct);
597 		if (order > 0) {
598 			*rp = np;
599 		} else {
600 			srp = tp->status_rec_list->status_record;
601 			while (tp != sp &&
602 			    cmp_priority(severity, srp->severity, sec,
603 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
604 				tp = tp->next;
605 				srp = tp->status_rec_list->status_record;
606 			}
607 		}
608 		np->next = tp;
609 		np->prev = tp->prev;
610 		tp->prev->next = np;
611 		tp->prev = np;
612 	}
613 }
614 
615 static void
616 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
617     resource_list_t **rpp)
618 {
619 	int order;
620 	resource_list_t *np, *end;
621 	status_record_t *srp;
622 
623 	np = *rpp;
624 	end = np;
625 	while (np) {
626 		if (strcmp(fp->name, np->resource) == 0) {
627 			np->not_suppressed |= status_rec_p->not_suppressed;
628 			np->injected |= status_rec_p->injected;
629 			srp = np->status_rec_list->status_record;
630 			order = cmp_priority(status_rec_p->severity,
631 			    srp->severity, status_rec_p->uurec->sec,
632 			    srp->uurec->sec, fp->max_pct, np->max_pct);
633 			if (order > 0 && np != end) {
634 				/*
635 				 * remove from list and add again using
636 				 * new priority
637 				 */
638 				np->prev->next = np->next;
639 				np->next->prev = np->prev;
640 				add_resource(status_rec_p,
641 				    rpp, np);
642 			} else {
643 				add_rec_list(status_rec_p,
644 				    &np->status_rec_list);
645 			}
646 			break;
647 		}
648 		np = np->next;
649 		if (np == end) {
650 			np = NULL;
651 			break;
652 		}
653 	}
654 	if (np == NULL) {
655 		np = malloc(sizeof (resource_list_t));
656 		np->resource = fp->name;
657 		np->not_suppressed = status_rec_p->not_suppressed;
658 		np->injected = status_rec_p->injected;
659 		np->status_rec_list = NULL;
660 		np->max_pct = fp->max_pct;
661 		add_resource(status_rec_p, rpp, np);
662 	}
663 }
664 
665 static void
666 add_list(status_record_t *status_rec_p, name_list_t *listp,
667     resource_list_t **glistp)
668 {
669 	name_list_t *fp, *end;
670 
671 	fp = listp;
672 	end = fp;
673 	while (fp) {
674 		add_resource_list(status_rec_p, fp, glistp);
675 		fp = fp->next;
676 		if (fp == end)
677 			break;
678 	}
679 }
680 
681 /*
682  * add record to rec, fru and asru lists.
683  */
684 static void
685 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
686     name_list_t *fru, name_list_t *asru, name_list_t *resource,
687     name_list_t *serial, boolean_t not_suppressed,
688     hostid_t *hostid, boolean_t injected, boolean_t dummy_fru)
689 {
690 	status_record_t *status_rec_p;
691 
692 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
693 	    resource, serial, not_suppressed, hostid, injected);
694 	add_rec_list(status_rec_p, &status_rec_list);
695 	if (status_rec_p->fru && !dummy_fru)
696 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
697 	if (status_rec_p->asru)
698 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
699 }
700 
701 static void
702 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
703 {
704 	char *name;
705 	char *serial = NULL;
706 	char **lserial = NULL;
707 	uint64_t serint;
708 	name_list_t *nlp;
709 	int j;
710 	uint_t nelem;
711 	char buf[64];
712 
713 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
714 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
715 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
716 			    &serint) == 0) {
717 				(void) snprintf(buf, sizeof (buf), "%llX",
718 				    serint);
719 				nlp = alloc_name_list(buf, pct);
720 				(void) merge_name_list(serial_p, nlp, 1);
721 			}
722 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
723 			if (nvlist_lookup_string_array(nvl,
724 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
725 				nlp = alloc_name_list(lserial[0], pct);
726 				for (j = 1; j < nelem; j++) {
727 					name_list_t *n1lp;
728 					n1lp = alloc_name_list(lserial[j], pct);
729 					(void) merge_name_list(&nlp, n1lp, 1);
730 				}
731 				(void) merge_name_list(serial_p, nlp, 1);
732 			}
733 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
734 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
735 			    &serial) == 0) {
736 				nlp = alloc_name_list(serial, pct);
737 				(void) merge_name_list(serial_p, nlp, 1);
738 			}
739 		}
740 	}
741 }
742 
743 static void
744 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
745     name_list_t **fru_p, name_list_t **serial_p, name_list_t **resource_p,
746     name_list_t **asru_p, boolean_t *dummy_fru, uint8_t status)
747 {
748 	nvlist_t *lfru, *lasru, *rsrc;
749 	name_list_t *nlp;
750 	char *name;
751 	uint8_t lpct = 0;
752 	char *lclass = NULL;
753 	char *label;
754 
755 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
756 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
757 		nlp = alloc_name_list(lclass, lpct);
758 		(void) merge_name_list(class_p, nlp, 1);
759 	}
760 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
761 		name = get_nvl2str_topo(lfru);
762 		if (name != NULL) {
763 			nlp = alloc_name_list(name, lpct);
764 			nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
765 			    FM_SUSPECT_DEGRADED);
766 			free(name);
767 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
768 			    &label) == 0)
769 				nlp->label = strdup(label);
770 			(void) merge_name_list(fru_p, nlp, 1);
771 		}
772 		get_serial_no(lfru, serial_p, lpct);
773 	} else if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) != 0) {
774 		/*
775 		 * No FRU or resource. But we want to display the repair status
776 		 * somehow, so create a dummy FRU field.
777 		 */
778 		*dummy_fru = 1;
779 		nlp = alloc_name_list(dgettext("FMD", "None"), lpct);
780 		nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
781 		    FM_SUSPECT_DEGRADED);
782 		(void) merge_name_list(fru_p, nlp, 1);
783 	}
784 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
785 		name = get_nvl2str_topo(lasru);
786 		if (name != NULL) {
787 			nlp = alloc_name_list(name, lpct);
788 			nlp->status = status & ~(FM_SUSPECT_NOT_PRESENT |
789 			    FM_SUSPECT_REPAIRED | FM_SUSPECT_REPLACED |
790 			    FM_SUSPECT_ACQUITTED);
791 			free(name);
792 			(void) merge_name_list(asru_p, nlp, 1);
793 		}
794 		get_serial_no(lasru, serial_p, lpct);
795 	}
796 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
797 		name = get_nvl2str_topo(rsrc);
798 		if (name != NULL) {
799 			nlp = alloc_name_list(name, lpct);
800 			nlp->status = status;
801 			free(name);
802 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
803 			    &label) == 0)
804 				nlp->label = strdup(label);
805 			(void) merge_name_list(resource_p, nlp, 1);
806 		}
807 	}
808 }
809 
810 static void
811 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid)
812 {
813 	char *msgid = "-";
814 	uint_t i, size = 0;
815 	name_list_t *class = NULL, *resource = NULL;
816 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
817 	nvlist_t **nva;
818 	uint8_t *ba;
819 	uurec_t *uurec_p;
820 	hostid_t *host;
821 	boolean_t not_suppressed = 1;
822 	boolean_t any_present = 0;
823 	boolean_t injected = 0;
824 	boolean_t dummy_fru = 0;
825 
826 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
827 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
828 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
829 	    &not_suppressed);
830 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_INJECTED, &injected);
831 
832 	if (size != 0) {
833 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
834 		    &nva, &size);
835 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
836 		    &ba, &size);
837 		for (i = 0; i < size; i++) {
838 			extract_record_info(nva[i], &class, &fru, &serial,
839 			    &resource, &asru, &dummy_fru, ba[i]);
840 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
841 			    (ba[i] & FM_SUSPECT_FAULTY))
842 				any_present = 1;
843 		}
844 		/*
845 		 * also suppress if no resources present
846 		 */
847 		if (any_present == 0)
848 			not_suppressed = 0;
849 	}
850 
851 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
852 	uurec_p->uuid = strdup(uuid);
853 	uurec_p->sec = sec;
854 	uurec_p->ari_uuid_list = NULL;
855 	uurec_p->event = NULL;
856 	(void) nvlist_dup(nvl, &uurec_p->event, 0);
857 	host = find_hostid(nvl);
858 	catalog_new_record(uurec_p, msgid, class, fru, asru,
859 	    resource, serial, not_suppressed, host, injected, dummy_fru);
860 }
861 
862 static void
863 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
864 {
865 	sr_list_t *srp;
866 	uurec_t *uurp;
867 	ari_list_t *ari_list;
868 
869 	srp = status_rec_list;
870 	if (srp) {
871 		for (;;) {
872 			uurp = srp->status_record->uurec;
873 			while (uurp) {
874 				if (strcmp(uuid, uurp->uuid) == 0) {
875 					ari_list = (ari_list_t *)
876 					    malloc(sizeof (ari_list_t));
877 					ari_list->ari_uuid = strdup(ari_uuid);
878 					ari_list->next = uurp->ari_uuid_list;
879 					uurp->ari_uuid_list = ari_list;
880 					return;
881 				}
882 				uurp = uurp->next;
883 			}
884 			if (srp->next == status_rec_list)
885 				break;
886 			srp = srp->next;
887 		}
888 	}
889 }
890 
891 static void
892 print_line(char *label, char *buf)
893 {
894 	char *cp, *ep, *wp;
895 	char c;
896 	int i;
897 	int lsz;
898 	char *padding;
899 
900 	lsz = strlen(label);
901 	padding = malloc(lsz + 1);
902 	for (i = 0; i < lsz; i++)
903 		padding[i] = ' ';
904 	padding[i] = 0;
905 	cp = buf;
906 	ep = buf;
907 	c = *ep;
908 	(void) printf("\n");
909 	while (c) {
910 		i = lsz;
911 		wp = NULL;
912 		while ((c = *ep) != '\0' && (wp == NULL || i < 80)) {
913 			if (c == ' ')
914 				wp = ep;
915 			else if (c == '\n') {
916 				i = 0;
917 				*ep = 0;
918 				do {
919 					ep++;
920 				} while ((c = *ep) != '\0' && c == ' ');
921 				break;
922 			}
923 			ep++;
924 			i++;
925 		}
926 		if (i >= 80 && wp) {
927 			*wp = 0;
928 			ep = wp + 1;
929 			c = *ep;
930 		}
931 		(void) printf("%s%s\n", label, cp);
932 		cp = ep;
933 		label = padding;
934 	}
935 	free(padding);
936 }
937 
938 static void
939 print_dict_info_line(nvlist_t *e, fmd_msg_item_t what, const char *linehdr)
940 {
941 	char *cp = fmd_msg_getitem_nv(fmadm_msghdl, NULL, e, what);
942 
943 	if (cp) {
944 		print_line(dgettext("FMD", linehdr), cp);
945 		free(cp);
946 	}
947 }
948 
949 static void
950 print_dict_info(nvlist_t *nvl)
951 {
952 	print_dict_info_line(nvl, FMD_MSG_ITEM_DESC, "Description : ");
953 	print_dict_info_line(nvl, FMD_MSG_ITEM_RESPONSE, "Response    : ");
954 	print_dict_info_line(nvl, FMD_MSG_ITEM_IMPACT, "Impact      : ");
955 	print_dict_info_line(nvl, FMD_MSG_ITEM_ACTION, "Action      : ");
956 }
957 
958 static void
959 print_name(name_list_t *list, char *padding, int *np, int pct, int full)
960 {
961 	char *name;
962 
963 	name = list->name;
964 	if (list->label) {
965 		(void) printf("%s \"%s\" (%s)", padding, list->label, name);
966 		*np += 1;
967 	} else {
968 		(void) printf("%s %s", padding, name);
969 		*np += 1;
970 	}
971 	if (list->pct && pct > 0 && pct < 100) {
972 		if (list->count > 1) {
973 			if (full) {
974 				(void) printf(" %d @ %s %d%%\n", list->count,
975 				    dgettext("FMD", "max"),
976 				    list->max_pct);
977 			} else {
978 				(void) printf(" %s %d%%\n",
979 				    dgettext("FMD", "max"),
980 				    list->max_pct);
981 			}
982 		} else {
983 			(void) printf(" %d%%\n", list->pct);
984 		}
985 	} else {
986 		(void) printf("\n");
987 	}
988 }
989 
990 static void
991 print_asru_status(int status, char *label)
992 {
993 	char *msg = NULL;
994 
995 	switch (status) {
996 	case 0:
997 		msg = dgettext("FMD", "ok and in service");
998 		break;
999 	case FM_SUSPECT_DEGRADED:
1000 		msg = dgettext("FMD", "service degraded, "
1001 		    "but associated components no longer faulty");
1002 		break;
1003 	case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1004 		msg = dgettext("FMD", "faulted but still "
1005 		    "providing degraded service");
1006 		break;
1007 	case FM_SUSPECT_FAULTY:
1008 		msg = dgettext("FMD", "faulted but still in service");
1009 		break;
1010 	case FM_SUSPECT_UNUSABLE:
1011 		msg = dgettext("FMD", "out of service, "
1012 		    "but associated components no longer faulty");
1013 		break;
1014 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1015 		msg = dgettext("FMD", "faulted and taken out of service");
1016 		break;
1017 	default:
1018 		break;
1019 	}
1020 	if (msg) {
1021 		(void) printf("%s     %s\n", label, msg);
1022 	}
1023 }
1024 
1025 static void
1026 print_fru_status(int status, char *label)
1027 {
1028 	char *msg = NULL;
1029 
1030 	if (status & FM_SUSPECT_NOT_PRESENT)
1031 		msg = dgettext("FMD", "not present");
1032 	else if (status & FM_SUSPECT_FAULTY)
1033 		msg = dgettext("FMD", "faulty");
1034 	else if (status & FM_SUSPECT_REPLACED)
1035 		msg = dgettext("FMD", "replaced");
1036 	else if (status & FM_SUSPECT_REPAIRED)
1037 		msg = dgettext("FMD", "repair attempted");
1038 	else if (status & FM_SUSPECT_ACQUITTED)
1039 		msg = dgettext("FMD", "acquitted");
1040 	else
1041 		msg = dgettext("FMD", "removed");
1042 	(void) printf("%s     %s\n", label, msg);
1043 }
1044 
1045 static void
1046 print_rsrc_status(int status, char *label)
1047 {
1048 	char *msg = "";
1049 
1050 	if (status & FM_SUSPECT_NOT_PRESENT)
1051 		msg = dgettext("FMD", "not present");
1052 	else if (status & FM_SUSPECT_FAULTY) {
1053 		if (status & FM_SUSPECT_DEGRADED)
1054 			msg = dgettext("FMD",
1055 			    "faulted but still providing degraded service");
1056 		else if (status & FM_SUSPECT_UNUSABLE)
1057 			msg = dgettext("FMD",
1058 			    "faulted and taken out of service");
1059 		else
1060 			msg = dgettext("FMD", "faulted but still in service");
1061 	} else if (status & FM_SUSPECT_REPLACED)
1062 		msg = dgettext("FMD", "replaced");
1063 	else if (status & FM_SUSPECT_REPAIRED)
1064 		msg = dgettext("FMD", "repair attempted");
1065 	else if (status & FM_SUSPECT_ACQUITTED)
1066 		msg = dgettext("FMD", "acquitted");
1067 	else
1068 		msg = dgettext("FMD", "removed");
1069 	(void) printf("%s     %s\n", label, msg);
1070 }
1071 
1072 static void
1073 print_name_list(name_list_t *list, char *label,
1074     int limit, int pct, void (func1)(int, char *), int full)
1075 {
1076 	char *name;
1077 	char *padding;
1078 	int i, j, l, n;
1079 	name_list_t *end = list;
1080 
1081 	l = strlen(label);
1082 	padding = malloc(l + 1);
1083 	for (i = 0; i < l; i++)
1084 		padding[i] = ' ';
1085 	padding[l] = 0;
1086 	(void) printf("%s", label);
1087 	name = list->name;
1088 	if (list->label)
1089 		(void) printf(" \"%s\" (%s)", list->label, name);
1090 	else
1091 		(void) printf(" %s", name);
1092 	if (list->pct && pct > 0 && pct < 100) {
1093 		if (list->count > 1) {
1094 			if (full) {
1095 				(void) printf(" %d @ %s %d%%\n", list->count,
1096 				    dgettext("FMD", "max"), list->max_pct);
1097 			} else {
1098 				(void) printf(" %s %d%%\n",
1099 				    dgettext("FMD", "max"), list->max_pct);
1100 			}
1101 		} else {
1102 			(void) printf(" %d%%\n", list->pct);
1103 		}
1104 	} else {
1105 		(void) printf("\n");
1106 	}
1107 	if (func1)
1108 		func1(list->status, padding);
1109 	n = 1;
1110 	j = 0;
1111 	while ((list = list->next) != end) {
1112 		if (limit == 0 || n < limit) {
1113 			print_name(list, padding, &n, pct, full);
1114 			if (func1)
1115 				func1(list->status, padding);
1116 		} else
1117 			j++;
1118 	}
1119 	if (j == 1) {
1120 		print_name(list->prev, padding, &n, pct, full);
1121 	} else if (j > 1) {
1122 		(void) printf("%s... %d %s\n", padding, j,
1123 		    dgettext("FMD", "more entries suppressed,"
1124 		    " use -v option for full list"));
1125 	}
1126 	free(padding);
1127 }
1128 
1129 static int
1130 asru_same_status(name_list_t *list)
1131 {
1132 	name_list_t *end = list;
1133 	int status = list->status;
1134 
1135 	while ((list = list->next) != end) {
1136 		if (status == -1) {
1137 			status = list->status;
1138 			continue;
1139 		}
1140 		if (list->status != -1 && status != list->status) {
1141 			status = -1;
1142 			break;
1143 		}
1144 	}
1145 	return (status);
1146 }
1147 
1148 static int
1149 serial_in_fru(name_list_t *fru, name_list_t *serial)
1150 {
1151 	name_list_t *sp = serial;
1152 	name_list_t *fp;
1153 	int nserial = 0;
1154 	int found = 0;
1155 	char buf[128];
1156 
1157 	while (sp) {
1158 		fp = fru;
1159 		nserial++;
1160 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1161 		buf[sizeof (buf) - 1] = 0;
1162 		while (fp) {
1163 			if (strstr(fp->name, buf) != NULL) {
1164 				found++;
1165 				break;
1166 			}
1167 			fp = fp->next;
1168 			if (fp == fru)
1169 				break;
1170 		}
1171 		sp = sp->next;
1172 		if (sp == serial)
1173 			break;
1174 	}
1175 	return (found == nserial ? 1 : 0);
1176 }
1177 
1178 static void
1179 print_sup_record(status_record_t *srp, int opt_i, int full)
1180 {
1181 	char buf[32];
1182 	uurec_t *uurp = srp->uurec;
1183 	int n, j, k, max;
1184 	int status;
1185 	ari_list_t *ari_list;
1186 
1187 	n = 0;
1188 	max = max_fault;
1189 	if (max < 0) {
1190 		max = 0;
1191 	}
1192 	j = max / 2;
1193 	max -= j;
1194 	k = srp->nrecs - max;
1195 	while ((uurp = uurp->next) != NULL) {
1196 		if (full || n < j || n >= k || max_fault == 0 ||
1197 		    srp->nrecs == max_fault+1) {
1198 			if (opt_i) {
1199 				ari_list = uurp->ari_uuid_list;
1200 				while (ari_list) {
1201 					(void) printf("%-15s %s\n",
1202 					    format_date(buf, sizeof (buf),
1203 					    uurp->sec), ari_list->ari_uuid);
1204 					ari_list = ari_list->next;
1205 				}
1206 			} else {
1207 				(void) printf("%-15s %s\n",
1208 				    format_date(buf, sizeof (buf), uurp->sec),
1209 				    uurp->uuid);
1210 			}
1211 		} else if (n == j)
1212 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1213 			    dgettext("FMD", "more entries suppressed"));
1214 		n++;
1215 	}
1216 	(void) printf("\n");
1217 	(void) printf("%s %s", dgettext("FMD", "Host        :"),
1218 	    srp->host->server);
1219 	if (srp->host->domain)
1220 		(void) printf("\t%s %s", dgettext("FMD", "Domain      :"),
1221 		    srp->host->domain);
1222 	(void) printf("\n%s %s", dgettext("FMD", "Platform    :"),
1223 	    srp->host->platform);
1224 	(void) printf("\t%s %s", dgettext("FMD", "Chassis_id  :"),
1225 	    srp->host->chassis ? srp->host->chassis : "");
1226 	(void) printf("\n%s %s\n\n", dgettext("FMD", "Product_sn  :"),
1227 	    srp->host->product_sn? srp->host->product_sn : "");
1228 	if (srp->class)
1229 		print_name_list(srp->class,
1230 		    dgettext("FMD", "Fault class :"), 0, srp->class->pct,
1231 		    NULL, full);
1232 	if (srp->asru) {
1233 		status = asru_same_status(srp->asru);
1234 		if (status != -1) {
1235 			print_name_list(srp->asru,
1236 			    dgettext("FMD", "Affects     :"),
1237 			    full ? 0 : max_display, 0, NULL, full);
1238 			print_asru_status(status, "             ");
1239 		} else
1240 			print_name_list(srp->asru,
1241 			    dgettext("FMD", "Affects     :"),
1242 			    full ? 0 : max_display, 0, print_asru_status, full);
1243 	}
1244 	if (full || srp->fru == NULL || srp->asru == NULL) {
1245 		if (srp->resource) {
1246 			status = asru_same_status(srp->resource);
1247 			if (status != -1) {
1248 				print_name_list(srp->resource,
1249 				    dgettext("FMD", "Problem in  :"),
1250 				    full ? 0 : max_display, 0, NULL, full);
1251 				print_rsrc_status(status, "             ");
1252 			} else
1253 				print_name_list(srp->resource,
1254 				    dgettext("FMD", "Problem in  :"),
1255 				    full ? 0 : max_display, 0,
1256 				    print_rsrc_status, full);
1257 		}
1258 	}
1259 	if (srp->fru) {
1260 		status = asru_same_status(srp->fru);
1261 		if (status != -1) {
1262 			print_name_list(srp->fru, dgettext("FMD",
1263 			    "FRU         :"), 0,
1264 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1265 			    NULL, full);
1266 			print_fru_status(status, "             ");
1267 		} else
1268 			print_name_list(srp->fru, dgettext("FMD",
1269 			    "FRU         :"), 0,
1270 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1271 			    print_fru_status, full);
1272 	}
1273 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1274 	    !serial_in_fru(srp->asru, srp->serial)) {
1275 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1276 		    0, 0, NULL, full);
1277 	}
1278 	print_dict_info(srp->uurec->event);
1279 	(void) printf("\n");
1280 }
1281 
1282 static void
1283 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1284 {
1285 	char buf[32];
1286 	uurec_t *uurp = srp->uurec;
1287 	static int header = 0;
1288 	char *head;
1289 	ari_list_t *ari_list;
1290 
1291 	if (!summary || !header) {
1292 		if (opt_i) {
1293 			head = "--------------- "
1294 			    "------------------------------------  "
1295 			    "-------------- ---------\n"
1296 			    "TIME            CACHE-ID"
1297 			    "                              MSG-ID"
1298 			    "         SEVERITY\n--------------- "
1299 			    "------------------------------------ "
1300 			    " -------------- ---------";
1301 		} else {
1302 			head = "--------------- "
1303 			    "------------------------------------  "
1304 			    "-------------- ---------\n"
1305 			    "TIME            EVENT-ID"
1306 			    "                              MSG-ID"
1307 			    "         SEVERITY\n--------------- "
1308 			    "------------------------------------ "
1309 			    " -------------- ---------";
1310 		}
1311 		(void) printf("%s\n", dgettext("FMD", head));
1312 		header = 1;
1313 	}
1314 	if (opt_i) {
1315 		ari_list = uurp->ari_uuid_list;
1316 		while (ari_list) {
1317 			(void) printf("%-15s %-37s %-14s %-9s %s\n",
1318 			    format_date(buf, sizeof (buf), uurp->sec),
1319 			    ari_list->ari_uuid, srp->msgid, srp->severity,
1320 			    srp->injected ? dgettext("FMD", "injected") : "");
1321 			ari_list = ari_list->next;
1322 		}
1323 	} else {
1324 		(void) printf("%-15s %-37s %-14s %-9s %s\n",
1325 		    format_date(buf, sizeof (buf), uurp->sec),
1326 		    uurp->uuid, srp->msgid, srp->severity,
1327 		    srp->injected ? dgettext("FMD", "injected") : "");
1328 	}
1329 
1330 	if (!summary)
1331 		print_sup_record(srp, opt_i, full);
1332 }
1333 
1334 static void
1335 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1336 {
1337 	status_record_t *srp;
1338 	sr_list_t *slp;
1339 
1340 	slp = status_rec_list;
1341 	if (slp) {
1342 		for (;;) {
1343 			srp = slp->status_record;
1344 			if (opt_a || srp->not_suppressed) {
1345 				if (page_feed)
1346 					(void) printf("\f\n");
1347 				print_status_record(srp, summary, opt_i, full);
1348 			}
1349 			if (slp->next == status_rec_list)
1350 				break;
1351 			slp = slp->next;
1352 		}
1353 	}
1354 }
1355 
1356 static name_list_t *
1357 find_fru(status_record_t *srp, char *resource)
1358 {
1359 	name_list_t *rt = NULL;
1360 	name_list_t *fru = srp->fru;
1361 
1362 	while (fru) {
1363 		if (strcmp(resource, fru->name) == 0) {
1364 			rt = fru;
1365 			break;
1366 		}
1367 		fru = fru->next;
1368 		if (fru == srp->fru)
1369 			break;
1370 	}
1371 	return (rt);
1372 }
1373 
1374 static void
1375 print_fru_line(name_list_t *fru, char *uuid)
1376 {
1377 	if (fru->pct == 100) {
1378 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1379 		    dgettext("FMD", "suspects in this FRU total certainty"),
1380 		    100);
1381 	} else {
1382 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1383 		    dgettext("FMD", "suspects in this FRU max certainty"),
1384 		    fru->max_pct);
1385 	}
1386 }
1387 
1388 static void
1389 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1390 {
1391 	resource_list_t *tp = status_fru_list;
1392 	status_record_t *srp;
1393 	sr_list_t *slp, *end;
1394 	uurec_t *uurp;
1395 	name_list_t *fru;
1396 	int status;
1397 	ari_list_t *ari_list;
1398 
1399 	while (tp) {
1400 		if (opt_a || tp->not_suppressed) {
1401 			if (page_feed)
1402 				(void) printf("\f\n");
1403 			if (!summary)
1404 				(void) printf("-----------------------------"
1405 				    "---------------------------------------"
1406 				    "----------\n");
1407 			slp = tp->status_rec_list;
1408 			end = slp;
1409 			do {
1410 				srp = slp->status_record;
1411 				if (!srp->not_suppressed) {
1412 					slp = slp->next;
1413 					continue;
1414 				}
1415 				fru = find_fru(srp, tp->resource);
1416 				if (fru) {
1417 					if (fru->label)
1418 						(void) printf("\"%s\" (%s) ",
1419 						    fru->label, fru->name);
1420 					else
1421 						(void) printf("%s ",
1422 						    fru->name);
1423 					break;
1424 				}
1425 				slp = slp->next;
1426 			} while (slp != end);
1427 
1428 			slp = tp->status_rec_list;
1429 			end = slp;
1430 			status = 0;
1431 			do {
1432 				srp = slp->status_record;
1433 				if (!srp->not_suppressed) {
1434 					slp = slp->next;
1435 					continue;
1436 				}
1437 				fru = srp->fru;
1438 				while (fru) {
1439 					if (strcmp(tp->resource,
1440 					    fru->name) == 0)
1441 						status |= fru->status;
1442 					fru = fru->next;
1443 					if (fru == srp->fru)
1444 						break;
1445 				}
1446 				slp = slp->next;
1447 			} while (slp != end);
1448 			if (status & FM_SUSPECT_NOT_PRESENT)
1449 				(void) printf(dgettext("FMD", "not present"));
1450 			else if (status & FM_SUSPECT_FAULTY)
1451 				(void) printf(dgettext("FMD", "faulty"));
1452 			else if (status & FM_SUSPECT_REPLACED)
1453 				(void) printf(dgettext("FMD", "replaced"));
1454 			else if (status & FM_SUSPECT_REPAIRED)
1455 				(void) printf(dgettext("FMD",
1456 				    "repair attempted"));
1457 			else if (status & FM_SUSPECT_ACQUITTED)
1458 				(void) printf(dgettext("FMD", "acquitted"));
1459 			else
1460 				(void) printf(dgettext("FMD", "removed"));
1461 
1462 			if (tp->injected)
1463 				(void) printf(dgettext("FMD", " injected\n"));
1464 			else
1465 				(void) printf(dgettext("FMD", "\n"));
1466 
1467 			slp = tp->status_rec_list;
1468 			end = slp;
1469 			do {
1470 				srp = slp->status_record;
1471 				if (!srp->not_suppressed) {
1472 					slp = slp->next;
1473 					continue;
1474 				}
1475 				uurp = srp->uurec;
1476 				fru = find_fru(srp, tp->resource);
1477 				if (fru) {
1478 					if (opt_i) {
1479 						ari_list = uurp->ari_uuid_list;
1480 						while (ari_list) {
1481 							print_fru_line(fru,
1482 							    ari_list->ari_uuid);
1483 							ari_list =
1484 							    ari_list->next;
1485 						}
1486 					} else {
1487 						print_fru_line(fru, uurp->uuid);
1488 					}
1489 				}
1490 				slp = slp->next;
1491 			} while (slp != end);
1492 			if (!summary) {
1493 				slp = tp->status_rec_list;
1494 				end = slp;
1495 				do {
1496 					srp = slp->status_record;
1497 					if (!srp->not_suppressed) {
1498 						slp = slp->next;
1499 						continue;
1500 					}
1501 					if (srp->serial &&
1502 					    !serial_in_fru(srp->fru,
1503 					    srp->serial)) {
1504 						print_name_list(srp->serial,
1505 						    dgettext("FMD",
1506 						    "Serial ID.  :"),
1507 						    0, 0, NULL, 1);
1508 						break;
1509 					}
1510 					slp = slp->next;
1511 				} while (slp != end);
1512 			}
1513 		}
1514 		tp = tp->next;
1515 		if (tp == status_fru_list)
1516 			break;
1517 	}
1518 }
1519 
1520 static void
1521 print_asru(int opt_a)
1522 {
1523 	resource_list_t *tp = status_asru_list;
1524 	status_record_t *srp;
1525 	sr_list_t *slp, *end;
1526 	char *msg;
1527 	int status;
1528 	name_list_t *asru;
1529 
1530 	while (tp) {
1531 		if (opt_a || tp->not_suppressed) {
1532 			status = 0;
1533 			slp = tp->status_rec_list;
1534 			end = slp;
1535 			do {
1536 				srp = slp->status_record;
1537 				if (!srp->not_suppressed) {
1538 					slp = slp->next;
1539 					continue;
1540 				}
1541 				asru = srp->asru;
1542 				while (asru) {
1543 					if (strcmp(tp->resource,
1544 					    asru->name) == 0)
1545 						status |= asru->status;
1546 					asru = asru->next;
1547 					if (asru == srp->asru)
1548 						break;
1549 				}
1550 				slp = slp->next;
1551 			} while (slp != end);
1552 			switch (status) {
1553 			case 0:
1554 				msg = dgettext("FMD", "ok");
1555 				break;
1556 			case FM_SUSPECT_DEGRADED:
1557 				msg = dgettext("FMD", "degraded");
1558 				break;
1559 			case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1560 				msg = dgettext("FMD", "degraded");
1561 				break;
1562 			case FM_SUSPECT_FAULTY:
1563 				msg = dgettext("FMD", "degraded");
1564 				break;
1565 			case FM_SUSPECT_UNUSABLE:
1566 				msg = dgettext("FMD", "unknown");
1567 				break;
1568 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1569 				msg = dgettext("FMD", "faulted");
1570 				break;
1571 			default:
1572 				msg = "";
1573 				break;
1574 			}
1575 			(void) printf("%-69s %s", tp->resource, msg);
1576 			if (tp->injected)
1577 				(void) printf(dgettext("FMD", " injected\n"));
1578 			else
1579 				(void) printf(dgettext("FMD", "\n"));
1580 		}
1581 		tp = tp->next;
1582 		if (tp == status_asru_list)
1583 			break;
1584 	}
1585 }
1586 
1587 static int
1588 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1589 {
1590 	while (uurecp) {
1591 		if (strcmp(uuid, uurecp->uuid) == 0)
1592 			return (1);
1593 		uurecp = uurecp->next;
1594 	}
1595 	return (0);
1596 }
1597 
1598 static int
1599 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1600 {
1601 	int64_t *diag_time;
1602 	uint_t nelem;
1603 	int rt = 0;
1604 	char *uuid = "-";
1605 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1606 
1607 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1608 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1609 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1610 		    &uuid);
1611 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1612 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1613 			    uuid);
1614 	} else {
1615 		rt = -1;
1616 	}
1617 	return (rt);
1618 }
1619 
1620 /*ARGSUSED*/
1621 static int
1622 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1623 {
1624 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1625 	return (0);
1626 }
1627 
1628 static int
1629 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1630 {
1631 	int rt = FMADM_EXIT_SUCCESS;
1632 
1633 	/*
1634 	 * These calls may fail with Protocol error if message payload is
1635 	 * too big
1636 	 */
1637 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1638 		die("failed to get case list from fmd");
1639 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1640 		die("failed to get case status from fmd");
1641 	return (rt);
1642 }
1643 
1644 /*
1645  * fmadm faulty command
1646  *
1647  *	-a		show hidden fault records
1648  *	-f		show faulty fru's
1649  *	-g		force grouping of similar faults on the same fru
1650  *	-n		number of fault records to display
1651  *	-p		pipe output through pager
1652  *	-r		show faulty asru's
1653  *	-s		print summary of first fault
1654  *	-u		print listed uuid's only
1655  *	-v		full output
1656  */
1657 
1658 int
1659 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1660 {
1661 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1662 	int opt_i = 0;
1663 	char *pager;
1664 	FILE *fp;
1665 	int rt, c, stat;
1666 	uurec_select_t *tp;
1667 	uurec_select_t *uurecp = NULL;
1668 
1669 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1670 		switch (c) {
1671 		case 'a':
1672 			opt_a++;
1673 			break;
1674 		case 'f':
1675 			opt_f++;
1676 			break;
1677 		case 'g':
1678 			opt_g++;
1679 			break;
1680 		case 'i':
1681 			opt_i++;
1682 			break;
1683 		case 'n':
1684 			max_fault = atoi(optarg);
1685 			break;
1686 		case 'p':
1687 			opt_p++;
1688 			break;
1689 		case 'r':
1690 			opt_r++;
1691 			break;
1692 		case 's':
1693 			opt_s++;
1694 			break;
1695 		case 'u':
1696 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1697 			tp->uuid = optarg;
1698 			tp->next = uurecp;
1699 			uurecp = tp;
1700 			opt_a = 1;
1701 			break;
1702 		case 'v':
1703 			opt_v++;
1704 			break;
1705 		default:
1706 			return (FMADM_EXIT_USAGE);
1707 		}
1708 	}
1709 	if (optind < argc)
1710 		return (FMADM_EXIT_USAGE);
1711 
1712 	if ((fmadm_msghdl = fmd_msg_init(NULL, FMD_MSG_VERSION)) == NULL)
1713 		return (FMADM_EXIT_ERROR);
1714 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1715 	if (opt_p) {
1716 		if ((pager = getenv("PAGER")) == NULL)
1717 			pager = "/usr/bin/more";
1718 		fp = popen(pager, "w");
1719 		if (fp == NULL) {
1720 			rt = FMADM_EXIT_ERROR;
1721 			opt_p = 0;
1722 		} else {
1723 			(void) dup2(fileno(fp), 1);
1724 			setbuf(stdout, NULL);
1725 			(void) fclose(fp);
1726 		}
1727 	}
1728 	max_display = max_fault;
1729 	if (opt_f)
1730 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1731 	if (opt_r)
1732 		print_asru(opt_a);
1733 	if (opt_f == 0 && opt_r == 0)
1734 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1735 	fmd_msg_fini(fmadm_msghdl);
1736 	if (topo_handle)
1737 		topo_close(topo_handle);
1738 	if (opt_p) {
1739 		(void) fclose(stdout);
1740 		(void) wait(&stat);
1741 	}
1742 	return (rt);
1743 }
1744 
1745 int
1746 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1747 {
1748 	int i, status = FMADM_EXIT_SUCCESS;
1749 
1750 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1751 		return (FMADM_EXIT_USAGE);
1752 
1753 	for (i = 1; i < argc; i++) {
1754 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1755 			warn("failed to flush %s", argv[i]);
1756 			status = FMADM_EXIT_ERROR;
1757 		} else
1758 			note("flushed resource history for %s\n", argv[i]);
1759 	}
1760 
1761 	return (status);
1762 }
1763 
1764 int
1765 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
1766 {
1767 	int err;
1768 
1769 	if (getopt(argc, argv, "") != EOF)
1770 		return (FMADM_EXIT_USAGE);
1771 
1772 	if (argc - optind != 1)
1773 		return (FMADM_EXIT_USAGE);
1774 
1775 	/*
1776 	 * argument could be a uuid, an fmri (asru, fru or resource)
1777 	 * or a label. Try uuid first, If that fails try the others.
1778 	 */
1779 	err = fmd_adm_case_repair(adm, argv[optind]);
1780 	if (err != 0)
1781 		err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1782 
1783 	if (err != 0)
1784 		die("failed to record repair to %s", argv[optind]);
1785 
1786 	note("recorded repair to %s\n", argv[optind]);
1787 	return (FMADM_EXIT_SUCCESS);
1788 }
1789 
1790 int
1791 cmd_repaired(fmd_adm_t *adm, int argc, char *argv[])
1792 {
1793 	int err;
1794 
1795 	if (getopt(argc, argv, "") != EOF)
1796 		return (FMADM_EXIT_USAGE);
1797 
1798 	if (argc - optind != 1)
1799 		return (FMADM_EXIT_USAGE);
1800 
1801 	/*
1802 	 * argument could be an fmri (asru, fru or resource) or a label.
1803 	 */
1804 	err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1805 	if (err != 0)
1806 		die("failed to record repair to %s", argv[optind]);
1807 
1808 	note("recorded repair to of %s\n", argv[optind]);
1809 	return (FMADM_EXIT_SUCCESS);
1810 }
1811 
1812 int
1813 cmd_replaced(fmd_adm_t *adm, int argc, char *argv[])
1814 {
1815 	int err;
1816 
1817 	if (getopt(argc, argv, "") != EOF)
1818 		return (FMADM_EXIT_USAGE);
1819 
1820 	if (argc - optind != 1)
1821 		return (FMADM_EXIT_USAGE);
1822 
1823 	/*
1824 	 * argument could be an fmri (asru, fru or resource) or a label.
1825 	 */
1826 	err = fmd_adm_rsrc_replaced(adm, argv[optind]);
1827 	if (err != 0)
1828 		die("failed to record replacement of %s", argv[optind]);
1829 
1830 	note("recorded replacement of %s\n", argv[optind]);
1831 	return (FMADM_EXIT_SUCCESS);
1832 }
1833 
1834 int
1835 cmd_acquit(fmd_adm_t *adm, int argc, char *argv[])
1836 {
1837 	int err;
1838 
1839 	if (getopt(argc, argv, "") != EOF)
1840 		return (FMADM_EXIT_USAGE);
1841 
1842 	if (argc - optind != 1 && argc - optind != 2)
1843 		return (FMADM_EXIT_USAGE);
1844 
1845 	/*
1846 	 * argument could be a uuid, an fmri (asru, fru or resource)
1847 	 * or a label. Or it could be a uuid and an fmri or label.
1848 	 */
1849 	if (argc - optind == 2) {
1850 		err = fmd_adm_rsrc_acquit(adm, argv[optind], argv[optind + 1]);
1851 		if (err != 0)
1852 			err = fmd_adm_rsrc_acquit(adm, argv[optind + 1],
1853 			    argv[optind]);
1854 	} else {
1855 		err = fmd_adm_case_acquit(adm, argv[optind]);
1856 		if (err != 0)
1857 			err = fmd_adm_rsrc_acquit(adm, argv[optind], "");
1858 	}
1859 
1860 	if (err != 0)
1861 		die("failed to record acquital of %s", argv[optind]);
1862 
1863 	note("recorded acquital of %s\n", argv[optind]);
1864 	return (FMADM_EXIT_SUCCESS);
1865 }
1866