xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 5750ef5c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <fmadm.h>
28 #include <errno.h>
29 #include <limits.h>
30 #include <strings.h>
31 #include <stdio.h>
32 #include <unistd.h>
33 #include <sys/wait.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <fm/fmd_log.h>
37 #include <sys/fm/protocol.h>
38 #include <fm/libtopo.h>
39 #include <fm/fmd_adm.h>
40 #include <fm/fmd_msg.h>
41 #include <dlfcn.h>
42 #include <sys/systeminfo.h>
43 #include <sys/utsname.h>
44 #include <libintl.h>
45 #include <locale.h>
46 #include <sys/smbios.h>
47 #include <libdevinfo.h>
48 #include <stdlib.h>
49 
50 #define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))
51 
52 /*
53  * Fault records are added to catalog by calling add_fault_record_to_catalog()
54  * records are stored in order of importance to the system.
55  * If -g flag is set or not_suppressed is not set and the class fru, fault,
56  * type are the same then details are merged into an existing record, with uuid
57  * records are stored in time order.
58  * For each record information is extracted from nvlist and merged into linked
59  * list each is checked for identical records for which percentage certainty are
60  * added together.
61  * print_catalog() is called to print out catalog and release external resources
62  *
63  *                         /---------------\
64  *	status_rec_list -> |               | -|
65  *                         \---------------/
66  *                                \/
67  *                         /---------------\    /-------\    /-------\
68  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
69  *            \/           |               | |- |       | <- |       |
70  *      /-------------\    |               |    \-------/    \-------/
71  *      |             | -> |               |       \/           \/
72  *      \-------------/    |               |    /-------\    /-------\
73  *            \/           |               | -> | asru  | -> | asru  |
74  *            ---          |               |    |       | <- |       |
75  *                         |               |    \-------/    \-------/
76  *      status_asru_list   |  class        |
77  *            \/           |  resource     |    /-------\    /-------\
78  *      /-------------\    |  fru          | -> | list  | -> | list  |
79  *      |             | -> |  serial       |    |       | <- |       |
80  *      \-------------/    |               |    \-------/    \-------/
81  *            \/           \---------------/
82  *            ---               \/    /\
83  *                         /---------------\
84  *                         | status_record |
85  *                         \---------------/
86  *
87  * Fmadm faulty takes a number of options which affect the format of the
88  * output displayed. By default, the display reports the FRU and ASRU along
89  * with other information on per-case basis as in the example below.
90  *
91  * --------------- ------------------------------------  -------------- -------
92  * TIME            EVENT-ID                              MSG-ID         SEVERITY
93  * --------------- ------------------------------------  -------------- -------
94  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
95  *
96  * Fault class	: fault.memory.dimm_sb
97  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
98  *		    faulted but still in service
99  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
100  *		    faulty
101  *
102  * Description	: The number of errors associated with this memory module has
103  *		exceeded acceptable levels.  Refer to
104  *		http://sun.com/msg/AMD-8000-2F for more information.
105  *
106  * Response	: Pages of memory associated with this memory module are being
107  *		removed from service as errors are reported.
108  *
109  * Impact	: Total system memory capacity will be reduced as pages are
110  *		retired.
111  *
112  * Action	: Schedule a repair procedure to replace the affected memory
113  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
114  *
115  * The -v flag is similar, but adds some additonal information such as the
116  * resource. The -s flag is also similar but just gives the top line summary.
117  * All these options (ie without the -f or -r flags) use the print_catalog()
118  * function to do the display.
119  *
120  * The -f flag changes the output so that it appears sorted on a per-fru basis.
121  * The output is somewhat cut down compared to the default output. If -f is
122  * used, then print_fru() is used to print the output.
123  *
124  * -----------------------------------------------------------------------------
125  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
126  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
127  *
128  * Description	: A problem was detected for a PCI device.
129  *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
130  *
131  * Response	: One or more device instances may be disabled
132  *
133  * Impact	: Possible loss of services provided by the device instances
134  *		associated with this fault
135  *
136  * Action	: Schedule a repair procedure to replace the affected device.
137  * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
138  *		Sun for support.
139  *
140  * The -r flag changes the output so that it appears sorted on a per-asru basis.
141  * The output is very much cut down compared to the default output, just giving
142  * the asru fmri and state. Here print_asru() is used to print the output.
143  *
144  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
145  *
146  * For all fmadm faulty options, the sequence of events is
147  *
148  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
149  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
150  * This will extract the data from the nvlist and call catalog_new_record() to
151  * save the data away in various linked lists in the catalogue.
152  *
153  * 2) Once this is done, the data can be supplemented by using
154  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
155  *
156  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
157  * appropriate to display the information from the catalogue sorted in the
158  * requested way.
159  *
160  */
161 
162 typedef struct name_list {
163 	struct name_list *next;
164 	struct name_list *prev;
165 	char *name;
166 	uint8_t pct;
167 	uint8_t max_pct;
168 	ushort_t count;
169 	int status;
170 	char *label;
171 } name_list_t;
172 
173 typedef struct ari_list {
174 	char *ari_uuid;
175 	struct ari_list *next;
176 } ari_list_t;
177 
178 typedef struct uurec {
179 	struct uurec *next;
180 	struct uurec *prev;
181 	char *uuid;
182 	ari_list_t *ari_uuid_list;
183 	name_list_t *asru;
184 	uint64_t sec;
185 	nvlist_t *event;
186 } uurec_t;
187 
188 typedef struct uurec_select {
189 	struct uurec_select *next;
190 	char *uuid;
191 } uurec_select_t;
192 
193 typedef struct host_id {
194 	char *chassis;
195 	char *server;
196 	char *platform;
197 	char *domain;
198 	char *product_sn;
199 } hostid_t;
200 
201 typedef struct host_id_list {
202 	hostid_t hostid;
203 	struct host_id_list *next;
204 } host_id_list_t;
205 
206 typedef struct status_record {
207 	hostid_t *host;
208 	int nrecs;
209 	uurec_t *uurec;
210 	char *severity;			/* in C locale */
211 	char *msgid;
212 	name_list_t *class;
213 	name_list_t *resource;
214 	name_list_t *asru;
215 	name_list_t *fru;
216 	name_list_t *serial;
217 	uint8_t not_suppressed;
218 } status_record_t;
219 
220 typedef struct sr_list {
221 	struct sr_list *next;
222 	struct sr_list *prev;
223 	struct status_record *status_record;
224 } sr_list_t;
225 
226 typedef struct resource_list {
227 	struct resource_list *next;
228 	struct resource_list *prev;
229 	sr_list_t *status_rec_list;
230 	char *resource;
231 	uint8_t not_suppressed;
232 	uint8_t max_pct;
233 } resource_list_t;
234 
235 typedef struct tgetlabel_data {
236 	char *label;
237 	char *fru;
238 } tgetlabel_data_t;
239 
240 sr_list_t *status_rec_list;
241 resource_list_t *status_fru_list;
242 resource_list_t *status_asru_list;
243 
244 static int max_display;
245 static int max_fault = 0;
246 static topo_hdl_t *topo_handle;
247 static char *topo_handle_uuid;
248 static host_id_list_t *host_list;
249 static int n_server;
250 static int opt_g;
251 static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */
252 
253 static char *
254 format_date(char *buf, size_t len, uint64_t sec)
255 {
256 	if (sec > LONG_MAX) {
257 		(void) fprintf(stderr,
258 		    "record time is too large for 32-bit utility\n");
259 		(void) snprintf(buf, len, "0x%llx", sec);
260 	} else {
261 		time_t tod = (time_t)sec;
262 		(void) strftime(buf, len, "%b %d %T", localtime(&tod));
263 	}
264 
265 	return (buf);
266 }
267 
268 static hostid_t *
269 find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
270     char *product_sn)
271 {
272 	hostid_t *rt = NULL;
273 	host_id_list_t *hostp;
274 
275 	if (platform == NULL)
276 		platform = "-";
277 	if (server == NULL)
278 		server = "-";
279 	hostp = host_list;
280 	while (hostp) {
281 		if (hostp->hostid.platform &&
282 		    strcmp(hostp->hostid.platform, platform) == 0 &&
283 		    hostp->hostid.server &&
284 		    strcmp(hostp->hostid.server, server) == 0 &&
285 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
286 		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
287 		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
288 		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
289 		    (domain == NULL || hostp->hostid.domain == NULL ||
290 		    strcmp(domain, hostp->hostid.domain) == 0)) {
291 			rt = &hostp->hostid;
292 			break;
293 		}
294 		hostp = hostp->next;
295 	}
296 	if (rt == NULL) {
297 		hostp = malloc(sizeof (host_id_list_t));
298 		hostp->hostid.platform = strdup(platform);
299 		hostp->hostid.product_sn =
300 		    product_sn ? strdup(product_sn) : NULL;
301 		hostp->hostid.server = strdup(server);
302 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
303 		hostp->hostid.domain = domain ? strdup(domain) : NULL;
304 		hostp->next = host_list;
305 		host_list = hostp;
306 		rt = &hostp->hostid;
307 		n_server++;
308 	}
309 	return (rt);
310 }
311 
312 static hostid_t *
313 find_hostid(nvlist_t *nvl)
314 {
315 	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
316 	char *product_sn = NULL;
317 	nvlist_t *auth, *fmri;
318 	hostid_t *rt = NULL;
319 
320 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
321 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
322 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
323 		    &platform);
324 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
325 		    &product_sn);
326 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
327 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
328 		    &chassis);
329 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
330 		rt = find_hostid_in_list(platform, chassis, server,
331 		    domain, product_sn);
332 	}
333 	return (rt);
334 }
335 
336 /*
337  * compare two fru strings which are made up of substrings seperated by '/'
338  * return true if every substring is the same in the two strings, or if a
339  * substring is null in one.
340  */
341 
342 static int
343 frucmp(char *f1, char *f2)
344 {
345 	char c1, c2;
346 	int i = 0;
347 
348 	for (;;) {
349 		c1 = *f1;
350 		c2 = *f2;
351 		if (c1 == c2) {
352 			i = (c1 == '/') ? 0 : i + 1;
353 		} else if (i == 0) {
354 			if (c1 == '/') {
355 				do {
356 					f2++;
357 				} while ((c2 = *f2) != 0 && c2 != '/');
358 				if (c2 == NULL)
359 					break;
360 			} else if (c2 == '/') {
361 				do {
362 					f1++;
363 				} while ((c1 = *f1) != 0 && c1 != '/');
364 				if (c1 == NULL)
365 					break;
366 			} else
367 				break;
368 		} else
369 			break;
370 		if (c1 == NULL)
371 			return (0);
372 		f1++;
373 		f2++;
374 	}
375 	return (1);
376 }
377 
378 static int
379 tgetlabel(topo_hdl_t *thp, tnode_t *node, void *arg)
380 {
381 	int err;
382 	char *fru_name, *lname;
383 	nvlist_t *fru = NULL;
384 	int rt = TOPO_WALK_NEXT;
385 	tgetlabel_data_t *tdp = (tgetlabel_data_t *)arg;
386 
387 	if (topo_node_fru(node, &fru, NULL, &err) == 0) {
388 		if (topo_fmri_nvl2str(thp, fru, &fru_name, &err) == 0) {
389 			if (frucmp(tdp->fru, fru_name) == 0 &&
390 			    topo_node_label(node, &lname, &err) == 0) {
391 				tdp->label = strdup(lname);
392 				topo_hdl_strfree(thp, lname);
393 				rt = TOPO_WALK_TERMINATE;
394 			}
395 			topo_hdl_strfree(thp, fru_name);
396 		}
397 		nvlist_free(fru);
398 	}
399 	return (rt);
400 }
401 
402 static void
403 label_get_topo(void)
404 {
405 	int err;
406 
407 	topo_handle = topo_open(TOPO_VERSION, 0, &err);
408 	if (topo_handle) {
409 		topo_handle_uuid = topo_snap_hold(topo_handle, NULL, &err);
410 	}
411 }
412 
413 static void
414 label_release_topo(void)
415 {
416 	if (topo_handle_uuid)
417 		topo_hdl_strfree(topo_handle, topo_handle_uuid);
418 	if (topo_handle) {
419 		topo_snap_release(topo_handle);
420 		topo_close(topo_handle);
421 	}
422 }
423 
424 static char *
425 get_fmri_label(char *fru)
426 {
427 	topo_walk_t *twp;
428 	tgetlabel_data_t td;
429 	int err;
430 
431 	td.label = NULL;
432 	td.fru = fru;
433 	if (topo_handle == NULL)
434 		label_get_topo();
435 	if (topo_handle_uuid) {
436 		twp = topo_walk_init(topo_handle, FM_FMRI_SCHEME_HC,
437 		    tgetlabel, &td, &err);
438 		if (twp) {
439 			topo_walk_step(twp, TOPO_WALK_CHILD);
440 			topo_walk_fini(twp);
441 		}
442 	}
443 	return (td.label);
444 }
445 
446 static char *
447 get_nvl2str_topo(nvlist_t *nvl)
448 {
449 	char *name = NULL;
450 	char *tname;
451 	int err;
452 	char *scheme = NULL;
453 	char *mod_name = NULL;
454 	char buf[128];
455 
456 	if (topo_handle == NULL)
457 		label_get_topo();
458 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
459 		name = strdup(tname);
460 		topo_hdl_strfree(topo_handle, tname);
461 	} else {
462 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
463 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
464 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
465 		    mod_name) {
466 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
467 			    scheme, mod_name);
468 			name = strdup(buf);
469 		}
470 	}
471 	return (name);
472 }
473 
474 static int
475 set_priority(char *s)
476 {
477 	int rt = 0;
478 
479 	if (s) {
480 		if (strcmp(s, "Minor") == 0)
481 			rt = 1;
482 		else if (strcmp(s, "Major") == 0)
483 			rt = 10;
484 		else if (strcmp(s, "Critical") == 0)
485 			rt = 100;
486 	}
487 	return (rt);
488 }
489 
490 static int
491 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
492     uint8_t p2)
493 {
494 	int r1, r2;
495 	int rt;
496 
497 	r1 = set_priority(s1);
498 	r2 = set_priority(s2);
499 	rt = r1 - r2;
500 	if (rt == 0) {
501 		if (t1 > t2)
502 			rt = 1;
503 		else if (t1 < t2)
504 			rt = -1;
505 		else
506 			rt = p1 - p2;
507 	}
508 	return (rt);
509 }
510 
511 /*
512  * merge two lists into one, by comparing enties in new and moving into list if
513  * name is not there or free off memory for names which are already there
514  * add_pct indicates if pct is the sum or highest pct
515  */
516 static name_list_t *
517 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
518 {
519 	name_list_t *lp, *np, *sp, *rt = NULL;
520 	int max_pct;
521 
522 	rt = *list;
523 	np = new;
524 	while (np) {
525 		lp = *list;
526 		while (lp) {
527 			if (strcmp(lp->name, np->name) == 0)
528 				break;
529 			lp = lp->next;
530 			if (lp == *list)
531 				lp = NULL;
532 		}
533 		if (np->next == new)
534 			sp = NULL;
535 		else
536 			sp = np->next;
537 		if (lp) {
538 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
539 			if (add_pct) {
540 				lp->pct += np->pct;
541 				lp->count += np->count;
542 			} else if (np->pct > lp->pct) {
543 				lp->pct = np->pct;
544 			}
545 			max_pct = np->max_pct;
546 			if (np->label)
547 				free(np->label);
548 			free(np->name);
549 			free(np);
550 			np = NULL;
551 			if (max_pct > lp->max_pct) {
552 				lp->max_pct = max_pct;
553 				if (lp->max_pct > lp->prev->max_pct &&
554 				    lp != *list) {
555 					lp->prev->next = lp->next;
556 					lp->next->prev = lp->prev;
557 					np = lp;
558 				}
559 			}
560 		}
561 		if (np) {
562 			lp = *list;
563 			if (lp) {
564 				if (np->max_pct > lp->max_pct) {
565 					np->next = lp;
566 					np->prev = lp->prev;
567 					lp->prev->next = np;
568 					lp->prev = np;
569 					*list = np;
570 					rt = np;
571 				} else {
572 					lp = lp->next;
573 					while (lp != *list &&
574 					    np->max_pct < lp->max_pct) {
575 						lp = lp->next;
576 					}
577 					np->next = lp;
578 					np->prev = lp->prev;
579 					lp->prev->next = np;
580 					lp->prev = np;
581 				}
582 			} else {
583 				*list = np;
584 				np->next = np;
585 				np->prev = np;
586 				rt = np;
587 			}
588 		}
589 		np = sp;
590 	}
591 	return (rt);
592 }
593 
594 static name_list_t *
595 alloc_name_list(char *name, uint8_t pct)
596 {
597 	name_list_t *nlp;
598 
599 	nlp = malloc(sizeof (*nlp));
600 	nlp->name = strdup(name);
601 	nlp->pct = pct;
602 	nlp->max_pct = pct;
603 	nlp->count = 1;
604 	nlp->next = nlp;
605 	nlp->prev = nlp;
606 	nlp->status = 0;
607 	nlp->label = NULL;
608 	return (nlp);
609 }
610 
611 static status_record_t *
612 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
613     name_list_t *fru, name_list_t *asru, name_list_t *resource,
614     name_list_t *serial, boolean_t not_suppressed,
615     hostid_t *hostid)
616 {
617 	status_record_t *status_rec_p;
618 
619 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
620 	status_rec_p->nrecs = 1;
621 	status_rec_p->host = hostid;
622 	status_rec_p->uurec = uurec_p;
623 	uurec_p->next = NULL;
624 	uurec_p->prev = NULL;
625 	uurec_p->asru = asru;
626 	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
627 	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
628 		status_rec_p->severity = strdup("unknown");
629 	status_rec_p->class = class;
630 	status_rec_p->fru = fru;
631 	status_rec_p->asru = asru;
632 	status_rec_p->resource = resource;
633 	status_rec_p->serial = serial;
634 	status_rec_p->msgid = strdup(msgid);
635 	status_rec_p->not_suppressed = not_suppressed;
636 	return (status_rec_p);
637 }
638 
639 /*
640  * add record to given list maintaining order higher priority first.
641  */
642 static void
643 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
644 {
645 	sr_list_t *tp, *np, *sp;
646 	int order;
647 	uint64_t sec;
648 
649 	np = malloc(sizeof (sr_list_t));
650 	np->status_record = status_rec_p;
651 	sec = status_rec_p->uurec->sec;
652 	if ((sp = *list_pp) == NULL) {
653 		*list_pp = np;
654 		np->next = np;
655 		np->prev = np;
656 	} else {
657 		/* insert new record in front of lower priority */
658 		tp = sp;
659 		order = cmp_priority(status_rec_p->severity,
660 		    sp->status_record->severity, sec,
661 		    tp->status_record->uurec->sec, 0, 0);
662 		if (order > 0) {
663 			*list_pp = np;
664 		} else {
665 			tp = sp->next;
666 			while (tp != sp &&
667 			    cmp_priority(status_rec_p->severity,
668 			    tp->status_record->severity, sec,
669 			    tp->status_record->uurec->sec, 0, 0)) {
670 				tp = tp->next;
671 			}
672 		}
673 		np->next = tp;
674 		np->prev = tp->prev;
675 		tp->prev->next = np;
676 		tp->prev = np;
677 	}
678 }
679 
680 static void
681 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
682     resource_list_t *np)
683 {
684 	int order;
685 	uint64_t sec;
686 	resource_list_t *sp, *tp;
687 	status_record_t *srp;
688 	char *severity = status_rec_p->severity;
689 
690 	add_rec_list(status_rec_p, &np->status_rec_list);
691 	if ((sp = *rp) == NULL) {
692 		np->next = np;
693 		np->prev = np;
694 		*rp = np;
695 	} else {
696 		/*
697 		 * insert new record in front of lower priority
698 		 */
699 		tp = sp->next;
700 		srp = sp->status_rec_list->status_record;
701 		sec = status_rec_p->uurec->sec;
702 		order = cmp_priority(severity, srp->severity, sec,
703 		    srp->uurec->sec, np->max_pct, sp->max_pct);
704 		if (order > 0) {
705 			*rp = np;
706 		} else {
707 			srp = tp->status_rec_list->status_record;
708 			while (tp != sp &&
709 			    cmp_priority(severity, srp->severity, sec,
710 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
711 				tp = tp->next;
712 				srp = tp->status_rec_list->status_record;
713 			}
714 		}
715 		np->next = tp;
716 		np->prev = tp->prev;
717 		tp->prev->next = np;
718 		tp->prev = np;
719 	}
720 }
721 
722 static void
723 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
724     resource_list_t **rpp)
725 {
726 	int order;
727 	resource_list_t *np, *end;
728 	status_record_t *srp;
729 
730 	np = *rpp;
731 	end = np;
732 	while (np) {
733 		if (strcmp(fp->name, np->resource) == 0) {
734 			np->not_suppressed |= status_rec_p->not_suppressed;
735 			srp = np->status_rec_list->status_record;
736 			order = cmp_priority(status_rec_p->severity,
737 			    srp->severity, status_rec_p->uurec->sec,
738 			    srp->uurec->sec, fp->max_pct, np->max_pct);
739 			if (order > 0 && np != end) {
740 				/*
741 				 * remove from list and add again using
742 				 * new priority
743 				 */
744 				np->prev->next = np->next;
745 				np->next->prev = np->prev;
746 				add_resource(status_rec_p,
747 				    rpp, np);
748 			} else {
749 				add_rec_list(status_rec_p,
750 				    &np->status_rec_list);
751 			}
752 			break;
753 		}
754 		np = np->next;
755 		if (np == end) {
756 			np = NULL;
757 			break;
758 		}
759 	}
760 	if (np == NULL) {
761 		np = malloc(sizeof (resource_list_t));
762 		np->resource = fp->name;
763 		np->not_suppressed = status_rec_p->not_suppressed;
764 		np->status_rec_list = NULL;
765 		np->max_pct = fp->max_pct;
766 		add_resource(status_rec_p, rpp, np);
767 	}
768 }
769 
770 static void
771 add_list(status_record_t *status_rec_p, name_list_t *listp,
772     resource_list_t **glistp)
773 {
774 	name_list_t *fp, *end;
775 
776 	fp = listp;
777 	end = fp;
778 	while (fp) {
779 		add_resource_list(status_rec_p, fp, glistp);
780 		fp = fp->next;
781 		if (fp == end)
782 			break;
783 	}
784 }
785 
786 /*
787  * add record to rec, fru and asru lists.
788  */
789 static void
790 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
791     name_list_t *fru, name_list_t *asru, name_list_t *resource,
792     name_list_t *serial, boolean_t not_suppressed,
793     hostid_t *hostid)
794 {
795 	status_record_t *status_rec_p;
796 
797 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
798 	    resource, serial, not_suppressed, hostid);
799 	add_rec_list(status_rec_p, &status_rec_list);
800 	if (status_rec_p->fru)
801 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
802 	if (status_rec_p->asru)
803 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
804 }
805 
806 static void
807 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
808 {
809 	char *name;
810 	char *serial = NULL;
811 	char **lserial = NULL;
812 	uint64_t serint;
813 	name_list_t *nlp;
814 	int j;
815 	uint_t nelem;
816 	char buf[64];
817 
818 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
819 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
820 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
821 			    &serint) == 0) {
822 				(void) snprintf(buf, sizeof (buf), "%llX",
823 				    serint);
824 				nlp = alloc_name_list(buf, pct);
825 				(void) merge_name_list(serial_p, nlp, 1);
826 			}
827 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
828 			if (nvlist_lookup_string_array(nvl,
829 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
830 				nlp = alloc_name_list(lserial[0], pct);
831 				for (j = 1; j < nelem; j++) {
832 					name_list_t *n1lp;
833 					n1lp = alloc_name_list(lserial[j], pct);
834 					(void) merge_name_list(&nlp, n1lp, 1);
835 				}
836 				(void) merge_name_list(serial_p, nlp, 1);
837 			}
838 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
839 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
840 			    &serial) == 0) {
841 				nlp = alloc_name_list(serial, pct);
842 				(void) merge_name_list(serial_p, nlp, 1);
843 			}
844 		}
845 	}
846 }
847 
848 static void
849 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
850     name_list_t **fru_p, name_list_t **serial_p,
851     name_list_t **resource_p, name_list_t **asru_p, uint8_t status)
852 {
853 	nvlist_t *lfru, *lasru, *rsrc;
854 	name_list_t *nlp;
855 	char *name;
856 	uint8_t lpct = 0;
857 	char *lclass = NULL;
858 	char *label;
859 
860 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
861 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
862 		nlp = alloc_name_list(lclass, lpct);
863 		(void) merge_name_list(class_p, nlp, 1);
864 	}
865 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
866 		name = get_nvl2str_topo(lfru);
867 		if (name != NULL) {
868 			nlp = alloc_name_list(name, lpct);
869 			nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
870 			    FM_SUSPECT_DEGRADED);
871 			free(name);
872 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
873 			    &label) == 0)
874 				nlp->label = strdup(label);
875 			(void) merge_name_list(fru_p, nlp, 1);
876 		}
877 		get_serial_no(lfru, serial_p, lpct);
878 	} else if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) != 0) {
879 		/*
880 		 * No FRU or resource. But we want to display the repair status
881 		 * somehow, so create a dummy FRU field.
882 		 */
883 		nlp = alloc_name_list(dgettext("FMD", "None"), lpct);
884 		nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
885 		    FM_SUSPECT_DEGRADED);
886 		(void) merge_name_list(fru_p, nlp, 1);
887 	}
888 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
889 		name = get_nvl2str_topo(lasru);
890 		if (name != NULL) {
891 			nlp = alloc_name_list(name, lpct);
892 			nlp->status = status & ~(FM_SUSPECT_NOT_PRESENT |
893 			    FM_SUSPECT_REPAIRED | FM_SUSPECT_REPLACED |
894 			    FM_SUSPECT_ACQUITTED);
895 			free(name);
896 			(void) merge_name_list(asru_p, nlp, 1);
897 		}
898 		get_serial_no(lasru, serial_p, lpct);
899 	}
900 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
901 		name = get_nvl2str_topo(rsrc);
902 		if (name != NULL) {
903 			nlp = alloc_name_list(name, lpct);
904 			nlp->status = status;
905 			free(name);
906 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
907 			    &label) == 0)
908 				nlp->label = strdup(label);
909 			(void) merge_name_list(resource_p, nlp, 1);
910 		}
911 	}
912 }
913 
914 static void
915 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid)
916 {
917 	char *msgid = "-";
918 	uint_t i, size = 0;
919 	name_list_t *class = NULL, *resource = NULL;
920 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
921 	nvlist_t **nva;
922 	uint8_t *ba;
923 	uurec_t *uurec_p;
924 	hostid_t *host;
925 	boolean_t not_suppressed = 1;
926 	boolean_t any_present = 0;
927 
928 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
929 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
930 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
931 	    &not_suppressed);
932 
933 	if (size != 0) {
934 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
935 		    &nva, &size);
936 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
937 		    &ba, &size);
938 		for (i = 0; i < size; i++) {
939 			extract_record_info(nva[i], &class, &fru, &serial,
940 			    &resource, &asru, ba[i]);
941 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
942 			    (ba[i] & FM_SUSPECT_FAULTY))
943 				any_present = 1;
944 		}
945 		/*
946 		 * also suppress if no resources present
947 		 */
948 		if (any_present == 0)
949 			not_suppressed = 0;
950 	}
951 
952 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
953 	uurec_p->uuid = strdup(uuid);
954 	uurec_p->sec = sec;
955 	uurec_p->ari_uuid_list = NULL;
956 	uurec_p->event = NULL;
957 	(void) nvlist_dup(nvl, &uurec_p->event, 0);
958 	host = find_hostid(nvl);
959 	catalog_new_record(uurec_p, msgid, class, fru, asru,
960 	    resource, serial, not_suppressed, host);
961 }
962 
963 static void
964 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
965 {
966 	sr_list_t *srp;
967 	uurec_t *uurp;
968 	ari_list_t *ari_list;
969 
970 	srp = status_rec_list;
971 	if (srp) {
972 		for (;;) {
973 			uurp = srp->status_record->uurec;
974 			while (uurp) {
975 				if (strcmp(uuid, uurp->uuid) == 0) {
976 					ari_list = (ari_list_t *)
977 					    malloc(sizeof (ari_list_t));
978 					ari_list->ari_uuid = strdup(ari_uuid);
979 					ari_list->next = uurp->ari_uuid_list;
980 					uurp->ari_uuid_list = ari_list;
981 					return;
982 				}
983 				uurp = uurp->next;
984 			}
985 			if (srp->next == status_rec_list)
986 				break;
987 			srp = srp->next;
988 		}
989 	}
990 }
991 
992 static void
993 print_line(char *label, char *buf)
994 {
995 	char *cp, *ep, *wp;
996 	char c;
997 	int i;
998 	int lsz;
999 	char *padding;
1000 
1001 	lsz = strlen(label);
1002 	padding = malloc(lsz + 1);
1003 	for (i = 0; i < lsz; i++)
1004 		padding[i] = ' ';
1005 	padding[i] = 0;
1006 	cp = buf;
1007 	ep = buf;
1008 	c = *ep;
1009 	(void) printf("\n");
1010 	while (c) {
1011 		i = lsz;
1012 		wp = NULL;
1013 		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
1014 			if (c == ' ')
1015 				wp = ep;
1016 			else if (c == '\n') {
1017 				i = 0;
1018 				*ep = 0;
1019 				do {
1020 					ep++;
1021 				} while ((c = *ep) != NULL && c == ' ');
1022 				break;
1023 			}
1024 			ep++;
1025 			i++;
1026 		}
1027 		if (i >= 80 && wp) {
1028 			*wp = 0;
1029 			ep = wp + 1;
1030 			c = *ep;
1031 		}
1032 		(void) printf("%s%s\n", label, cp);
1033 		cp = ep;
1034 		label = padding;
1035 	}
1036 	free(padding);
1037 }
1038 
1039 static void
1040 print_dict_info_line(nvlist_t *e, fmd_msg_item_t what, const char *linehdr)
1041 {
1042 	char *cp = fmd_msg_getitem_nv(fmadm_msghdl, NULL, e, what);
1043 
1044 	if (cp) {
1045 		print_line(dgettext("FMD", linehdr), cp);
1046 		free(cp);
1047 	}
1048 }
1049 
1050 static void
1051 print_dict_info(nvlist_t *nvl)
1052 {
1053 	print_dict_info_line(nvl, FMD_MSG_ITEM_DESC, "Description : ");
1054 	print_dict_info_line(nvl, FMD_MSG_ITEM_RESPONSE, "Response    : ");
1055 	print_dict_info_line(nvl, FMD_MSG_ITEM_IMPACT, "Impact      : ");
1056 	print_dict_info_line(nvl, FMD_MSG_ITEM_ACTION, "Action      : ");
1057 }
1058 
1059 static void
1060 print_name(name_list_t *list, char *(func)(char *), char *padding, int *np,
1061     int pct, int full)
1062 {
1063 	char *name, *fru_label = NULL;
1064 
1065 	name = list->name;
1066 	if (list->label) {
1067 		(void) printf("%s \"%s\" (%s)", padding, list->label, name);
1068 		*np += 1;
1069 	} else if (func && (fru_label = func(list->name)) != NULL) {
1070 		(void) printf("%s \"%s\" (%s)", padding, fru_label, name);
1071 		*np += 1;
1072 		free(fru_label);
1073 	} else {
1074 		(void) printf("%s %s", padding, name);
1075 		*np += 1;
1076 	}
1077 	if (list->pct && pct > 0 && pct < 100) {
1078 		if (list->count > 1) {
1079 			if (full) {
1080 				(void) printf(" %d @ %s %d%%\n", list->count,
1081 				    dgettext("FMD", "max"),
1082 				    list->max_pct);
1083 			} else {
1084 				(void) printf(" %s %d%%\n",
1085 				    dgettext("FMD", "max"),
1086 				    list->max_pct);
1087 			}
1088 		} else {
1089 			(void) printf(" %d%%\n", list->pct);
1090 		}
1091 	} else {
1092 		(void) printf("\n");
1093 	}
1094 }
1095 
1096 static void
1097 print_asru_status(int status, char *label)
1098 {
1099 	char *msg = NULL;
1100 
1101 	switch (status) {
1102 	case 0:
1103 		msg = dgettext("FMD", "ok and in service");
1104 		break;
1105 	case FM_SUSPECT_DEGRADED:
1106 		msg = dgettext("FMD", "service degraded, "
1107 		    "but associated components no longer faulty");
1108 		break;
1109 	case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1110 		msg = dgettext("FMD", "faulted but still "
1111 		    "providing degraded service");
1112 		break;
1113 	case FM_SUSPECT_FAULTY:
1114 		msg = dgettext("FMD", "faulted but still in service");
1115 		break;
1116 	case FM_SUSPECT_UNUSABLE:
1117 		msg = dgettext("FMD", "out of service, "
1118 		    "but associated components no longer faulty");
1119 		break;
1120 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1121 		msg = dgettext("FMD", "faulted and taken out of service");
1122 		break;
1123 	default:
1124 		break;
1125 	}
1126 	if (msg) {
1127 		(void) printf("%s     %s\n", label, msg);
1128 	}
1129 }
1130 
1131 static void
1132 print_fru_status(int status, char *label)
1133 {
1134 	char *msg = NULL;
1135 
1136 	if (status & FM_SUSPECT_NOT_PRESENT)
1137 		msg = dgettext("FMD", "not present");
1138 	else if (status & FM_SUSPECT_FAULTY)
1139 		msg = dgettext("FMD", "faulty");
1140 	else if (status & FM_SUSPECT_REPLACED)
1141 		msg = dgettext("FMD", "replaced");
1142 	else if (status & FM_SUSPECT_REPAIRED)
1143 		msg = dgettext("FMD", "repair attempted");
1144 	else if (status & FM_SUSPECT_ACQUITTED)
1145 		msg = dgettext("FMD", "acquitted");
1146 	else
1147 		msg = dgettext("FMD", "removed");
1148 	(void) printf("%s     %s\n", label, msg);
1149 }
1150 
1151 static void
1152 print_rsrc_status(int status, char *label)
1153 {
1154 	char *msg = "";
1155 
1156 	if (status & FM_SUSPECT_NOT_PRESENT)
1157 		msg = dgettext("FMD", "not present");
1158 	else if (status & FM_SUSPECT_FAULTY) {
1159 		if (status & FM_SUSPECT_DEGRADED)
1160 			msg = dgettext("FMD",
1161 			    "faulted but still providing degraded service");
1162 		else if (status & FM_SUSPECT_UNUSABLE)
1163 			msg = dgettext("FMD",
1164 			    "faulted and taken out of service");
1165 		else
1166 			msg = dgettext("FMD", "faulted but still in service");
1167 	} else if (status & FM_SUSPECT_REPLACED)
1168 		msg = dgettext("FMD", "replaced");
1169 	else if (status & FM_SUSPECT_REPAIRED)
1170 		msg = dgettext("FMD", "repair attempted");
1171 	else if (status & FM_SUSPECT_ACQUITTED)
1172 		msg = dgettext("FMD", "acquitted");
1173 	else
1174 		msg = dgettext("FMD", "removed");
1175 	(void) printf("%s     %s\n", label, msg);
1176 }
1177 
1178 static void
1179 print_name_list(name_list_t *list, char *label, char *(func)(char *),
1180     int limit, int pct, void (func1)(int, char *), int full)
1181 {
1182 	char *name, *fru_label = NULL;
1183 	char *padding;
1184 	int i, j, l, n;
1185 	name_list_t *end = list;
1186 
1187 	l = strlen(label);
1188 	padding = malloc(l + 1);
1189 	for (i = 0; i < l; i++)
1190 		padding[i] = ' ';
1191 	padding[l] = 0;
1192 	(void) printf("%s", label);
1193 	name = list->name;
1194 	if (list->label)
1195 		(void) printf(" \"%s\" (%s)", list->label, name);
1196 	else if (func && (fru_label = func(list->name)) != NULL) {
1197 		(void) printf(" \"%s\" (%s)", fru_label, name);
1198 		free(fru_label);
1199 	} else
1200 		(void) printf(" %s", name);
1201 	if (list->pct && pct > 0 && pct < 100) {
1202 		if (list->count > 1) {
1203 			if (full) {
1204 				(void) printf(" %d @ %s %d%%\n", list->count,
1205 				    dgettext("FMD", "max"), list->max_pct);
1206 			} else {
1207 				(void) printf(" %s %d%%\n",
1208 				    dgettext("FMD", "max"), list->max_pct);
1209 			}
1210 		} else {
1211 			(void) printf(" %d%%\n", list->pct);
1212 		}
1213 	} else {
1214 		(void) printf("\n");
1215 	}
1216 	if (func1)
1217 		func1(list->status, padding);
1218 	n = 1;
1219 	j = 0;
1220 	while ((list = list->next) != end) {
1221 		if (limit == 0 || n < limit) {
1222 			print_name(list, func, padding, &n, pct, full);
1223 			if (func1)
1224 				func1(list->status, padding);
1225 		} else
1226 			j++;
1227 	}
1228 	if (j == 1) {
1229 		print_name(list->prev, func, padding, &n, pct, full);
1230 	} else if (j > 1) {
1231 		(void) printf("%s... %d %s\n", padding, j,
1232 		    dgettext("FMD", "more entries suppressed,"
1233 		    " use -v option for full list"));
1234 	}
1235 	free(padding);
1236 }
1237 
1238 static int
1239 asru_same_status(name_list_t *list)
1240 {
1241 	name_list_t *end = list;
1242 	int status = list->status;
1243 
1244 	while ((list = list->next) != end) {
1245 		if (status == -1) {
1246 			status = list->status;
1247 			continue;
1248 		}
1249 		if (list->status != -1 && status != list->status) {
1250 			status = -1;
1251 			break;
1252 		}
1253 	}
1254 	return (status);
1255 }
1256 
1257 static int
1258 serial_in_fru(name_list_t *fru, name_list_t *serial)
1259 {
1260 	name_list_t *sp = serial;
1261 	name_list_t *fp;
1262 	int nserial = 0;
1263 	int found = 0;
1264 	char buf[128];
1265 
1266 	while (sp) {
1267 		fp = fru;
1268 		nserial++;
1269 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1270 		buf[sizeof (buf) - 1] = 0;
1271 		while (fp) {
1272 			if (strstr(fp->name, buf) != NULL) {
1273 				found++;
1274 				break;
1275 			}
1276 			fp = fp->next;
1277 			if (fp == fru)
1278 				break;
1279 		}
1280 		sp = sp->next;
1281 		if (sp == serial)
1282 			break;
1283 	}
1284 	return (found == nserial ? 1 : 0);
1285 }
1286 
1287 static void
1288 print_sup_record(status_record_t *srp, int opt_i, int full)
1289 {
1290 	char buf[32];
1291 	uurec_t *uurp = srp->uurec;
1292 	int n, j, k, max;
1293 	int status;
1294 	ari_list_t *ari_list;
1295 
1296 	n = 0;
1297 	max = max_fault;
1298 	if (max < 0) {
1299 		max = 0;
1300 	}
1301 	j = max / 2;
1302 	max -= j;
1303 	k = srp->nrecs - max;
1304 	while ((uurp = uurp->next) != NULL) {
1305 		if (full || n < j || n >= k || max_fault == 0 ||
1306 		    srp->nrecs == max_fault+1) {
1307 			if (opt_i) {
1308 				ari_list = uurp->ari_uuid_list;
1309 				while (ari_list) {
1310 					(void) printf("%-15s %s\n",
1311 					    format_date(buf, sizeof (buf),
1312 					    uurp->sec), ari_list->ari_uuid);
1313 					ari_list = ari_list->next;
1314 				}
1315 			} else {
1316 				(void) printf("%-15s %s\n",
1317 				    format_date(buf, sizeof (buf), uurp->sec),
1318 				    uurp->uuid);
1319 			}
1320 		} else if (n == j)
1321 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1322 			    dgettext("FMD", "more entries suppressed"));
1323 		n++;
1324 	}
1325 	(void) printf("\n");
1326 	(void) printf("%s %s", dgettext("FMD", "Host        :"),
1327 	    srp->host->server);
1328 	if (srp->host->domain)
1329 		(void) printf("\t%s %s", dgettext("FMD", "Domain      :"),
1330 		    srp->host->domain);
1331 	(void) printf("\n%s %s", dgettext("FMD", "Platform    :"),
1332 	    srp->host->platform);
1333 	(void) printf("\t%s %s", dgettext("FMD", "Chassis_id  :"),
1334 	    srp->host->chassis ? srp->host->chassis : "");
1335 	(void) printf("\n%s %s\n\n", dgettext("FMD", "Product_sn  :"),
1336 	    srp->host->product_sn? srp->host->product_sn : "");
1337 	if (srp->class)
1338 		print_name_list(srp->class,
1339 		    dgettext("FMD", "Fault class :"), NULL, 0, srp->class->pct,
1340 		    NULL, full);
1341 	if (srp->asru) {
1342 		status = asru_same_status(srp->asru);
1343 		if (status != -1) {
1344 			print_name_list(srp->asru,
1345 			    dgettext("FMD", "Affects     :"), NULL,
1346 			    full ? 0 : max_display, 0, NULL, full);
1347 			print_asru_status(status, "             ");
1348 		} else
1349 			print_name_list(srp->asru,
1350 			    dgettext("FMD", "Affects     :"), NULL,
1351 			    full ? 0 : max_display, 0, print_asru_status, full);
1352 	}
1353 	if (full || srp->fru == NULL || srp->asru == NULL) {
1354 		if (srp->resource) {
1355 			status = asru_same_status(srp->resource);
1356 			if (status != -1) {
1357 				print_name_list(srp->resource,
1358 				    dgettext("FMD", "Problem in  :"), NULL,
1359 				    full ? 0 : max_display, 0, NULL, full);
1360 				print_rsrc_status(status, "             ");
1361 			} else
1362 				print_name_list(srp->resource,
1363 				    dgettext("FMD", "Problem in  :"),
1364 				    NULL, full ? 0 : max_display, 0,
1365 				    print_rsrc_status, full);
1366 		}
1367 	}
1368 	if (srp->fru) {
1369 		status = asru_same_status(srp->fru);
1370 		if (status != -1) {
1371 			print_name_list(srp->fru, dgettext("FMD",
1372 			    "FRU         :"), get_fmri_label, 0,
1373 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1374 			    NULL, full);
1375 			print_fru_status(status, "             ");
1376 		} else
1377 			print_name_list(srp->fru, dgettext("FMD",
1378 			    "FRU         :"), get_fmri_label, 0,
1379 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1380 			    print_fru_status, full);
1381 	}
1382 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1383 	    !serial_in_fru(srp->asru, srp->serial)) {
1384 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1385 		    NULL, 0, 0, NULL, full);
1386 	}
1387 	print_dict_info(srp->uurec->event);
1388 	(void) printf("\n");
1389 }
1390 
1391 static void
1392 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1393 {
1394 	char buf[32];
1395 	uurec_t *uurp = srp->uurec;
1396 	static int header = 0;
1397 	char *head;
1398 	ari_list_t *ari_list;
1399 
1400 	if (!summary || !header) {
1401 		if (opt_i) {
1402 			head = "--------------- "
1403 			    "------------------------------------  "
1404 			    "-------------- ---------\n"
1405 			    "TIME            CACHE-ID"
1406 			    "                              MSG-ID"
1407 			    "         SEVERITY\n--------------- "
1408 			    "------------------------------------ "
1409 			    " -------------- ---------";
1410 		} else {
1411 			head = "--------------- "
1412 			    "------------------------------------  "
1413 			    "-------------- ---------\n"
1414 			    "TIME            EVENT-ID"
1415 			    "                              MSG-ID"
1416 			    "         SEVERITY\n--------------- "
1417 			    "------------------------------------ "
1418 			    " -------------- ---------";
1419 		}
1420 		(void) printf("%s\n", dgettext("FMD", head));
1421 		header = 1;
1422 	}
1423 	if (opt_i) {
1424 		ari_list = uurp->ari_uuid_list;
1425 		while (ari_list) {
1426 			(void) printf("%-15s %-37s %-14s %-9s\n",
1427 			    format_date(buf, sizeof (buf), uurp->sec),
1428 			    ari_list->ari_uuid, srp->msgid, srp->severity);
1429 			ari_list = ari_list->next;
1430 		}
1431 	} else {
1432 		(void) printf("%-15s %-37s %-14s %-9s\n",
1433 		    format_date(buf, sizeof (buf), uurp->sec),
1434 		    uurp->uuid, srp->msgid, srp->severity);
1435 	}
1436 
1437 	if (!summary)
1438 		print_sup_record(srp, opt_i, full);
1439 }
1440 
1441 static void
1442 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1443 {
1444 	status_record_t *srp;
1445 	sr_list_t *slp;
1446 
1447 	slp = status_rec_list;
1448 	if (slp) {
1449 		for (;;) {
1450 			srp = slp->status_record;
1451 			if (opt_a || srp->not_suppressed) {
1452 				if (page_feed)
1453 					(void) printf("\f\n");
1454 				print_status_record(srp, summary, opt_i, full);
1455 			}
1456 			if (slp->next == status_rec_list)
1457 				break;
1458 			slp = slp->next;
1459 		}
1460 	}
1461 }
1462 
1463 static name_list_t *
1464 find_fru(status_record_t *srp, char *resource)
1465 {
1466 	name_list_t *rt = NULL;
1467 	name_list_t *fru = srp->fru;
1468 
1469 	while (fru) {
1470 		if (strcmp(resource, fru->name) == 0) {
1471 			rt = fru;
1472 			break;
1473 		}
1474 		fru = fru->next;
1475 		if (fru == srp->fru)
1476 			break;
1477 	}
1478 	return (rt);
1479 }
1480 
1481 static void
1482 print_fru_line(name_list_t *fru, char *uuid)
1483 {
1484 	if (fru->pct == 100) {
1485 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1486 		    dgettext("FMD", "suspects in this FRU total certainty"),
1487 		    100);
1488 	} else {
1489 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1490 		    dgettext("FMD", "suspects in this FRU max certainty"),
1491 		    fru->max_pct);
1492 	}
1493 }
1494 
1495 static void
1496 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1497 {
1498 	resource_list_t *tp = status_fru_list;
1499 	status_record_t *srp;
1500 	sr_list_t *slp, *end;
1501 	char *msgid, *fru_label;
1502 	uurec_t *uurp;
1503 	name_list_t *fru;
1504 	int status;
1505 	ari_list_t *ari_list;
1506 
1507 	while (tp) {
1508 		if (opt_a || tp->not_suppressed) {
1509 			if (page_feed)
1510 				(void) printf("\f\n");
1511 			if (!summary)
1512 				(void) printf("-----------------------------"
1513 				    "---------------------------------------"
1514 				    "----------\n");
1515 			slp = tp->status_rec_list;
1516 			end = slp;
1517 			do {
1518 				srp = slp->status_record;
1519 				fru = find_fru(srp, tp->resource);
1520 				if (fru) {
1521 					if (fru->label)
1522 						(void) printf("\"%s\" (%s) ",
1523 						    fru->label, fru->name);
1524 					else if ((fru_label = get_fmri_label(
1525 					    fru->name)) != NULL) {
1526 						(void) printf("\"%s\" (%s) ",
1527 						    fru_label, fru->name);
1528 						free(fru_label);
1529 					} else
1530 						(void) printf("%s ",
1531 						    fru->name);
1532 					break;
1533 				}
1534 				slp = slp->next;
1535 			} while (slp != end);
1536 
1537 			slp = tp->status_rec_list;
1538 			end = slp;
1539 			status = 0;
1540 			do {
1541 				srp = slp->status_record;
1542 				fru = srp->fru;
1543 				while (fru) {
1544 					if (strcmp(tp->resource,
1545 					    fru->name) == 0)
1546 						status |= fru->status;
1547 					fru = fru->next;
1548 					if (fru == srp->fru)
1549 						break;
1550 				}
1551 				slp = slp->next;
1552 			} while (slp != end);
1553 			if (status & FM_SUSPECT_NOT_PRESENT)
1554 				(void) printf(dgettext("FMD", "not present\n"));
1555 			else if (status & FM_SUSPECT_FAULTY)
1556 				(void) printf(dgettext("FMD", "faulty\n"));
1557 			else if (status & FM_SUSPECT_REPLACED)
1558 				(void) printf(dgettext("FMD", "replaced\n"));
1559 			else if (status & FM_SUSPECT_REPAIRED)
1560 				(void) printf(dgettext("FMD",
1561 				    "repair attempted\n"));
1562 			else if (status & FM_SUSPECT_ACQUITTED)
1563 				(void) printf(dgettext("FMD", "acquitted\n"));
1564 			else
1565 				(void) printf(dgettext("FMD", "removed\n"));
1566 
1567 			slp = tp->status_rec_list;
1568 			end = slp;
1569 			do {
1570 				srp = slp->status_record;
1571 				uurp = srp->uurec;
1572 				fru = find_fru(srp, tp->resource);
1573 				if (fru) {
1574 					if (opt_i) {
1575 						ari_list = uurp->ari_uuid_list;
1576 						while (ari_list) {
1577 							print_fru_line(fru,
1578 							    ari_list->ari_uuid);
1579 							ari_list =
1580 							    ari_list->next;
1581 						}
1582 					} else {
1583 						print_fru_line(fru, uurp->uuid);
1584 					}
1585 				}
1586 				slp = slp->next;
1587 			} while (slp != end);
1588 			if (!summary) {
1589 				slp = tp->status_rec_list;
1590 				end = slp;
1591 				srp = slp->status_record;
1592 				if (srp->serial &&
1593 				    !serial_in_fru(srp->fru, srp->serial)) {
1594 					print_name_list(srp->serial,
1595 					    dgettext("FMD", "Serial ID.  :"),
1596 					    NULL, 0, 0, NULL, 1);
1597 				}
1598 				msgid = NULL;
1599 				do {
1600 					if (msgid == NULL ||
1601 					    strcmp(msgid, srp->msgid) != 0) {
1602 						msgid = srp->msgid;
1603 						print_dict_info(uurp->event);
1604 					}
1605 					slp = slp->next;
1606 				} while (slp != end);
1607 			}
1608 		}
1609 		tp = tp->next;
1610 		if (tp == status_fru_list)
1611 			break;
1612 	}
1613 }
1614 
1615 static void
1616 print_asru(int opt_a)
1617 {
1618 	resource_list_t *tp = status_asru_list;
1619 	status_record_t *srp;
1620 	sr_list_t *slp, *end;
1621 	char *msg;
1622 	int status;
1623 	name_list_t *asru;
1624 
1625 	while (tp) {
1626 		if (opt_a || tp->not_suppressed) {
1627 			status = 0;
1628 			slp = tp->status_rec_list;
1629 			end = slp;
1630 			do {
1631 				srp = slp->status_record;
1632 				asru = srp->asru;
1633 				while (asru) {
1634 					if (strcmp(tp->resource,
1635 					    asru->name) == 0)
1636 						status |= asru->status;
1637 					asru = asru->next;
1638 					if (asru == srp->asru)
1639 						break;
1640 				}
1641 				slp = slp->next;
1642 			} while (slp != end);
1643 			switch (status) {
1644 			case 0:
1645 				msg = dgettext("FMD", "ok");
1646 				break;
1647 			case FM_SUSPECT_DEGRADED:
1648 				msg = dgettext("FMD", "degraded");
1649 				break;
1650 			case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1651 				msg = dgettext("FMD", "degraded");
1652 				break;
1653 			case FM_SUSPECT_FAULTY:
1654 				msg = dgettext("FMD", "degraded");
1655 				break;
1656 			case FM_SUSPECT_UNUSABLE:
1657 				msg = dgettext("FMD", "unknown");
1658 				break;
1659 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1660 				msg = dgettext("FMD", "faulted");
1661 				break;
1662 			default:
1663 				msg = "";
1664 				break;
1665 			}
1666 			(void) printf("%-69s %s\n", tp->resource, msg);
1667 		}
1668 		tp = tp->next;
1669 		if (tp == status_asru_list)
1670 			break;
1671 	}
1672 }
1673 
1674 static int
1675 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1676 {
1677 	while (uurecp) {
1678 		if (strcmp(uuid, uurecp->uuid) == 0)
1679 			return (1);
1680 		uurecp = uurecp->next;
1681 	}
1682 	return (0);
1683 }
1684 
1685 static int
1686 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1687 {
1688 	int64_t *diag_time;
1689 	uint_t nelem;
1690 	int rt = 0;
1691 	char *uuid = "-";
1692 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1693 
1694 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1695 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1696 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1697 		    &uuid);
1698 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1699 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1700 			    uuid);
1701 	} else {
1702 		rt = -1;
1703 	}
1704 	return (rt);
1705 }
1706 
1707 /*ARGSUSED*/
1708 static int
1709 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1710 {
1711 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1712 	return (0);
1713 }
1714 
1715 static int
1716 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1717 {
1718 	int rt = FMADM_EXIT_SUCCESS;
1719 
1720 	/*
1721 	 * These calls may fail with Protocol error if message payload is
1722 	 * too big
1723 	 */
1724 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1725 		die("failed to get case list from fmd");
1726 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1727 		die("failed to get case status from fmd");
1728 	return (rt);
1729 }
1730 
1731 /*
1732  * fmadm faulty command
1733  *
1734  *	-a		show hidden fault records
1735  *	-f		show faulty fru's
1736  *	-g		force grouping of similar faults on the same fru
1737  *	-n		number of fault records to display
1738  *	-p		pipe output through pager
1739  *	-r		show faulty asru's
1740  *	-s		print summary of first fault
1741  *	-u		print listed uuid's only
1742  *	-v		full output
1743  */
1744 
1745 int
1746 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1747 {
1748 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1749 	int opt_i = 0;
1750 	char *pager;
1751 	FILE *fp;
1752 	int rt, c, stat;
1753 	uurec_select_t *tp;
1754 	uurec_select_t *uurecp = NULL;
1755 
1756 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1757 		switch (c) {
1758 		case 'a':
1759 			opt_a++;
1760 			break;
1761 		case 'f':
1762 			opt_f++;
1763 			break;
1764 		case 'g':
1765 			opt_g++;
1766 			break;
1767 		case 'i':
1768 			opt_i++;
1769 			break;
1770 		case 'n':
1771 			max_fault = atoi(optarg);
1772 			break;
1773 		case 'p':
1774 			opt_p++;
1775 			break;
1776 		case 'r':
1777 			opt_r++;
1778 			break;
1779 		case 's':
1780 			opt_s++;
1781 			break;
1782 		case 'u':
1783 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1784 			tp->uuid = optarg;
1785 			tp->next = uurecp;
1786 			uurecp = tp;
1787 			opt_a = 1;
1788 			break;
1789 		case 'v':
1790 			opt_v++;
1791 			break;
1792 		default:
1793 			return (FMADM_EXIT_USAGE);
1794 		}
1795 	}
1796 	if (optind < argc)
1797 		return (FMADM_EXIT_USAGE);
1798 
1799 	if ((fmadm_msghdl = fmd_msg_init(NULL, FMD_MSG_VERSION)) == NULL)
1800 		return (FMADM_EXIT_ERROR);
1801 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1802 	if (opt_p) {
1803 		if ((pager = getenv("PAGER")) == NULL)
1804 			pager = "/usr/bin/more";
1805 		fp = popen(pager, "w");
1806 		if (fp == NULL) {
1807 			rt = FMADM_EXIT_ERROR;
1808 			opt_p = 0;
1809 		} else {
1810 			dup2(fileno(fp), 1);
1811 			setbuf(stdout, NULL);
1812 			(void) fclose(fp);
1813 		}
1814 	}
1815 	max_display = max_fault;
1816 	if (opt_f)
1817 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1818 	if (opt_r)
1819 		print_asru(opt_a);
1820 	if (opt_f == 0 && opt_r == 0)
1821 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1822 	fmd_msg_fini(fmadm_msghdl);
1823 	label_release_topo();
1824 	if (opt_p) {
1825 		(void) fclose(stdout);
1826 		(void) wait(&stat);
1827 	}
1828 	return (rt);
1829 }
1830 
1831 int
1832 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1833 {
1834 	int i, status = FMADM_EXIT_SUCCESS;
1835 
1836 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1837 		return (FMADM_EXIT_USAGE);
1838 
1839 	for (i = 1; i < argc; i++) {
1840 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1841 			warn("failed to flush %s", argv[i]);
1842 			status = FMADM_EXIT_ERROR;
1843 		} else
1844 			note("flushed resource history for %s\n", argv[i]);
1845 	}
1846 
1847 	return (status);
1848 }
1849 
1850 int
1851 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
1852 {
1853 	int err;
1854 
1855 	if (getopt(argc, argv, "") != EOF)
1856 		return (FMADM_EXIT_USAGE);
1857 
1858 	if (argc - optind != 1)
1859 		return (FMADM_EXIT_USAGE);
1860 
1861 	/*
1862 	 * argument could be a uuid, an fmri (asru, fru or resource)
1863 	 * or a label. Try uuid first, If that fails try the others.
1864 	 */
1865 	err = fmd_adm_case_repair(adm, argv[optind]);
1866 	if (err != 0)
1867 		err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1868 
1869 	if (err != 0)
1870 		die("failed to record repair to %s", argv[optind]);
1871 
1872 	note("recorded repair to %s\n", argv[optind]);
1873 	return (FMADM_EXIT_SUCCESS);
1874 }
1875 
1876 int
1877 cmd_repaired(fmd_adm_t *adm, int argc, char *argv[])
1878 {
1879 	int err;
1880 
1881 	if (getopt(argc, argv, "") != EOF)
1882 		return (FMADM_EXIT_USAGE);
1883 
1884 	if (argc - optind != 1)
1885 		return (FMADM_EXIT_USAGE);
1886 
1887 	/*
1888 	 * argument could be an fmri (asru, fru or resource) or a label.
1889 	 */
1890 	err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1891 	if (err != 0)
1892 		die("failed to record repair to %s", argv[optind]);
1893 
1894 	note("recorded repair to of %s\n", argv[optind]);
1895 	return (FMADM_EXIT_SUCCESS);
1896 }
1897 
1898 int
1899 cmd_replaced(fmd_adm_t *adm, int argc, char *argv[])
1900 {
1901 	int err;
1902 
1903 	if (getopt(argc, argv, "") != EOF)
1904 		return (FMADM_EXIT_USAGE);
1905 
1906 	if (argc - optind != 1)
1907 		return (FMADM_EXIT_USAGE);
1908 
1909 	/*
1910 	 * argument could be an fmri (asru, fru or resource) or a label.
1911 	 */
1912 	err = fmd_adm_rsrc_replaced(adm, argv[optind]);
1913 	if (err != 0)
1914 		die("failed to record replacement of %s", argv[optind]);
1915 
1916 	note("recorded replacement of %s\n", argv[optind]);
1917 	return (FMADM_EXIT_SUCCESS);
1918 }
1919 
1920 int
1921 cmd_acquit(fmd_adm_t *adm, int argc, char *argv[])
1922 {
1923 	int err;
1924 
1925 	if (getopt(argc, argv, "") != EOF)
1926 		return (FMADM_EXIT_USAGE);
1927 
1928 	if (argc - optind != 1 && argc - optind != 2)
1929 		return (FMADM_EXIT_USAGE);
1930 
1931 	/*
1932 	 * argument could be a uuid, an fmri (asru, fru or resource)
1933 	 * or a label. Or it could be a uuid and an fmri or label.
1934 	 */
1935 	if (argc - optind == 2) {
1936 		err = fmd_adm_rsrc_acquit(adm, argv[optind], argv[optind + 1]);
1937 		if (err != 0)
1938 			err = fmd_adm_rsrc_acquit(adm, argv[optind + 1],
1939 			    argv[optind]);
1940 	} else {
1941 		err = fmd_adm_case_acquit(adm, argv[optind]);
1942 		if (err != 0)
1943 			err = fmd_adm_rsrc_acquit(adm, argv[optind], "");
1944 	}
1945 
1946 	if (err != 0)
1947 		die("failed to record acquital of %s", argv[optind]);
1948 
1949 	note("recorded acquital of %s\n", argv[optind]);
1950 	return (FMADM_EXIT_SUCCESS);
1951 }
1952