xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision b6955755)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <fmadm.h>
28 #include <errno.h>
29 #include <limits.h>
30 #include <strings.h>
31 #include <stdio.h>
32 #include <unistd.h>
33 #include <sys/wait.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <fm/fmd_log.h>
37 #include <sys/fm/protocol.h>
38 #include <fm/libtopo.h>
39 #include <fm/fmd_adm.h>
40 #include <fm/fmd_msg.h>
41 #include <dlfcn.h>
42 #include <sys/systeminfo.h>
43 #include <sys/utsname.h>
44 #include <libintl.h>
45 #include <locale.h>
46 #include <sys/smbios.h>
47 #include <libdevinfo.h>
48 #include <stdlib.h>
49 
50 #define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))
51 
52 /*
53  * Fault records are added to catalog by calling add_fault_record_to_catalog()
54  * records are stored in order of importance to the system.
55  * If -g flag is set or not_suppressed is not set and the class fru, fault,
56  * type are the same then details are merged into an existing record, with uuid
57  * records are stored in time order.
58  * For each record information is extracted from nvlist and merged into linked
59  * list each is checked for identical records for which percentage certainty are
60  * added together.
61  * print_catalog() is called to print out catalog and release external resources
62  *
63  *                         /---------------\
64  *	status_rec_list -> |               | -|
65  *                         \---------------/
66  *                                \/
67  *                         /---------------\    /-------\    /-------\
68  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
69  *            \/           |               | |- |       | <- |       |
70  *      /-------------\    |               |    \-------/    \-------/
71  *      |             | -> |               |       \/           \/
72  *      \-------------/    |               |    /-------\    /-------\
73  *            \/           |               | -> | asru  | -> | asru  |
74  *            ---          |               |    |       | <- |       |
75  *                         |               |    \-------/    \-------/
76  *      status_asru_list   |  class        |
77  *            \/           |  resource     |    /-------\    /-------\
78  *      /-------------\    |  fru          | -> | list  | -> | list  |
79  *      |             | -> |  serial       |    |       | <- |       |
80  *      \-------------/    |               |    \-------/    \-------/
81  *            \/           \---------------/
82  *            ---               \/    /\
83  *                         /---------------\
84  *                         | status_record |
85  *                         \---------------/
86  *
87  * Fmadm faulty takes a number of options which affect the format of the
88  * output displayed. By default, the display reports the FRU and ASRU along
89  * with other information on per-case basis as in the example below.
90  *
91  * --------------- ------------------------------------  -------------- -------
92  * TIME            EVENT-ID                              MSG-ID         SEVERITY
93  * --------------- ------------------------------------  -------------- -------
94  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
95  *
96  * Fault class	: fault.memory.dimm_sb
97  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
98  *		    faulted but still in service
99  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
100  *		    faulty
101  *
102  * Description	: The number of errors associated with this memory module has
103  *		exceeded acceptable levels.  Refer to
104  *		http://sun.com/msg/AMD-8000-2F for more information.
105  *
106  * Response	: Pages of memory associated with this memory module are being
107  *		removed from service as errors are reported.
108  *
109  * Impact	: Total system memory capacity will be reduced as pages are
110  *		retired.
111  *
112  * Action	: Schedule a repair procedure to replace the affected memory
113  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
114  *
115  * The -v flag is similar, but adds some additonal information such as the
116  * resource. The -s flag is also similar but just gives the top line summary.
117  * All these options (ie without the -f or -r flags) use the print_catalog()
118  * function to do the display.
119  *
120  * The -f flag changes the output so that it appears sorted on a per-fru basis.
121  * The output is somewhat cut down compared to the default output. If -f is
122  * used, then print_fru() is used to print the output.
123  *
124  * -----------------------------------------------------------------------------
125  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
126  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
127  *
128  * Description	: A problem was detected for a PCI device.
129  *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
130  *
131  * Response	: One or more device instances may be disabled
132  *
133  * Impact	: Possible loss of services provided by the device instances
134  *		associated with this fault
135  *
136  * Action	: Schedule a repair procedure to replace the affected device.
137  * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
138  *		Sun for support.
139  *
140  * The -r flag changes the output so that it appears sorted on a per-asru basis.
141  * The output is very much cut down compared to the default output, just giving
142  * the asru fmri and state. Here print_asru() is used to print the output.
143  *
144  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
145  *
146  * For all fmadm faulty options, the sequence of events is
147  *
148  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
149  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
150  * This will extract the data from the nvlist and call catalog_new_record() to
151  * save the data away in various linked lists in the catalogue.
152  *
153  * 2) Once this is done, the data can be supplemented by using
154  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
155  *
156  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
157  * appropriate to display the information from the catalogue sorted in the
158  * requested way.
159  *
160  */
161 
162 typedef struct name_list {
163 	struct name_list *next;
164 	struct name_list *prev;
165 	char *name;
166 	uint8_t pct;
167 	uint8_t max_pct;
168 	ushort_t count;
169 	int status;
170 	char *label;
171 } name_list_t;
172 
173 typedef struct ari_list {
174 	char *ari_uuid;
175 	struct ari_list *next;
176 } ari_list_t;
177 
178 typedef struct uurec {
179 	struct uurec *next;
180 	struct uurec *prev;
181 	char *uuid;
182 	ari_list_t *ari_uuid_list;
183 	name_list_t *asru;
184 	uint64_t sec;
185 } uurec_t;
186 
187 typedef struct uurec_select {
188 	struct uurec_select *next;
189 	char *uuid;
190 } uurec_select_t;
191 
192 typedef struct host_id {
193 	char *chassis;
194 	char *server;
195 	char *platform;
196 	char *domain;
197 } hostid_t;
198 
199 typedef struct host_id_list {
200 	hostid_t hostid;
201 	struct host_id_list *next;
202 } host_id_list_t;
203 
204 typedef struct status_record {
205 	hostid_t *host;
206 	int nrecs;
207 	uurec_t *uurec;
208 	char *severity;			/* in C locale */
209 	char *msgid;
210 	name_list_t *class;
211 	name_list_t *resource;
212 	name_list_t *asru;
213 	name_list_t *fru;
214 	name_list_t *serial;
215 	uint8_t not_suppressed;
216 } status_record_t;
217 
218 typedef struct sr_list {
219 	struct sr_list *next;
220 	struct sr_list *prev;
221 	struct status_record *status_record;
222 } sr_list_t;
223 
224 typedef struct resource_list {
225 	struct resource_list *next;
226 	struct resource_list *prev;
227 	sr_list_t *status_rec_list;
228 	char *resource;
229 	uint8_t not_suppressed;
230 	uint8_t max_pct;
231 } resource_list_t;
232 
233 typedef struct tgetlabel_data {
234 	char *label;
235 	char *fru;
236 } tgetlabel_data_t;
237 
238 sr_list_t *status_rec_list;
239 resource_list_t *status_fru_list;
240 resource_list_t *status_asru_list;
241 
242 static int max_display;
243 static int max_fault = 0;
244 static topo_hdl_t *topo_handle;
245 static char *topo_handle_uuid;
246 static host_id_list_t *host_list;
247 static int n_server;
248 static int opt_g;
249 static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */
250 
251 static char *
252 format_date(char *buf, size_t len, uint64_t sec)
253 {
254 	if (sec > LONG_MAX) {
255 		(void) fprintf(stderr,
256 		    "record time is too large for 32-bit utility\n");
257 		(void) snprintf(buf, len, "0x%llx", sec);
258 	} else {
259 		time_t tod = (time_t)sec;
260 		(void) strftime(buf, len, "%b %d %T", localtime(&tod));
261 	}
262 
263 	return (buf);
264 }
265 
266 static hostid_t *
267 find_hostid_in_list(char *platform, char *chassis, char *server, char *domain)
268 {
269 	hostid_t *rt = NULL;
270 	host_id_list_t *hostp;
271 
272 	if (platform == NULL)
273 		platform = "-";
274 	if (server == NULL)
275 		server = "-";
276 	hostp = host_list;
277 	while (hostp) {
278 		if (hostp->hostid.platform &&
279 		    strcmp(hostp->hostid.platform, platform) == 0 &&
280 		    hostp->hostid.server &&
281 		    strcmp(hostp->hostid.server, server) == 0 &&
282 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
283 		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
284 		    (domain == NULL || hostp->hostid.domain == NULL ||
285 		    strcmp(domain, hostp->hostid.domain) == 0)) {
286 			rt = &hostp->hostid;
287 			break;
288 		}
289 		hostp = hostp->next;
290 	}
291 	if (rt == NULL) {
292 		hostp = malloc(sizeof (host_id_list_t));
293 		hostp->hostid.platform = strdup(platform);
294 		hostp->hostid.server = strdup(server);
295 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
296 		hostp->hostid.domain = domain ? strdup(domain) : NULL;
297 		hostp->next = host_list;
298 		host_list = hostp;
299 		rt = &hostp->hostid;
300 		n_server++;
301 	}
302 	return (rt);
303 }
304 
305 static hostid_t *
306 find_hostid(nvlist_t *nvl)
307 {
308 	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
309 	nvlist_t *auth, *fmri;
310 	hostid_t *rt = NULL;
311 
312 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
313 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
314 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
315 		    &platform);
316 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
317 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
318 		    &chassis);
319 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
320 		rt = find_hostid_in_list(platform, chassis, server, domain);
321 	}
322 	return (rt);
323 }
324 
325 /*
326  * compare two fru strings which are made up of substrings seperated by '/'
327  * return true if every substring is the same in the two strings, or if a
328  * substring is null in one.
329  */
330 
331 static int
332 frucmp(char *f1, char *f2)
333 {
334 	char c1, c2;
335 	int i = 0;
336 
337 	for (;;) {
338 		c1 = *f1;
339 		c2 = *f2;
340 		if (c1 == c2) {
341 			i = (c1 == '/') ? 0 : i + 1;
342 		} else if (i == 0) {
343 			if (c1 == '/') {
344 				do {
345 					f2++;
346 				} while ((c2 = *f2) != 0 && c2 != '/');
347 				if (c2 == NULL)
348 					break;
349 			} else if (c2 == '/') {
350 				do {
351 					f1++;
352 				} while ((c1 = *f1) != 0 && c1 != '/');
353 				if (c1 == NULL)
354 					break;
355 			} else
356 				break;
357 		} else
358 			break;
359 		if (c1 == NULL)
360 			return (0);
361 		f1++;
362 		f2++;
363 	}
364 	return (1);
365 }
366 
367 static int
368 tgetlabel(topo_hdl_t *thp, tnode_t *node, void *arg)
369 {
370 	int err;
371 	char *fru_name, *lname;
372 	nvlist_t *fru = NULL;
373 	int rt = TOPO_WALK_NEXT;
374 	tgetlabel_data_t *tdp = (tgetlabel_data_t *)arg;
375 
376 	if (topo_node_fru(node, &fru, NULL, &err) == 0) {
377 		if (topo_fmri_nvl2str(thp, fru, &fru_name, &err) == 0) {
378 			if (frucmp(tdp->fru, fru_name) == 0 &&
379 			    topo_node_label(node, &lname, &err) == 0) {
380 				tdp->label = strdup(lname);
381 				topo_hdl_strfree(thp, lname);
382 				rt = TOPO_WALK_TERMINATE;
383 			}
384 			topo_hdl_strfree(thp, fru_name);
385 		}
386 		nvlist_free(fru);
387 	}
388 	return (rt);
389 }
390 
391 static void
392 label_get_topo(void)
393 {
394 	int err;
395 
396 	topo_handle = topo_open(TOPO_VERSION, 0, &err);
397 	if (topo_handle) {
398 		topo_handle_uuid = topo_snap_hold(topo_handle, NULL, &err);
399 	}
400 }
401 
402 static void
403 label_release_topo(void)
404 {
405 	if (topo_handle_uuid)
406 		topo_hdl_strfree(topo_handle, topo_handle_uuid);
407 	if (topo_handle) {
408 		topo_snap_release(topo_handle);
409 		topo_close(topo_handle);
410 	}
411 }
412 
413 static char *
414 get_fmri_label(char *fru)
415 {
416 	topo_walk_t *twp;
417 	tgetlabel_data_t td;
418 	int err;
419 
420 	td.label = NULL;
421 	td.fru = fru;
422 	if (topo_handle == NULL)
423 		label_get_topo();
424 	if (topo_handle_uuid) {
425 		twp = topo_walk_init(topo_handle, FM_FMRI_SCHEME_HC,
426 		    tgetlabel, &td, &err);
427 		if (twp) {
428 			topo_walk_step(twp, TOPO_WALK_CHILD);
429 			topo_walk_fini(twp);
430 		}
431 	}
432 	return (td.label);
433 }
434 
435 static char *
436 get_nvl2str_topo(nvlist_t *nvl)
437 {
438 	char *name = NULL;
439 	char *tname;
440 	int err;
441 	char *scheme = NULL;
442 	char *mod_name = NULL;
443 	char buf[128];
444 
445 	if (topo_handle == NULL)
446 		label_get_topo();
447 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
448 		name = strdup(tname);
449 		topo_hdl_strfree(topo_handle, tname);
450 	} else {
451 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
452 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
453 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
454 		    mod_name) {
455 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
456 			    scheme, mod_name);
457 			name = strdup(buf);
458 		}
459 	}
460 	return (name);
461 }
462 
463 static int
464 set_priority(char *s)
465 {
466 	int rt = 0;
467 
468 	if (s) {
469 		if (strcmp(s, "Minor") == 0)
470 			rt = 1;
471 		else if (strcmp(s, "Major") == 0)
472 			rt = 10;
473 		else if (strcmp(s, "Critical") == 0)
474 			rt = 100;
475 	}
476 	return (rt);
477 }
478 
479 static int
480 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
481     uint8_t p2)
482 {
483 	int r1, r2;
484 	int rt;
485 
486 	r1 = set_priority(s1);
487 	r2 = set_priority(s2);
488 	rt = r1 - r2;
489 	if (rt == 0) {
490 		if (t1 > t2)
491 			rt = 1;
492 		else if (t1 < t2)
493 			rt = -1;
494 		else
495 			rt = p1 - p2;
496 	}
497 	return (rt);
498 }
499 
500 /*
501  * merge two lists into one, by comparing enties in new and moving into list if
502  * name is not there or free off memory for names which are already there
503  * add_pct indicates if pct is the sum or highest pct
504  */
505 static name_list_t *
506 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
507 {
508 	name_list_t *lp, *np, *sp, *rt = NULL;
509 	int max_pct;
510 
511 	rt = *list;
512 	np = new;
513 	while (np) {
514 		lp = *list;
515 		while (lp) {
516 			if (strcmp(lp->name, np->name) == 0)
517 				break;
518 			lp = lp->next;
519 			if (lp == *list)
520 				lp = NULL;
521 		}
522 		if (np->next == new)
523 			sp = NULL;
524 		else
525 			sp = np->next;
526 		if (lp) {
527 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
528 			if (add_pct) {
529 				lp->pct += np->pct;
530 				lp->count += np->count;
531 			} else if (np->pct > lp->pct) {
532 				lp->pct = np->pct;
533 			}
534 			max_pct = np->max_pct;
535 			if (np->label)
536 				free(np->label);
537 			free(np->name);
538 			free(np);
539 			np = NULL;
540 			if (max_pct > lp->max_pct) {
541 				lp->max_pct = max_pct;
542 				if (lp->max_pct > lp->prev->max_pct &&
543 				    lp != *list) {
544 					lp->prev->next = lp->next;
545 					lp->next->prev = lp->prev;
546 					np = lp;
547 				}
548 			}
549 		}
550 		if (np) {
551 			lp = *list;
552 			if (lp) {
553 				if (np->max_pct > lp->max_pct) {
554 					np->next = lp;
555 					np->prev = lp->prev;
556 					lp->prev->next = np;
557 					lp->prev = np;
558 					*list = np;
559 					rt = np;
560 				} else {
561 					lp = lp->next;
562 					while (lp != *list &&
563 					    np->max_pct < lp->max_pct) {
564 						lp = lp->next;
565 					}
566 					np->next = lp;
567 					np->prev = lp->prev;
568 					lp->prev->next = np;
569 					lp->prev = np;
570 				}
571 			} else {
572 				*list = np;
573 				np->next = np;
574 				np->prev = np;
575 				rt = np;
576 			}
577 		}
578 		np = sp;
579 	}
580 	return (rt);
581 }
582 
583 /*
584  * compare entries in two lists return true if the two lists have identical
585  * content. The two lists may not have entries in the same order, so we compare
586  * the size of the list as well as trying to find every entry from one list in
587  * the other.
588  */
589 static int
590 cmp_name_list(name_list_t *lxp1, name_list_t *lxp2)
591 {
592 	name_list_t *lp1, *lp2;
593 	int l1 = 0, l2 = 0, common = 0;
594 
595 	lp2 = lxp2;
596 	while (lp2) {
597 		l2++;
598 		lp2 = lp2->next;
599 		if (lp2 == lxp2)
600 			break;
601 	}
602 	lp1 = lxp1;
603 	while (lp1) {
604 		l1++;
605 		lp2 = lxp2;
606 		while (lp2) {
607 			if (strcmp(lp2->name, lp1->name) == 0) {
608 				common++;
609 				break;
610 			}
611 			lp2 = lp2->next;
612 			if (lp2 == lxp2)
613 				break;
614 		}
615 		lp1 = lp1->next;
616 		if (lp1 == lxp1)
617 			break;
618 	}
619 	if (l1 == l2 && l2 == common)
620 		return (0);
621 	else
622 		return (1);
623 }
624 
625 static name_list_t *
626 alloc_name_list(char *name, uint8_t pct)
627 {
628 	name_list_t *nlp;
629 
630 	nlp = malloc(sizeof (*nlp));
631 	nlp->name = strdup(name);
632 	nlp->pct = pct;
633 	nlp->max_pct = pct;
634 	nlp->count = 1;
635 	nlp->next = nlp;
636 	nlp->prev = nlp;
637 	nlp->status = 0;
638 	nlp->label = NULL;
639 	return (nlp);
640 }
641 
642 static void
643 free_name_list(name_list_t *list)
644 {
645 	name_list_t *next = list;
646 	name_list_t *lp;
647 
648 	if (list) {
649 		do {
650 			lp = next;
651 			next = lp->next;
652 			if (lp->label)
653 				free(lp->label);
654 			free(lp->name);
655 			free(lp);
656 		} while (next != list);
657 	}
658 }
659 
660 static status_record_t *
661 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
662     name_list_t *fru, name_list_t *asru, name_list_t *resource,
663     name_list_t *serial, boolean_t not_suppressed,
664     hostid_t *hostid)
665 {
666 	status_record_t *status_rec_p;
667 
668 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
669 	status_rec_p->nrecs = 1;
670 	status_rec_p->host = hostid;
671 	status_rec_p->uurec = uurec_p;
672 	uurec_p->next = NULL;
673 	uurec_p->prev = NULL;
674 	uurec_p->asru = asru;
675 	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
676 	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
677 		status_rec_p->severity = strdup("unknown");
678 	status_rec_p->class = class;
679 	status_rec_p->fru = fru;
680 	status_rec_p->asru = asru;
681 	status_rec_p->resource = resource;
682 	status_rec_p->serial = serial;
683 	status_rec_p->msgid = strdup(msgid);
684 	status_rec_p->not_suppressed = not_suppressed;
685 	return (status_rec_p);
686 }
687 
688 /*
689  * add record to given list maintaining order higher priority first.
690  */
691 static void
692 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
693 {
694 	sr_list_t *tp, *np, *sp;
695 	int order;
696 	uint64_t sec;
697 
698 	np = malloc(sizeof (sr_list_t));
699 	np->status_record = status_rec_p;
700 	sec = status_rec_p->uurec->sec;
701 	if ((sp = *list_pp) == NULL) {
702 		*list_pp = np;
703 		np->next = np;
704 		np->prev = np;
705 	} else {
706 		/* insert new record in front of lower priority */
707 		tp = sp;
708 		order = cmp_priority(status_rec_p->severity,
709 		    sp->status_record->severity, sec,
710 		    tp->status_record->uurec->sec, 0, 0);
711 		if (order > 0) {
712 			*list_pp = np;
713 		} else {
714 			tp = sp->next;
715 			while (tp != sp &&
716 			    cmp_priority(status_rec_p->severity,
717 			    tp->status_record->severity, sec,
718 			    tp->status_record->uurec->sec, 0, 0)) {
719 				tp = tp->next;
720 			}
721 		}
722 		np->next = tp;
723 		np->prev = tp->prev;
724 		tp->prev->next = np;
725 		tp->prev = np;
726 	}
727 }
728 
729 static void
730 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
731     resource_list_t *np)
732 {
733 	int order;
734 	uint64_t sec;
735 	resource_list_t *sp, *tp;
736 	status_record_t *srp;
737 	char *severity = status_rec_p->severity;
738 
739 	add_rec_list(status_rec_p, &np->status_rec_list);
740 	if ((sp = *rp) == NULL) {
741 		np->next = np;
742 		np->prev = np;
743 		*rp = np;
744 	} else {
745 		/*
746 		 * insert new record in front of lower priority
747 		 */
748 		tp = sp->next;
749 		srp = sp->status_rec_list->status_record;
750 		sec = status_rec_p->uurec->sec;
751 		order = cmp_priority(severity, srp->severity, sec,
752 		    srp->uurec->sec, np->max_pct, sp->max_pct);
753 		if (order > 0) {
754 			*rp = np;
755 		} else {
756 			srp = tp->status_rec_list->status_record;
757 			while (tp != sp &&
758 			    cmp_priority(severity, srp->severity, sec,
759 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
760 				tp = tp->next;
761 				srp = tp->status_rec_list->status_record;
762 			}
763 		}
764 		np->next = tp;
765 		np->prev = tp->prev;
766 		tp->prev->next = np;
767 		tp->prev = np;
768 	}
769 }
770 
771 static void
772 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
773     resource_list_t **rpp)
774 {
775 	int order;
776 	resource_list_t *np, *end;
777 	status_record_t *srp;
778 
779 	np = *rpp;
780 	end = np;
781 	while (np) {
782 		if (strcmp(fp->name, np->resource) == 0) {
783 			np->not_suppressed |= status_rec_p->not_suppressed;
784 			srp = np->status_rec_list->status_record;
785 			order = cmp_priority(status_rec_p->severity,
786 			    srp->severity, status_rec_p->uurec->sec,
787 			    srp->uurec->sec, fp->max_pct, np->max_pct);
788 			if (order > 0 && np != end) {
789 				/*
790 				 * remove from list and add again using
791 				 * new priority
792 				 */
793 				np->prev->next = np->next;
794 				np->next->prev = np->prev;
795 				add_resource(status_rec_p,
796 				    rpp, np);
797 			} else {
798 				add_rec_list(status_rec_p,
799 				    &np->status_rec_list);
800 			}
801 			break;
802 		}
803 		np = np->next;
804 		if (np == end) {
805 			np = NULL;
806 			break;
807 		}
808 	}
809 	if (np == NULL) {
810 		np = malloc(sizeof (resource_list_t));
811 		np->resource = fp->name;
812 		np->not_suppressed = status_rec_p->not_suppressed;
813 		np->status_rec_list = NULL;
814 		np->max_pct = fp->max_pct;
815 		add_resource(status_rec_p, rpp, np);
816 	}
817 }
818 
819 static void
820 add_list(status_record_t *status_rec_p, name_list_t *listp,
821     resource_list_t **glistp)
822 {
823 	name_list_t *fp, *end;
824 
825 	fp = listp;
826 	end = fp;
827 	while (fp) {
828 		add_resource_list(status_rec_p, fp, glistp);
829 		fp = fp->next;
830 		if (fp == end)
831 			break;
832 	}
833 }
834 
835 /*
836  * add record to rec, fru and asru lists.
837  */
838 static void
839 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
840     name_list_t *fru, name_list_t *asru, name_list_t *resource,
841     name_list_t *serial, boolean_t not_suppressed,
842     hostid_t *hostid)
843 {
844 	status_record_t *status_rec_p;
845 
846 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
847 	    resource, serial, not_suppressed, hostid);
848 	add_rec_list(status_rec_p, &status_rec_list);
849 	if (status_rec_p->fru)
850 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
851 	if (status_rec_p->asru)
852 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
853 }
854 
855 /*
856  * add uuid and diagnoses time to an existing record for similar fault on the
857  * same fru
858  */
859 static void
860 catalog_merge_record(status_record_t *status_rec_p, uurec_t *uurec_p,
861     name_list_t *asru, name_list_t *resource, name_list_t *serial,
862     boolean_t not_suppressed)
863 {
864 	uurec_t *uurec1_p;
865 
866 	status_rec_p->nrecs++;
867 	/* add uurec in time order */
868 	if (status_rec_p->uurec->sec > uurec_p->sec) {
869 		uurec_p->next = status_rec_p->uurec;
870 		uurec_p->prev = NULL;
871 		status_rec_p->uurec = uurec_p;
872 	} else {
873 		uurec1_p = status_rec_p->uurec;
874 		while (uurec1_p->next && uurec1_p->next->sec <= uurec_p->sec)
875 			uurec1_p = uurec1_p->next;
876 		if (uurec1_p->next)
877 			uurec1_p->next->prev = uurec_p;
878 		uurec_p->next = uurec1_p->next;
879 		uurec_p->prev = uurec1_p;
880 		uurec1_p->next = uurec_p;
881 	}
882 	status_rec_p->not_suppressed |= not_suppressed;
883 	uurec_p->asru = merge_name_list(&status_rec_p->asru, asru, 0);
884 	(void) merge_name_list(&status_rec_p->resource, resource, 0);
885 	(void) merge_name_list(&status_rec_p->serial, serial, 0);
886 }
887 
888 static status_record_t *
889 record_in_catalog(name_list_t *class, name_list_t *fru,
890     char *msgid, hostid_t *host)
891 {
892 	sr_list_t *status_rec_p;
893 	status_record_t *srp = NULL;
894 
895 	status_rec_p = status_rec_list;
896 	while (status_rec_p) {
897 		srp = status_rec_p->status_record;
898 		if (host == srp->host &&
899 		    cmp_name_list(class, srp->class) == 0 &&
900 		    cmp_name_list(fru, srp->fru) == 0 &&
901 		    strcmp(msgid, srp->msgid) == 0)
902 			break;
903 		if (status_rec_p->next == status_rec_list) {
904 			srp = NULL;
905 			break;
906 		} else {
907 			status_rec_p = status_rec_p->next;
908 		}
909 	}
910 	return (srp);
911 }
912 
913 static void
914 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
915 {
916 	char *name;
917 	char *serial = NULL;
918 	char **lserial = NULL;
919 	uint64_t serint;
920 	name_list_t *nlp;
921 	int j;
922 	uint_t nelem;
923 	char buf[64];
924 
925 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
926 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
927 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
928 			    &serint) == 0) {
929 				(void) snprintf(buf, sizeof (buf), "%llX",
930 				    serint);
931 				nlp = alloc_name_list(buf, pct);
932 				(void) merge_name_list(serial_p, nlp, 1);
933 			}
934 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
935 			if (nvlist_lookup_string_array(nvl,
936 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
937 				nlp = alloc_name_list(lserial[0], pct);
938 				for (j = 1; j < nelem; j++) {
939 					name_list_t *n1lp;
940 					n1lp = alloc_name_list(lserial[j], pct);
941 					(void) merge_name_list(&nlp, n1lp, 1);
942 				}
943 				(void) merge_name_list(serial_p, nlp, 1);
944 			}
945 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
946 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
947 			    &serial) == 0) {
948 				nlp = alloc_name_list(serial, pct);
949 				(void) merge_name_list(serial_p, nlp, 1);
950 			}
951 		}
952 	}
953 }
954 
955 static void
956 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
957     name_list_t **fru_p, name_list_t **serial_p,
958     name_list_t **resource_p, name_list_t **asru_p, uint8_t status)
959 {
960 	nvlist_t *lfru, *lasru, *rsrc;
961 	name_list_t *nlp;
962 	char *name;
963 	uint8_t lpct = 0;
964 	char *lclass = NULL;
965 	char *label;
966 
967 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
968 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
969 		nlp = alloc_name_list(lclass, lpct);
970 		(void) merge_name_list(class_p, nlp, 1);
971 	}
972 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
973 		name = get_nvl2str_topo(lfru);
974 		if (name != NULL) {
975 			nlp = alloc_name_list(name, lpct);
976 			nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
977 			    FM_SUSPECT_DEGRADED);
978 			free(name);
979 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
980 			    &label) == 0)
981 				nlp->label = strdup(label);
982 			(void) merge_name_list(fru_p, nlp, 1);
983 		}
984 		get_serial_no(lfru, serial_p, lpct);
985 	}
986 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
987 		name = get_nvl2str_topo(lasru);
988 		if (name != NULL) {
989 			nlp = alloc_name_list(name, lpct);
990 			nlp->status = status & ~(FM_SUSPECT_NOT_PRESENT |
991 			    FM_SUSPECT_REPAIRED | FM_SUSPECT_REPLACED |
992 			    FM_SUSPECT_ACQUITTED);
993 			free(name);
994 			(void) merge_name_list(asru_p, nlp, 1);
995 		}
996 		get_serial_no(lasru, serial_p, lpct);
997 	}
998 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
999 		name = get_nvl2str_topo(rsrc);
1000 		if (name != NULL) {
1001 			nlp = alloc_name_list(name, lpct);
1002 			nlp->status = status;
1003 			free(name);
1004 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
1005 			    &label) == 0)
1006 				nlp->label = strdup(label);
1007 			(void) merge_name_list(resource_p, nlp, 1);
1008 		}
1009 	}
1010 }
1011 
1012 static void
1013 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid)
1014 {
1015 	char *msgid = "-";
1016 	uint_t i, size = 0;
1017 	name_list_t *class = NULL, *resource = NULL;
1018 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
1019 	nvlist_t **nva;
1020 	uint8_t *ba;
1021 	status_record_t *status_rec_p;
1022 	uurec_t *uurec_p;
1023 	hostid_t *host;
1024 	boolean_t not_suppressed = 1;
1025 	boolean_t any_present = 0;
1026 
1027 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
1028 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
1029 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
1030 	    &not_suppressed);
1031 
1032 	if (size != 0) {
1033 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
1034 		    &nva, &size);
1035 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
1036 		    &ba, &size);
1037 		for (i = 0; i < size; i++) {
1038 			extract_record_info(nva[i], &class, &fru, &serial,
1039 			    &resource, &asru, ba[i]);
1040 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
1041 			    (ba[i] & FM_SUSPECT_FAULTY))
1042 				any_present = 1;
1043 		}
1044 		/*
1045 		 * also suppress if no resources present
1046 		 */
1047 		if (any_present == 0)
1048 			not_suppressed = 0;
1049 	}
1050 
1051 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
1052 	uurec_p->uuid = strdup(uuid);
1053 	uurec_p->sec = sec;
1054 	uurec_p->ari_uuid_list = NULL;
1055 	host = find_hostid(nvl);
1056 	if (not_suppressed && !opt_g)
1057 		status_rec_p = NULL;
1058 	else
1059 		status_rec_p = record_in_catalog(class, fru, msgid, host);
1060 	if (status_rec_p) {
1061 		catalog_merge_record(status_rec_p, uurec_p, asru, resource,
1062 		    serial, not_suppressed);
1063 		free_name_list(class);
1064 		free_name_list(fru);
1065 	} else {
1066 		catalog_new_record(uurec_p, msgid, class, fru, asru,
1067 		    resource, serial, not_suppressed, host);
1068 	}
1069 }
1070 
1071 static void
1072 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
1073 {
1074 	sr_list_t *srp;
1075 	uurec_t *uurp;
1076 	ari_list_t *ari_list;
1077 
1078 	srp = status_rec_list;
1079 	if (srp) {
1080 		for (;;) {
1081 			uurp = srp->status_record->uurec;
1082 			while (uurp) {
1083 				if (strcmp(uuid, uurp->uuid) == 0) {
1084 					ari_list = (ari_list_t *)
1085 					    malloc(sizeof (ari_list_t));
1086 					ari_list->ari_uuid = strdup(ari_uuid);
1087 					ari_list->next = uurp->ari_uuid_list;
1088 					uurp->ari_uuid_list = ari_list;
1089 					return;
1090 				}
1091 				uurp = uurp->next;
1092 			}
1093 			if (srp->next == status_rec_list)
1094 				break;
1095 			srp = srp->next;
1096 		}
1097 	}
1098 }
1099 
1100 static void
1101 print_line(char *label, char *buf)
1102 {
1103 	char *cp, *ep, *wp;
1104 	char c;
1105 	int i;
1106 	int lsz;
1107 	char *padding;
1108 
1109 	lsz = strlen(label);
1110 	padding = malloc(lsz + 1);
1111 	for (i = 0; i < lsz; i++)
1112 		padding[i] = ' ';
1113 	padding[i] = 0;
1114 	cp = buf;
1115 	ep = buf;
1116 	c = *ep;
1117 	(void) printf("\n");
1118 	while (c) {
1119 		i = lsz;
1120 		wp = NULL;
1121 		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
1122 			if (c == ' ')
1123 				wp = ep;
1124 			else if (c == '\n') {
1125 				i = 0;
1126 				*ep = 0;
1127 				do {
1128 					ep++;
1129 				} while ((c = *ep) != NULL && c == ' ');
1130 				break;
1131 			}
1132 			ep++;
1133 			i++;
1134 		}
1135 		if (i >= 80 && wp) {
1136 			*wp = 0;
1137 			ep = wp + 1;
1138 			c = *ep;
1139 		}
1140 		(void) printf("%s%s\n", label, cp);
1141 		cp = ep;
1142 		label = padding;
1143 	}
1144 	free(padding);
1145 }
1146 
1147 static void
1148 print_dict_info_line(char *msgid, fmd_msg_item_t what, const char *linehdr)
1149 {
1150 	char *cp = fmd_msg_getitem_id(fmadm_msghdl, NULL, msgid, what);
1151 
1152 	if (cp) {
1153 		print_line(dgettext("FMD", linehdr), cp);
1154 		free(cp);
1155 	}
1156 }
1157 
1158 static void
1159 print_dict_info(char *msgid)
1160 {
1161 	print_dict_info_line(msgid, FMD_MSG_ITEM_DESC, "Description : ");
1162 	print_dict_info_line(msgid, FMD_MSG_ITEM_RESPONSE, "Response    : ");
1163 	print_dict_info_line(msgid, FMD_MSG_ITEM_IMPACT, "Impact      : ");
1164 	print_dict_info_line(msgid, FMD_MSG_ITEM_ACTION, "Action      : ");
1165 }
1166 
1167 static void
1168 print_name(name_list_t *list, char *(func)(char *), char *padding, int *np,
1169     int pct, int full)
1170 {
1171 	char *name, *fru_label = NULL;
1172 
1173 	name = list->name;
1174 	if (list->label) {
1175 		(void) printf("%s \"%s\" (%s)", padding, list->label, name);
1176 		*np += 1;
1177 	} else if (func && (fru_label = func(list->name)) != NULL) {
1178 		(void) printf("%s \"%s\" (%s)", padding, fru_label, name);
1179 		*np += 1;
1180 		free(fru_label);
1181 	} else {
1182 		(void) printf("%s %s", padding, name);
1183 		*np += 1;
1184 	}
1185 	if (list->pct && pct > 0 && pct < 100) {
1186 		if (list->count > 1) {
1187 			if (full) {
1188 				(void) printf(" %d @ %s %d%%\n", list->count,
1189 				    dgettext("FMD", "max"),
1190 				    list->max_pct);
1191 			} else {
1192 				(void) printf(" %s %d%%\n",
1193 				    dgettext("FMD", "max"),
1194 				    list->max_pct);
1195 			}
1196 		} else {
1197 			(void) printf(" %d%%\n", list->pct);
1198 		}
1199 	} else {
1200 		(void) printf("\n");
1201 	}
1202 }
1203 
1204 static void
1205 print_asru_status(int status, char *label)
1206 {
1207 	char *msg = NULL;
1208 
1209 	switch (status) {
1210 	case 0:
1211 		msg = dgettext("FMD", "ok and in service");
1212 		break;
1213 	case FM_SUSPECT_DEGRADED:
1214 		msg = dgettext("FMD", "service degraded, "
1215 		    "but associated components no longer faulty");
1216 		break;
1217 	case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1218 		msg = dgettext("FMD", "faulted but still "
1219 		    "providing degraded service");
1220 		break;
1221 	case FM_SUSPECT_FAULTY:
1222 		msg = dgettext("FMD", "faulted but still in service");
1223 		break;
1224 	case FM_SUSPECT_UNUSABLE:
1225 		msg = dgettext("FMD", "out of service, "
1226 		    "but associated components no longer faulty");
1227 		break;
1228 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1229 		msg = dgettext("FMD", "faulted and taken out of service");
1230 		break;
1231 	default:
1232 		break;
1233 	}
1234 	if (msg) {
1235 		(void) printf("%s     %s\n", label, msg);
1236 	}
1237 }
1238 
1239 static void
1240 print_fru_status(int status, char *label)
1241 {
1242 	char *msg = NULL;
1243 
1244 	if (status & FM_SUSPECT_NOT_PRESENT)
1245 		msg = dgettext("FMD", "not present");
1246 	else if (status & FM_SUSPECT_FAULTY)
1247 		msg = dgettext("FMD", "faulty");
1248 	else if (status & FM_SUSPECT_REPLACED)
1249 		msg = dgettext("FMD", "replaced");
1250 	else if (status & FM_SUSPECT_REPAIRED)
1251 		msg = dgettext("FMD", "repair attempted");
1252 	else if (status & FM_SUSPECT_ACQUITTED)
1253 		msg = dgettext("FMD", "acquitted");
1254 	else
1255 		msg = dgettext("FMD", "removed");
1256 	(void) printf("%s     %s\n", label, msg);
1257 }
1258 
1259 static void
1260 print_rsrc_status(int status, char *label)
1261 {
1262 	char *msg = "";
1263 
1264 	if (status & FM_SUSPECT_NOT_PRESENT)
1265 		msg = dgettext("FMD", "not present");
1266 	else if (status & FM_SUSPECT_FAULTY) {
1267 		if (status & FM_SUSPECT_DEGRADED)
1268 			msg = dgettext("FMD",
1269 			    "faulted but still providing degraded service");
1270 		else if (status & FM_SUSPECT_UNUSABLE)
1271 			msg = dgettext("FMD",
1272 			    "faulted and taken out of service");
1273 		else
1274 			msg = dgettext("FMD", "faulted but still in service");
1275 	} else if (status & FM_SUSPECT_REPLACED)
1276 		msg = dgettext("FMD", "replaced");
1277 	else if (status & FM_SUSPECT_REPAIRED)
1278 		msg = dgettext("FMD", "repair attempted");
1279 	else if (status & FM_SUSPECT_ACQUITTED)
1280 		msg = dgettext("FMD", "acquitted");
1281 	else
1282 		msg = dgettext("FMD", "removed");
1283 	(void) printf("%s     %s\n", label, msg);
1284 }
1285 
1286 static void
1287 print_name_list(name_list_t *list, char *label, char *(func)(char *),
1288     int limit, int pct, void (func1)(int, char *), int full)
1289 {
1290 	char *name, *fru_label = NULL;
1291 	char *padding;
1292 	int i, j, l, n;
1293 	name_list_t *end = list;
1294 
1295 	l = strlen(label);
1296 	padding = malloc(l + 1);
1297 	for (i = 0; i < l; i++)
1298 		padding[i] = ' ';
1299 	padding[l] = 0;
1300 	(void) printf("%s", label);
1301 	name = list->name;
1302 	if (list->label)
1303 		(void) printf(" \"%s\" (%s)", list->label, name);
1304 	else if (func && (fru_label = func(list->name)) != NULL) {
1305 		(void) printf(" \"%s\" (%s)", fru_label, name);
1306 		free(fru_label);
1307 	} else
1308 		(void) printf(" %s", name);
1309 	if (list->pct && pct > 0 && pct < 100) {
1310 		if (list->count > 1) {
1311 			if (full) {
1312 				(void) printf(" %d @ %s %d%%\n", list->count,
1313 				    dgettext("FMD", "max"), list->max_pct);
1314 			} else {
1315 				(void) printf(" %s %d%%\n",
1316 				    dgettext("FMD", "max"), list->max_pct);
1317 			}
1318 		} else {
1319 			(void) printf(" %d%%\n", list->pct);
1320 		}
1321 	} else {
1322 		(void) printf("\n");
1323 	}
1324 	if (func1)
1325 		func1(list->status, padding);
1326 	n = 1;
1327 	j = 0;
1328 	while ((list = list->next) != end) {
1329 		if (limit == 0 || n < limit) {
1330 			print_name(list, func, padding, &n, pct, full);
1331 			if (func1)
1332 				func1(list->status, padding);
1333 		} else
1334 			j++;
1335 	}
1336 	if (j == 1) {
1337 		print_name(list->prev, func, padding, &n, pct, full);
1338 	} else if (j > 1) {
1339 		(void) printf("%s... %d %s\n", padding, j,
1340 		    dgettext("FMD", "more entries suppressed,"
1341 		    " use -v option for full list"));
1342 	}
1343 	free(padding);
1344 }
1345 
1346 static int
1347 asru_same_status(name_list_t *list)
1348 {
1349 	name_list_t *end = list;
1350 	int status = list->status;
1351 
1352 	while ((list = list->next) != end) {
1353 		if (status == -1) {
1354 			status = list->status;
1355 			continue;
1356 		}
1357 		if (list->status != -1 && status != list->status) {
1358 			status = -1;
1359 			break;
1360 		}
1361 	}
1362 	return (status);
1363 }
1364 
1365 static int
1366 serial_in_fru(name_list_t *fru, name_list_t *serial)
1367 {
1368 	name_list_t *sp = serial;
1369 	name_list_t *fp;
1370 	int nserial = 0;
1371 	int found = 0;
1372 	char buf[128];
1373 
1374 	while (sp) {
1375 		fp = fru;
1376 		nserial++;
1377 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1378 		buf[sizeof (buf) - 1] = 0;
1379 		while (fp) {
1380 			if (strstr(fp->name, buf) != NULL) {
1381 				found++;
1382 				break;
1383 			}
1384 			fp = fp->next;
1385 			if (fp == fru)
1386 				break;
1387 		}
1388 		sp = sp->next;
1389 		if (sp == serial)
1390 			break;
1391 	}
1392 	return (found == nserial ? 1 : 0);
1393 }
1394 
1395 static void
1396 print_sup_record(status_record_t *srp, int opt_i, int full)
1397 {
1398 	char buf[32];
1399 	uurec_t *uurp = srp->uurec;
1400 	int n, j, k, max;
1401 	int status;
1402 	ari_list_t *ari_list;
1403 
1404 	n = 0;
1405 	max = max_fault;
1406 	if (max < 0) {
1407 		max = 0;
1408 	}
1409 	j = max / 2;
1410 	max -= j;
1411 	k = srp->nrecs - max;
1412 	while ((uurp = uurp->next) != NULL) {
1413 		if (full || n < j || n >= k || max_fault == 0 ||
1414 		    srp->nrecs == max_fault+1) {
1415 			if (opt_i) {
1416 				ari_list = uurp->ari_uuid_list;
1417 				while (ari_list) {
1418 					(void) printf("%-15s %s\n",
1419 					    format_date(buf, sizeof (buf),
1420 					    uurp->sec), ari_list->ari_uuid);
1421 					ari_list = ari_list->next;
1422 				}
1423 			} else {
1424 				(void) printf("%-15s %s\n",
1425 				    format_date(buf, sizeof (buf), uurp->sec),
1426 				    uurp->uuid);
1427 			}
1428 		} else if (n == j)
1429 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1430 			    dgettext("FMD", "more entries suppressed"));
1431 		n++;
1432 	}
1433 	(void) printf("\n");
1434 	(void) printf("%s %s", dgettext("FMD", "Host        :"),
1435 	    srp->host->server);
1436 	if (srp->host->domain)
1437 		(void) printf("\t%s %s", dgettext("FMD", "Domain      :"),
1438 		    srp->host->domain);
1439 	(void) printf("\n%s %s", dgettext("FMD", "Platform    :"),
1440 	    srp->host->platform);
1441 	(void) printf("\t%s %s\n\n", dgettext("FMD", "Chassis_id  :"),
1442 	    srp->host->chassis ? srp->host->chassis : "");
1443 	if (srp->class)
1444 		print_name_list(srp->class,
1445 		    dgettext("FMD", "Fault class :"), NULL, 0, srp->class->pct,
1446 		    NULL, full);
1447 	if (srp->asru) {
1448 		status = asru_same_status(srp->asru);
1449 		if (status != -1) {
1450 			print_name_list(srp->asru,
1451 			    dgettext("FMD", "Affects     :"), NULL,
1452 			    full ? 0 : max_display, 0, NULL, full);
1453 			print_asru_status(status, "             ");
1454 		} else
1455 			print_name_list(srp->asru,
1456 			    dgettext("FMD", "Affects     :"), NULL,
1457 			    full ? 0 : max_display, 0, print_asru_status, full);
1458 	}
1459 	if (full || srp->fru == NULL || srp->asru == NULL) {
1460 		if (srp->resource) {
1461 			print_name_list(srp->resource,
1462 			    dgettext("FMD", "Problem in  :"),
1463 			    NULL, full ? 0 : max_display, 0, print_rsrc_status,
1464 			    full);
1465 		}
1466 	}
1467 	if (srp->fru) {
1468 		status = asru_same_status(srp->fru);
1469 		if (status != -1) {
1470 			print_name_list(srp->fru, dgettext("FMD",
1471 			    "FRU         :"), get_fmri_label, 0,
1472 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1473 			    NULL, full);
1474 			print_fru_status(status, "             ");
1475 		} else
1476 			print_name_list(srp->fru, dgettext("FMD",
1477 			    "FRU         :"), get_fmri_label, 0,
1478 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1479 			    print_fru_status, full);
1480 	}
1481 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1482 	    !serial_in_fru(srp->asru, srp->serial)) {
1483 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1484 		    NULL, 0, 0, NULL, full);
1485 	}
1486 	print_dict_info(srp->msgid);
1487 	(void) printf("\n");
1488 }
1489 
1490 static void
1491 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1492 {
1493 	char buf[32];
1494 	uurec_t *uurp = srp->uurec;
1495 	static int header = 0;
1496 	char *head;
1497 	ari_list_t *ari_list;
1498 
1499 	if (!summary || !header) {
1500 		if (opt_i) {
1501 			head = "--------------- "
1502 			    "------------------------------------  "
1503 			    "-------------- ---------\n"
1504 			    "TIME            CACHE-ID"
1505 			    "                              MSG-ID"
1506 			    "         SEVERITY\n--------------- "
1507 			    "------------------------------------ "
1508 			    " -------------- ---------";
1509 		} else {
1510 			head = "--------------- "
1511 			    "------------------------------------  "
1512 			    "-------------- ---------\n"
1513 			    "TIME            EVENT-ID"
1514 			    "                              MSG-ID"
1515 			    "         SEVERITY\n--------------- "
1516 			    "------------------------------------ "
1517 			    " -------------- ---------";
1518 		}
1519 		(void) printf("%s\n", dgettext("FMD", head));
1520 		header = 1;
1521 	}
1522 	if (opt_i) {
1523 		ari_list = uurp->ari_uuid_list;
1524 		while (ari_list) {
1525 			(void) printf("%-15s %-37s %-14s %-9s\n",
1526 			    format_date(buf, sizeof (buf), uurp->sec),
1527 			    ari_list->ari_uuid, srp->msgid, srp->severity);
1528 			ari_list = ari_list->next;
1529 		}
1530 	} else {
1531 		(void) printf("%-15s %-37s %-14s %-9s\n",
1532 		    format_date(buf, sizeof (buf), uurp->sec),
1533 		    uurp->uuid, srp->msgid, srp->severity);
1534 	}
1535 
1536 	if (!summary)
1537 		print_sup_record(srp, opt_i, full);
1538 }
1539 
1540 static void
1541 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1542 {
1543 	status_record_t *srp;
1544 	sr_list_t *slp;
1545 
1546 	slp = status_rec_list;
1547 	if (slp) {
1548 		for (;;) {
1549 			srp = slp->status_record;
1550 			if (opt_a || srp->not_suppressed) {
1551 				if (page_feed)
1552 					(void) printf("\f\n");
1553 				print_status_record(srp, summary, opt_i, full);
1554 			}
1555 			if (slp->next == status_rec_list)
1556 				break;
1557 			slp = slp->next;
1558 		}
1559 	}
1560 }
1561 
1562 static name_list_t *
1563 find_fru(status_record_t *srp, char *resource)
1564 {
1565 	name_list_t *rt = NULL;
1566 	name_list_t *fru = srp->fru;
1567 
1568 	while (fru) {
1569 		if (strcmp(resource, fru->name) == 0) {
1570 			rt = fru;
1571 			break;
1572 		}
1573 		fru = fru->next;
1574 		if (fru == srp->fru)
1575 			break;
1576 	}
1577 	return (rt);
1578 }
1579 
1580 static void
1581 print_fru_line(name_list_t *fru, char *uuid)
1582 {
1583 	if (fru->pct == 100) {
1584 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1585 		    dgettext("FMD", "suspects in this FRU total certainty"),
1586 		    100);
1587 	} else {
1588 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1589 		    dgettext("FMD", "suspects in this FRU max certainty"),
1590 		    fru->max_pct);
1591 	}
1592 }
1593 
1594 static void
1595 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1596 {
1597 	resource_list_t *tp = status_fru_list;
1598 	status_record_t *srp;
1599 	sr_list_t *slp, *end;
1600 	char *msgid, *fru_label;
1601 	uurec_t *uurp;
1602 	name_list_t *fru;
1603 	int status;
1604 	ari_list_t *ari_list;
1605 
1606 	while (tp) {
1607 		if (opt_a || tp->not_suppressed) {
1608 			if (page_feed)
1609 				(void) printf("\f\n");
1610 			if (!summary)
1611 				(void) printf("-----------------------------"
1612 				    "---------------------------------------"
1613 				    "----------\n");
1614 			slp = tp->status_rec_list;
1615 			end = slp;
1616 			do {
1617 				srp = slp->status_record;
1618 				fru = find_fru(srp, tp->resource);
1619 				if (fru) {
1620 					if (fru->label)
1621 						(void) printf("\"%s\" (%s) ",
1622 						    fru->label, fru->name);
1623 					else if ((fru_label = get_fmri_label(
1624 					    fru->name)) != NULL) {
1625 						(void) printf("\"%s\" (%s) ",
1626 						    fru_label, fru->name);
1627 						free(fru_label);
1628 					} else
1629 						(void) printf("%s ",
1630 						    fru->name);
1631 					break;
1632 				}
1633 				slp = slp->next;
1634 			} while (slp != end);
1635 
1636 			slp = tp->status_rec_list;
1637 			end = slp;
1638 			status = 0;
1639 			do {
1640 				srp = slp->status_record;
1641 				fru = srp->fru;
1642 				while (fru) {
1643 					if (strcmp(tp->resource,
1644 					    fru->name) == 0)
1645 						status |= fru->status;
1646 					fru = fru->next;
1647 					if (fru == srp->fru)
1648 						break;
1649 				}
1650 				slp = slp->next;
1651 			} while (slp != end);
1652 			if (status & FM_SUSPECT_NOT_PRESENT)
1653 				(void) printf(dgettext("FMD", "not present\n"));
1654 			else if (status & FM_SUSPECT_FAULTY)
1655 				(void) printf(dgettext("FMD", "faulty\n"));
1656 			else if (status & FM_SUSPECT_REPLACED)
1657 				(void) printf(dgettext("FMD", "replaced\n"));
1658 			else if (status & FM_SUSPECT_REPAIRED)
1659 				(void) printf(dgettext("FMD",
1660 				    "repair attempted\n"));
1661 			else if (status & FM_SUSPECT_ACQUITTED)
1662 				(void) printf(dgettext("FMD", "acquitted\n"));
1663 			else
1664 				(void) printf(dgettext("FMD", "removed\n"));
1665 
1666 			slp = tp->status_rec_list;
1667 			end = slp;
1668 			do {
1669 				srp = slp->status_record;
1670 				uurp = srp->uurec;
1671 				fru = find_fru(srp, tp->resource);
1672 				if (fru) {
1673 					if (opt_i) {
1674 						ari_list = uurp->ari_uuid_list;
1675 						while (ari_list) {
1676 							print_fru_line(fru,
1677 							    ari_list->ari_uuid);
1678 							ari_list =
1679 							    ari_list->next;
1680 						}
1681 					} else {
1682 						print_fru_line(fru, uurp->uuid);
1683 					}
1684 				}
1685 				slp = slp->next;
1686 			} while (slp != end);
1687 			if (!summary) {
1688 				slp = tp->status_rec_list;
1689 				end = slp;
1690 				srp = slp->status_record;
1691 				if (srp->serial &&
1692 				    !serial_in_fru(srp->fru, srp->serial)) {
1693 					print_name_list(srp->serial,
1694 					    dgettext("FMD", "Serial ID.  :"),
1695 					    NULL, 0, 0, NULL, 1);
1696 				}
1697 				msgid = NULL;
1698 				do {
1699 					if (msgid == NULL ||
1700 					    strcmp(msgid, srp->msgid) != 0) {
1701 						msgid = srp->msgid;
1702 						print_dict_info(srp->msgid);
1703 					}
1704 					slp = slp->next;
1705 				} while (slp != end);
1706 			}
1707 		}
1708 		tp = tp->next;
1709 		if (tp == status_fru_list)
1710 			break;
1711 	}
1712 }
1713 
1714 static void
1715 print_asru(int opt_a)
1716 {
1717 	resource_list_t *tp = status_asru_list;
1718 	status_record_t *srp;
1719 	sr_list_t *slp, *end;
1720 	char *msg;
1721 	int status;
1722 	name_list_t *asru;
1723 
1724 	while (tp) {
1725 		if (opt_a || tp->not_suppressed) {
1726 			status = 0;
1727 			slp = tp->status_rec_list;
1728 			end = slp;
1729 			do {
1730 				srp = slp->status_record;
1731 				asru = srp->asru;
1732 				while (asru) {
1733 					if (strcmp(tp->resource,
1734 					    asru->name) == 0)
1735 						status |= asru->status;
1736 					asru = asru->next;
1737 					if (asru == srp->asru)
1738 						break;
1739 				}
1740 				slp = slp->next;
1741 			} while (slp != end);
1742 			switch (status) {
1743 			case 0:
1744 				msg = dgettext("FMD", "ok");
1745 				break;
1746 			case FM_SUSPECT_DEGRADED:
1747 				msg = dgettext("FMD", "degraded");
1748 				break;
1749 			case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1750 				msg = dgettext("FMD", "degraded");
1751 				break;
1752 			case FM_SUSPECT_FAULTY:
1753 				msg = dgettext("FMD", "degraded");
1754 				break;
1755 			case FM_SUSPECT_UNUSABLE:
1756 				msg = dgettext("FMD", "unknown");
1757 				break;
1758 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1759 				msg = dgettext("FMD", "faulted");
1760 				break;
1761 			default:
1762 				msg = "";
1763 				break;
1764 			}
1765 			(void) printf("%-69s %s\n", tp->resource, msg);
1766 		}
1767 		tp = tp->next;
1768 		if (tp == status_asru_list)
1769 			break;
1770 	}
1771 }
1772 
1773 static int
1774 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1775 {
1776 	while (uurecp) {
1777 		if (strcmp(uuid, uurecp->uuid) == 0)
1778 			return (1);
1779 		uurecp = uurecp->next;
1780 	}
1781 	return (0);
1782 }
1783 
1784 static int
1785 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1786 {
1787 	int64_t *diag_time;
1788 	uint_t nelem;
1789 	int rt = 0;
1790 	char *uuid = "-";
1791 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1792 
1793 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1794 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1795 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1796 		    &uuid);
1797 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1798 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1799 			    uuid);
1800 	} else {
1801 		rt = -1;
1802 	}
1803 	return (rt);
1804 }
1805 
1806 /*ARGSUSED*/
1807 static int
1808 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1809 {
1810 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1811 	return (0);
1812 }
1813 
1814 static int
1815 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1816 {
1817 	int rt = FMADM_EXIT_SUCCESS;
1818 
1819 	/*
1820 	 * These calls may fail with Protocol error if message payload is to big
1821 	 */
1822 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1823 		die("failed to get case list from fmd");
1824 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1825 		die("failed to get case status from fmd");
1826 	return (rt);
1827 }
1828 
1829 /*
1830  * fmadm faulty command
1831  *
1832  *	-a		show hidden fault records
1833  *	-f		show faulty fru's
1834  *	-g		force grouping of similar faults on the same fru
1835  *	-n		number of fault records to display
1836  *	-p		pipe output through pager
1837  *	-r		show faulty asru's
1838  *	-s		print summary of first fault
1839  *	-u		print listed uuid's only
1840  *	-v		full output
1841  */
1842 
1843 int
1844 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1845 {
1846 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1847 	int opt_i = 0;
1848 	char *pager;
1849 	FILE *fp;
1850 	int rt, c, stat;
1851 	uurec_select_t *tp;
1852 	uurec_select_t *uurecp = NULL;
1853 
1854 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1855 		switch (c) {
1856 		case 'a':
1857 			opt_a++;
1858 			break;
1859 		case 'f':
1860 			opt_f++;
1861 			break;
1862 		case 'g':
1863 			opt_g++;
1864 			break;
1865 		case 'i':
1866 			opt_i++;
1867 			break;
1868 		case 'n':
1869 			max_fault = atoi(optarg);
1870 			break;
1871 		case 'p':
1872 			opt_p++;
1873 			break;
1874 		case 'r':
1875 			opt_r++;
1876 			break;
1877 		case 's':
1878 			opt_s++;
1879 			break;
1880 		case 'u':
1881 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1882 			tp->uuid = optarg;
1883 			tp->next = uurecp;
1884 			uurecp = tp;
1885 			opt_a = 1;
1886 			break;
1887 		case 'v':
1888 			opt_v++;
1889 			break;
1890 		default:
1891 			return (FMADM_EXIT_USAGE);
1892 		}
1893 	}
1894 	if (optind < argc)
1895 		return (FMADM_EXIT_USAGE);
1896 
1897 	if ((fmadm_msghdl = fmd_msg_init(NULL, FMD_MSG_VERSION)) == NULL)
1898 		return (FMADM_EXIT_ERROR);
1899 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1900 	if (opt_p) {
1901 		if ((pager = getenv("PAGER")) == NULL)
1902 			pager = "/usr/bin/more";
1903 		fp = popen(pager, "w");
1904 		if (fp == NULL) {
1905 			rt = FMADM_EXIT_ERROR;
1906 			opt_p = 0;
1907 		} else {
1908 			dup2(fileno(fp), 1);
1909 			setbuf(stdout, NULL);
1910 			(void) fclose(fp);
1911 		}
1912 	}
1913 	max_display = max_fault;
1914 	if (opt_f)
1915 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1916 	if (opt_r)
1917 		print_asru(opt_a);
1918 	if (opt_f == 0 && opt_r == 0)
1919 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1920 	fmd_msg_fini(fmadm_msghdl);
1921 	label_release_topo();
1922 	if (opt_p) {
1923 		(void) fclose(stdout);
1924 		(void) wait(&stat);
1925 	}
1926 	return (rt);
1927 }
1928 
1929 int
1930 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1931 {
1932 	int i, status = FMADM_EXIT_SUCCESS;
1933 
1934 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1935 		return (FMADM_EXIT_USAGE);
1936 
1937 	for (i = 1; i < argc; i++) {
1938 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1939 			warn("failed to flush %s", argv[i]);
1940 			status = FMADM_EXIT_ERROR;
1941 		} else
1942 			note("flushed resource history for %s\n", argv[i]);
1943 	}
1944 
1945 	return (status);
1946 }
1947 
1948 int
1949 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
1950 {
1951 	int err;
1952 
1953 	if (getopt(argc, argv, "") != EOF)
1954 		return (FMADM_EXIT_USAGE);
1955 
1956 	if (argc - optind != 1)
1957 		return (FMADM_EXIT_USAGE);
1958 
1959 	/*
1960 	 * argument could be a uuid, an fmri (asru, fru or resource)
1961 	 * or a label. Try uuid first, If that fails try the others.
1962 	 */
1963 	err = fmd_adm_case_repair(adm, argv[optind]);
1964 	if (err != 0)
1965 		err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1966 
1967 	if (err != 0)
1968 		die("failed to record repair to %s", argv[optind]);
1969 
1970 	note("recorded repair to %s\n", argv[optind]);
1971 	return (FMADM_EXIT_SUCCESS);
1972 }
1973 
1974 int
1975 cmd_repaired(fmd_adm_t *adm, int argc, char *argv[])
1976 {
1977 	int err;
1978 
1979 	if (getopt(argc, argv, "") != EOF)
1980 		return (FMADM_EXIT_USAGE);
1981 
1982 	if (argc - optind != 1)
1983 		return (FMADM_EXIT_USAGE);
1984 
1985 	/*
1986 	 * argument could be an fmri (asru, fru or resource) or a label.
1987 	 */
1988 	err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1989 	if (err != 0)
1990 		die("failed to record repair to %s", argv[optind]);
1991 
1992 	note("recorded repair to of %s\n", argv[optind]);
1993 	return (FMADM_EXIT_SUCCESS);
1994 }
1995 
1996 int
1997 cmd_replaced(fmd_adm_t *adm, int argc, char *argv[])
1998 {
1999 	int err;
2000 
2001 	if (getopt(argc, argv, "") != EOF)
2002 		return (FMADM_EXIT_USAGE);
2003 
2004 	if (argc - optind != 1)
2005 		return (FMADM_EXIT_USAGE);
2006 
2007 	/*
2008 	 * argument could be an fmri (asru, fru or resource) or a label.
2009 	 */
2010 	err = fmd_adm_rsrc_replaced(adm, argv[optind]);
2011 	if (err != 0)
2012 		die("failed to record replacement of %s", argv[optind]);
2013 
2014 	note("recorded replacement of %s\n", argv[optind]);
2015 	return (FMADM_EXIT_SUCCESS);
2016 }
2017 
2018 int
2019 cmd_acquit(fmd_adm_t *adm, int argc, char *argv[])
2020 {
2021 	int err;
2022 
2023 	if (getopt(argc, argv, "") != EOF)
2024 		return (FMADM_EXIT_USAGE);
2025 
2026 	if (argc - optind != 1 && argc - optind != 2)
2027 		return (FMADM_EXIT_USAGE);
2028 
2029 	/*
2030 	 * argument could be a uuid, an fmri (asru, fru or resource)
2031 	 * or a label. Or it could be a uuid and an fmri or label.
2032 	 */
2033 	if (argc - optind == 2) {
2034 		err = fmd_adm_rsrc_acquit(adm, argv[optind], argv[optind + 1]);
2035 		if (err != 0)
2036 			err = fmd_adm_rsrc_acquit(adm, argv[optind + 1],
2037 			    argv[optind]);
2038 	} else {
2039 		err = fmd_adm_case_acquit(adm, argv[optind]);
2040 		if (err != 0)
2041 			err = fmd_adm_rsrc_acquit(adm, argv[optind], "");
2042 	}
2043 
2044 	if (err != 0)
2045 		die("failed to record acquital of %s", argv[optind]);
2046 
2047 	note("recorded acquital of %s\n", argv[optind]);
2048 	return (FMADM_EXIT_SUCCESS);
2049 }
2050