xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 540db9a9)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <fmadm.h>
28 #include <errno.h>
29 #include <limits.h>
30 #include <strings.h>
31 #include <stdio.h>
32 #include <unistd.h>
33 #include <sys/wait.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <fm/fmd_log.h>
37 #include <sys/fm/protocol.h>
38 #include <fm/libtopo.h>
39 #include <fm/fmd_adm.h>
40 #include <fm/fmd_msg.h>
41 #include <dlfcn.h>
42 #include <sys/systeminfo.h>
43 #include <sys/utsname.h>
44 #include <libintl.h>
45 #include <locale.h>
46 #include <sys/smbios.h>
47 #include <libdevinfo.h>
48 #include <stdlib.h>
49 
50 #define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))
51 
52 /*
53  * Fault records are added to catalog by calling add_fault_record_to_catalog()
54  * records are stored in order of importance to the system.
55  * If -g flag is set or not_suppressed is not set and the class fru, fault,
56  * type are the same then details are merged into an existing record, with uuid
57  * records are stored in time order.
58  * For each record information is extracted from nvlist and merged into linked
59  * list each is checked for identical records for which percentage certainty are
60  * added together.
61  * print_catalog() is called to print out catalog and release external resources
62  *
63  *                         /---------------\
64  *	status_rec_list -> |               | -|
65  *                         \---------------/
66  *                                \/
67  *                         /---------------\    /-------\    /-------\
68  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
69  *            \/           |               | |- |       | <- |       |
70  *      /-------------\    |               |    \-------/    \-------/
71  *      |             | -> |               |       \/           \/
72  *      \-------------/    |               |    /-------\    /-------\
73  *            \/           |               | -> | asru  | -> | asru  |
74  *            ---          |               |    |       | <- |       |
75  *                         |               |    \-------/    \-------/
76  *      status_asru_list   |  class        |
77  *            \/           |  resource     |    /-------\    /-------\
78  *      /-------------\    |  fru          | -> | list  | -> | list  |
79  *      |             | -> |  serial       |    |       | <- |       |
80  *      \-------------/    |               |    \-------/    \-------/
81  *            \/           \---------------/
82  *            ---               \/    /\
83  *                         /---------------\
84  *                         | status_record |
85  *                         \---------------/
86  *
87  * Fmadm faulty takes a number of options which affect the format of the
88  * output displayed. By default, the display reports the FRU and ASRU along
89  * with other information on per-case basis as in the example below.
90  *
91  * --------------- ------------------------------------  -------------- -------
92  * TIME            EVENT-ID                              MSG-ID         SEVERITY
93  * --------------- ------------------------------------  -------------- -------
94  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
95  *
96  * Fault class	: fault.memory.dimm_sb
97  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
98  *		    faulted but still in service
99  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
100  *		    faulty
101  *
102  * Description	: The number of errors associated with this memory module has
103  *		exceeded acceptable levels.  Refer to
104  *		http://sun.com/msg/AMD-8000-2F for more information.
105  *
106  * Response	: Pages of memory associated with this memory module are being
107  *		removed from service as errors are reported.
108  *
109  * Impact	: Total system memory capacity will be reduced as pages are
110  *		retired.
111  *
112  * Action	: Schedule a repair procedure to replace the affected memory
113  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
114  *
115  * The -v flag is similar, but adds some additonal information such as the
116  * resource. The -s flag is also similar but just gives the top line summary.
117  * All these options (ie without the -f or -r flags) use the print_catalog()
118  * function to do the display.
119  *
120  * The -f flag changes the output so that it appears sorted on a per-fru basis.
121  * The output is somewhat cut down compared to the default output. If -f is
122  * used, then print_fru() is used to print the output.
123  *
124  * -----------------------------------------------------------------------------
125  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
126  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
127  *
128  * Description	: A problem was detected for a PCI device.
129  *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
130  *
131  * Response	: One or more device instances may be disabled
132  *
133  * Impact	: Possible loss of services provided by the device instances
134  *		associated with this fault
135  *
136  * Action	: Schedule a repair procedure to replace the affected device.
137  * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
138  *		Sun for support.
139  *
140  * The -r flag changes the output so that it appears sorted on a per-asru basis.
141  * The output is very much cut down compared to the default output, just giving
142  * the asru fmri and state. Here print_asru() is used to print the output.
143  *
144  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
145  *
146  * For all fmadm faulty options, the sequence of events is
147  *
148  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
149  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
150  * This will extract the data from the nvlist and call catalog_new_record() to
151  * save the data away in various linked lists in the catalogue.
152  *
153  * 2) Once this is done, the data can be supplemented by using
154  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
155  *
156  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
157  * appropriate to display the information from the catalogue sorted in the
158  * requested way.
159  *
160  */
161 
162 typedef struct name_list {
163 	struct name_list *next;
164 	struct name_list *prev;
165 	char *name;
166 	uint8_t pct;
167 	uint8_t max_pct;
168 	ushort_t count;
169 	int status;
170 	char *label;
171 } name_list_t;
172 
173 typedef struct ari_list {
174 	char *ari_uuid;
175 	struct ari_list *next;
176 } ari_list_t;
177 
178 typedef struct uurec {
179 	struct uurec *next;
180 	struct uurec *prev;
181 	char *uuid;
182 	ari_list_t *ari_uuid_list;
183 	name_list_t *asru;
184 	uint64_t sec;
185 	nvlist_t *event;
186 } uurec_t;
187 
188 typedef struct uurec_select {
189 	struct uurec_select *next;
190 	char *uuid;
191 } uurec_select_t;
192 
193 typedef struct host_id {
194 	char *chassis;
195 	char *server;
196 	char *platform;
197 	char *domain;
198 	char *product_sn;
199 } hostid_t;
200 
201 typedef struct host_id_list {
202 	hostid_t hostid;
203 	struct host_id_list *next;
204 } host_id_list_t;
205 
206 typedef struct status_record {
207 	hostid_t *host;
208 	int nrecs;
209 	uurec_t *uurec;
210 	char *severity;			/* in C locale */
211 	char *msgid;
212 	name_list_t *class;
213 	name_list_t *resource;
214 	name_list_t *asru;
215 	name_list_t *fru;
216 	name_list_t *serial;
217 	uint8_t not_suppressed;
218 	uint8_t injected;
219 } status_record_t;
220 
221 typedef struct sr_list {
222 	struct sr_list *next;
223 	struct sr_list *prev;
224 	struct status_record *status_record;
225 } sr_list_t;
226 
227 typedef struct resource_list {
228 	struct resource_list *next;
229 	struct resource_list *prev;
230 	sr_list_t *status_rec_list;
231 	char *resource;
232 	uint8_t not_suppressed;
233 	uint8_t injected;
234 	uint8_t max_pct;
235 } resource_list_t;
236 
237 typedef struct tgetlabel_data {
238 	char *label;
239 	char *fru;
240 } tgetlabel_data_t;
241 
242 sr_list_t *status_rec_list;
243 resource_list_t *status_fru_list;
244 resource_list_t *status_asru_list;
245 
246 static int max_display;
247 static int max_fault = 0;
248 static topo_hdl_t *topo_handle;
249 static char *topo_handle_uuid;
250 static host_id_list_t *host_list;
251 static int n_server;
252 static int opt_g;
253 static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */
254 
255 static char *
256 format_date(char *buf, size_t len, uint64_t sec)
257 {
258 	if (sec > LONG_MAX) {
259 		(void) fprintf(stderr,
260 		    "record time is too large for 32-bit utility\n");
261 		(void) snprintf(buf, len, "0x%llx", sec);
262 	} else {
263 		time_t tod = (time_t)sec;
264 		time_t now = time(NULL);
265 		if (tod > now+60 ||
266 		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
267 			(void) strftime(buf, len, "%b %d %Y    ",
268 			    localtime(&tod));
269 		} else {
270 			(void) strftime(buf, len, "%b %d %T", localtime(&tod));
271 		}
272 	}
273 
274 	return (buf);
275 }
276 
277 static hostid_t *
278 find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
279     char *product_sn)
280 {
281 	hostid_t *rt = NULL;
282 	host_id_list_t *hostp;
283 
284 	if (platform == NULL)
285 		platform = "-";
286 	if (server == NULL)
287 		server = "-";
288 	hostp = host_list;
289 	while (hostp) {
290 		if (hostp->hostid.platform &&
291 		    strcmp(hostp->hostid.platform, platform) == 0 &&
292 		    hostp->hostid.server &&
293 		    strcmp(hostp->hostid.server, server) == 0 &&
294 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
295 		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
296 		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
297 		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
298 		    (domain == NULL || hostp->hostid.domain == NULL ||
299 		    strcmp(domain, hostp->hostid.domain) == 0)) {
300 			rt = &hostp->hostid;
301 			break;
302 		}
303 		hostp = hostp->next;
304 	}
305 	if (rt == NULL) {
306 		hostp = malloc(sizeof (host_id_list_t));
307 		hostp->hostid.platform = strdup(platform);
308 		hostp->hostid.product_sn =
309 		    product_sn ? strdup(product_sn) : NULL;
310 		hostp->hostid.server = strdup(server);
311 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
312 		hostp->hostid.domain = domain ? strdup(domain) : NULL;
313 		hostp->next = host_list;
314 		host_list = hostp;
315 		rt = &hostp->hostid;
316 		n_server++;
317 	}
318 	return (rt);
319 }
320 
321 static hostid_t *
322 find_hostid(nvlist_t *nvl)
323 {
324 	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
325 	char *product_sn = NULL;
326 	nvlist_t *auth, *fmri;
327 	hostid_t *rt = NULL;
328 
329 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
330 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
331 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
332 		    &platform);
333 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
334 		    &product_sn);
335 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
336 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
337 		    &chassis);
338 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
339 		rt = find_hostid_in_list(platform, chassis, server,
340 		    domain, product_sn);
341 	}
342 	return (rt);
343 }
344 
345 /*
346  * compare two fru strings which are made up of substrings seperated by '/'
347  * return true if every substring is the same in the two strings, or if a
348  * substring is null in one.
349  */
350 
351 static int
352 frucmp(char *f1, char *f2)
353 {
354 	char c1, c2;
355 	int i = 0;
356 
357 	for (;;) {
358 		c1 = *f1;
359 		c2 = *f2;
360 		if (c1 == c2) {
361 			i = (c1 == '/') ? 0 : i + 1;
362 		} else if (i == 0) {
363 			if (c1 == '/') {
364 				do {
365 					f2++;
366 				} while ((c2 = *f2) != 0 && c2 != '/');
367 				if (c2 == NULL)
368 					break;
369 			} else if (c2 == '/') {
370 				do {
371 					f1++;
372 				} while ((c1 = *f1) != 0 && c1 != '/');
373 				if (c1 == NULL)
374 					break;
375 			} else
376 				break;
377 		} else
378 			break;
379 		if (c1 == NULL)
380 			return (0);
381 		f1++;
382 		f2++;
383 	}
384 	return (1);
385 }
386 
387 static int
388 tgetlabel(topo_hdl_t *thp, tnode_t *node, void *arg)
389 {
390 	int err;
391 	char *fru_name, *lname;
392 	nvlist_t *fru = NULL;
393 	int rt = TOPO_WALK_NEXT;
394 	tgetlabel_data_t *tdp = (tgetlabel_data_t *)arg;
395 
396 	if (topo_node_fru(node, &fru, NULL, &err) == 0) {
397 		if (topo_fmri_nvl2str(thp, fru, &fru_name, &err) == 0) {
398 			if (frucmp(tdp->fru, fru_name) == 0 &&
399 			    topo_node_label(node, &lname, &err) == 0) {
400 				tdp->label = strdup(lname);
401 				topo_hdl_strfree(thp, lname);
402 				rt = TOPO_WALK_TERMINATE;
403 			}
404 			topo_hdl_strfree(thp, fru_name);
405 		}
406 		nvlist_free(fru);
407 	}
408 	return (rt);
409 }
410 
411 static void
412 label_get_topo(void)
413 {
414 	int err;
415 
416 	topo_handle = topo_open(TOPO_VERSION, 0, &err);
417 	if (topo_handle) {
418 		topo_handle_uuid = topo_snap_hold(topo_handle, NULL, &err);
419 	}
420 }
421 
422 static void
423 label_release_topo(void)
424 {
425 	if (topo_handle_uuid)
426 		topo_hdl_strfree(topo_handle, topo_handle_uuid);
427 	if (topo_handle) {
428 		topo_snap_release(topo_handle);
429 		topo_close(topo_handle);
430 	}
431 }
432 
433 static char *
434 get_fmri_label(char *fru)
435 {
436 	topo_walk_t *twp;
437 	tgetlabel_data_t td;
438 	int err;
439 
440 	td.label = NULL;
441 	td.fru = fru;
442 	if (topo_handle == NULL)
443 		label_get_topo();
444 	if (topo_handle_uuid) {
445 		twp = topo_walk_init(topo_handle, FM_FMRI_SCHEME_HC,
446 		    tgetlabel, &td, &err);
447 		if (twp) {
448 			topo_walk_step(twp, TOPO_WALK_CHILD);
449 			topo_walk_fini(twp);
450 		}
451 	}
452 	return (td.label);
453 }
454 
455 static char *
456 get_nvl2str_topo(nvlist_t *nvl)
457 {
458 	char *name = NULL;
459 	char *tname;
460 	int err;
461 	char *scheme = NULL;
462 	char *mod_name = NULL;
463 	char buf[128];
464 
465 	if (topo_handle == NULL)
466 		label_get_topo();
467 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
468 		name = strdup(tname);
469 		topo_hdl_strfree(topo_handle, tname);
470 	} else {
471 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
472 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
473 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
474 		    mod_name) {
475 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
476 			    scheme, mod_name);
477 			name = strdup(buf);
478 		}
479 	}
480 	return (name);
481 }
482 
483 static int
484 set_priority(char *s)
485 {
486 	int rt = 0;
487 
488 	if (s) {
489 		if (strcmp(s, "Minor") == 0)
490 			rt = 1;
491 		else if (strcmp(s, "Major") == 0)
492 			rt = 10;
493 		else if (strcmp(s, "Critical") == 0)
494 			rt = 100;
495 	}
496 	return (rt);
497 }
498 
499 static int
500 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
501     uint8_t p2)
502 {
503 	int r1, r2;
504 	int rt;
505 
506 	r1 = set_priority(s1);
507 	r2 = set_priority(s2);
508 	rt = r1 - r2;
509 	if (rt == 0) {
510 		if (t1 > t2)
511 			rt = 1;
512 		else if (t1 < t2)
513 			rt = -1;
514 		else
515 			rt = p1 - p2;
516 	}
517 	return (rt);
518 }
519 
520 /*
521  * merge two lists into one, by comparing enties in new and moving into list if
522  * name is not there or free off memory for names which are already there
523  * add_pct indicates if pct is the sum or highest pct
524  */
525 static name_list_t *
526 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
527 {
528 	name_list_t *lp, *np, *sp, *rt = NULL;
529 	int max_pct;
530 
531 	rt = *list;
532 	np = new;
533 	while (np) {
534 		lp = *list;
535 		while (lp) {
536 			if (strcmp(lp->name, np->name) == 0)
537 				break;
538 			lp = lp->next;
539 			if (lp == *list)
540 				lp = NULL;
541 		}
542 		if (np->next == new)
543 			sp = NULL;
544 		else
545 			sp = np->next;
546 		if (lp) {
547 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
548 			if (add_pct) {
549 				lp->pct += np->pct;
550 				lp->count += np->count;
551 			} else if (np->pct > lp->pct) {
552 				lp->pct = np->pct;
553 			}
554 			max_pct = np->max_pct;
555 			if (np->label)
556 				free(np->label);
557 			free(np->name);
558 			free(np);
559 			np = NULL;
560 			if (max_pct > lp->max_pct) {
561 				lp->max_pct = max_pct;
562 				if (lp->max_pct > lp->prev->max_pct &&
563 				    lp != *list) {
564 					lp->prev->next = lp->next;
565 					lp->next->prev = lp->prev;
566 					np = lp;
567 				}
568 			}
569 		}
570 		if (np) {
571 			lp = *list;
572 			if (lp) {
573 				if (np->max_pct > lp->max_pct) {
574 					np->next = lp;
575 					np->prev = lp->prev;
576 					lp->prev->next = np;
577 					lp->prev = np;
578 					*list = np;
579 					rt = np;
580 				} else {
581 					lp = lp->next;
582 					while (lp != *list &&
583 					    np->max_pct < lp->max_pct) {
584 						lp = lp->next;
585 					}
586 					np->next = lp;
587 					np->prev = lp->prev;
588 					lp->prev->next = np;
589 					lp->prev = np;
590 				}
591 			} else {
592 				*list = np;
593 				np->next = np;
594 				np->prev = np;
595 				rt = np;
596 			}
597 		}
598 		np = sp;
599 	}
600 	return (rt);
601 }
602 
603 static name_list_t *
604 alloc_name_list(char *name, uint8_t pct)
605 {
606 	name_list_t *nlp;
607 
608 	nlp = malloc(sizeof (*nlp));
609 	nlp->name = strdup(name);
610 	nlp->pct = pct;
611 	nlp->max_pct = pct;
612 	nlp->count = 1;
613 	nlp->next = nlp;
614 	nlp->prev = nlp;
615 	nlp->status = 0;
616 	nlp->label = NULL;
617 	return (nlp);
618 }
619 
620 static status_record_t *
621 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
622     name_list_t *fru, name_list_t *asru, name_list_t *resource,
623     name_list_t *serial, boolean_t not_suppressed,
624     hostid_t *hostid, boolean_t injected)
625 {
626 	status_record_t *status_rec_p;
627 
628 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
629 	status_rec_p->nrecs = 1;
630 	status_rec_p->host = hostid;
631 	status_rec_p->uurec = uurec_p;
632 	uurec_p->next = NULL;
633 	uurec_p->prev = NULL;
634 	uurec_p->asru = asru;
635 	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
636 	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
637 		status_rec_p->severity = strdup("unknown");
638 	status_rec_p->class = class;
639 	status_rec_p->fru = fru;
640 	status_rec_p->asru = asru;
641 	status_rec_p->resource = resource;
642 	status_rec_p->serial = serial;
643 	status_rec_p->msgid = strdup(msgid);
644 	status_rec_p->not_suppressed = not_suppressed;
645 	status_rec_p->injected = injected;
646 	return (status_rec_p);
647 }
648 
649 /*
650  * add record to given list maintaining order higher priority first.
651  */
652 static void
653 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
654 {
655 	sr_list_t *tp, *np, *sp;
656 	int order;
657 	uint64_t sec;
658 
659 	np = malloc(sizeof (sr_list_t));
660 	np->status_record = status_rec_p;
661 	sec = status_rec_p->uurec->sec;
662 	if ((sp = *list_pp) == NULL) {
663 		*list_pp = np;
664 		np->next = np;
665 		np->prev = np;
666 	} else {
667 		/* insert new record in front of lower priority */
668 		tp = sp;
669 		order = cmp_priority(status_rec_p->severity,
670 		    sp->status_record->severity, sec,
671 		    tp->status_record->uurec->sec, 0, 0);
672 		if (order > 0) {
673 			*list_pp = np;
674 		} else {
675 			tp = sp->next;
676 			while (tp != sp &&
677 			    cmp_priority(status_rec_p->severity,
678 			    tp->status_record->severity, sec,
679 			    tp->status_record->uurec->sec, 0, 0)) {
680 				tp = tp->next;
681 			}
682 		}
683 		np->next = tp;
684 		np->prev = tp->prev;
685 		tp->prev->next = np;
686 		tp->prev = np;
687 	}
688 }
689 
690 static void
691 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
692     resource_list_t *np)
693 {
694 	int order;
695 	uint64_t sec;
696 	resource_list_t *sp, *tp;
697 	status_record_t *srp;
698 	char *severity = status_rec_p->severity;
699 
700 	add_rec_list(status_rec_p, &np->status_rec_list);
701 	if ((sp = *rp) == NULL) {
702 		np->next = np;
703 		np->prev = np;
704 		*rp = np;
705 	} else {
706 		/*
707 		 * insert new record in front of lower priority
708 		 */
709 		tp = sp->next;
710 		srp = sp->status_rec_list->status_record;
711 		sec = status_rec_p->uurec->sec;
712 		order = cmp_priority(severity, srp->severity, sec,
713 		    srp->uurec->sec, np->max_pct, sp->max_pct);
714 		if (order > 0) {
715 			*rp = np;
716 		} else {
717 			srp = tp->status_rec_list->status_record;
718 			while (tp != sp &&
719 			    cmp_priority(severity, srp->severity, sec,
720 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
721 				tp = tp->next;
722 				srp = tp->status_rec_list->status_record;
723 			}
724 		}
725 		np->next = tp;
726 		np->prev = tp->prev;
727 		tp->prev->next = np;
728 		tp->prev = np;
729 	}
730 }
731 
732 static void
733 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
734     resource_list_t **rpp)
735 {
736 	int order;
737 	resource_list_t *np, *end;
738 	status_record_t *srp;
739 
740 	np = *rpp;
741 	end = np;
742 	while (np) {
743 		if (strcmp(fp->name, np->resource) == 0) {
744 			np->not_suppressed |= status_rec_p->not_suppressed;
745 			np->injected |= status_rec_p->injected;
746 			srp = np->status_rec_list->status_record;
747 			order = cmp_priority(status_rec_p->severity,
748 			    srp->severity, status_rec_p->uurec->sec,
749 			    srp->uurec->sec, fp->max_pct, np->max_pct);
750 			if (order > 0 && np != end) {
751 				/*
752 				 * remove from list and add again using
753 				 * new priority
754 				 */
755 				np->prev->next = np->next;
756 				np->next->prev = np->prev;
757 				add_resource(status_rec_p,
758 				    rpp, np);
759 			} else {
760 				add_rec_list(status_rec_p,
761 				    &np->status_rec_list);
762 			}
763 			break;
764 		}
765 		np = np->next;
766 		if (np == end) {
767 			np = NULL;
768 			break;
769 		}
770 	}
771 	if (np == NULL) {
772 		np = malloc(sizeof (resource_list_t));
773 		np->resource = fp->name;
774 		np->not_suppressed = status_rec_p->not_suppressed;
775 		np->injected = status_rec_p->injected;
776 		np->status_rec_list = NULL;
777 		np->max_pct = fp->max_pct;
778 		add_resource(status_rec_p, rpp, np);
779 	}
780 }
781 
782 static void
783 add_list(status_record_t *status_rec_p, name_list_t *listp,
784     resource_list_t **glistp)
785 {
786 	name_list_t *fp, *end;
787 
788 	fp = listp;
789 	end = fp;
790 	while (fp) {
791 		add_resource_list(status_rec_p, fp, glistp);
792 		fp = fp->next;
793 		if (fp == end)
794 			break;
795 	}
796 }
797 
798 /*
799  * add record to rec, fru and asru lists.
800  */
801 static void
802 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
803     name_list_t *fru, name_list_t *asru, name_list_t *resource,
804     name_list_t *serial, boolean_t not_suppressed,
805     hostid_t *hostid, boolean_t injected)
806 {
807 	status_record_t *status_rec_p;
808 
809 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
810 	    resource, serial, not_suppressed, hostid, injected);
811 	add_rec_list(status_rec_p, &status_rec_list);
812 	if (status_rec_p->fru)
813 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
814 	if (status_rec_p->asru)
815 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
816 }
817 
818 static void
819 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
820 {
821 	char *name;
822 	char *serial = NULL;
823 	char **lserial = NULL;
824 	uint64_t serint;
825 	name_list_t *nlp;
826 	int j;
827 	uint_t nelem;
828 	char buf[64];
829 
830 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
831 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
832 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
833 			    &serint) == 0) {
834 				(void) snprintf(buf, sizeof (buf), "%llX",
835 				    serint);
836 				nlp = alloc_name_list(buf, pct);
837 				(void) merge_name_list(serial_p, nlp, 1);
838 			}
839 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
840 			if (nvlist_lookup_string_array(nvl,
841 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
842 				nlp = alloc_name_list(lserial[0], pct);
843 				for (j = 1; j < nelem; j++) {
844 					name_list_t *n1lp;
845 					n1lp = alloc_name_list(lserial[j], pct);
846 					(void) merge_name_list(&nlp, n1lp, 1);
847 				}
848 				(void) merge_name_list(serial_p, nlp, 1);
849 			}
850 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
851 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
852 			    &serial) == 0) {
853 				nlp = alloc_name_list(serial, pct);
854 				(void) merge_name_list(serial_p, nlp, 1);
855 			}
856 		}
857 	}
858 }
859 
860 static void
861 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
862     name_list_t **fru_p, name_list_t **serial_p,
863     name_list_t **resource_p, name_list_t **asru_p, uint8_t status)
864 {
865 	nvlist_t *lfru, *lasru, *rsrc;
866 	name_list_t *nlp;
867 	char *name;
868 	uint8_t lpct = 0;
869 	char *lclass = NULL;
870 	char *label;
871 
872 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
873 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
874 		nlp = alloc_name_list(lclass, lpct);
875 		(void) merge_name_list(class_p, nlp, 1);
876 	}
877 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
878 		name = get_nvl2str_topo(lfru);
879 		if (name != NULL) {
880 			nlp = alloc_name_list(name, lpct);
881 			nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
882 			    FM_SUSPECT_DEGRADED);
883 			free(name);
884 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
885 			    &label) == 0)
886 				nlp->label = strdup(label);
887 			(void) merge_name_list(fru_p, nlp, 1);
888 		}
889 		get_serial_no(lfru, serial_p, lpct);
890 	} else if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) != 0) {
891 		/*
892 		 * No FRU or resource. But we want to display the repair status
893 		 * somehow, so create a dummy FRU field.
894 		 */
895 		nlp = alloc_name_list(dgettext("FMD", "None"), lpct);
896 		nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
897 		    FM_SUSPECT_DEGRADED);
898 		(void) merge_name_list(fru_p, nlp, 1);
899 	}
900 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
901 		name = get_nvl2str_topo(lasru);
902 		if (name != NULL) {
903 			nlp = alloc_name_list(name, lpct);
904 			nlp->status = status & ~(FM_SUSPECT_NOT_PRESENT |
905 			    FM_SUSPECT_REPAIRED | FM_SUSPECT_REPLACED |
906 			    FM_SUSPECT_ACQUITTED);
907 			free(name);
908 			(void) merge_name_list(asru_p, nlp, 1);
909 		}
910 		get_serial_no(lasru, serial_p, lpct);
911 	}
912 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
913 		name = get_nvl2str_topo(rsrc);
914 		if (name != NULL) {
915 			nlp = alloc_name_list(name, lpct);
916 			nlp->status = status;
917 			free(name);
918 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
919 			    &label) == 0)
920 				nlp->label = strdup(label);
921 			(void) merge_name_list(resource_p, nlp, 1);
922 		}
923 	}
924 }
925 
926 static void
927 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid)
928 {
929 	char *msgid = "-";
930 	uint_t i, size = 0;
931 	name_list_t *class = NULL, *resource = NULL;
932 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
933 	nvlist_t **nva;
934 	uint8_t *ba;
935 	uurec_t *uurec_p;
936 	hostid_t *host;
937 	boolean_t not_suppressed = 1;
938 	boolean_t any_present = 0;
939 	boolean_t injected = 0;
940 
941 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
942 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
943 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
944 	    &not_suppressed);
945 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_INJECTED, &injected);
946 
947 	if (size != 0) {
948 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
949 		    &nva, &size);
950 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
951 		    &ba, &size);
952 		for (i = 0; i < size; i++) {
953 			extract_record_info(nva[i], &class, &fru, &serial,
954 			    &resource, &asru, ba[i]);
955 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
956 			    (ba[i] & FM_SUSPECT_FAULTY))
957 				any_present = 1;
958 		}
959 		/*
960 		 * also suppress if no resources present
961 		 */
962 		if (any_present == 0)
963 			not_suppressed = 0;
964 	}
965 
966 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
967 	uurec_p->uuid = strdup(uuid);
968 	uurec_p->sec = sec;
969 	uurec_p->ari_uuid_list = NULL;
970 	uurec_p->event = NULL;
971 	(void) nvlist_dup(nvl, &uurec_p->event, 0);
972 	host = find_hostid(nvl);
973 	catalog_new_record(uurec_p, msgid, class, fru, asru,
974 	    resource, serial, not_suppressed, host, injected);
975 }
976 
977 static void
978 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
979 {
980 	sr_list_t *srp;
981 	uurec_t *uurp;
982 	ari_list_t *ari_list;
983 
984 	srp = status_rec_list;
985 	if (srp) {
986 		for (;;) {
987 			uurp = srp->status_record->uurec;
988 			while (uurp) {
989 				if (strcmp(uuid, uurp->uuid) == 0) {
990 					ari_list = (ari_list_t *)
991 					    malloc(sizeof (ari_list_t));
992 					ari_list->ari_uuid = strdup(ari_uuid);
993 					ari_list->next = uurp->ari_uuid_list;
994 					uurp->ari_uuid_list = ari_list;
995 					return;
996 				}
997 				uurp = uurp->next;
998 			}
999 			if (srp->next == status_rec_list)
1000 				break;
1001 			srp = srp->next;
1002 		}
1003 	}
1004 }
1005 
1006 static void
1007 print_line(char *label, char *buf)
1008 {
1009 	char *cp, *ep, *wp;
1010 	char c;
1011 	int i;
1012 	int lsz;
1013 	char *padding;
1014 
1015 	lsz = strlen(label);
1016 	padding = malloc(lsz + 1);
1017 	for (i = 0; i < lsz; i++)
1018 		padding[i] = ' ';
1019 	padding[i] = 0;
1020 	cp = buf;
1021 	ep = buf;
1022 	c = *ep;
1023 	(void) printf("\n");
1024 	while (c) {
1025 		i = lsz;
1026 		wp = NULL;
1027 		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
1028 			if (c == ' ')
1029 				wp = ep;
1030 			else if (c == '\n') {
1031 				i = 0;
1032 				*ep = 0;
1033 				do {
1034 					ep++;
1035 				} while ((c = *ep) != NULL && c == ' ');
1036 				break;
1037 			}
1038 			ep++;
1039 			i++;
1040 		}
1041 		if (i >= 80 && wp) {
1042 			*wp = 0;
1043 			ep = wp + 1;
1044 			c = *ep;
1045 		}
1046 		(void) printf("%s%s\n", label, cp);
1047 		cp = ep;
1048 		label = padding;
1049 	}
1050 	free(padding);
1051 }
1052 
1053 static void
1054 print_dict_info_line(nvlist_t *e, fmd_msg_item_t what, const char *linehdr)
1055 {
1056 	char *cp = fmd_msg_getitem_nv(fmadm_msghdl, NULL, e, what);
1057 
1058 	if (cp) {
1059 		print_line(dgettext("FMD", linehdr), cp);
1060 		free(cp);
1061 	}
1062 }
1063 
1064 static void
1065 print_dict_info(nvlist_t *nvl)
1066 {
1067 	print_dict_info_line(nvl, FMD_MSG_ITEM_DESC, "Description : ");
1068 	print_dict_info_line(nvl, FMD_MSG_ITEM_RESPONSE, "Response    : ");
1069 	print_dict_info_line(nvl, FMD_MSG_ITEM_IMPACT, "Impact      : ");
1070 	print_dict_info_line(nvl, FMD_MSG_ITEM_ACTION, "Action      : ");
1071 }
1072 
1073 static void
1074 print_name(name_list_t *list, char *(func)(char *), char *padding, int *np,
1075     int pct, int full)
1076 {
1077 	char *name, *fru_label = NULL;
1078 
1079 	name = list->name;
1080 	if (list->label) {
1081 		(void) printf("%s \"%s\" (%s)", padding, list->label, name);
1082 		*np += 1;
1083 	} else if (func && (fru_label = func(list->name)) != NULL) {
1084 		(void) printf("%s \"%s\" (%s)", padding, fru_label, name);
1085 		*np += 1;
1086 		free(fru_label);
1087 	} else {
1088 		(void) printf("%s %s", padding, name);
1089 		*np += 1;
1090 	}
1091 	if (list->pct && pct > 0 && pct < 100) {
1092 		if (list->count > 1) {
1093 			if (full) {
1094 				(void) printf(" %d @ %s %d%%\n", list->count,
1095 				    dgettext("FMD", "max"),
1096 				    list->max_pct);
1097 			} else {
1098 				(void) printf(" %s %d%%\n",
1099 				    dgettext("FMD", "max"),
1100 				    list->max_pct);
1101 			}
1102 		} else {
1103 			(void) printf(" %d%%\n", list->pct);
1104 		}
1105 	} else {
1106 		(void) printf("\n");
1107 	}
1108 }
1109 
1110 static void
1111 print_asru_status(int status, char *label)
1112 {
1113 	char *msg = NULL;
1114 
1115 	switch (status) {
1116 	case 0:
1117 		msg = dgettext("FMD", "ok and in service");
1118 		break;
1119 	case FM_SUSPECT_DEGRADED:
1120 		msg = dgettext("FMD", "service degraded, "
1121 		    "but associated components no longer faulty");
1122 		break;
1123 	case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1124 		msg = dgettext("FMD", "faulted but still "
1125 		    "providing degraded service");
1126 		break;
1127 	case FM_SUSPECT_FAULTY:
1128 		msg = dgettext("FMD", "faulted but still in service");
1129 		break;
1130 	case FM_SUSPECT_UNUSABLE:
1131 		msg = dgettext("FMD", "out of service, "
1132 		    "but associated components no longer faulty");
1133 		break;
1134 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1135 		msg = dgettext("FMD", "faulted and taken out of service");
1136 		break;
1137 	default:
1138 		break;
1139 	}
1140 	if (msg) {
1141 		(void) printf("%s     %s\n", label, msg);
1142 	}
1143 }
1144 
1145 static void
1146 print_fru_status(int status, char *label)
1147 {
1148 	char *msg = NULL;
1149 
1150 	if (status & FM_SUSPECT_NOT_PRESENT)
1151 		msg = dgettext("FMD", "not present");
1152 	else if (status & FM_SUSPECT_FAULTY)
1153 		msg = dgettext("FMD", "faulty");
1154 	else if (status & FM_SUSPECT_REPLACED)
1155 		msg = dgettext("FMD", "replaced");
1156 	else if (status & FM_SUSPECT_REPAIRED)
1157 		msg = dgettext("FMD", "repair attempted");
1158 	else if (status & FM_SUSPECT_ACQUITTED)
1159 		msg = dgettext("FMD", "acquitted");
1160 	else
1161 		msg = dgettext("FMD", "removed");
1162 	(void) printf("%s     %s\n", label, msg);
1163 }
1164 
1165 static void
1166 print_rsrc_status(int status, char *label)
1167 {
1168 	char *msg = "";
1169 
1170 	if (status & FM_SUSPECT_NOT_PRESENT)
1171 		msg = dgettext("FMD", "not present");
1172 	else if (status & FM_SUSPECT_FAULTY) {
1173 		if (status & FM_SUSPECT_DEGRADED)
1174 			msg = dgettext("FMD",
1175 			    "faulted but still providing degraded service");
1176 		else if (status & FM_SUSPECT_UNUSABLE)
1177 			msg = dgettext("FMD",
1178 			    "faulted and taken out of service");
1179 		else
1180 			msg = dgettext("FMD", "faulted but still in service");
1181 	} else if (status & FM_SUSPECT_REPLACED)
1182 		msg = dgettext("FMD", "replaced");
1183 	else if (status & FM_SUSPECT_REPAIRED)
1184 		msg = dgettext("FMD", "repair attempted");
1185 	else if (status & FM_SUSPECT_ACQUITTED)
1186 		msg = dgettext("FMD", "acquitted");
1187 	else
1188 		msg = dgettext("FMD", "removed");
1189 	(void) printf("%s     %s\n", label, msg);
1190 }
1191 
1192 static void
1193 print_name_list(name_list_t *list, char *label, char *(func)(char *),
1194     int limit, int pct, void (func1)(int, char *), int full)
1195 {
1196 	char *name, *fru_label = NULL;
1197 	char *padding;
1198 	int i, j, l, n;
1199 	name_list_t *end = list;
1200 
1201 	l = strlen(label);
1202 	padding = malloc(l + 1);
1203 	for (i = 0; i < l; i++)
1204 		padding[i] = ' ';
1205 	padding[l] = 0;
1206 	(void) printf("%s", label);
1207 	name = list->name;
1208 	if (list->label)
1209 		(void) printf(" \"%s\" (%s)", list->label, name);
1210 	else if (func && (fru_label = func(list->name)) != NULL) {
1211 		(void) printf(" \"%s\" (%s)", fru_label, name);
1212 		free(fru_label);
1213 	} else
1214 		(void) printf(" %s", name);
1215 	if (list->pct && pct > 0 && pct < 100) {
1216 		if (list->count > 1) {
1217 			if (full) {
1218 				(void) printf(" %d @ %s %d%%\n", list->count,
1219 				    dgettext("FMD", "max"), list->max_pct);
1220 			} else {
1221 				(void) printf(" %s %d%%\n",
1222 				    dgettext("FMD", "max"), list->max_pct);
1223 			}
1224 		} else {
1225 			(void) printf(" %d%%\n", list->pct);
1226 		}
1227 	} else {
1228 		(void) printf("\n");
1229 	}
1230 	if (func1)
1231 		func1(list->status, padding);
1232 	n = 1;
1233 	j = 0;
1234 	while ((list = list->next) != end) {
1235 		if (limit == 0 || n < limit) {
1236 			print_name(list, func, padding, &n, pct, full);
1237 			if (func1)
1238 				func1(list->status, padding);
1239 		} else
1240 			j++;
1241 	}
1242 	if (j == 1) {
1243 		print_name(list->prev, func, padding, &n, pct, full);
1244 	} else if (j > 1) {
1245 		(void) printf("%s... %d %s\n", padding, j,
1246 		    dgettext("FMD", "more entries suppressed,"
1247 		    " use -v option for full list"));
1248 	}
1249 	free(padding);
1250 }
1251 
1252 static int
1253 asru_same_status(name_list_t *list)
1254 {
1255 	name_list_t *end = list;
1256 	int status = list->status;
1257 
1258 	while ((list = list->next) != end) {
1259 		if (status == -1) {
1260 			status = list->status;
1261 			continue;
1262 		}
1263 		if (list->status != -1 && status != list->status) {
1264 			status = -1;
1265 			break;
1266 		}
1267 	}
1268 	return (status);
1269 }
1270 
1271 static int
1272 serial_in_fru(name_list_t *fru, name_list_t *serial)
1273 {
1274 	name_list_t *sp = serial;
1275 	name_list_t *fp;
1276 	int nserial = 0;
1277 	int found = 0;
1278 	char buf[128];
1279 
1280 	while (sp) {
1281 		fp = fru;
1282 		nserial++;
1283 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1284 		buf[sizeof (buf) - 1] = 0;
1285 		while (fp) {
1286 			if (strstr(fp->name, buf) != NULL) {
1287 				found++;
1288 				break;
1289 			}
1290 			fp = fp->next;
1291 			if (fp == fru)
1292 				break;
1293 		}
1294 		sp = sp->next;
1295 		if (sp == serial)
1296 			break;
1297 	}
1298 	return (found == nserial ? 1 : 0);
1299 }
1300 
1301 static void
1302 print_sup_record(status_record_t *srp, int opt_i, int full)
1303 {
1304 	char buf[32];
1305 	uurec_t *uurp = srp->uurec;
1306 	int n, j, k, max;
1307 	int status;
1308 	ari_list_t *ari_list;
1309 
1310 	n = 0;
1311 	max = max_fault;
1312 	if (max < 0) {
1313 		max = 0;
1314 	}
1315 	j = max / 2;
1316 	max -= j;
1317 	k = srp->nrecs - max;
1318 	while ((uurp = uurp->next) != NULL) {
1319 		if (full || n < j || n >= k || max_fault == 0 ||
1320 		    srp->nrecs == max_fault+1) {
1321 			if (opt_i) {
1322 				ari_list = uurp->ari_uuid_list;
1323 				while (ari_list) {
1324 					(void) printf("%-15s %s\n",
1325 					    format_date(buf, sizeof (buf),
1326 					    uurp->sec), ari_list->ari_uuid);
1327 					ari_list = ari_list->next;
1328 				}
1329 			} else {
1330 				(void) printf("%-15s %s\n",
1331 				    format_date(buf, sizeof (buf), uurp->sec),
1332 				    uurp->uuid);
1333 			}
1334 		} else if (n == j)
1335 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1336 			    dgettext("FMD", "more entries suppressed"));
1337 		n++;
1338 	}
1339 	(void) printf("\n");
1340 	(void) printf("%s %s", dgettext("FMD", "Host        :"),
1341 	    srp->host->server);
1342 	if (srp->host->domain)
1343 		(void) printf("\t%s %s", dgettext("FMD", "Domain      :"),
1344 		    srp->host->domain);
1345 	(void) printf("\n%s %s", dgettext("FMD", "Platform    :"),
1346 	    srp->host->platform);
1347 	(void) printf("\t%s %s", dgettext("FMD", "Chassis_id  :"),
1348 	    srp->host->chassis ? srp->host->chassis : "");
1349 	(void) printf("\n%s %s\n\n", dgettext("FMD", "Product_sn  :"),
1350 	    srp->host->product_sn? srp->host->product_sn : "");
1351 	if (srp->class)
1352 		print_name_list(srp->class,
1353 		    dgettext("FMD", "Fault class :"), NULL, 0, srp->class->pct,
1354 		    NULL, full);
1355 	if (srp->asru) {
1356 		status = asru_same_status(srp->asru);
1357 		if (status != -1) {
1358 			print_name_list(srp->asru,
1359 			    dgettext("FMD", "Affects     :"), NULL,
1360 			    full ? 0 : max_display, 0, NULL, full);
1361 			print_asru_status(status, "             ");
1362 		} else
1363 			print_name_list(srp->asru,
1364 			    dgettext("FMD", "Affects     :"), NULL,
1365 			    full ? 0 : max_display, 0, print_asru_status, full);
1366 	}
1367 	if (full || srp->fru == NULL || srp->asru == NULL) {
1368 		if (srp->resource) {
1369 			status = asru_same_status(srp->resource);
1370 			if (status != -1) {
1371 				print_name_list(srp->resource,
1372 				    dgettext("FMD", "Problem in  :"), NULL,
1373 				    full ? 0 : max_display, 0, NULL, full);
1374 				print_rsrc_status(status, "             ");
1375 			} else
1376 				print_name_list(srp->resource,
1377 				    dgettext("FMD", "Problem in  :"),
1378 				    NULL, full ? 0 : max_display, 0,
1379 				    print_rsrc_status, full);
1380 		}
1381 	}
1382 	if (srp->fru) {
1383 		status = asru_same_status(srp->fru);
1384 		if (status != -1) {
1385 			print_name_list(srp->fru, dgettext("FMD",
1386 			    "FRU         :"), get_fmri_label, 0,
1387 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1388 			    NULL, full);
1389 			print_fru_status(status, "             ");
1390 		} else
1391 			print_name_list(srp->fru, dgettext("FMD",
1392 			    "FRU         :"), get_fmri_label, 0,
1393 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1394 			    print_fru_status, full);
1395 	}
1396 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1397 	    !serial_in_fru(srp->asru, srp->serial)) {
1398 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1399 		    NULL, 0, 0, NULL, full);
1400 	}
1401 	print_dict_info(srp->uurec->event);
1402 	(void) printf("\n");
1403 }
1404 
1405 static void
1406 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1407 {
1408 	char buf[32];
1409 	uurec_t *uurp = srp->uurec;
1410 	static int header = 0;
1411 	char *head;
1412 	ari_list_t *ari_list;
1413 
1414 	if (!summary || !header) {
1415 		if (opt_i) {
1416 			head = "--------------- "
1417 			    "------------------------------------  "
1418 			    "-------------- ---------\n"
1419 			    "TIME            CACHE-ID"
1420 			    "                              MSG-ID"
1421 			    "         SEVERITY\n--------------- "
1422 			    "------------------------------------ "
1423 			    " -------------- ---------";
1424 		} else {
1425 			head = "--------------- "
1426 			    "------------------------------------  "
1427 			    "-------------- ---------\n"
1428 			    "TIME            EVENT-ID"
1429 			    "                              MSG-ID"
1430 			    "         SEVERITY\n--------------- "
1431 			    "------------------------------------ "
1432 			    " -------------- ---------";
1433 		}
1434 		(void) printf("%s\n", dgettext("FMD", head));
1435 		header = 1;
1436 	}
1437 	if (opt_i) {
1438 		ari_list = uurp->ari_uuid_list;
1439 		while (ari_list) {
1440 			(void) printf("%-15s %-37s %-14s %-9s %s\n",
1441 			    format_date(buf, sizeof (buf), uurp->sec),
1442 			    ari_list->ari_uuid, srp->msgid, srp->severity,
1443 			    srp->injected ? dgettext("FMD", "injected") : "");
1444 			ari_list = ari_list->next;
1445 		}
1446 	} else {
1447 		(void) printf("%-15s %-37s %-14s %-9s %s\n",
1448 		    format_date(buf, sizeof (buf), uurp->sec),
1449 		    uurp->uuid, srp->msgid, srp->severity,
1450 		    srp->injected ? dgettext("FMD", "injected") : "");
1451 	}
1452 
1453 	if (!summary)
1454 		print_sup_record(srp, opt_i, full);
1455 }
1456 
1457 static void
1458 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1459 {
1460 	status_record_t *srp;
1461 	sr_list_t *slp;
1462 
1463 	slp = status_rec_list;
1464 	if (slp) {
1465 		for (;;) {
1466 			srp = slp->status_record;
1467 			if (opt_a || srp->not_suppressed) {
1468 				if (page_feed)
1469 					(void) printf("\f\n");
1470 				print_status_record(srp, summary, opt_i, full);
1471 			}
1472 			if (slp->next == status_rec_list)
1473 				break;
1474 			slp = slp->next;
1475 		}
1476 	}
1477 }
1478 
1479 static name_list_t *
1480 find_fru(status_record_t *srp, char *resource)
1481 {
1482 	name_list_t *rt = NULL;
1483 	name_list_t *fru = srp->fru;
1484 
1485 	while (fru) {
1486 		if (strcmp(resource, fru->name) == 0) {
1487 			rt = fru;
1488 			break;
1489 		}
1490 		fru = fru->next;
1491 		if (fru == srp->fru)
1492 			break;
1493 	}
1494 	return (rt);
1495 }
1496 
1497 static void
1498 print_fru_line(name_list_t *fru, char *uuid)
1499 {
1500 	if (fru->pct == 100) {
1501 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1502 		    dgettext("FMD", "suspects in this FRU total certainty"),
1503 		    100);
1504 	} else {
1505 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1506 		    dgettext("FMD", "suspects in this FRU max certainty"),
1507 		    fru->max_pct);
1508 	}
1509 }
1510 
1511 static void
1512 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1513 {
1514 	resource_list_t *tp = status_fru_list;
1515 	status_record_t *srp;
1516 	sr_list_t *slp, *end;
1517 	char *msgid, *fru_label;
1518 	uurec_t *uurp;
1519 	name_list_t *fru;
1520 	int status;
1521 	ari_list_t *ari_list;
1522 
1523 	while (tp) {
1524 		if (opt_a || tp->not_suppressed) {
1525 			if (page_feed)
1526 				(void) printf("\f\n");
1527 			if (!summary)
1528 				(void) printf("-----------------------------"
1529 				    "---------------------------------------"
1530 				    "----------\n");
1531 			slp = tp->status_rec_list;
1532 			end = slp;
1533 			do {
1534 				srp = slp->status_record;
1535 				fru = find_fru(srp, tp->resource);
1536 				if (fru) {
1537 					if (fru->label)
1538 						(void) printf("\"%s\" (%s) ",
1539 						    fru->label, fru->name);
1540 					else if ((fru_label = get_fmri_label(
1541 					    fru->name)) != NULL) {
1542 						(void) printf("\"%s\" (%s) ",
1543 						    fru_label, fru->name);
1544 						free(fru_label);
1545 					} else
1546 						(void) printf("%s ",
1547 						    fru->name);
1548 					break;
1549 				}
1550 				slp = slp->next;
1551 			} while (slp != end);
1552 
1553 			slp = tp->status_rec_list;
1554 			end = slp;
1555 			status = 0;
1556 			do {
1557 				srp = slp->status_record;
1558 				fru = srp->fru;
1559 				while (fru) {
1560 					if (strcmp(tp->resource,
1561 					    fru->name) == 0)
1562 						status |= fru->status;
1563 					fru = fru->next;
1564 					if (fru == srp->fru)
1565 						break;
1566 				}
1567 				slp = slp->next;
1568 			} while (slp != end);
1569 			if (status & FM_SUSPECT_NOT_PRESENT)
1570 				(void) printf(dgettext("FMD", "not present"));
1571 			else if (status & FM_SUSPECT_FAULTY)
1572 				(void) printf(dgettext("FMD", "faulty"));
1573 			else if (status & FM_SUSPECT_REPLACED)
1574 				(void) printf(dgettext("FMD", "replaced"));
1575 			else if (status & FM_SUSPECT_REPAIRED)
1576 				(void) printf(dgettext("FMD",
1577 				    "repair attempted"));
1578 			else if (status & FM_SUSPECT_ACQUITTED)
1579 				(void) printf(dgettext("FMD", "acquitted"));
1580 			else
1581 				(void) printf(dgettext("FMD", "removed"));
1582 
1583 			if (tp->injected)
1584 				(void) printf(dgettext("FMD", " injected\n"));
1585 			else
1586 				(void) printf(dgettext("FMD", "\n"));
1587 
1588 			slp = tp->status_rec_list;
1589 			end = slp;
1590 			do {
1591 				srp = slp->status_record;
1592 				uurp = srp->uurec;
1593 				fru = find_fru(srp, tp->resource);
1594 				if (fru) {
1595 					if (opt_i) {
1596 						ari_list = uurp->ari_uuid_list;
1597 						while (ari_list) {
1598 							print_fru_line(fru,
1599 							    ari_list->ari_uuid);
1600 							ari_list =
1601 							    ari_list->next;
1602 						}
1603 					} else {
1604 						print_fru_line(fru, uurp->uuid);
1605 					}
1606 				}
1607 				slp = slp->next;
1608 			} while (slp != end);
1609 			if (!summary) {
1610 				slp = tp->status_rec_list;
1611 				end = slp;
1612 				srp = slp->status_record;
1613 				if (srp->serial &&
1614 				    !serial_in_fru(srp->fru, srp->serial)) {
1615 					print_name_list(srp->serial,
1616 					    dgettext("FMD", "Serial ID.  :"),
1617 					    NULL, 0, 0, NULL, 1);
1618 				}
1619 				msgid = NULL;
1620 				do {
1621 					if (msgid == NULL ||
1622 					    strcmp(msgid, srp->msgid) != 0) {
1623 						msgid = srp->msgid;
1624 						print_dict_info(uurp->event);
1625 					}
1626 					slp = slp->next;
1627 				} while (slp != end);
1628 			}
1629 		}
1630 		tp = tp->next;
1631 		if (tp == status_fru_list)
1632 			break;
1633 	}
1634 }
1635 
1636 static void
1637 print_asru(int opt_a)
1638 {
1639 	resource_list_t *tp = status_asru_list;
1640 	status_record_t *srp;
1641 	sr_list_t *slp, *end;
1642 	char *msg;
1643 	int status;
1644 	name_list_t *asru;
1645 
1646 	while (tp) {
1647 		if (opt_a || tp->not_suppressed) {
1648 			status = 0;
1649 			slp = tp->status_rec_list;
1650 			end = slp;
1651 			do {
1652 				srp = slp->status_record;
1653 				asru = srp->asru;
1654 				while (asru) {
1655 					if (strcmp(tp->resource,
1656 					    asru->name) == 0)
1657 						status |= asru->status;
1658 					asru = asru->next;
1659 					if (asru == srp->asru)
1660 						break;
1661 				}
1662 				slp = slp->next;
1663 			} while (slp != end);
1664 			switch (status) {
1665 			case 0:
1666 				msg = dgettext("FMD", "ok");
1667 				break;
1668 			case FM_SUSPECT_DEGRADED:
1669 				msg = dgettext("FMD", "degraded");
1670 				break;
1671 			case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1672 				msg = dgettext("FMD", "degraded");
1673 				break;
1674 			case FM_SUSPECT_FAULTY:
1675 				msg = dgettext("FMD", "degraded");
1676 				break;
1677 			case FM_SUSPECT_UNUSABLE:
1678 				msg = dgettext("FMD", "unknown");
1679 				break;
1680 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1681 				msg = dgettext("FMD", "faulted");
1682 				break;
1683 			default:
1684 				msg = "";
1685 				break;
1686 			}
1687 			(void) printf("%-69s %s", tp->resource, msg);
1688 			if (tp->injected)
1689 				(void) printf(dgettext("FMD", " injected\n"));
1690 			else
1691 				(void) printf(dgettext("FMD", "\n"));
1692 		}
1693 		tp = tp->next;
1694 		if (tp == status_asru_list)
1695 			break;
1696 	}
1697 }
1698 
1699 static int
1700 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1701 {
1702 	while (uurecp) {
1703 		if (strcmp(uuid, uurecp->uuid) == 0)
1704 			return (1);
1705 		uurecp = uurecp->next;
1706 	}
1707 	return (0);
1708 }
1709 
1710 static int
1711 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1712 {
1713 	int64_t *diag_time;
1714 	uint_t nelem;
1715 	int rt = 0;
1716 	char *uuid = "-";
1717 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1718 
1719 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1720 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1721 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1722 		    &uuid);
1723 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1724 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1725 			    uuid);
1726 	} else {
1727 		rt = -1;
1728 	}
1729 	return (rt);
1730 }
1731 
1732 /*ARGSUSED*/
1733 static int
1734 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1735 {
1736 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1737 	return (0);
1738 }
1739 
1740 static int
1741 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1742 {
1743 	int rt = FMADM_EXIT_SUCCESS;
1744 
1745 	/*
1746 	 * These calls may fail with Protocol error if message payload is
1747 	 * too big
1748 	 */
1749 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1750 		die("failed to get case list from fmd");
1751 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1752 		die("failed to get case status from fmd");
1753 	return (rt);
1754 }
1755 
1756 /*
1757  * fmadm faulty command
1758  *
1759  *	-a		show hidden fault records
1760  *	-f		show faulty fru's
1761  *	-g		force grouping of similar faults on the same fru
1762  *	-n		number of fault records to display
1763  *	-p		pipe output through pager
1764  *	-r		show faulty asru's
1765  *	-s		print summary of first fault
1766  *	-u		print listed uuid's only
1767  *	-v		full output
1768  */
1769 
1770 int
1771 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1772 {
1773 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1774 	int opt_i = 0;
1775 	char *pager;
1776 	FILE *fp;
1777 	int rt, c, stat;
1778 	uurec_select_t *tp;
1779 	uurec_select_t *uurecp = NULL;
1780 
1781 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1782 		switch (c) {
1783 		case 'a':
1784 			opt_a++;
1785 			break;
1786 		case 'f':
1787 			opt_f++;
1788 			break;
1789 		case 'g':
1790 			opt_g++;
1791 			break;
1792 		case 'i':
1793 			opt_i++;
1794 			break;
1795 		case 'n':
1796 			max_fault = atoi(optarg);
1797 			break;
1798 		case 'p':
1799 			opt_p++;
1800 			break;
1801 		case 'r':
1802 			opt_r++;
1803 			break;
1804 		case 's':
1805 			opt_s++;
1806 			break;
1807 		case 'u':
1808 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1809 			tp->uuid = optarg;
1810 			tp->next = uurecp;
1811 			uurecp = tp;
1812 			opt_a = 1;
1813 			break;
1814 		case 'v':
1815 			opt_v++;
1816 			break;
1817 		default:
1818 			return (FMADM_EXIT_USAGE);
1819 		}
1820 	}
1821 	if (optind < argc)
1822 		return (FMADM_EXIT_USAGE);
1823 
1824 	if ((fmadm_msghdl = fmd_msg_init(NULL, FMD_MSG_VERSION)) == NULL)
1825 		return (FMADM_EXIT_ERROR);
1826 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1827 	if (opt_p) {
1828 		if ((pager = getenv("PAGER")) == NULL)
1829 			pager = "/usr/bin/more";
1830 		fp = popen(pager, "w");
1831 		if (fp == NULL) {
1832 			rt = FMADM_EXIT_ERROR;
1833 			opt_p = 0;
1834 		} else {
1835 			dup2(fileno(fp), 1);
1836 			setbuf(stdout, NULL);
1837 			(void) fclose(fp);
1838 		}
1839 	}
1840 	max_display = max_fault;
1841 	if (opt_f)
1842 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1843 	if (opt_r)
1844 		print_asru(opt_a);
1845 	if (opt_f == 0 && opt_r == 0)
1846 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1847 	fmd_msg_fini(fmadm_msghdl);
1848 	label_release_topo();
1849 	if (opt_p) {
1850 		(void) fclose(stdout);
1851 		(void) wait(&stat);
1852 	}
1853 	return (rt);
1854 }
1855 
1856 int
1857 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1858 {
1859 	int i, status = FMADM_EXIT_SUCCESS;
1860 
1861 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1862 		return (FMADM_EXIT_USAGE);
1863 
1864 	for (i = 1; i < argc; i++) {
1865 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1866 			warn("failed to flush %s", argv[i]);
1867 			status = FMADM_EXIT_ERROR;
1868 		} else
1869 			note("flushed resource history for %s\n", argv[i]);
1870 	}
1871 
1872 	return (status);
1873 }
1874 
1875 int
1876 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
1877 {
1878 	int err;
1879 
1880 	if (getopt(argc, argv, "") != EOF)
1881 		return (FMADM_EXIT_USAGE);
1882 
1883 	if (argc - optind != 1)
1884 		return (FMADM_EXIT_USAGE);
1885 
1886 	/*
1887 	 * argument could be a uuid, an fmri (asru, fru or resource)
1888 	 * or a label. Try uuid first, If that fails try the others.
1889 	 */
1890 	err = fmd_adm_case_repair(adm, argv[optind]);
1891 	if (err != 0)
1892 		err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1893 
1894 	if (err != 0)
1895 		die("failed to record repair to %s", argv[optind]);
1896 
1897 	note("recorded repair to %s\n", argv[optind]);
1898 	return (FMADM_EXIT_SUCCESS);
1899 }
1900 
1901 int
1902 cmd_repaired(fmd_adm_t *adm, int argc, char *argv[])
1903 {
1904 	int err;
1905 
1906 	if (getopt(argc, argv, "") != EOF)
1907 		return (FMADM_EXIT_USAGE);
1908 
1909 	if (argc - optind != 1)
1910 		return (FMADM_EXIT_USAGE);
1911 
1912 	/*
1913 	 * argument could be an fmri (asru, fru or resource) or a label.
1914 	 */
1915 	err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1916 	if (err != 0)
1917 		die("failed to record repair to %s", argv[optind]);
1918 
1919 	note("recorded repair to of %s\n", argv[optind]);
1920 	return (FMADM_EXIT_SUCCESS);
1921 }
1922 
1923 int
1924 cmd_replaced(fmd_adm_t *adm, int argc, char *argv[])
1925 {
1926 	int err;
1927 
1928 	if (getopt(argc, argv, "") != EOF)
1929 		return (FMADM_EXIT_USAGE);
1930 
1931 	if (argc - optind != 1)
1932 		return (FMADM_EXIT_USAGE);
1933 
1934 	/*
1935 	 * argument could be an fmri (asru, fru or resource) or a label.
1936 	 */
1937 	err = fmd_adm_rsrc_replaced(adm, argv[optind]);
1938 	if (err != 0)
1939 		die("failed to record replacement of %s", argv[optind]);
1940 
1941 	note("recorded replacement of %s\n", argv[optind]);
1942 	return (FMADM_EXIT_SUCCESS);
1943 }
1944 
1945 int
1946 cmd_acquit(fmd_adm_t *adm, int argc, char *argv[])
1947 {
1948 	int err;
1949 
1950 	if (getopt(argc, argv, "") != EOF)
1951 		return (FMADM_EXIT_USAGE);
1952 
1953 	if (argc - optind != 1 && argc - optind != 2)
1954 		return (FMADM_EXIT_USAGE);
1955 
1956 	/*
1957 	 * argument could be a uuid, an fmri (asru, fru or resource)
1958 	 * or a label. Or it could be a uuid and an fmri or label.
1959 	 */
1960 	if (argc - optind == 2) {
1961 		err = fmd_adm_rsrc_acquit(adm, argv[optind], argv[optind + 1]);
1962 		if (err != 0)
1963 			err = fmd_adm_rsrc_acquit(adm, argv[optind + 1],
1964 			    argv[optind]);
1965 	} else {
1966 		err = fmd_adm_case_acquit(adm, argv[optind]);
1967 		if (err != 0)
1968 			err = fmd_adm_rsrc_acquit(adm, argv[optind], "");
1969 	}
1970 
1971 	if (err != 0)
1972 		die("failed to record acquital of %s", argv[optind]);
1973 
1974 	note("recorded acquital of %s\n", argv[optind]);
1975 	return (FMADM_EXIT_SUCCESS);
1976 }
1977