xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 44743693)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <fmadm.h>
30 #include <errno.h>
31 #include <limits.h>
32 #include <strings.h>
33 #include <stdio.h>
34 #include <unistd.h>
35 #include <sys/wait.h>
36 #include <sys/stat.h>
37 #include <fcntl.h>
38 #include <fm/fmd_log.h>
39 #include <sys/fm/protocol.h>
40 #include <fm/libtopo.h>
41 #include <fm/fmd_adm.h>
42 #include <dlfcn.h>
43 #include <sys/systeminfo.h>
44 #include <sys/utsname.h>
45 #include <libintl.h>
46 #include <locale.h>
47 #include <sys/smbios.h>
48 #include <libdevinfo.h>
49 #include <stdlib.h>
50 
51 #define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))
52 
53 /*
54  * catalog_setup() must be called to setup support functions.
55  * Fault records are added to catalog by calling add_fault_record_to_catalog()
56  * records are stored in order of importance to the system.
57  * If -g flag is set or not_suppressed is not set and the class fru, fault,
58  * type are the same then details are merged into an existing record, with uuid
59  * records are stored in time order.
60  * For each record information is extracted from nvlist and merged into linked
61  * list each is checked for identical records for which percentage certainty are
62  * added together.
63  * print_catalog() is called to print out catalog and release external resources
64  *
65  *                         /---------------\
66  *	status_rec_list -> |               | -|
67  *                         \---------------/
68  *                                \/
69  *                         /---------------\    /-------\    /-------\
70  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
71  *            \/           |               | |- |       | <- |       |
72  *      /-------------\    |               |    \-------/    \-------/
73  *      |             | -> |               |       \/           \/
74  *      \-------------/    |               |    /-------\    /-------\
75  *            \/           |               | -> | asru  | -> | asru  |
76  *            ---          |               |    |       | <- |       |
77  *                         |               |    \-------/    \-------/
78  *      status_asru_list   |  class        |
79  *            \/           |  resource     |    /-------\    /-------\
80  *      /-------------\    |  fru          | -> | list  | -> | list  |
81  *      |             | -> |  serial       |    |       | <- |       |
82  *      \-------------/    |               |    \-------/    \-------/
83  *            \/           \---------------/
84  *            ---               \/    /\
85  *                         /---------------\
86  *                         | status_record |
87  *                         \---------------/
88  *
89  * Fmadm faulty takes a number of options which affect the format of the
90  * output displayed. By default, the display reports the FRU and ASRU along
91  * with other information on per-case basis as in the example below.
92  *
93  * --------------- ------------------------------------  -------------- -------
94  * TIME            EVENT-ID                              MSG-ID         SEVERITY
95  * --------------- ------------------------------------  -------------- -------
96  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
97  *
98  * Fault class	: fault.memory.dimm_sb
99  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
100  *		    degraded but still in service
101  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
102  *		    faulty
103  *
104  * Description	: The number of errors associated with this memory module has
105  *		exceeded acceptable levels.  Refer to
106  *		http://sun.com/msg/AMD-8000-2F for more information.
107  *
108  * Response	: Pages of memory associated with this memory module are being
109  *		removed from service as errors are reported.
110  *
111  * Impact	: Total system memory capacity will be reduced as pages are
112  *		retired.
113  *
114  * Action	: Schedule a repair procedure to replace the affected memory
115  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
116  *
117  * The -v flag is similar, but adds some additonal information such as the
118  * resource. The -s flag is also similar but just gives the top line summary.
119  * All these options (ie without the -f or -r flags) use the print_catalog()
120  * function to do the display.
121  *
122  * The -f flag changes the output so that it appears sorted on a per-fru basis.
123  * The output is somewhat cut down compared to the default output. If -f is
124  * used, then print_fru() is used to print the output.
125  *
126  * -----------------------------------------------------------------------------
127  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
128  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
129  *
130  * Description	: A problem was detected for a PCI device.
131  *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
132  *
133  * Response	: One or more device instances may be disabled
134  *
135  * Impact	: Possible loss of services provided by the device instances
136  *		associated with this fault
137  *
138  * Action	: Schedule a repair procedure to replace the affected device.
139  * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
140  *		Sun for support.
141  *
142  * The -r flag changes the output so that it appears sorted on a per-asru basis.
143  * The output is very much cut down compared to the default output, just giving
144  * the asru fmri and state. Here print_asru() is used to print the output.
145  *
146  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
147  *
148  * For all fmadm faulty options, the sequence of events is
149  *
150  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
151  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
152  * This will extract the data from the nvlist and call catalog_new_record() to
153  * save the data away in various linked lists in the catalogue.
154  *
155  * 2) Once this is done, the data can be supplemented by using
156  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
157  *
158  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
159  * appropriate to display the information from the catalogue sorted in the
160  * requested way.
161  *
162  */
163 
164 typedef struct name_list {
165 	struct name_list *next;
166 	struct name_list *prev;
167 	char *name;
168 	uint8_t pct;
169 	uint8_t max_pct;
170 	ushort_t count;
171 	int status;
172 } name_list_t;
173 
174 typedef struct ari_list {
175 	char *ari_uuid;
176 	struct ari_list *next;
177 } ari_list_t;
178 
179 typedef struct uurec {
180 	struct uurec *next;
181 	struct uurec *prev;
182 	char *uuid;
183 	ari_list_t *ari_uuid_list;
184 	name_list_t *asru;
185 	uint64_t sec;
186 } uurec_t;
187 
188 typedef struct uurec_select {
189 	struct uurec_select *next;
190 	char *uuid;
191 } uurec_select_t;
192 
193 typedef struct host_id {
194 	char *chassis;
195 	char *server;
196 	char *platform;
197 } hostid_t;
198 
199 typedef struct host_id_list {
200 	hostid_t hostid;
201 	struct host_id_list *next;
202 } host_id_list_t;
203 
204 typedef struct status_record {
205 	hostid_t *host;
206 	int nrecs;
207 	uurec_t *uurec;
208 	char *severity;			/* in C locale */
209 	char *msgid;
210 	name_list_t *class;
211 	name_list_t *resource;
212 	name_list_t *asru;
213 	name_list_t *fru;
214 	name_list_t *serial;
215 	char *url;
216 	uint8_t not_suppressed;
217 } status_record_t;
218 
219 typedef struct sr_list {
220 	struct sr_list *next;
221 	struct sr_list *prev;
222 	struct status_record *status_record;
223 } sr_list_t;
224 
225 typedef struct resource_list {
226 	struct resource_list *next;
227 	struct resource_list *prev;
228 	sr_list_t *status_rec_list;
229 	char *resource;
230 	uint8_t not_suppressed;
231 	uint8_t max_pct;
232 } resource_list_t;
233 
234 typedef struct tgetlabel_data {
235 	char *label;
236 	char *fru;
237 } tgetlabel_data_t;
238 
239 sr_list_t *status_rec_list;
240 resource_list_t *status_fru_list;
241 resource_list_t *status_asru_list;
242 
243 static char *locale;
244 static char *nlspath;
245 static int max_display;
246 static int max_fault = 0;
247 static topo_hdl_t *topo_handle;
248 static char *topo_handle_uuid;
249 static host_id_list_t *host_list;
250 static int n_server;
251 static int opt_g;
252 
253 static char *
254 format_date(char *buf, size_t len, uint64_t sec)
255 {
256 	if (sec > LONG_MAX) {
257 		(void) fprintf(stderr,
258 		    "record time is too large for 32-bit utility\n");
259 		(void) snprintf(buf, len, "0x%llx", sec);
260 	} else {
261 		time_t tod = (time_t)sec;
262 		(void) strftime(buf, len, "%b %d %T", localtime(&tod));
263 	}
264 
265 	return (buf);
266 }
267 
268 static hostid_t *
269 find_hostid_in_list(char *platform, char *chassis, char *server)
270 {
271 	hostid_t *rt = NULL;
272 	host_id_list_t *hostp;
273 
274 	if (platform == NULL)
275 		platform = "-";
276 	if (server == NULL)
277 		server = "-";
278 	hostp = host_list;
279 	while (hostp) {
280 		if (hostp->hostid.platform &&
281 		    strcmp(hostp->hostid.platform, platform) == 0 &&
282 		    hostp->hostid.server &&
283 		    strcmp(hostp->hostid.server, server) == 0 &&
284 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
285 		    strcmp(chassis, hostp->hostid.chassis) == 0)) {
286 			rt = &hostp->hostid;
287 			break;
288 		}
289 		hostp = hostp->next;
290 	}
291 	if (rt == NULL) {
292 		hostp = malloc(sizeof (host_id_list_t));
293 		hostp->hostid.platform = strdup(platform);
294 		hostp->hostid.server = strdup(server);
295 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
296 		hostp->next = host_list;
297 		host_list = hostp;
298 		rt = &hostp->hostid;
299 		n_server++;
300 	}
301 	return (rt);
302 }
303 
304 static hostid_t *
305 find_hostid(nvlist_t *nvl)
306 {
307 	char *platform = NULL, *chassis = NULL, *server = NULL;
308 	nvlist_t *auth, *fmri;
309 	hostid_t *rt = NULL;
310 
311 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
312 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
313 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
314 		    &platform);
315 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
316 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
317 		    &chassis);
318 		rt = find_hostid_in_list(platform, chassis, server);
319 	}
320 	return (rt);
321 }
322 
323 static void
324 catalog_setup(void)
325 {
326 	char *tp;
327 	int pl;
328 
329 	/*
330 	 * All FMA event dictionaries use msgfmt(1) message objects to produce
331 	 * messages, even for the C locale.  We therefore want to use dgettext
332 	 * for all message lookups, but its defined behavior in the C locale is
333 	 * to return the input string.  Since our input strings are event codes
334 	 * and not format strings, this doesn't help us.  We resolve this nit
335 	 * by setting NLSPATH to a non-existent file: the presence of NLSPATH
336 	 * is defined to force dgettext(3C) to do a full lookup even for C.
337 	 */
338 	nlspath = getenv("NLSPATH");
339 	if (nlspath == NULL)
340 		putenv("NLSPATH=/usr/lib/fm/fmd/fmd.cat");
341 	else {
342 		pl = strlen(nlspath) + sizeof ("NLSPATH=") + 1;
343 		tp = malloc(pl);
344 		(void) snprintf(tp, pl, "NLSPATH=%s", nlspath);
345 		nlspath = tp;
346 	}
347 
348 	locale = setlocale(LC_MESSAGES, "");
349 }
350 
351 static char *
352 get_dict_url(char *id)
353 {
354 	char *url = "http://sun.com/msg/";
355 	int msz = sizeof (url) + strlen(id) + 1;
356 	char *cp;
357 
358 	cp = malloc(msz);
359 	(void) snprintf(cp, msz, "%s%s", url, id);
360 	return (cp);
361 }
362 
363 static char *
364 get_dict_msg(char *id, char *idx, int unknown, int translate)
365 {
366 	char mbuf[128];
367 	char *msg;
368 	char dbuf[32];
369 	char *p;
370 	int restore_env = 0;
371 	int restore_locale = 0;
372 
373 	p = strchr(id, '-');
374 	if (p == NULL || p == id || (p - id) >= 32) {
375 		msg = mbuf;
376 	} else {
377 		strncpy(dbuf, id, (size_t)(p - id));
378 		dbuf[(size_t)(p - id)] = 0;
379 
380 		(void) snprintf(mbuf, sizeof (mbuf), "%s.%s", id, idx);
381 		if (translate == 0 || nlspath == NULL) {
382 			(void) setlocale(LC_MESSAGES, "C");
383 			restore_locale = 1;
384 		}
385 		bindtextdomain("FMD", "/usr/lib/locale");
386 		msg = dgettext(dbuf, mbuf);
387 		if (msg == mbuf) {
388 			(void) setlocale(LC_MESSAGES, "C");
389 			restore_locale = 1;
390 			msg = dgettext(dbuf, mbuf);
391 		}
392 		if (msg == mbuf) {
393 			putenv("NLSPATH=/usr/lib/fm/fmd/fmd.cat");
394 			restore_env = 1;
395 			(void) setlocale(LC_MESSAGES, "C");
396 			msg = dgettext(dbuf, mbuf);
397 		}
398 		if (restore_locale)
399 			(void) setlocale(LC_MESSAGES, locale);
400 		if (restore_env && nlspath)
401 			putenv(nlspath);
402 	}
403 	if (msg == mbuf) {
404 		if (unknown)
405 			msg = "unknown";
406 		else
407 			msg = NULL;
408 	}
409 	return (msg);
410 }
411 
412 /*
413  * compare two fru strings which are made up of substrings seperated by '/'
414  * return true if every substring is the same in the two strings, or if a
415  * substring is null in one.
416  */
417 
418 static int
419 frucmp(char *f1, char *f2)
420 {
421 	char c1, c2;
422 	int i = 0;
423 
424 	for (;;) {
425 		c1 = *f1;
426 		c2 = *f2;
427 		if (c1 == c2) {
428 			i = (c1 == '/') ? 0 : i + 1;
429 		} else if (i == 0) {
430 			if (c1 == '/') {
431 				do {
432 					f2++;
433 				} while ((c2 = *f2) != 0 && c2 != '/');
434 				if (c2 == NULL)
435 					break;
436 			} else if (c2 == '/') {
437 				do {
438 					f1++;
439 				} while ((c1 = *f1) != 0 && c1 != '/');
440 				if (c1 == NULL)
441 					break;
442 			} else
443 				break;
444 		} else
445 			break;
446 		if (c1 == NULL)
447 			return (0);
448 		f1++;
449 		f2++;
450 	}
451 	return (1);
452 }
453 
454 static int
455 tgetlabel(topo_hdl_t *thp, tnode_t *node, void *arg)
456 {
457 	int err;
458 	char *fru_name, *lname;
459 	nvlist_t *fru = NULL;
460 	int rt = TOPO_WALK_NEXT;
461 	tgetlabel_data_t *tdp = (tgetlabel_data_t *)arg;
462 
463 	if (topo_node_fru(node, &fru, NULL, &err) == 0) {
464 		if (topo_fmri_nvl2str(thp, fru, &fru_name, &err) == 0) {
465 			if (frucmp(tdp->fru, fru_name) == 0 &&
466 			    topo_node_label(node, &lname, &err) == 0) {
467 				tdp->label = strdup(lname);
468 				topo_hdl_strfree(thp, lname);
469 				rt = TOPO_WALK_TERMINATE;
470 			}
471 			topo_hdl_strfree(thp, fru_name);
472 		}
473 		nvlist_free(fru);
474 	}
475 	return (rt);
476 }
477 
478 static void
479 label_get_topo(void)
480 {
481 	int err;
482 
483 	topo_handle = topo_open(TOPO_VERSION, 0, &err);
484 	if (topo_handle) {
485 		topo_handle_uuid = topo_snap_hold(topo_handle, NULL, &err);
486 	}
487 }
488 
489 static void
490 label_release_topo(void)
491 {
492 	if (topo_handle_uuid)
493 		topo_hdl_strfree(topo_handle, topo_handle_uuid);
494 	if (topo_handle) {
495 		topo_snap_release(topo_handle);
496 		topo_close(topo_handle);
497 	}
498 }
499 
500 static char *
501 get_fmri_label(char *fru)
502 {
503 	topo_walk_t *twp;
504 	tgetlabel_data_t td;
505 	int err;
506 
507 	td.label = NULL;
508 	td.fru = fru;
509 	if (topo_handle == NULL)
510 		label_get_topo();
511 	if (topo_handle_uuid) {
512 		twp = topo_walk_init(topo_handle, FM_FMRI_SCHEME_HC,
513 		    tgetlabel, &td, &err);
514 		if (twp) {
515 			topo_walk_step(twp, TOPO_WALK_CHILD);
516 			topo_walk_fini(twp);
517 		}
518 	}
519 	return (td.label);
520 }
521 
522 static char *
523 get_nvl2str_topo(nvlist_t *nvl)
524 {
525 	char *name = NULL;
526 	char *tname;
527 	int err;
528 	char *scheme = NULL;
529 	char *mod_name = NULL;
530 	char buf[128];
531 
532 	if (topo_handle == NULL)
533 		label_get_topo();
534 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
535 		name = strdup(tname);
536 		topo_hdl_strfree(topo_handle, tname);
537 	} else {
538 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
539 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
540 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
541 		    mod_name) {
542 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
543 			    scheme, mod_name);
544 			name = strdup(buf);
545 		}
546 	}
547 	return (name);
548 }
549 
550 static int
551 set_priority(char *s)
552 {
553 	int rt = 0;
554 
555 	if (s) {
556 		if (strcmp(s, "Minor") == 0)
557 			rt = 1;
558 		else if (strcmp(s, "Major") == 0)
559 			rt = 10;
560 		else if (strcmp(s, "Critical") == 0)
561 			rt = 100;
562 	}
563 	return (rt);
564 }
565 
566 static int
567 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
568     uint8_t p2)
569 {
570 	int r1, r2;
571 	int rt;
572 
573 	r1 = set_priority(s1);
574 	r2 = set_priority(s2);
575 	rt = r1 - r2;
576 	if (rt == 0) {
577 		if (t1 > t2)
578 			rt = 1;
579 		else if (t1 < t2)
580 			rt = -1;
581 		else
582 			rt = p1 - p2;
583 	}
584 	return (rt);
585 }
586 
587 /*
588  * merge two lists into one, by comparing enties in new and moving into list if
589  * name is not there or free off memory for names which are already there
590  * add_pct indicates if pct is the sum or highest pct
591  */
592 static name_list_t *
593 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
594 {
595 	name_list_t *lp, *np, *sp, *rt = NULL;
596 	int max_pct;
597 
598 	rt = *list;
599 	np = new;
600 	while (np) {
601 		lp = *list;
602 		while (lp) {
603 			if (strcmp(lp->name, np->name) == 0)
604 				break;
605 			lp = lp->next;
606 			if (lp == *list)
607 				lp = NULL;
608 		}
609 		if (np->next == new)
610 			sp = NULL;
611 		else
612 			sp = np->next;
613 		if (lp) {
614 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
615 			if (add_pct) {
616 				lp->pct += np->pct;
617 				lp->count += np->count;
618 			} else if (np->pct > lp->pct) {
619 				lp->pct = np->pct;
620 			}
621 			max_pct = np->max_pct;
622 			free(np->name);
623 			free(np);
624 			np = NULL;
625 			if (max_pct > lp->max_pct) {
626 				lp->max_pct = max_pct;
627 				if (lp->max_pct > lp->prev->max_pct &&
628 				    lp != *list) {
629 					lp->prev->next = lp->next;
630 					lp->next->prev = lp->prev;
631 					np = lp;
632 				}
633 			}
634 		}
635 		if (np) {
636 			lp = *list;
637 			if (lp) {
638 				if (np->max_pct > lp->max_pct) {
639 					np->next = lp;
640 					np->prev = lp->prev;
641 					lp->prev->next = np;
642 					lp->prev = np;
643 					*list = np;
644 					rt = np;
645 				} else {
646 					lp = lp->next;
647 					while (lp != *list &&
648 					    np->max_pct < lp->max_pct) {
649 						lp = lp->next;
650 					}
651 					np->next = lp;
652 					np->prev = lp->prev;
653 					lp->prev->next = np;
654 					lp->prev = np;
655 				}
656 			} else {
657 				*list = np;
658 				np->next = np;
659 				np->prev = np;
660 				rt = np;
661 			}
662 		}
663 		np = sp;
664 	}
665 	return (rt);
666 }
667 
668 /*
669  * compare entries in two lists return true if the two lists have identical
670  * content. The two lists may not have entries in the same order, so we compare
671  * the size of the list as well as trying to find every entry from one list in
672  * the other.
673  */
674 static int
675 cmp_name_list(name_list_t *lxp1, name_list_t *lxp2)
676 {
677 	name_list_t *lp1, *lp2;
678 	int l1 = 0, l2 = 0, common = 0;
679 
680 	lp2 = lxp2;
681 	while (lp2) {
682 		l2++;
683 		lp2 = lp2->next;
684 		if (lp2 == lxp2)
685 			break;
686 	}
687 	lp1 = lxp1;
688 	while (lp1) {
689 		l1++;
690 		lp2 = lxp2;
691 		while (lp2) {
692 			if (strcmp(lp2->name, lp1->name) == 0) {
693 				common++;
694 				break;
695 			}
696 			lp2 = lp2->next;
697 			if (lp2 == lxp2)
698 				break;
699 		}
700 		lp1 = lp1->next;
701 		if (lp1 == lxp1)
702 			break;
703 	}
704 	if (l1 == l2 && l2 == common)
705 		return (0);
706 	else
707 		return (1);
708 }
709 
710 static name_list_t *
711 alloc_name_list(char *name, uint8_t pct)
712 {
713 	name_list_t *nlp;
714 
715 	nlp = malloc(sizeof (*nlp));
716 	nlp->name = strdup(name);
717 	nlp->pct = pct;
718 	nlp->max_pct = pct;
719 	nlp->count = 1;
720 	nlp->next = nlp;
721 	nlp->prev = nlp;
722 	nlp->status = 0;
723 	return (nlp);
724 }
725 
726 static void
727 free_name_list(name_list_t *list)
728 {
729 	name_list_t *next = list;
730 	name_list_t *lp;
731 
732 	if (list) {
733 		do {
734 			lp = next;
735 			next = lp->next;
736 			free(lp->name);
737 			free(lp);
738 		} while (next != list);
739 	}
740 }
741 
742 static status_record_t *
743 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
744     name_list_t *fru, name_list_t *asru, name_list_t *resource,
745     name_list_t *serial, const char *url, boolean_t not_suppressed,
746     hostid_t *hostid)
747 {
748 	status_record_t *status_rec_p;
749 
750 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
751 	status_rec_p->nrecs = 1;
752 	status_rec_p->host = hostid;
753 	status_rec_p->uurec = uurec_p;
754 	uurec_p->next = NULL;
755 	uurec_p->prev = NULL;
756 	uurec_p->asru = asru;
757 	status_rec_p->severity = get_dict_msg(msgid, "severity", 1, 0);
758 	status_rec_p->class = class;
759 	status_rec_p->fru = fru;
760 	status_rec_p->asru = asru;
761 	status_rec_p->resource = resource;
762 	status_rec_p->serial = serial;
763 	status_rec_p->url = url ? strdup(url) : NULL;
764 	status_rec_p->msgid = strdup(msgid);
765 	status_rec_p->not_suppressed = not_suppressed;
766 	return (status_rec_p);
767 }
768 
769 /*
770  * add record to given list maintaining order higher priority first.
771  */
772 static void
773 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
774 {
775 	sr_list_t *tp, *np, *sp;
776 	int order;
777 	uint64_t sec;
778 
779 	np = malloc(sizeof (sr_list_t));
780 	np->status_record = status_rec_p;
781 	sec = status_rec_p->uurec->sec;
782 	if ((sp = *list_pp) == NULL) {
783 		*list_pp = np;
784 		np->next = np;
785 		np->prev = np;
786 	} else {
787 		/* insert new record in front of lower priority */
788 		tp = sp;
789 		order = cmp_priority(status_rec_p->severity,
790 		    sp->status_record->severity, sec,
791 		    tp->status_record->uurec->sec, 0, 0);
792 		if (order > 0) {
793 			*list_pp = np;
794 		} else {
795 			tp = sp->next;
796 			while (tp != sp &&
797 			    cmp_priority(status_rec_p->severity,
798 			    tp->status_record->severity, sec,
799 			    tp->status_record->uurec->sec, 0, 0)) {
800 				tp = tp->next;
801 			}
802 		}
803 		np->next = tp;
804 		np->prev = tp->prev;
805 		tp->prev->next = np;
806 		tp->prev = np;
807 	}
808 }
809 
810 static void
811 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
812     resource_list_t *np)
813 {
814 	int order;
815 	uint64_t sec;
816 	resource_list_t *sp, *tp;
817 	status_record_t *srp;
818 	char *severity = status_rec_p->severity;
819 
820 	add_rec_list(status_rec_p, &np->status_rec_list);
821 	if ((sp = *rp) == NULL) {
822 		np->next = np;
823 		np->prev = np;
824 		*rp = np;
825 	} else {
826 		/*
827 		 * insert new record in front of lower priority
828 		 */
829 		tp = sp->next;
830 		srp = sp->status_rec_list->status_record;
831 		sec = status_rec_p->uurec->sec;
832 		order = cmp_priority(severity, srp->severity, sec,
833 		    srp->uurec->sec, np->max_pct, sp->max_pct);
834 		if (order > 0) {
835 			*rp = np;
836 		} else {
837 			srp = tp->status_rec_list->status_record;
838 			while (tp != sp &&
839 			    cmp_priority(severity, srp->severity, sec,
840 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
841 				tp = tp->next;
842 				srp = tp->status_rec_list->status_record;
843 			}
844 		}
845 		np->next = tp;
846 		np->prev = tp->prev;
847 		tp->prev->next = np;
848 		tp->prev = np;
849 	}
850 }
851 
852 static void
853 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
854     resource_list_t **rpp)
855 {
856 	int order;
857 	resource_list_t *np, *end;
858 	status_record_t *srp;
859 
860 	np = *rpp;
861 	end = np;
862 	while (np) {
863 		if (strcmp(fp->name, np->resource) == 0) {
864 			np->not_suppressed |= status_rec_p->not_suppressed;
865 			srp = np->status_rec_list->status_record;
866 			order = cmp_priority(status_rec_p->severity,
867 			    srp->severity, status_rec_p->uurec->sec,
868 			    srp->uurec->sec, fp->max_pct, np->max_pct);
869 			if (order > 0 && np != end) {
870 				/*
871 				 * remove from list and add again using
872 				 * new priority
873 				 */
874 				np->prev->next = np->next;
875 				np->next->prev = np->prev;
876 				add_resource(status_rec_p,
877 				    rpp, np);
878 			} else {
879 				add_rec_list(status_rec_p,
880 				    &np->status_rec_list);
881 			}
882 			break;
883 		}
884 		np = np->next;
885 		if (np == end) {
886 			np = NULL;
887 			break;
888 		}
889 	}
890 	if (np == NULL) {
891 		np = malloc(sizeof (resource_list_t));
892 		np->resource = fp->name;
893 		np->not_suppressed = status_rec_p->not_suppressed;
894 		np->status_rec_list = NULL;
895 		np->max_pct = fp->max_pct;
896 		add_resource(status_rec_p, rpp, np);
897 	}
898 }
899 
900 static void
901 add_list(status_record_t *status_rec_p, name_list_t *listp,
902     resource_list_t **glistp)
903 {
904 	name_list_t *fp, *end;
905 
906 	fp = listp;
907 	end = fp;
908 	while (fp) {
909 		add_resource_list(status_rec_p, fp, glistp);
910 		fp = fp->next;
911 		if (fp == end)
912 			break;
913 	}
914 }
915 
916 /*
917  * add record to rec, fru and asru lists.
918  */
919 static void
920 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
921     name_list_t *fru, name_list_t *asru, name_list_t *resource,
922     name_list_t *serial, const char *url, boolean_t not_suppressed,
923     hostid_t *hostid)
924 {
925 	status_record_t *status_rec_p;
926 
927 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
928 	    resource, serial, url, not_suppressed, hostid);
929 	add_rec_list(status_rec_p, &status_rec_list);
930 	if (status_rec_p->fru)
931 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
932 	if (status_rec_p->asru)
933 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
934 }
935 
936 /*
937  * add uuid and diagnoses time to an existing record for similar fault on the
938  * same fru
939  */
940 static void
941 catalog_merge_record(status_record_t *status_rec_p, uurec_t *uurec_p,
942     name_list_t *asru, name_list_t *resource, name_list_t *serial,
943     const char *url, boolean_t not_suppressed)
944 {
945 	uurec_t *uurec1_p;
946 
947 	status_rec_p->nrecs++;
948 	/* add uurec in time order */
949 	if (status_rec_p->uurec->sec > uurec_p->sec) {
950 		uurec_p->next = status_rec_p->uurec;
951 		uurec_p->prev = NULL;
952 		status_rec_p->uurec = uurec_p;
953 	} else {
954 		uurec1_p = status_rec_p->uurec;
955 		while (uurec1_p->next && uurec1_p->next->sec <= uurec_p->sec)
956 			uurec1_p = uurec1_p->next;
957 		if (uurec1_p->next)
958 			uurec1_p->next->prev = uurec_p;
959 		uurec_p->next = uurec1_p->next;
960 		uurec_p->prev = uurec1_p;
961 		uurec1_p->next = uurec_p;
962 	}
963 	if (status_rec_p->url == NULL && url != NULL)
964 		status_rec_p->url = strdup(url);
965 	status_rec_p->not_suppressed |= not_suppressed;
966 	uurec_p->asru = merge_name_list(&status_rec_p->asru, asru, 0);
967 	(void) merge_name_list(&status_rec_p->resource, resource, 0);
968 	(void) merge_name_list(&status_rec_p->serial, serial, 0);
969 }
970 
971 static status_record_t *
972 record_in_catalog(name_list_t *class, name_list_t *fru,
973     char *msgid, hostid_t *host)
974 {
975 	sr_list_t *status_rec_p;
976 	status_record_t *srp = NULL;
977 
978 	status_rec_p = status_rec_list;
979 	while (status_rec_p) {
980 		srp = status_rec_p->status_record;
981 		if (host == srp->host &&
982 		    cmp_name_list(class, srp->class) == 0 &&
983 		    cmp_name_list(fru, srp->fru) == 0 &&
984 		    strcmp(msgid, srp->msgid) == 0)
985 			break;
986 		if (status_rec_p->next == status_rec_list) {
987 			srp = NULL;
988 			break;
989 		} else {
990 			status_rec_p = status_rec_p->next;
991 		}
992 	}
993 	return (srp);
994 }
995 
996 static void
997 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
998 {
999 	char *name;
1000 	char *serial = NULL;
1001 	char **lserial = NULL;
1002 	uint64_t serint;
1003 	name_list_t *nlp;
1004 	int j;
1005 	uint_t nelem;
1006 	char buf[64];
1007 
1008 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
1009 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
1010 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
1011 			    &serint) == 0) {
1012 				(void) snprintf(buf, sizeof (buf), "%llX",
1013 				    serint);
1014 				nlp = alloc_name_list(buf, pct);
1015 				(void) merge_name_list(serial_p, nlp, 1);
1016 			}
1017 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
1018 			if (nvlist_lookup_string_array(nvl,
1019 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
1020 				nlp = alloc_name_list(lserial[0], pct);
1021 				for (j = 1; j < nelem; j++) {
1022 					name_list_t *n1lp;
1023 					n1lp = alloc_name_list(lserial[j], pct);
1024 					(void) merge_name_list(&nlp, n1lp, 1);
1025 				}
1026 				(void) merge_name_list(serial_p, nlp, 1);
1027 			}
1028 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
1029 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
1030 			    &serial) == 0) {
1031 				nlp = alloc_name_list(serial, pct);
1032 				(void) merge_name_list(serial_p, nlp, 1);
1033 			}
1034 		}
1035 	}
1036 }
1037 
1038 static void
1039 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
1040     name_list_t **fru_p, name_list_t **serial_p,
1041     name_list_t **resource_p, name_list_t **asru_p, uint8_t status)
1042 {
1043 	nvlist_t *lfru, *lasru, *rsrc;
1044 	name_list_t *nlp;
1045 	char *name;
1046 	uint8_t lpct = 0;
1047 	char *lclass = NULL;
1048 
1049 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
1050 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
1051 		nlp = alloc_name_list(lclass, lpct);
1052 		(void) merge_name_list(class_p, nlp, 1);
1053 	}
1054 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
1055 		name = get_nvl2str_topo(lfru);
1056 		if (name != NULL) {
1057 			nlp = alloc_name_list(name, lpct);
1058 			free(name);
1059 			(void) merge_name_list(fru_p, nlp, 1);
1060 		}
1061 		get_serial_no(lfru, serial_p, lpct);
1062 	}
1063 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
1064 		name = get_nvl2str_topo(lasru);
1065 		if (name != NULL) {
1066 			nlp = alloc_name_list(name, lpct);
1067 			nlp->status = status & ~FM_SUSPECT_NOT_PRESENT;
1068 			free(name);
1069 			(void) merge_name_list(asru_p, nlp, 1);
1070 		}
1071 		get_serial_no(lasru, serial_p, lpct);
1072 	}
1073 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
1074 		name = get_nvl2str_topo(rsrc);
1075 		if (name != NULL) {
1076 			nlp = alloc_name_list(name, lpct);
1077 			free(name);
1078 			(void) merge_name_list(resource_p, nlp, 1);
1079 		}
1080 	}
1081 }
1082 
1083 static void
1084 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid,
1085     const char *url)
1086 {
1087 	char *msgid = "-";
1088 	uint_t i, size = 0;
1089 	name_list_t *class = NULL, *resource = NULL;
1090 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
1091 	nvlist_t **nva;
1092 	uint8_t *ba;
1093 	status_record_t *status_rec_p;
1094 	uurec_t *uurec_p;
1095 	hostid_t *host;
1096 	boolean_t not_suppressed = 1;
1097 	boolean_t any_present = 0;
1098 
1099 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
1100 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
1101 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
1102 	    &not_suppressed);
1103 
1104 	if (size != 0) {
1105 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
1106 		    &nva, &size);
1107 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
1108 		    &ba, &size);
1109 		for (i = 0; i < size; i++) {
1110 			extract_record_info(nva[i], &class, &fru, &serial,
1111 			    &resource, &asru, ba[i]);
1112 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
1113 			    (ba[i] & FM_SUSPECT_FAULTY))
1114 				any_present = 1;
1115 		}
1116 		/*
1117 		 * also suppress if no resources present
1118 		 */
1119 		if (any_present == 0)
1120 			not_suppressed = 0;
1121 	}
1122 
1123 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
1124 	uurec_p->uuid = strdup(uuid);
1125 	uurec_p->sec = sec;
1126 	uurec_p->ari_uuid_list = NULL;
1127 	host = find_hostid(nvl);
1128 	if (not_suppressed && !opt_g)
1129 		status_rec_p = NULL;
1130 	else
1131 		status_rec_p = record_in_catalog(class, fru, msgid, host);
1132 	if (status_rec_p) {
1133 		catalog_merge_record(status_rec_p, uurec_p, asru, resource,
1134 		    serial, url, not_suppressed);
1135 		free_name_list(class);
1136 		free_name_list(fru);
1137 	} else {
1138 		catalog_new_record(uurec_p, msgid, class, fru, asru,
1139 		    resource, serial, url, not_suppressed, host);
1140 	}
1141 }
1142 
1143 static void
1144 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
1145 {
1146 	sr_list_t *srp;
1147 	uurec_t *uurp;
1148 	ari_list_t *ari_list;
1149 
1150 	srp = status_rec_list;
1151 	if (srp) {
1152 		for (;;) {
1153 			uurp = srp->status_record->uurec;
1154 			while (uurp) {
1155 				if (strcmp(uuid, uurp->uuid) == 0) {
1156 					ari_list = (ari_list_t *)
1157 					    malloc(sizeof (ari_list_t));
1158 					ari_list->ari_uuid = strdup(ari_uuid);
1159 					ari_list->next = uurp->ari_uuid_list;
1160 					uurp->ari_uuid_list = ari_list;
1161 					return;
1162 				}
1163 				uurp = uurp->next;
1164 			}
1165 			if (srp->next == status_rec_list)
1166 				break;
1167 			srp = srp->next;
1168 		}
1169 	}
1170 }
1171 
1172 static void
1173 print_line(char *label, char *buf)
1174 {
1175 	char *cp, *ep, *wp;
1176 	char c;
1177 	int i;
1178 	int lsz;
1179 	char *padding;
1180 
1181 	lsz = strlen(label);
1182 	padding = malloc(lsz + 1);
1183 	for (i = 0; i < lsz; i++)
1184 		padding[i] = ' ';
1185 	padding[i] = 0;
1186 	cp = buf;
1187 	ep = buf;
1188 	c = *ep;
1189 	(void) printf("\n");
1190 	while (c) {
1191 		i = lsz;
1192 		wp = NULL;
1193 		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
1194 			if (c == ' ')
1195 				wp = ep;
1196 			else if (c == '\n') {
1197 				i = 0;
1198 				*ep = 0;
1199 				do {
1200 					ep++;
1201 				} while ((c = *ep) != NULL && c == ' ');
1202 				break;
1203 			}
1204 			ep++;
1205 			i++;
1206 		}
1207 		if (i >= 80 && wp) {
1208 			*wp = 0;
1209 			ep = wp + 1;
1210 			c = *ep;
1211 		}
1212 		(void) printf("%s%s\n", label, cp);
1213 		cp = ep;
1214 		label = padding;
1215 	}
1216 	free(padding);
1217 }
1218 
1219 static void
1220 print_dict_info(char *msgid, char *url)
1221 {
1222 	const char *cp;
1223 	char *l_url;
1224 	char *buf;
1225 	int bufsz;
1226 
1227 	cp = get_dict_msg(msgid, "description", 0, 1);
1228 	if (cp) {
1229 		if (url)
1230 			l_url = url;
1231 		else
1232 			l_url = get_dict_url(msgid);
1233 		bufsz = strlen(cp) + strlen(l_url) + 1;
1234 		buf = malloc(bufsz);
1235 		(void) snprintf(buf, bufsz, cp, l_url);
1236 		print_line(dgettext("FMD", "Description : "), buf);
1237 		free(buf);
1238 		if (!url)
1239 			free(l_url);
1240 	}
1241 	cp = get_dict_msg(msgid, "response", 0, 1);
1242 	if (cp) {
1243 		buf = strdup(cp);
1244 		print_line(dgettext("FMD", "Response    : "), buf);
1245 		free(buf);
1246 	}
1247 	cp = get_dict_msg(msgid, "impact", 0, 1);
1248 	if (cp) {
1249 		buf = strdup(cp);
1250 		print_line(dgettext("FMD", "Impact      : "), buf);
1251 		free(buf);
1252 	}
1253 	cp = get_dict_msg(msgid, "action", 0, 1);
1254 	if (cp) {
1255 		buf = strdup(cp);
1256 		print_line(dgettext("FMD", "Action      : "), buf);
1257 		free(buf);
1258 	}
1259 }
1260 
1261 static void
1262 print_name(name_list_t *list, char *(func)(char *), char *padding, int *np,
1263     int pct, int full)
1264 {
1265 	char *name, *fru = NULL;
1266 
1267 	name = list->name;
1268 	if (func)
1269 		fru = func(list->name);
1270 	if (fru) {
1271 		(void) printf("%s \"%s\" (%s)", padding, fru, name);
1272 		*np += 1;
1273 		free(fru);
1274 	} else {
1275 		(void) printf("%s %s", padding, name);
1276 		*np += 1;
1277 	}
1278 	if (list->pct && pct > 0 && pct < 100) {
1279 		if (list->count > 1) {
1280 			if (full) {
1281 				(void) printf(" %d @ %s %d%%\n", list->count,
1282 				    dgettext("FMD", "max"),
1283 				    list->max_pct);
1284 			} else {
1285 				(void) printf(" %s %d%%\n",
1286 				    dgettext("FMD", "max"),
1287 				    list->max_pct);
1288 			}
1289 		} else {
1290 			(void) printf(" %d%%\n", list->pct);
1291 		}
1292 	} else {
1293 		(void) printf("\n");
1294 	}
1295 }
1296 
1297 static void
1298 print_asru_status(int status, char *label)
1299 {
1300 	char *msg = NULL;
1301 
1302 	switch (status) {
1303 	case 0:
1304 		msg = dgettext("FMD", "ok and in service");
1305 		break;
1306 	case FM_SUSPECT_FAULTY:
1307 		msg = dgettext("FMD", "degraded but still in service");
1308 		break;
1309 	case FM_SUSPECT_UNUSABLE:
1310 		msg = dgettext("FMD", "unknown, not present or disabled");
1311 		break;
1312 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1313 		msg = dgettext("FMD", "faulted and taken out of service");
1314 		break;
1315 	default:
1316 		break;
1317 	}
1318 	if (msg) {
1319 		(void) printf("%s     %s\n", label, msg);
1320 	}
1321 }
1322 
1323 static void
1324 print_name_list(name_list_t *list, char *label, char *(func)(char *),
1325     int limit, int pct, void (func1)(int, char *), int full)
1326 {
1327 	char *name, *fru = NULL;
1328 	char *padding;
1329 	int i, j, l, n;
1330 	name_list_t *end = list;
1331 
1332 	l = strlen(label);
1333 	padding = malloc(l + 1);
1334 	for (i = 0; i < l; i++)
1335 		padding[i] = ' ';
1336 	padding[l] = 0;
1337 	(void) printf("%s", label);
1338 	name = list->name;
1339 	if (func)
1340 		fru = func(list->name);
1341 	if (fru) {
1342 		(void) printf(" \"%s\" (%s)", fru, name);
1343 		free(fru);
1344 	} else {
1345 		(void) printf(" %s", name);
1346 	}
1347 	if (list->pct && pct > 0 && pct < 100) {
1348 		if (list->count > 1) {
1349 			if (full) {
1350 				(void) printf(" %d @ %s %d%%\n", list->count,
1351 				    dgettext("FMD", "max"), list->max_pct);
1352 			} else {
1353 				(void) printf(" %s %d%%\n",
1354 				    dgettext("FMD", "max"), list->max_pct);
1355 			}
1356 		} else {
1357 			(void) printf(" %d%%\n", list->pct);
1358 		}
1359 	} else {
1360 		(void) printf("\n");
1361 	}
1362 	if (func1)
1363 		func1(list->status, padding);
1364 	n = 1;
1365 	j = 0;
1366 	while ((list = list->next) != end) {
1367 		if (limit == 0 || n < limit) {
1368 			print_name(list, func, padding, &n, pct, full);
1369 			if (func1)
1370 				func1(list->status, padding);
1371 		} else
1372 			j++;
1373 	}
1374 	if (j == 1) {
1375 		print_name(list->prev, func, padding, &n, pct, full);
1376 	} else if (j > 1) {
1377 		(void) printf("%s... %d %s\n", padding, j,
1378 		    dgettext("FMD", "more entries suppressed,"
1379 		    " use -v option for full list"));
1380 	}
1381 	free(padding);
1382 }
1383 
1384 static int
1385 asru_same_status(name_list_t *list)
1386 {
1387 	name_list_t *end = list;
1388 	int status = list->status;
1389 
1390 	while ((list = list->next) != end) {
1391 		if (status == -1) {
1392 			status = list->status;
1393 			continue;
1394 		}
1395 		if (list->status != -1 && status != list->status) {
1396 			status = -1;
1397 			break;
1398 		}
1399 	}
1400 	return (status);
1401 }
1402 
1403 static int
1404 serial_in_fru(name_list_t *fru, name_list_t *serial)
1405 {
1406 	name_list_t *sp = serial;
1407 	name_list_t *fp;
1408 	int nserial = 0;
1409 	int found = 0;
1410 	char buf[128];
1411 
1412 	while (sp) {
1413 		fp = fru;
1414 		nserial++;
1415 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1416 		buf[sizeof (buf) - 1] = 0;
1417 		while (fp) {
1418 			if (strstr(fp->name, buf) != NULL) {
1419 				found++;
1420 				break;
1421 			}
1422 			fp = fp->next;
1423 			if (fp == fru)
1424 				break;
1425 		}
1426 		sp = sp->next;
1427 		if (sp == serial)
1428 			break;
1429 	}
1430 	return (found == nserial ? 1 : 0);
1431 }
1432 
1433 static void
1434 print_server_name(hostid_t *host, char *label)
1435 {
1436 	(void) printf("%s %s %s %s\n", label, host->server, host->platform,
1437 	    host->chassis ? host->chassis : "");
1438 }
1439 
1440 static void
1441 print_sup_record(status_record_t *srp, int opt_i, int full)
1442 {
1443 	char buf[32];
1444 	uurec_t *uurp = srp->uurec;
1445 	int n, j, k, max;
1446 	int status;
1447 	ari_list_t *ari_list;
1448 
1449 	n = 0;
1450 	max = max_fault;
1451 	if (max < 0) {
1452 		max = 0;
1453 	}
1454 	j = max / 2;
1455 	max -= j;
1456 	k = srp->nrecs - max;
1457 	while ((uurp = uurp->next) != NULL) {
1458 		if (full || n < j || n >= k || max_fault == 0 ||
1459 		    srp->nrecs == max_fault+1) {
1460 			if (opt_i) {
1461 				ari_list = uurp->ari_uuid_list;
1462 				while (ari_list) {
1463 					(void) printf("%-15s %s\n",
1464 					    format_date(buf, sizeof (buf),
1465 					    uurp->sec), ari_list->ari_uuid);
1466 					ari_list = ari_list->next;
1467 				}
1468 			} else {
1469 				(void) printf("%-15s %s\n",
1470 				    format_date(buf, sizeof (buf), uurp->sec),
1471 				    uurp->uuid);
1472 			}
1473 		} else if (n == j)
1474 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1475 			    dgettext("FMD", "more entries suppressed"));
1476 		n++;
1477 	}
1478 	(void) printf("\n");
1479 	if (n_server > 1)
1480 		print_server_name(srp->host, dgettext("FMD", "Host        :"));
1481 	if (srp->class)
1482 		print_name_list(srp->class,
1483 		    dgettext("FMD", "Fault class :"), NULL, 0, srp->class->pct,
1484 		    NULL, full);
1485 	if (srp->asru) {
1486 		status = asru_same_status(srp->asru);
1487 		if (status != -1) {
1488 			print_name_list(srp->asru,
1489 			    dgettext("FMD", "Affects     :"), NULL,
1490 			    full ? 0 : max_display, 0, NULL, full);
1491 			print_asru_status(status, "             ");
1492 		} else
1493 			print_name_list(srp->asru,
1494 			    dgettext("FMD", "Affects     :"), NULL,
1495 			    full ? 0 : max_display, 0, print_asru_status, full);
1496 	}
1497 	if (full || srp->fru == NULL) {
1498 		if (srp->resource) {
1499 			print_name_list(srp->resource,
1500 			    dgettext("FMD", "Problem in  :"),
1501 			    NULL, full ? 0 : max_display, 0, NULL, full);
1502 		}
1503 	}
1504 	if (srp->fru) {
1505 		print_name_list(srp->fru, dgettext("FMD", "FRU         :"),
1506 		    get_fmri_label, 0,
1507 		    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1508 		    NULL, full);
1509 	}
1510 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1511 	    !serial_in_fru(srp->asru, srp->serial)) {
1512 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1513 		    NULL, 0, 0, NULL, full);
1514 	}
1515 	print_dict_info(srp->msgid, srp->url);
1516 	(void) printf("\n");
1517 }
1518 
1519 static void
1520 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1521 {
1522 	char buf[32];
1523 	uurec_t *uurp = srp->uurec;
1524 	char *severity;
1525 	static int header = 0;
1526 	char *head;
1527 	ari_list_t *ari_list;
1528 
1529 	if (nlspath)
1530 		severity = get_dict_msg(srp->msgid, "severity", 1, 1);
1531 	else
1532 		severity = srp->severity;
1533 
1534 	if (!summary || !header) {
1535 		if (opt_i) {
1536 			head = "--------------- "
1537 			    "------------------------------------  "
1538 			    "-------------- ---------\n"
1539 			    "TIME            CACHE-ID"
1540 			    "                              MSG-ID"
1541 			    "         SEVERITY\n--------------- "
1542 			    "------------------------------------ "
1543 			    " -------------- ---------";
1544 		} else {
1545 			head = "--------------- "
1546 			    "------------------------------------  "
1547 			    "-------------- ---------\n"
1548 			    "TIME            EVENT-ID"
1549 			    "                              MSG-ID"
1550 			    "         SEVERITY\n--------------- "
1551 			    "------------------------------------ "
1552 			    " -------------- ---------";
1553 		}
1554 		(void) printf("%s\n", dgettext("FMD", head));
1555 		header = 1;
1556 	}
1557 	if (opt_i) {
1558 		ari_list = uurp->ari_uuid_list;
1559 		while (ari_list) {
1560 			(void) printf("%-15s %-37s %-14s %-9s\n",
1561 			    format_date(buf, sizeof (buf), uurp->sec),
1562 			    ari_list->ari_uuid, srp->msgid, severity);
1563 			ari_list = ari_list->next;
1564 		}
1565 	} else {
1566 		(void) printf("%-15s %-37s %-14s %-9s\n",
1567 		    format_date(buf, sizeof (buf), uurp->sec),
1568 		    uurp->uuid, srp->msgid, severity);
1569 	}
1570 
1571 	if (!summary)
1572 		print_sup_record(srp, opt_i, full);
1573 }
1574 
1575 static void
1576 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1577 {
1578 	status_record_t *srp;
1579 	sr_list_t *slp;
1580 
1581 	slp = status_rec_list;
1582 	if (slp) {
1583 		for (;;) {
1584 			srp = slp->status_record;
1585 			if (opt_a || srp->not_suppressed) {
1586 				if (page_feed)
1587 					(void) printf("\f\n");
1588 				print_status_record(srp, summary, opt_i, full);
1589 			}
1590 			if (slp->next == status_rec_list)
1591 				break;
1592 			slp = slp->next;
1593 		}
1594 	}
1595 }
1596 
1597 static name_list_t *
1598 find_fru(status_record_t *srp, char *resource)
1599 {
1600 	name_list_t *rt = NULL;
1601 	name_list_t *fru = srp->fru;
1602 
1603 	while (fru) {
1604 		if (strcmp(resource, fru->name) == 0) {
1605 			rt = fru;
1606 			break;
1607 		}
1608 		fru = fru->next;
1609 		if (fru == srp->fru)
1610 			break;
1611 	}
1612 	return (rt);
1613 }
1614 
1615 static void
1616 print_fru_line(name_list_t *fru, char *uuid)
1617 {
1618 	if (fru->pct == 100) {
1619 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1620 		    dgettext("FMD", "suspects in this FRU total certainty"),
1621 		    100);
1622 	} else {
1623 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1624 		    dgettext("FMD", "suspects in this FRU max certainty"),
1625 		    fru->max_pct);
1626 	}
1627 }
1628 
1629 static void
1630 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1631 {
1632 	resource_list_t *tp = status_fru_list;
1633 	status_record_t *srp;
1634 	sr_list_t *slp, *end;
1635 	char *msgid, *fru_label;
1636 	uurec_t *uurp;
1637 	name_list_t *fru;
1638 	ari_list_t *ari_list;
1639 
1640 	while (tp) {
1641 		if (opt_a || tp->not_suppressed) {
1642 			if (page_feed)
1643 				(void) printf("\f\n");
1644 			if (!summary)
1645 				(void) printf("-----------------------------"
1646 				    "---------------------------------------"
1647 				    "----------\n");
1648 			fru_label = get_fmri_label(tp->resource);
1649 			if (fru_label) {
1650 				(void) printf("\"%s\" (%s)\n", fru_label,
1651 				    tp->resource);
1652 				free(fru_label);
1653 			} else {
1654 				(void) printf("%s\n", tp->resource);
1655 			}
1656 			slp = tp->status_rec_list;
1657 			end = slp;
1658 			do {
1659 				srp = slp->status_record;
1660 				uurp = srp->uurec;
1661 				fru = find_fru(srp, tp->resource);
1662 				if (fru) {
1663 					if (opt_i) {
1664 						ari_list = uurp->ari_uuid_list;
1665 						while (ari_list) {
1666 							print_fru_line(fru,
1667 							    ari_list->ari_uuid);
1668 							ari_list =
1669 							    ari_list->next;
1670 						}
1671 					} else {
1672 						print_fru_line(fru, uurp->uuid);
1673 					}
1674 				}
1675 				slp = slp->next;
1676 			} while (slp != end);
1677 			if (!summary) {
1678 				slp = tp->status_rec_list;
1679 				end = slp;
1680 				srp = slp->status_record;
1681 				if (srp->serial &&
1682 				    !serial_in_fru(srp->fru, srp->serial)) {
1683 					print_name_list(srp->serial,
1684 					    dgettext("FMD", "Serial ID.  :"),
1685 					    NULL, 0, 0, NULL, 1);
1686 				}
1687 				msgid = NULL;
1688 				do {
1689 					if (msgid == NULL ||
1690 					    strcmp(msgid, srp->msgid) != 0) {
1691 						msgid = srp->msgid;
1692 						print_dict_info(srp->msgid,
1693 						    srp->url);
1694 					}
1695 					slp = slp->next;
1696 				} while (slp != end);
1697 			}
1698 		}
1699 		tp = tp->next;
1700 		if (tp == status_fru_list)
1701 			break;
1702 	}
1703 }
1704 
1705 static void
1706 print_asru(int opt_a)
1707 {
1708 	resource_list_t *tp = status_asru_list;
1709 	status_record_t *srp;
1710 	sr_list_t *slp, *end;
1711 	char *msg;
1712 	int status;
1713 	name_list_t *asru;
1714 
1715 	while (tp) {
1716 		if (opt_a || tp->not_suppressed) {
1717 			status = 0;
1718 			slp = tp->status_rec_list;
1719 			end = slp;
1720 			do {
1721 				srp = slp->status_record;
1722 				asru = srp->asru;
1723 				while (asru) {
1724 					if (strcmp(tp->resource,
1725 					    asru->name) == 0)
1726 						status |= asru->status;
1727 					asru = asru->next;
1728 					if (asru == srp->asru)
1729 						break;
1730 				}
1731 				slp = slp->next;
1732 			} while (slp != end);
1733 			switch (status) {
1734 			case 0:
1735 				msg = dgettext("FMD", "ok");
1736 				break;
1737 			case FM_SUSPECT_FAULTY:
1738 				msg = dgettext("FMD", "degraded");
1739 				break;
1740 			case FM_SUSPECT_UNUSABLE:
1741 				msg = dgettext("FMD", "unknown");
1742 				break;
1743 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1744 				msg = dgettext("FMD", "faulted");
1745 				break;
1746 			default:
1747 				msg = "";
1748 				break;
1749 			}
1750 			(void) printf("%-69s %s\n", tp->resource, msg);
1751 		}
1752 		tp = tp->next;
1753 		if (tp == status_asru_list)
1754 			break;
1755 	}
1756 }
1757 
1758 static int
1759 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1760 {
1761 	while (uurecp) {
1762 		if (strcmp(uuid, uurecp->uuid) == 0)
1763 			return (1);
1764 		uurecp = uurecp->next;
1765 	}
1766 	return (0);
1767 }
1768 
1769 static int
1770 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1771 {
1772 	int64_t *diag_time;
1773 	uint_t nelem;
1774 	int rt = 0;
1775 	char *uuid = "-";
1776 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1777 
1778 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1779 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1780 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1781 		    &uuid);
1782 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1783 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1784 			    uuid, acp->aci_url);
1785 	} else {
1786 		rt = -1;
1787 	}
1788 	return (rt);
1789 }
1790 
1791 /*ARGSUSED*/
1792 static int
1793 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1794 {
1795 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1796 	return (0);
1797 }
1798 
1799 static int
1800 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1801 {
1802 	int rt = FMADM_EXIT_SUCCESS;
1803 
1804 	/*
1805 	 * These calls may fail with Protocol error if message payload is to big
1806 	 */
1807 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1808 		die("failed to get case list from fmd");
1809 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1810 		die("failed to get case status from fmd");
1811 	return (rt);
1812 }
1813 
1814 /*
1815  * fmadm faulty command
1816  *
1817  *	-a		show hidden fault records
1818  *	-f		show faulty fru's
1819  *	-g		force grouping of similar faults on the same fru
1820  *	-n		number of fault records to display
1821  *	-p		pipe output through pager
1822  *	-r		show faulty asru's
1823  *	-s		print summary of first fault
1824  *	-u		print listed uuid's only
1825  *	-v		full output
1826  */
1827 
1828 int
1829 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1830 {
1831 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1832 	int opt_i = 0;
1833 	char *pager;
1834 	FILE *fp;
1835 	int rt, c, stat;
1836 	uurec_select_t *tp;
1837 	uurec_select_t *uurecp = NULL;
1838 
1839 	catalog_setup();
1840 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1841 		switch (c) {
1842 		case 'a':
1843 			opt_a++;
1844 			break;
1845 		case 'f':
1846 			opt_f++;
1847 			break;
1848 		case 'g':
1849 			opt_g++;
1850 			break;
1851 		case 'i':
1852 			opt_i++;
1853 			break;
1854 		case 'n':
1855 			max_fault = atoi(optarg);
1856 			break;
1857 		case 'p':
1858 			opt_p++;
1859 			break;
1860 		case 'r':
1861 			opt_r++;
1862 			break;
1863 		case 's':
1864 			opt_s++;
1865 			break;
1866 		case 'u':
1867 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1868 			tp->uuid = optarg;
1869 			tp->next = uurecp;
1870 			uurecp = tp;
1871 			opt_a = 1;
1872 			break;
1873 		case 'v':
1874 			opt_v++;
1875 			break;
1876 		default:
1877 			return (FMADM_EXIT_USAGE);
1878 		}
1879 	}
1880 	if (optind < argc)
1881 		return (FMADM_EXIT_USAGE);
1882 
1883 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1884 	if (opt_p) {
1885 		if ((pager = getenv("PAGER")) == NULL)
1886 			pager = "/usr/bin/more";
1887 		fp = popen(pager, "w");
1888 		if (fp == NULL) {
1889 			rt = FMADM_EXIT_ERROR;
1890 			opt_p = 0;
1891 		} else {
1892 			dup2(fileno(fp), 1);
1893 			setbuf(stdout, NULL);
1894 			(void) fclose(fp);
1895 		}
1896 	}
1897 	max_display = max_fault;
1898 	if (opt_f)
1899 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1900 	if (opt_r)
1901 		print_asru(opt_a);
1902 	if (opt_f == 0 && opt_r == 0)
1903 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1904 	label_release_topo();
1905 	if (opt_p) {
1906 		(void) fclose(stdout);
1907 		(void) wait(&stat);
1908 	}
1909 	return (rt);
1910 }
1911 
1912 int
1913 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1914 {
1915 	int i, status = FMADM_EXIT_SUCCESS;
1916 
1917 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1918 		return (FMADM_EXIT_USAGE);
1919 
1920 	for (i = 1; i < argc; i++) {
1921 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1922 			warn("failed to flush %s", argv[i]);
1923 			status = FMADM_EXIT_ERROR;
1924 		} else
1925 			note("flushed resource history for %s\n", argv[i]);
1926 	}
1927 
1928 	return (status);
1929 }
1930 
1931 int
1932 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
1933 {
1934 	int err;
1935 
1936 	if (getopt(argc, argv, "") != EOF)
1937 		return (FMADM_EXIT_USAGE);
1938 
1939 	if (argc - optind != 1)
1940 		return (FMADM_EXIT_USAGE);
1941 
1942 	/*
1943 	 * For now, we assume that if the input string contains a colon, it is
1944 	 * an FMRI and if it does not it is a UUID.  If things get more complex
1945 	 * in the future with multiple UUID formats, an FMRI parser can be
1946 	 * added here to differentiate the input argument appropriately.
1947 	 */
1948 	if (strchr(argv[optind], ':') != NULL)
1949 		err = fmd_adm_rsrc_repair(adm, argv[optind]);
1950 	else
1951 		err = fmd_adm_case_repair(adm, argv[optind]);
1952 
1953 	if (err != 0)
1954 		die("failed to record repair to %s", argv[optind]);
1955 
1956 	note("recorded repair to %s\n", argv[optind]);
1957 	return (FMADM_EXIT_SUCCESS);
1958 }
1959