xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 567cc2e6)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <fmadm.h>
30 #include <errno.h>
31 #include <limits.h>
32 #include <strings.h>
33 #include <stdio.h>
34 #include <unistd.h>
35 #include <sys/wait.h>
36 #include <sys/stat.h>
37 #include <fcntl.h>
38 #include <fm/fmd_log.h>
39 #include <sys/fm/protocol.h>
40 #include <fm/libtopo.h>
41 #include <fm/fmd_adm.h>
42 #include <dlfcn.h>
43 #include <sys/systeminfo.h>
44 #include <sys/utsname.h>
45 #include <libintl.h>
46 #include <locale.h>
47 #include <sys/smbios.h>
48 #include <libdevinfo.h>
49 #include <stdlib.h>
50 
51 #define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))
52 
53 /*
54  * catalog_setup() must be called to setup support functions.
55  * Fault records are added to catalog by calling add_fault_record_to_catalog()
56  * records are stored in order of importance to the system.
57  * If -g flag is set or not_suppressed is not set and the class fru, fault,
58  * type are the same then details are merged into an existing record, with uuid
59  * records are stored in time order.
60  * For each record information is extracted from nvlist and merged into linked
61  * list each is checked for identical records for which percentage certainty are
62  * added together.
63  * print_catalog() is called to print out catalog and release external resources
64  *
65  *                         /---------------\
66  *	status_rec_list -> |               | -|
67  *                         \---------------/
68  *                                \/
69  *                         /---------------\    /-------\    /-------\
70  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
71  *            \/           |               | |- |       | <- |       |
72  *      /-------------\    |               |    \-------/    \-------/
73  *      |             | -> |               |       \/           \/
74  *      \-------------/    |               |    /-------\    /-------\
75  *            \/           |               | -> | asru  | -> | asru  |
76  *            ---          |               |    |       | <- |       |
77  *                         |               |    \-------/    \-------/
78  *      status_asru_list   |  class        |
79  *            \/           |  resource     |    /-------\    /-------\
80  *      /-------------\    |  fru          | -> | list  | -> | list  |
81  *      |             | -> |  serial       |    |       | <- |       |
82  *      \-------------/    |               |    \-------/    \-------/
83  *            \/           \---------------/
84  *            ---               \/    /\
85  *                         /---------------\
86  *                         | status_record |
87  *                         \---------------/
88  *
89  * Fmadm faulty takes a number of options which affect the format of the
90  * output displayed. By default, the display reports the FRU and ASRU along
91  * with other information on per-case basis as in the example below.
92  *
93  * --------------- ------------------------------------  -------------- -------
94  * TIME            EVENT-ID                              MSG-ID         SEVERITY
95  * --------------- ------------------------------------  -------------- -------
96  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
97  *
98  * Fault class	: fault.memory.dimm_sb
99  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
100  *		    degraded but still in service
101  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
102  *		    faulty
103  *
104  * Description	: The number of errors associated with this memory module has
105  *		exceeded acceptable levels.  Refer to
106  *		http://sun.com/msg/AMD-8000-2F for more information.
107  *
108  * Response	: Pages of memory associated with this memory module are being
109  *		removed from service as errors are reported.
110  *
111  * Impact	: Total system memory capacity will be reduced as pages are
112  *		retired.
113  *
114  * Action	: Schedule a repair procedure to replace the affected memory
115  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
116  *
117  * The -v flag is similar, but adds some additonal information such as the
118  * resource. The -s flag is also similar but just gives the top line summary.
119  * All these options (ie without the -f or -r flags) use the print_catalog()
120  * function to do the display.
121  *
122  * The -f flag changes the output so that it appears sorted on a per-fru basis.
123  * The output is somewhat cut down compared to the default output. If -f is
124  * used, then print_fru() is used to print the output.
125  *
126  * -----------------------------------------------------------------------------
127  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
128  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
129  *
130  * Description	: A problem was detected for a PCI device.
131  *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
132  *
133  * Response	: One or more device instances may be disabled
134  *
135  * Impact	: Possible loss of services provided by the device instances
136  *		associated with this fault
137  *
138  * Action	: Schedule a repair procedure to replace the affected device.
139  * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
140  *		Sun for support.
141  *
142  * The -r flag changes the output so that it appears sorted on a per-asru basis.
143  * The output is very much cut down compared to the default output, just giving
144  * the asru fmri and state. Here print_asru() is used to print the output.
145  *
146  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
147  *
148  * For all fmadm faulty options, the sequence of events is
149  *
150  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
151  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
152  * This will extract the data from the nvlist and call catalog_new_record() to
153  * save the data away in various linked lists in the catalogue.
154  *
155  * 2) Once this is done, the data can be supplemented by using
156  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
157  *
158  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
159  * appropriate to display the information from the catalogue sorted in the
160  * requested way.
161  *
162  */
163 
164 typedef struct name_list {
165 	struct name_list *next;
166 	struct name_list *prev;
167 	char *name;
168 	uint8_t pct;
169 	uint8_t max_pct;
170 	ushort_t count;
171 	int status;
172 	char *label;
173 } name_list_t;
174 
175 typedef struct ari_list {
176 	char *ari_uuid;
177 	struct ari_list *next;
178 } ari_list_t;
179 
180 typedef struct uurec {
181 	struct uurec *next;
182 	struct uurec *prev;
183 	char *uuid;
184 	ari_list_t *ari_uuid_list;
185 	name_list_t *asru;
186 	uint64_t sec;
187 } uurec_t;
188 
189 typedef struct uurec_select {
190 	struct uurec_select *next;
191 	char *uuid;
192 } uurec_select_t;
193 
194 typedef struct host_id {
195 	char *chassis;
196 	char *server;
197 	char *platform;
198 } hostid_t;
199 
200 typedef struct host_id_list {
201 	hostid_t hostid;
202 	struct host_id_list *next;
203 } host_id_list_t;
204 
205 typedef struct status_record {
206 	hostid_t *host;
207 	int nrecs;
208 	uurec_t *uurec;
209 	char *severity;			/* in C locale */
210 	char *msgid;
211 	name_list_t *class;
212 	name_list_t *resource;
213 	name_list_t *asru;
214 	name_list_t *fru;
215 	name_list_t *serial;
216 	char *url;
217 	uint8_t not_suppressed;
218 } status_record_t;
219 
220 typedef struct sr_list {
221 	struct sr_list *next;
222 	struct sr_list *prev;
223 	struct status_record *status_record;
224 } sr_list_t;
225 
226 typedef struct resource_list {
227 	struct resource_list *next;
228 	struct resource_list *prev;
229 	sr_list_t *status_rec_list;
230 	char *resource;
231 	uint8_t not_suppressed;
232 	uint8_t max_pct;
233 } resource_list_t;
234 
235 typedef struct tgetlabel_data {
236 	char *label;
237 	char *fru;
238 } tgetlabel_data_t;
239 
240 sr_list_t *status_rec_list;
241 resource_list_t *status_fru_list;
242 resource_list_t *status_asru_list;
243 
244 static char *locale;
245 static char *nlspath;
246 static int max_display;
247 static int max_fault = 0;
248 static topo_hdl_t *topo_handle;
249 static char *topo_handle_uuid;
250 static host_id_list_t *host_list;
251 static int n_server;
252 static int opt_g;
253 
254 static char *
255 format_date(char *buf, size_t len, uint64_t sec)
256 {
257 	if (sec > LONG_MAX) {
258 		(void) fprintf(stderr,
259 		    "record time is too large for 32-bit utility\n");
260 		(void) snprintf(buf, len, "0x%llx", sec);
261 	} else {
262 		time_t tod = (time_t)sec;
263 		(void) strftime(buf, len, "%b %d %T", localtime(&tod));
264 	}
265 
266 	return (buf);
267 }
268 
269 static hostid_t *
270 find_hostid_in_list(char *platform, char *chassis, char *server)
271 {
272 	hostid_t *rt = NULL;
273 	host_id_list_t *hostp;
274 
275 	if (platform == NULL)
276 		platform = "-";
277 	if (server == NULL)
278 		server = "-";
279 	hostp = host_list;
280 	while (hostp) {
281 		if (hostp->hostid.platform &&
282 		    strcmp(hostp->hostid.platform, platform) == 0 &&
283 		    hostp->hostid.server &&
284 		    strcmp(hostp->hostid.server, server) == 0 &&
285 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
286 		    strcmp(chassis, hostp->hostid.chassis) == 0)) {
287 			rt = &hostp->hostid;
288 			break;
289 		}
290 		hostp = hostp->next;
291 	}
292 	if (rt == NULL) {
293 		hostp = malloc(sizeof (host_id_list_t));
294 		hostp->hostid.platform = strdup(platform);
295 		hostp->hostid.server = strdup(server);
296 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
297 		hostp->next = host_list;
298 		host_list = hostp;
299 		rt = &hostp->hostid;
300 		n_server++;
301 	}
302 	return (rt);
303 }
304 
305 static hostid_t *
306 find_hostid(nvlist_t *nvl)
307 {
308 	char *platform = NULL, *chassis = NULL, *server = NULL;
309 	nvlist_t *auth, *fmri;
310 	hostid_t *rt = NULL;
311 
312 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
313 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
314 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
315 		    &platform);
316 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
317 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
318 		    &chassis);
319 		rt = find_hostid_in_list(platform, chassis, server);
320 	}
321 	return (rt);
322 }
323 
324 static void
325 catalog_setup(void)
326 {
327 	char *tp;
328 	int pl;
329 
330 	/*
331 	 * All FMA event dictionaries use msgfmt(1) message objects to produce
332 	 * messages, even for the C locale.  We therefore want to use dgettext
333 	 * for all message lookups, but its defined behavior in the C locale is
334 	 * to return the input string.  Since our input strings are event codes
335 	 * and not format strings, this doesn't help us.  We resolve this nit
336 	 * by setting NLSPATH to a non-existent file: the presence of NLSPATH
337 	 * is defined to force dgettext(3C) to do a full lookup even for C.
338 	 */
339 	nlspath = getenv("NLSPATH");
340 	if (nlspath == NULL)
341 		putenv("NLSPATH=/usr/lib/fm/fmd/fmd.cat");
342 	else {
343 		pl = strlen(nlspath) + sizeof ("NLSPATH=") + 1;
344 		tp = malloc(pl);
345 		(void) snprintf(tp, pl, "NLSPATH=%s", nlspath);
346 		nlspath = tp;
347 	}
348 
349 	locale = setlocale(LC_MESSAGES, "");
350 }
351 
352 static char *
353 get_dict_url(char *id)
354 {
355 	char *url = "http://sun.com/msg/";
356 	int msz = sizeof (url) + strlen(id) + 1;
357 	char *cp;
358 
359 	cp = malloc(msz);
360 	(void) snprintf(cp, msz, "%s%s", url, id);
361 	return (cp);
362 }
363 
364 static char *
365 get_dict_msg(char *id, char *idx, int unknown, int translate)
366 {
367 	char mbuf[128];
368 	char *msg;
369 	char dbuf[32];
370 	char *p;
371 	int restore_env = 0;
372 	int restore_locale = 0;
373 
374 	p = strchr(id, '-');
375 	if (p == NULL || p == id || (p - id) >= 32) {
376 		msg = mbuf;
377 	} else {
378 		strncpy(dbuf, id, (size_t)(p - id));
379 		dbuf[(size_t)(p - id)] = 0;
380 
381 		(void) snprintf(mbuf, sizeof (mbuf), "%s.%s", id, idx);
382 		if (translate == 0 || nlspath == NULL) {
383 			(void) setlocale(LC_MESSAGES, "C");
384 			restore_locale = 1;
385 		}
386 		bindtextdomain("FMD", "/usr/lib/locale");
387 		msg = dgettext(dbuf, mbuf);
388 		if (msg == mbuf) {
389 			(void) setlocale(LC_MESSAGES, "C");
390 			restore_locale = 1;
391 			msg = dgettext(dbuf, mbuf);
392 		}
393 		if (msg == mbuf) {
394 			putenv("NLSPATH=/usr/lib/fm/fmd/fmd.cat");
395 			restore_env = 1;
396 			(void) setlocale(LC_MESSAGES, "C");
397 			msg = dgettext(dbuf, mbuf);
398 		}
399 		if (restore_locale)
400 			(void) setlocale(LC_MESSAGES, locale);
401 		if (restore_env && nlspath)
402 			putenv(nlspath);
403 	}
404 	if (msg == mbuf) {
405 		if (unknown)
406 			msg = "unknown";
407 		else
408 			msg = NULL;
409 	}
410 	return (msg);
411 }
412 
413 /*
414  * compare two fru strings which are made up of substrings seperated by '/'
415  * return true if every substring is the same in the two strings, or if a
416  * substring is null in one.
417  */
418 
419 static int
420 frucmp(char *f1, char *f2)
421 {
422 	char c1, c2;
423 	int i = 0;
424 
425 	for (;;) {
426 		c1 = *f1;
427 		c2 = *f2;
428 		if (c1 == c2) {
429 			i = (c1 == '/') ? 0 : i + 1;
430 		} else if (i == 0) {
431 			if (c1 == '/') {
432 				do {
433 					f2++;
434 				} while ((c2 = *f2) != 0 && c2 != '/');
435 				if (c2 == NULL)
436 					break;
437 			} else if (c2 == '/') {
438 				do {
439 					f1++;
440 				} while ((c1 = *f1) != 0 && c1 != '/');
441 				if (c1 == NULL)
442 					break;
443 			} else
444 				break;
445 		} else
446 			break;
447 		if (c1 == NULL)
448 			return (0);
449 		f1++;
450 		f2++;
451 	}
452 	return (1);
453 }
454 
455 static int
456 tgetlabel(topo_hdl_t *thp, tnode_t *node, void *arg)
457 {
458 	int err;
459 	char *fru_name, *lname;
460 	nvlist_t *fru = NULL;
461 	int rt = TOPO_WALK_NEXT;
462 	tgetlabel_data_t *tdp = (tgetlabel_data_t *)arg;
463 
464 	if (topo_node_fru(node, &fru, NULL, &err) == 0) {
465 		if (topo_fmri_nvl2str(thp, fru, &fru_name, &err) == 0) {
466 			if (frucmp(tdp->fru, fru_name) == 0 &&
467 			    topo_node_label(node, &lname, &err) == 0) {
468 				tdp->label = strdup(lname);
469 				topo_hdl_strfree(thp, lname);
470 				rt = TOPO_WALK_TERMINATE;
471 			}
472 			topo_hdl_strfree(thp, fru_name);
473 		}
474 		nvlist_free(fru);
475 	}
476 	return (rt);
477 }
478 
479 static void
480 label_get_topo(void)
481 {
482 	int err;
483 
484 	topo_handle = topo_open(TOPO_VERSION, 0, &err);
485 	if (topo_handle) {
486 		topo_handle_uuid = topo_snap_hold(topo_handle, NULL, &err);
487 	}
488 }
489 
490 static void
491 label_release_topo(void)
492 {
493 	if (topo_handle_uuid)
494 		topo_hdl_strfree(topo_handle, topo_handle_uuid);
495 	if (topo_handle) {
496 		topo_snap_release(topo_handle);
497 		topo_close(topo_handle);
498 	}
499 }
500 
501 static char *
502 get_fmri_label(char *fru)
503 {
504 	topo_walk_t *twp;
505 	tgetlabel_data_t td;
506 	int err;
507 
508 	td.label = NULL;
509 	td.fru = fru;
510 	if (topo_handle == NULL)
511 		label_get_topo();
512 	if (topo_handle_uuid) {
513 		twp = topo_walk_init(topo_handle, FM_FMRI_SCHEME_HC,
514 		    tgetlabel, &td, &err);
515 		if (twp) {
516 			topo_walk_step(twp, TOPO_WALK_CHILD);
517 			topo_walk_fini(twp);
518 		}
519 	}
520 	return (td.label);
521 }
522 
523 static char *
524 get_nvl2str_topo(nvlist_t *nvl)
525 {
526 	char *name = NULL;
527 	char *tname;
528 	int err;
529 	char *scheme = NULL;
530 	char *mod_name = NULL;
531 	char buf[128];
532 
533 	if (topo_handle == NULL)
534 		label_get_topo();
535 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
536 		name = strdup(tname);
537 		topo_hdl_strfree(topo_handle, tname);
538 	} else {
539 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
540 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
541 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
542 		    mod_name) {
543 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
544 			    scheme, mod_name);
545 			name = strdup(buf);
546 		}
547 	}
548 	return (name);
549 }
550 
551 static int
552 set_priority(char *s)
553 {
554 	int rt = 0;
555 
556 	if (s) {
557 		if (strcmp(s, "Minor") == 0)
558 			rt = 1;
559 		else if (strcmp(s, "Major") == 0)
560 			rt = 10;
561 		else if (strcmp(s, "Critical") == 0)
562 			rt = 100;
563 	}
564 	return (rt);
565 }
566 
567 static int
568 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
569     uint8_t p2)
570 {
571 	int r1, r2;
572 	int rt;
573 
574 	r1 = set_priority(s1);
575 	r2 = set_priority(s2);
576 	rt = r1 - r2;
577 	if (rt == 0) {
578 		if (t1 > t2)
579 			rt = 1;
580 		else if (t1 < t2)
581 			rt = -1;
582 		else
583 			rt = p1 - p2;
584 	}
585 	return (rt);
586 }
587 
588 /*
589  * merge two lists into one, by comparing enties in new and moving into list if
590  * name is not there or free off memory for names which are already there
591  * add_pct indicates if pct is the sum or highest pct
592  */
593 static name_list_t *
594 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
595 {
596 	name_list_t *lp, *np, *sp, *rt = NULL;
597 	int max_pct;
598 
599 	rt = *list;
600 	np = new;
601 	while (np) {
602 		lp = *list;
603 		while (lp) {
604 			if (strcmp(lp->name, np->name) == 0)
605 				break;
606 			lp = lp->next;
607 			if (lp == *list)
608 				lp = NULL;
609 		}
610 		if (np->next == new)
611 			sp = NULL;
612 		else
613 			sp = np->next;
614 		if (lp) {
615 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
616 			if (add_pct) {
617 				lp->pct += np->pct;
618 				lp->count += np->count;
619 			} else if (np->pct > lp->pct) {
620 				lp->pct = np->pct;
621 			}
622 			max_pct = np->max_pct;
623 			if (np->label)
624 				free(np->label);
625 			free(np->name);
626 			free(np);
627 			np = NULL;
628 			if (max_pct > lp->max_pct) {
629 				lp->max_pct = max_pct;
630 				if (lp->max_pct > lp->prev->max_pct &&
631 				    lp != *list) {
632 					lp->prev->next = lp->next;
633 					lp->next->prev = lp->prev;
634 					np = lp;
635 				}
636 			}
637 		}
638 		if (np) {
639 			lp = *list;
640 			if (lp) {
641 				if (np->max_pct > lp->max_pct) {
642 					np->next = lp;
643 					np->prev = lp->prev;
644 					lp->prev->next = np;
645 					lp->prev = np;
646 					*list = np;
647 					rt = np;
648 				} else {
649 					lp = lp->next;
650 					while (lp != *list &&
651 					    np->max_pct < lp->max_pct) {
652 						lp = lp->next;
653 					}
654 					np->next = lp;
655 					np->prev = lp->prev;
656 					lp->prev->next = np;
657 					lp->prev = np;
658 				}
659 			} else {
660 				*list = np;
661 				np->next = np;
662 				np->prev = np;
663 				rt = np;
664 			}
665 		}
666 		np = sp;
667 	}
668 	return (rt);
669 }
670 
671 /*
672  * compare entries in two lists return true if the two lists have identical
673  * content. The two lists may not have entries in the same order, so we compare
674  * the size of the list as well as trying to find every entry from one list in
675  * the other.
676  */
677 static int
678 cmp_name_list(name_list_t *lxp1, name_list_t *lxp2)
679 {
680 	name_list_t *lp1, *lp2;
681 	int l1 = 0, l2 = 0, common = 0;
682 
683 	lp2 = lxp2;
684 	while (lp2) {
685 		l2++;
686 		lp2 = lp2->next;
687 		if (lp2 == lxp2)
688 			break;
689 	}
690 	lp1 = lxp1;
691 	while (lp1) {
692 		l1++;
693 		lp2 = lxp2;
694 		while (lp2) {
695 			if (strcmp(lp2->name, lp1->name) == 0) {
696 				common++;
697 				break;
698 			}
699 			lp2 = lp2->next;
700 			if (lp2 == lxp2)
701 				break;
702 		}
703 		lp1 = lp1->next;
704 		if (lp1 == lxp1)
705 			break;
706 	}
707 	if (l1 == l2 && l2 == common)
708 		return (0);
709 	else
710 		return (1);
711 }
712 
713 static name_list_t *
714 alloc_name_list(char *name, uint8_t pct)
715 {
716 	name_list_t *nlp;
717 
718 	nlp = malloc(sizeof (*nlp));
719 	nlp->name = strdup(name);
720 	nlp->pct = pct;
721 	nlp->max_pct = pct;
722 	nlp->count = 1;
723 	nlp->next = nlp;
724 	nlp->prev = nlp;
725 	nlp->status = 0;
726 	nlp->label = NULL;
727 	return (nlp);
728 }
729 
730 static void
731 free_name_list(name_list_t *list)
732 {
733 	name_list_t *next = list;
734 	name_list_t *lp;
735 
736 	if (list) {
737 		do {
738 			lp = next;
739 			next = lp->next;
740 			if (lp->label)
741 				free(lp->label);
742 			free(lp->name);
743 			free(lp);
744 		} while (next != list);
745 	}
746 }
747 
748 static status_record_t *
749 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
750     name_list_t *fru, name_list_t *asru, name_list_t *resource,
751     name_list_t *serial, const char *url, boolean_t not_suppressed,
752     hostid_t *hostid)
753 {
754 	status_record_t *status_rec_p;
755 
756 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
757 	status_rec_p->nrecs = 1;
758 	status_rec_p->host = hostid;
759 	status_rec_p->uurec = uurec_p;
760 	uurec_p->next = NULL;
761 	uurec_p->prev = NULL;
762 	uurec_p->asru = asru;
763 	status_rec_p->severity = get_dict_msg(msgid, "severity", 1, 0);
764 	status_rec_p->class = class;
765 	status_rec_p->fru = fru;
766 	status_rec_p->asru = asru;
767 	status_rec_p->resource = resource;
768 	status_rec_p->serial = serial;
769 	status_rec_p->url = url ? strdup(url) : NULL;
770 	status_rec_p->msgid = strdup(msgid);
771 	status_rec_p->not_suppressed = not_suppressed;
772 	return (status_rec_p);
773 }
774 
775 /*
776  * add record to given list maintaining order higher priority first.
777  */
778 static void
779 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
780 {
781 	sr_list_t *tp, *np, *sp;
782 	int order;
783 	uint64_t sec;
784 
785 	np = malloc(sizeof (sr_list_t));
786 	np->status_record = status_rec_p;
787 	sec = status_rec_p->uurec->sec;
788 	if ((sp = *list_pp) == NULL) {
789 		*list_pp = np;
790 		np->next = np;
791 		np->prev = np;
792 	} else {
793 		/* insert new record in front of lower priority */
794 		tp = sp;
795 		order = cmp_priority(status_rec_p->severity,
796 		    sp->status_record->severity, sec,
797 		    tp->status_record->uurec->sec, 0, 0);
798 		if (order > 0) {
799 			*list_pp = np;
800 		} else {
801 			tp = sp->next;
802 			while (tp != sp &&
803 			    cmp_priority(status_rec_p->severity,
804 			    tp->status_record->severity, sec,
805 			    tp->status_record->uurec->sec, 0, 0)) {
806 				tp = tp->next;
807 			}
808 		}
809 		np->next = tp;
810 		np->prev = tp->prev;
811 		tp->prev->next = np;
812 		tp->prev = np;
813 	}
814 }
815 
816 static void
817 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
818     resource_list_t *np)
819 {
820 	int order;
821 	uint64_t sec;
822 	resource_list_t *sp, *tp;
823 	status_record_t *srp;
824 	char *severity = status_rec_p->severity;
825 
826 	add_rec_list(status_rec_p, &np->status_rec_list);
827 	if ((sp = *rp) == NULL) {
828 		np->next = np;
829 		np->prev = np;
830 		*rp = np;
831 	} else {
832 		/*
833 		 * insert new record in front of lower priority
834 		 */
835 		tp = sp->next;
836 		srp = sp->status_rec_list->status_record;
837 		sec = status_rec_p->uurec->sec;
838 		order = cmp_priority(severity, srp->severity, sec,
839 		    srp->uurec->sec, np->max_pct, sp->max_pct);
840 		if (order > 0) {
841 			*rp = np;
842 		} else {
843 			srp = tp->status_rec_list->status_record;
844 			while (tp != sp &&
845 			    cmp_priority(severity, srp->severity, sec,
846 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
847 				tp = tp->next;
848 				srp = tp->status_rec_list->status_record;
849 			}
850 		}
851 		np->next = tp;
852 		np->prev = tp->prev;
853 		tp->prev->next = np;
854 		tp->prev = np;
855 	}
856 }
857 
858 static void
859 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
860     resource_list_t **rpp)
861 {
862 	int order;
863 	resource_list_t *np, *end;
864 	status_record_t *srp;
865 
866 	np = *rpp;
867 	end = np;
868 	while (np) {
869 		if (strcmp(fp->name, np->resource) == 0) {
870 			np->not_suppressed |= status_rec_p->not_suppressed;
871 			srp = np->status_rec_list->status_record;
872 			order = cmp_priority(status_rec_p->severity,
873 			    srp->severity, status_rec_p->uurec->sec,
874 			    srp->uurec->sec, fp->max_pct, np->max_pct);
875 			if (order > 0 && np != end) {
876 				/*
877 				 * remove from list and add again using
878 				 * new priority
879 				 */
880 				np->prev->next = np->next;
881 				np->next->prev = np->prev;
882 				add_resource(status_rec_p,
883 				    rpp, np);
884 			} else {
885 				add_rec_list(status_rec_p,
886 				    &np->status_rec_list);
887 			}
888 			break;
889 		}
890 		np = np->next;
891 		if (np == end) {
892 			np = NULL;
893 			break;
894 		}
895 	}
896 	if (np == NULL) {
897 		np = malloc(sizeof (resource_list_t));
898 		np->resource = fp->name;
899 		np->not_suppressed = status_rec_p->not_suppressed;
900 		np->status_rec_list = NULL;
901 		np->max_pct = fp->max_pct;
902 		add_resource(status_rec_p, rpp, np);
903 	}
904 }
905 
906 static void
907 add_list(status_record_t *status_rec_p, name_list_t *listp,
908     resource_list_t **glistp)
909 {
910 	name_list_t *fp, *end;
911 
912 	fp = listp;
913 	end = fp;
914 	while (fp) {
915 		add_resource_list(status_rec_p, fp, glistp);
916 		fp = fp->next;
917 		if (fp == end)
918 			break;
919 	}
920 }
921 
922 /*
923  * add record to rec, fru and asru lists.
924  */
925 static void
926 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
927     name_list_t *fru, name_list_t *asru, name_list_t *resource,
928     name_list_t *serial, const char *url, boolean_t not_suppressed,
929     hostid_t *hostid)
930 {
931 	status_record_t *status_rec_p;
932 
933 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
934 	    resource, serial, url, not_suppressed, hostid);
935 	add_rec_list(status_rec_p, &status_rec_list);
936 	if (status_rec_p->fru)
937 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
938 	if (status_rec_p->asru)
939 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
940 }
941 
942 /*
943  * add uuid and diagnoses time to an existing record for similar fault on the
944  * same fru
945  */
946 static void
947 catalog_merge_record(status_record_t *status_rec_p, uurec_t *uurec_p,
948     name_list_t *asru, name_list_t *resource, name_list_t *serial,
949     const char *url, boolean_t not_suppressed)
950 {
951 	uurec_t *uurec1_p;
952 
953 	status_rec_p->nrecs++;
954 	/* add uurec in time order */
955 	if (status_rec_p->uurec->sec > uurec_p->sec) {
956 		uurec_p->next = status_rec_p->uurec;
957 		uurec_p->prev = NULL;
958 		status_rec_p->uurec = uurec_p;
959 	} else {
960 		uurec1_p = status_rec_p->uurec;
961 		while (uurec1_p->next && uurec1_p->next->sec <= uurec_p->sec)
962 			uurec1_p = uurec1_p->next;
963 		if (uurec1_p->next)
964 			uurec1_p->next->prev = uurec_p;
965 		uurec_p->next = uurec1_p->next;
966 		uurec_p->prev = uurec1_p;
967 		uurec1_p->next = uurec_p;
968 	}
969 	if (status_rec_p->url == NULL && url != NULL)
970 		status_rec_p->url = strdup(url);
971 	status_rec_p->not_suppressed |= not_suppressed;
972 	uurec_p->asru = merge_name_list(&status_rec_p->asru, asru, 0);
973 	(void) merge_name_list(&status_rec_p->resource, resource, 0);
974 	(void) merge_name_list(&status_rec_p->serial, serial, 0);
975 }
976 
977 static status_record_t *
978 record_in_catalog(name_list_t *class, name_list_t *fru,
979     char *msgid, hostid_t *host)
980 {
981 	sr_list_t *status_rec_p;
982 	status_record_t *srp = NULL;
983 
984 	status_rec_p = status_rec_list;
985 	while (status_rec_p) {
986 		srp = status_rec_p->status_record;
987 		if (host == srp->host &&
988 		    cmp_name_list(class, srp->class) == 0 &&
989 		    cmp_name_list(fru, srp->fru) == 0 &&
990 		    strcmp(msgid, srp->msgid) == 0)
991 			break;
992 		if (status_rec_p->next == status_rec_list) {
993 			srp = NULL;
994 			break;
995 		} else {
996 			status_rec_p = status_rec_p->next;
997 		}
998 	}
999 	return (srp);
1000 }
1001 
1002 static void
1003 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
1004 {
1005 	char *name;
1006 	char *serial = NULL;
1007 	char **lserial = NULL;
1008 	uint64_t serint;
1009 	name_list_t *nlp;
1010 	int j;
1011 	uint_t nelem;
1012 	char buf[64];
1013 
1014 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
1015 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
1016 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
1017 			    &serint) == 0) {
1018 				(void) snprintf(buf, sizeof (buf), "%llX",
1019 				    serint);
1020 				nlp = alloc_name_list(buf, pct);
1021 				(void) merge_name_list(serial_p, nlp, 1);
1022 			}
1023 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
1024 			if (nvlist_lookup_string_array(nvl,
1025 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
1026 				nlp = alloc_name_list(lserial[0], pct);
1027 				for (j = 1; j < nelem; j++) {
1028 					name_list_t *n1lp;
1029 					n1lp = alloc_name_list(lserial[j], pct);
1030 					(void) merge_name_list(&nlp, n1lp, 1);
1031 				}
1032 				(void) merge_name_list(serial_p, nlp, 1);
1033 			}
1034 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
1035 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
1036 			    &serial) == 0) {
1037 				nlp = alloc_name_list(serial, pct);
1038 				(void) merge_name_list(serial_p, nlp, 1);
1039 			}
1040 		}
1041 	}
1042 }
1043 
1044 static void
1045 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
1046     name_list_t **fru_p, name_list_t **serial_p,
1047     name_list_t **resource_p, name_list_t **asru_p, uint8_t status)
1048 {
1049 	nvlist_t *lfru, *lasru, *rsrc;
1050 	name_list_t *nlp;
1051 	char *name;
1052 	uint8_t lpct = 0;
1053 	char *lclass = NULL;
1054 	char *label;
1055 
1056 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
1057 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
1058 		nlp = alloc_name_list(lclass, lpct);
1059 		(void) merge_name_list(class_p, nlp, 1);
1060 	}
1061 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
1062 		name = get_nvl2str_topo(lfru);
1063 		if (name != NULL) {
1064 			nlp = alloc_name_list(name, lpct);
1065 			nlp->status = status & ~FM_SUSPECT_UNUSABLE;
1066 			free(name);
1067 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
1068 			    &label) == 0)
1069 				nlp->label = strdup(label);
1070 			(void) merge_name_list(fru_p, nlp, 1);
1071 		}
1072 		get_serial_no(lfru, serial_p, lpct);
1073 	}
1074 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
1075 		name = get_nvl2str_topo(lasru);
1076 		if (name != NULL) {
1077 			nlp = alloc_name_list(name, lpct);
1078 			nlp->status = status & ~FM_SUSPECT_NOT_PRESENT;
1079 			free(name);
1080 			(void) merge_name_list(asru_p, nlp, 1);
1081 		}
1082 		get_serial_no(lasru, serial_p, lpct);
1083 	}
1084 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
1085 		name = get_nvl2str_topo(rsrc);
1086 		if (name != NULL) {
1087 			nlp = alloc_name_list(name, lpct);
1088 			nlp->status = status;
1089 			free(name);
1090 			(void) merge_name_list(resource_p, nlp, 1);
1091 		}
1092 	}
1093 }
1094 
1095 static void
1096 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid,
1097     const char *url)
1098 {
1099 	char *msgid = "-";
1100 	uint_t i, size = 0;
1101 	name_list_t *class = NULL, *resource = NULL;
1102 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
1103 	nvlist_t **nva;
1104 	uint8_t *ba;
1105 	status_record_t *status_rec_p;
1106 	uurec_t *uurec_p;
1107 	hostid_t *host;
1108 	boolean_t not_suppressed = 1;
1109 	boolean_t any_present = 0;
1110 
1111 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
1112 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
1113 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
1114 	    &not_suppressed);
1115 
1116 	if (size != 0) {
1117 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
1118 		    &nva, &size);
1119 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
1120 		    &ba, &size);
1121 		for (i = 0; i < size; i++) {
1122 			extract_record_info(nva[i], &class, &fru, &serial,
1123 			    &resource, &asru, ba[i]);
1124 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
1125 			    (ba[i] & FM_SUSPECT_FAULTY))
1126 				any_present = 1;
1127 		}
1128 		/*
1129 		 * also suppress if no resources present
1130 		 */
1131 		if (any_present == 0)
1132 			not_suppressed = 0;
1133 	}
1134 
1135 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
1136 	uurec_p->uuid = strdup(uuid);
1137 	uurec_p->sec = sec;
1138 	uurec_p->ari_uuid_list = NULL;
1139 	host = find_hostid(nvl);
1140 	if (not_suppressed && !opt_g)
1141 		status_rec_p = NULL;
1142 	else
1143 		status_rec_p = record_in_catalog(class, fru, msgid, host);
1144 	if (status_rec_p) {
1145 		catalog_merge_record(status_rec_p, uurec_p, asru, resource,
1146 		    serial, url, not_suppressed);
1147 		free_name_list(class);
1148 		free_name_list(fru);
1149 	} else {
1150 		catalog_new_record(uurec_p, msgid, class, fru, asru,
1151 		    resource, serial, url, not_suppressed, host);
1152 	}
1153 }
1154 
1155 static void
1156 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
1157 {
1158 	sr_list_t *srp;
1159 	uurec_t *uurp;
1160 	ari_list_t *ari_list;
1161 
1162 	srp = status_rec_list;
1163 	if (srp) {
1164 		for (;;) {
1165 			uurp = srp->status_record->uurec;
1166 			while (uurp) {
1167 				if (strcmp(uuid, uurp->uuid) == 0) {
1168 					ari_list = (ari_list_t *)
1169 					    malloc(sizeof (ari_list_t));
1170 					ari_list->ari_uuid = strdup(ari_uuid);
1171 					ari_list->next = uurp->ari_uuid_list;
1172 					uurp->ari_uuid_list = ari_list;
1173 					return;
1174 				}
1175 				uurp = uurp->next;
1176 			}
1177 			if (srp->next == status_rec_list)
1178 				break;
1179 			srp = srp->next;
1180 		}
1181 	}
1182 }
1183 
1184 static void
1185 print_line(char *label, char *buf)
1186 {
1187 	char *cp, *ep, *wp;
1188 	char c;
1189 	int i;
1190 	int lsz;
1191 	char *padding;
1192 
1193 	lsz = strlen(label);
1194 	padding = malloc(lsz + 1);
1195 	for (i = 0; i < lsz; i++)
1196 		padding[i] = ' ';
1197 	padding[i] = 0;
1198 	cp = buf;
1199 	ep = buf;
1200 	c = *ep;
1201 	(void) printf("\n");
1202 	while (c) {
1203 		i = lsz;
1204 		wp = NULL;
1205 		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
1206 			if (c == ' ')
1207 				wp = ep;
1208 			else if (c == '\n') {
1209 				i = 0;
1210 				*ep = 0;
1211 				do {
1212 					ep++;
1213 				} while ((c = *ep) != NULL && c == ' ');
1214 				break;
1215 			}
1216 			ep++;
1217 			i++;
1218 		}
1219 		if (i >= 80 && wp) {
1220 			*wp = 0;
1221 			ep = wp + 1;
1222 			c = *ep;
1223 		}
1224 		(void) printf("%s%s\n", label, cp);
1225 		cp = ep;
1226 		label = padding;
1227 	}
1228 	free(padding);
1229 }
1230 
1231 static void
1232 print_dict_info(char *msgid, char *url)
1233 {
1234 	const char *cp;
1235 	char *l_url;
1236 	char *buf;
1237 	int bufsz;
1238 
1239 	cp = get_dict_msg(msgid, "description", 0, 1);
1240 	if (cp) {
1241 		if (url)
1242 			l_url = url;
1243 		else
1244 			l_url = get_dict_url(msgid);
1245 		bufsz = strlen(cp) + strlen(l_url) + 1;
1246 		buf = malloc(bufsz);
1247 		(void) snprintf(buf, bufsz, cp, l_url);
1248 		print_line(dgettext("FMD", "Description : "), buf);
1249 		free(buf);
1250 		if (!url)
1251 			free(l_url);
1252 	}
1253 	cp = get_dict_msg(msgid, "response", 0, 1);
1254 	if (cp) {
1255 		buf = strdup(cp);
1256 		print_line(dgettext("FMD", "Response    : "), buf);
1257 		free(buf);
1258 	}
1259 	cp = get_dict_msg(msgid, "impact", 0, 1);
1260 	if (cp) {
1261 		buf = strdup(cp);
1262 		print_line(dgettext("FMD", "Impact      : "), buf);
1263 		free(buf);
1264 	}
1265 	cp = get_dict_msg(msgid, "action", 0, 1);
1266 	if (cp) {
1267 		buf = strdup(cp);
1268 		print_line(dgettext("FMD", "Action      : "), buf);
1269 		free(buf);
1270 	}
1271 }
1272 
1273 static void
1274 print_name(name_list_t *list, char *(func)(char *), char *padding, int *np,
1275     int pct, int full)
1276 {
1277 	char *name, *fru = NULL;
1278 
1279 	name = list->name;
1280 	if (func)
1281 		fru = func(list->name);
1282 	if (fru) {
1283 		(void) printf("%s \"%s\" (%s)", padding, fru, name);
1284 		*np += 1;
1285 		free(fru);
1286 	} else {
1287 		(void) printf("%s %s", padding, name);
1288 		*np += 1;
1289 	}
1290 	if (list->pct && pct > 0 && pct < 100) {
1291 		if (list->count > 1) {
1292 			if (full) {
1293 				(void) printf(" %d @ %s %d%%\n", list->count,
1294 				    dgettext("FMD", "max"),
1295 				    list->max_pct);
1296 			} else {
1297 				(void) printf(" %s %d%%\n",
1298 				    dgettext("FMD", "max"),
1299 				    list->max_pct);
1300 			}
1301 		} else {
1302 			(void) printf(" %d%%\n", list->pct);
1303 		}
1304 	} else {
1305 		(void) printf("\n");
1306 	}
1307 }
1308 
1309 static void
1310 print_asru_status(int status, char *label)
1311 {
1312 	char *msg = NULL;
1313 
1314 	switch (status) {
1315 	case 0:
1316 		msg = dgettext("FMD", "ok and in service");
1317 		break;
1318 	case FM_SUSPECT_FAULTY:
1319 		msg = dgettext("FMD", "degraded but still in service");
1320 		break;
1321 	case FM_SUSPECT_UNUSABLE:
1322 		msg = dgettext("FMD", "unknown, not present or disabled");
1323 		break;
1324 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1325 		msg = dgettext("FMD", "faulted and taken out of service");
1326 		break;
1327 	default:
1328 		break;
1329 	}
1330 	if (msg) {
1331 		(void) printf("%s     %s\n", label, msg);
1332 	}
1333 }
1334 
1335 static void
1336 print_fru_status(int status, char *label)
1337 {
1338 	char *msg = NULL;
1339 
1340 	if (status & FM_SUSPECT_NOT_PRESENT)
1341 		msg = dgettext("FMD", "not present");
1342 	else if (status & FM_SUSPECT_FAULTY)
1343 		msg = dgettext("FMD", "faulty");
1344 	else
1345 		msg = dgettext("FMD", "repaired");
1346 	(void) printf("%s     %s\n", label, msg);
1347 }
1348 
1349 static void
1350 print_name_list(name_list_t *list, char *label, char *(func)(char *),
1351     int limit, int pct, void (func1)(int, char *), int full)
1352 {
1353 	char *name, *fru = NULL;
1354 	char *padding;
1355 	int i, j, l, n;
1356 	name_list_t *end = list;
1357 
1358 	l = strlen(label);
1359 	padding = malloc(l + 1);
1360 	for (i = 0; i < l; i++)
1361 		padding[i] = ' ';
1362 	padding[l] = 0;
1363 	(void) printf("%s", label);
1364 	name = list->name;
1365 	if (func == NULL)
1366 		(void) printf(" %s", name);
1367 	else if (list->label)
1368 		(void) printf(" \"%s\" (%s)", list->label, name);
1369 	else {
1370 		fru = func(list->name);
1371 		if (fru) {
1372 			(void) printf(" \"%s\" (%s)", fru, name);
1373 			free(fru);
1374 		} else
1375 			(void) printf(" %s", name);
1376 	}
1377 	if (list->pct && pct > 0 && pct < 100) {
1378 		if (list->count > 1) {
1379 			if (full) {
1380 				(void) printf(" %d @ %s %d%%\n", list->count,
1381 				    dgettext("FMD", "max"), list->max_pct);
1382 			} else {
1383 				(void) printf(" %s %d%%\n",
1384 				    dgettext("FMD", "max"), list->max_pct);
1385 			}
1386 		} else {
1387 			(void) printf(" %d%%\n", list->pct);
1388 		}
1389 	} else {
1390 		(void) printf("\n");
1391 	}
1392 	if (func1)
1393 		func1(list->status, padding);
1394 	n = 1;
1395 	j = 0;
1396 	while ((list = list->next) != end) {
1397 		if (limit == 0 || n < limit) {
1398 			print_name(list, func, padding, &n, pct, full);
1399 			if (func1)
1400 				func1(list->status, padding);
1401 		} else
1402 			j++;
1403 	}
1404 	if (j == 1) {
1405 		print_name(list->prev, func, padding, &n, pct, full);
1406 	} else if (j > 1) {
1407 		(void) printf("%s... %d %s\n", padding, j,
1408 		    dgettext("FMD", "more entries suppressed,"
1409 		    " use -v option for full list"));
1410 	}
1411 	free(padding);
1412 }
1413 
1414 static int
1415 asru_same_status(name_list_t *list)
1416 {
1417 	name_list_t *end = list;
1418 	int status = list->status;
1419 
1420 	while ((list = list->next) != end) {
1421 		if (status == -1) {
1422 			status = list->status;
1423 			continue;
1424 		}
1425 		if (list->status != -1 && status != list->status) {
1426 			status = -1;
1427 			break;
1428 		}
1429 	}
1430 	return (status);
1431 }
1432 
1433 static int
1434 serial_in_fru(name_list_t *fru, name_list_t *serial)
1435 {
1436 	name_list_t *sp = serial;
1437 	name_list_t *fp;
1438 	int nserial = 0;
1439 	int found = 0;
1440 	char buf[128];
1441 
1442 	while (sp) {
1443 		fp = fru;
1444 		nserial++;
1445 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1446 		buf[sizeof (buf) - 1] = 0;
1447 		while (fp) {
1448 			if (strstr(fp->name, buf) != NULL) {
1449 				found++;
1450 				break;
1451 			}
1452 			fp = fp->next;
1453 			if (fp == fru)
1454 				break;
1455 		}
1456 		sp = sp->next;
1457 		if (sp == serial)
1458 			break;
1459 	}
1460 	return (found == nserial ? 1 : 0);
1461 }
1462 
1463 static void
1464 print_server_name(hostid_t *host, char *label)
1465 {
1466 	(void) printf("%s %s %s %s\n", label, host->server, host->platform,
1467 	    host->chassis ? host->chassis : "");
1468 }
1469 
1470 static void
1471 print_sup_record(status_record_t *srp, int opt_i, int full)
1472 {
1473 	char buf[32];
1474 	uurec_t *uurp = srp->uurec;
1475 	int n, j, k, max;
1476 	int status;
1477 	ari_list_t *ari_list;
1478 
1479 	n = 0;
1480 	max = max_fault;
1481 	if (max < 0) {
1482 		max = 0;
1483 	}
1484 	j = max / 2;
1485 	max -= j;
1486 	k = srp->nrecs - max;
1487 	while ((uurp = uurp->next) != NULL) {
1488 		if (full || n < j || n >= k || max_fault == 0 ||
1489 		    srp->nrecs == max_fault+1) {
1490 			if (opt_i) {
1491 				ari_list = uurp->ari_uuid_list;
1492 				while (ari_list) {
1493 					(void) printf("%-15s %s\n",
1494 					    format_date(buf, sizeof (buf),
1495 					    uurp->sec), ari_list->ari_uuid);
1496 					ari_list = ari_list->next;
1497 				}
1498 			} else {
1499 				(void) printf("%-15s %s\n",
1500 				    format_date(buf, sizeof (buf), uurp->sec),
1501 				    uurp->uuid);
1502 			}
1503 		} else if (n == j)
1504 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1505 			    dgettext("FMD", "more entries suppressed"));
1506 		n++;
1507 	}
1508 	(void) printf("\n");
1509 	if (n_server > 1)
1510 		print_server_name(srp->host, dgettext("FMD", "Host        :"));
1511 	if (srp->class)
1512 		print_name_list(srp->class,
1513 		    dgettext("FMD", "Fault class :"), NULL, 0, srp->class->pct,
1514 		    NULL, full);
1515 	if (srp->asru) {
1516 		status = asru_same_status(srp->asru);
1517 		if (status != -1) {
1518 			print_name_list(srp->asru,
1519 			    dgettext("FMD", "Affects     :"), NULL,
1520 			    full ? 0 : max_display, 0, NULL, full);
1521 			print_asru_status(status, "             ");
1522 		} else
1523 			print_name_list(srp->asru,
1524 			    dgettext("FMD", "Affects     :"), NULL,
1525 			    full ? 0 : max_display, 0, print_asru_status, full);
1526 	}
1527 	if (full || srp->fru == NULL) {
1528 		if (srp->resource) {
1529 			print_name_list(srp->resource,
1530 			    dgettext("FMD", "Problem in  :"),
1531 			    NULL, full ? 0 : max_display, 0, print_fru_status,
1532 			    full);
1533 		}
1534 	}
1535 	if (srp->fru) {
1536 		status = asru_same_status(srp->fru);
1537 		if (status != -1) {
1538 			print_name_list(srp->fru, dgettext("FMD",
1539 			    "FRU         :"), get_fmri_label, 0,
1540 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1541 			    NULL, full);
1542 			print_fru_status(status, "             ");
1543 		} else
1544 			print_name_list(srp->fru, dgettext("FMD",
1545 			    "FRU         :"), get_fmri_label, 0,
1546 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1547 			    print_fru_status, full);
1548 	}
1549 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1550 	    !serial_in_fru(srp->asru, srp->serial)) {
1551 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1552 		    NULL, 0, 0, NULL, full);
1553 	}
1554 	print_dict_info(srp->msgid, srp->url);
1555 	(void) printf("\n");
1556 }
1557 
1558 static void
1559 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1560 {
1561 	char buf[32];
1562 	uurec_t *uurp = srp->uurec;
1563 	char *severity;
1564 	static int header = 0;
1565 	char *head;
1566 	ari_list_t *ari_list;
1567 
1568 	if (nlspath)
1569 		severity = get_dict_msg(srp->msgid, "severity", 1, 1);
1570 	else
1571 		severity = srp->severity;
1572 
1573 	if (!summary || !header) {
1574 		if (opt_i) {
1575 			head = "--------------- "
1576 			    "------------------------------------  "
1577 			    "-------------- ---------\n"
1578 			    "TIME            CACHE-ID"
1579 			    "                              MSG-ID"
1580 			    "         SEVERITY\n--------------- "
1581 			    "------------------------------------ "
1582 			    " -------------- ---------";
1583 		} else {
1584 			head = "--------------- "
1585 			    "------------------------------------  "
1586 			    "-------------- ---------\n"
1587 			    "TIME            EVENT-ID"
1588 			    "                              MSG-ID"
1589 			    "         SEVERITY\n--------------- "
1590 			    "------------------------------------ "
1591 			    " -------------- ---------";
1592 		}
1593 		(void) printf("%s\n", dgettext("FMD", head));
1594 		header = 1;
1595 	}
1596 	if (opt_i) {
1597 		ari_list = uurp->ari_uuid_list;
1598 		while (ari_list) {
1599 			(void) printf("%-15s %-37s %-14s %-9s\n",
1600 			    format_date(buf, sizeof (buf), uurp->sec),
1601 			    ari_list->ari_uuid, srp->msgid, severity);
1602 			ari_list = ari_list->next;
1603 		}
1604 	} else {
1605 		(void) printf("%-15s %-37s %-14s %-9s\n",
1606 		    format_date(buf, sizeof (buf), uurp->sec),
1607 		    uurp->uuid, srp->msgid, severity);
1608 	}
1609 
1610 	if (!summary)
1611 		print_sup_record(srp, opt_i, full);
1612 }
1613 
1614 static void
1615 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1616 {
1617 	status_record_t *srp;
1618 	sr_list_t *slp;
1619 
1620 	slp = status_rec_list;
1621 	if (slp) {
1622 		for (;;) {
1623 			srp = slp->status_record;
1624 			if (opt_a || srp->not_suppressed) {
1625 				if (page_feed)
1626 					(void) printf("\f\n");
1627 				print_status_record(srp, summary, opt_i, full);
1628 			}
1629 			if (slp->next == status_rec_list)
1630 				break;
1631 			slp = slp->next;
1632 		}
1633 	}
1634 }
1635 
1636 static name_list_t *
1637 find_fru(status_record_t *srp, char *resource)
1638 {
1639 	name_list_t *rt = NULL;
1640 	name_list_t *fru = srp->fru;
1641 
1642 	while (fru) {
1643 		if (strcmp(resource, fru->name) == 0) {
1644 			rt = fru;
1645 			break;
1646 		}
1647 		fru = fru->next;
1648 		if (fru == srp->fru)
1649 			break;
1650 	}
1651 	return (rt);
1652 }
1653 
1654 static void
1655 print_fru_line(name_list_t *fru, char *uuid)
1656 {
1657 	if (fru->pct == 100) {
1658 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1659 		    dgettext("FMD", "suspects in this FRU total certainty"),
1660 		    100);
1661 	} else {
1662 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1663 		    dgettext("FMD", "suspects in this FRU max certainty"),
1664 		    fru->max_pct);
1665 	}
1666 }
1667 
1668 static void
1669 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1670 {
1671 	resource_list_t *tp = status_fru_list;
1672 	status_record_t *srp;
1673 	sr_list_t *slp, *end;
1674 	char *msgid, *fru_label;
1675 	uurec_t *uurp;
1676 	name_list_t *fru;
1677 	int status;
1678 	ari_list_t *ari_list;
1679 
1680 	while (tp) {
1681 		if (opt_a || tp->not_suppressed) {
1682 			if (page_feed)
1683 				(void) printf("\f\n");
1684 			if (!summary)
1685 				(void) printf("-----------------------------"
1686 				    "---------------------------------------"
1687 				    "----------\n");
1688 			slp = tp->status_rec_list;
1689 			end = slp;
1690 			do {
1691 				srp = slp->status_record;
1692 				fru = find_fru(srp, tp->resource);
1693 				if (fru) {
1694 					if (fru->label)
1695 						(void) printf("\"%s\" (%s) ",
1696 						    fru->label, fru->name);
1697 					else if ((fru_label = get_fmri_label(
1698 					    fru->name)) != NULL) {
1699 						(void) printf("\"%s\" (%s) ",
1700 						    fru_label, fru->name);
1701 						free(fru_label);
1702 					} else
1703 						(void) printf("%s ",
1704 						    fru->name);
1705 					break;
1706 				}
1707 				slp = slp->next;
1708 			} while (slp != end);
1709 
1710 			slp = tp->status_rec_list;
1711 			end = slp;
1712 			status = 0;
1713 			do {
1714 				srp = slp->status_record;
1715 				fru = srp->fru;
1716 				while (fru) {
1717 					if (strcmp(tp->resource,
1718 					    fru->name) == 0)
1719 						status |= fru->status;
1720 					fru = fru->next;
1721 					if (fru == srp->fru)
1722 						break;
1723 				}
1724 				slp = slp->next;
1725 			} while (slp != end);
1726 			if (status & FM_SUSPECT_NOT_PRESENT)
1727 				(void) printf(dgettext("FMD", "not present\n"));
1728 			else if (status & FM_SUSPECT_FAULTY)
1729 				(void) printf(dgettext("FMD", "faulty\n"));
1730 			else
1731 				(void) printf(dgettext("FMD", "repaired\n"));
1732 
1733 			slp = tp->status_rec_list;
1734 			end = slp;
1735 			do {
1736 				srp = slp->status_record;
1737 				uurp = srp->uurec;
1738 				fru = find_fru(srp, tp->resource);
1739 				if (fru) {
1740 					if (opt_i) {
1741 						ari_list = uurp->ari_uuid_list;
1742 						while (ari_list) {
1743 							print_fru_line(fru,
1744 							    ari_list->ari_uuid);
1745 							ari_list =
1746 							    ari_list->next;
1747 						}
1748 					} else {
1749 						print_fru_line(fru, uurp->uuid);
1750 					}
1751 				}
1752 				slp = slp->next;
1753 			} while (slp != end);
1754 			if (!summary) {
1755 				slp = tp->status_rec_list;
1756 				end = slp;
1757 				srp = slp->status_record;
1758 				if (srp->serial &&
1759 				    !serial_in_fru(srp->fru, srp->serial)) {
1760 					print_name_list(srp->serial,
1761 					    dgettext("FMD", "Serial ID.  :"),
1762 					    NULL, 0, 0, NULL, 1);
1763 				}
1764 				msgid = NULL;
1765 				do {
1766 					if (msgid == NULL ||
1767 					    strcmp(msgid, srp->msgid) != 0) {
1768 						msgid = srp->msgid;
1769 						print_dict_info(srp->msgid,
1770 						    srp->url);
1771 					}
1772 					slp = slp->next;
1773 				} while (slp != end);
1774 			}
1775 		}
1776 		tp = tp->next;
1777 		if (tp == status_fru_list)
1778 			break;
1779 	}
1780 }
1781 
1782 static void
1783 print_asru(int opt_a)
1784 {
1785 	resource_list_t *tp = status_asru_list;
1786 	status_record_t *srp;
1787 	sr_list_t *slp, *end;
1788 	char *msg;
1789 	int status;
1790 	name_list_t *asru;
1791 
1792 	while (tp) {
1793 		if (opt_a || tp->not_suppressed) {
1794 			status = 0;
1795 			slp = tp->status_rec_list;
1796 			end = slp;
1797 			do {
1798 				srp = slp->status_record;
1799 				asru = srp->asru;
1800 				while (asru) {
1801 					if (strcmp(tp->resource,
1802 					    asru->name) == 0)
1803 						status |= asru->status;
1804 					asru = asru->next;
1805 					if (asru == srp->asru)
1806 						break;
1807 				}
1808 				slp = slp->next;
1809 			} while (slp != end);
1810 			switch (status) {
1811 			case 0:
1812 				msg = dgettext("FMD", "ok");
1813 				break;
1814 			case FM_SUSPECT_FAULTY:
1815 				msg = dgettext("FMD", "degraded");
1816 				break;
1817 			case FM_SUSPECT_UNUSABLE:
1818 				msg = dgettext("FMD", "unknown");
1819 				break;
1820 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1821 				msg = dgettext("FMD", "faulted");
1822 				break;
1823 			default:
1824 				msg = "";
1825 				break;
1826 			}
1827 			(void) printf("%-69s %s\n", tp->resource, msg);
1828 		}
1829 		tp = tp->next;
1830 		if (tp == status_asru_list)
1831 			break;
1832 	}
1833 }
1834 
1835 static int
1836 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1837 {
1838 	while (uurecp) {
1839 		if (strcmp(uuid, uurecp->uuid) == 0)
1840 			return (1);
1841 		uurecp = uurecp->next;
1842 	}
1843 	return (0);
1844 }
1845 
1846 static int
1847 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1848 {
1849 	int64_t *diag_time;
1850 	uint_t nelem;
1851 	int rt = 0;
1852 	char *uuid = "-";
1853 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1854 
1855 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1856 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1857 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1858 		    &uuid);
1859 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1860 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1861 			    uuid, acp->aci_url);
1862 	} else {
1863 		rt = -1;
1864 	}
1865 	return (rt);
1866 }
1867 
1868 /*ARGSUSED*/
1869 static int
1870 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1871 {
1872 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1873 	return (0);
1874 }
1875 
1876 static int
1877 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1878 {
1879 	int rt = FMADM_EXIT_SUCCESS;
1880 
1881 	/*
1882 	 * These calls may fail with Protocol error if message payload is to big
1883 	 */
1884 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1885 		die("failed to get case list from fmd");
1886 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1887 		die("failed to get case status from fmd");
1888 	return (rt);
1889 }
1890 
1891 /*
1892  * fmadm faulty command
1893  *
1894  *	-a		show hidden fault records
1895  *	-f		show faulty fru's
1896  *	-g		force grouping of similar faults on the same fru
1897  *	-n		number of fault records to display
1898  *	-p		pipe output through pager
1899  *	-r		show faulty asru's
1900  *	-s		print summary of first fault
1901  *	-u		print listed uuid's only
1902  *	-v		full output
1903  */
1904 
1905 int
1906 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1907 {
1908 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1909 	int opt_i = 0;
1910 	char *pager;
1911 	FILE *fp;
1912 	int rt, c, stat;
1913 	uurec_select_t *tp;
1914 	uurec_select_t *uurecp = NULL;
1915 
1916 	catalog_setup();
1917 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1918 		switch (c) {
1919 		case 'a':
1920 			opt_a++;
1921 			break;
1922 		case 'f':
1923 			opt_f++;
1924 			break;
1925 		case 'g':
1926 			opt_g++;
1927 			break;
1928 		case 'i':
1929 			opt_i++;
1930 			break;
1931 		case 'n':
1932 			max_fault = atoi(optarg);
1933 			break;
1934 		case 'p':
1935 			opt_p++;
1936 			break;
1937 		case 'r':
1938 			opt_r++;
1939 			break;
1940 		case 's':
1941 			opt_s++;
1942 			break;
1943 		case 'u':
1944 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1945 			tp->uuid = optarg;
1946 			tp->next = uurecp;
1947 			uurecp = tp;
1948 			opt_a = 1;
1949 			break;
1950 		case 'v':
1951 			opt_v++;
1952 			break;
1953 		default:
1954 			return (FMADM_EXIT_USAGE);
1955 		}
1956 	}
1957 	if (optind < argc)
1958 		return (FMADM_EXIT_USAGE);
1959 
1960 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1961 	if (opt_p) {
1962 		if ((pager = getenv("PAGER")) == NULL)
1963 			pager = "/usr/bin/more";
1964 		fp = popen(pager, "w");
1965 		if (fp == NULL) {
1966 			rt = FMADM_EXIT_ERROR;
1967 			opt_p = 0;
1968 		} else {
1969 			dup2(fileno(fp), 1);
1970 			setbuf(stdout, NULL);
1971 			(void) fclose(fp);
1972 		}
1973 	}
1974 	max_display = max_fault;
1975 	if (opt_f)
1976 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1977 	if (opt_r)
1978 		print_asru(opt_a);
1979 	if (opt_f == 0 && opt_r == 0)
1980 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1981 	label_release_topo();
1982 	if (opt_p) {
1983 		(void) fclose(stdout);
1984 		(void) wait(&stat);
1985 	}
1986 	return (rt);
1987 }
1988 
1989 int
1990 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1991 {
1992 	int i, status = FMADM_EXIT_SUCCESS;
1993 
1994 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1995 		return (FMADM_EXIT_USAGE);
1996 
1997 	for (i = 1; i < argc; i++) {
1998 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1999 			warn("failed to flush %s", argv[i]);
2000 			status = FMADM_EXIT_ERROR;
2001 		} else
2002 			note("flushed resource history for %s\n", argv[i]);
2003 	}
2004 
2005 	return (status);
2006 }
2007 
2008 int
2009 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
2010 {
2011 	int err;
2012 
2013 	if (getopt(argc, argv, "") != EOF)
2014 		return (FMADM_EXIT_USAGE);
2015 
2016 	if (argc - optind != 1)
2017 		return (FMADM_EXIT_USAGE);
2018 
2019 	/*
2020 	 * argument could be a uuid, and fmri (asru, fru or resource)
2021 	 * or a label. Try uuid first, If that fails try the others.
2022 	 */
2023 	err = fmd_adm_case_repair(adm, argv[optind]);
2024 	if (err != 0)
2025 		err = fmd_adm_rsrc_repair(adm, argv[optind]);
2026 
2027 	if (err != 0)
2028 		die("failed to record repair to %s", argv[optind]);
2029 
2030 	note("recorded repair to %s\n", argv[optind]);
2031 	return (FMADM_EXIT_SUCCESS);
2032 }
2033