xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 7adb68a6)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
544743693Sstephh  * Common Development and Distribution License (the "License").
644743693Sstephh  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22e58a33b6SStephen Hanson  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23*7adb68a6SHans Rosenfeld  * Copyright 2020 Joyent, Inc.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
2644743693Sstephh #include <sys/types.h>
277c478bd9Sstevel@tonic-gate #include <fmadm.h>
2844743693Sstephh #include <errno.h>
2944743693Sstephh #include <limits.h>
3044743693Sstephh #include <strings.h>
3144743693Sstephh #include <stdio.h>
3244743693Sstephh #include <unistd.h>
3344743693Sstephh #include <sys/wait.h>
3444743693Sstephh #include <sys/stat.h>
3544743693Sstephh #include <fcntl.h>
3644743693Sstephh #include <fm/fmd_log.h>
3744743693Sstephh #include <sys/fm/protocol.h>
3844743693Sstephh #include <fm/libtopo.h>
3944743693Sstephh #include <fm/fmd_adm.h>
40b6955755SRobert Johnston #include <fm/fmd_msg.h>
4144743693Sstephh #include <dlfcn.h>
4244743693Sstephh #include <sys/systeminfo.h>
4344743693Sstephh #include <sys/utsname.h>
4444743693Sstephh #include <libintl.h>
4544743693Sstephh #include <locale.h>
4644743693Sstephh #include <sys/smbios.h>
4744743693Sstephh #include <libdevinfo.h>
4844743693Sstephh #include <stdlib.h>
495c5f1371SRichard Lowe #include <stddef.h>
5044743693Sstephh 
5144743693Sstephh /*
5244743693Sstephh  * Fault records are added to catalog by calling add_fault_record_to_catalog()
5344743693Sstephh  * records are stored in order of importance to the system.
5444743693Sstephh  * If -g flag is set or not_suppressed is not set and the class fru, fault,
5544743693Sstephh  * type are the same then details are merged into an existing record, with uuid
5644743693Sstephh  * records are stored in time order.
5744743693Sstephh  * For each record information is extracted from nvlist and merged into linked
5844743693Sstephh  * list each is checked for identical records for which percentage certainty are
5944743693Sstephh  * added together.
6044743693Sstephh  * print_catalog() is called to print out catalog and release external resources
6144743693Sstephh  *
6244743693Sstephh  *                         /---------------\
6344743693Sstephh  *	status_rec_list -> |               | -|
6444743693Sstephh  *                         \---------------/
6544743693Sstephh  *                                \/
6644743693Sstephh  *                         /---------------\    /-------\    /-------\
6744743693Sstephh  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
6844743693Sstephh  *            \/           |               | |- |       | <- |       |
6944743693Sstephh  *      /-------------\    |               |    \-------/    \-------/
7044743693Sstephh  *      |             | -> |               |       \/           \/
7144743693Sstephh  *      \-------------/    |               |    /-------\    /-------\
7244743693Sstephh  *            \/           |               | -> | asru  | -> | asru  |
7344743693Sstephh  *            ---          |               |    |       | <- |       |
7444743693Sstephh  *                         |               |    \-------/    \-------/
7544743693Sstephh  *      status_asru_list   |  class        |
7644743693Sstephh  *            \/           |  resource     |    /-------\    /-------\
7744743693Sstephh  *      /-------------\    |  fru          | -> | list  | -> | list  |
7844743693Sstephh  *      |             | -> |  serial       |    |       | <- |       |
7944743693Sstephh  *      \-------------/    |               |    \-------/    \-------/
8044743693Sstephh  *            \/           \---------------/
8144743693Sstephh  *            ---               \/    /\
8244743693Sstephh  *                         /---------------\
8344743693Sstephh  *                         | status_record |
8444743693Sstephh  *                         \---------------/
8544743693Sstephh  *
8644743693Sstephh  * Fmadm faulty takes a number of options which affect the format of the
8744743693Sstephh  * output displayed. By default, the display reports the FRU and ASRU along
8844743693Sstephh  * with other information on per-case basis as in the example below.
8944743693Sstephh  *
9044743693Sstephh  * --------------- ------------------------------------  -------------- -------
9144743693Sstephh  * TIME            EVENT-ID                              MSG-ID         SEVERITY
9244743693Sstephh  * --------------- ------------------------------------  -------------- -------
9344743693Sstephh  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
9444743693Sstephh  *
9544743693Sstephh  * Fault class	: fault.memory.dimm_sb
9644743693Sstephh  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
9725c6ff4bSstephh  *		    faulted but still in service
9844743693Sstephh  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
9944743693Sstephh  *		    faulty
10044743693Sstephh  *
10144743693Sstephh  * Description	: The number of errors associated with this memory module has
10244743693Sstephh  *		exceeded acceptable levels.  Refer to
103654b400cSJoshua M. Clulow  *		http://illumos.org/msg/AMD-8000-2F for more information.
10444743693Sstephh  *
10544743693Sstephh  * Response	: Pages of memory associated with this memory module are being
10644743693Sstephh  *		removed from service as errors are reported.
10744743693Sstephh  *
10844743693Sstephh  * Impact	: Total system memory capacity will be reduced as pages are
10944743693Sstephh  *		retired.
11044743693Sstephh  *
11144743693Sstephh  * Action	: Schedule a repair procedure to replace the affected memory
11244743693Sstephh  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
11344743693Sstephh  *
11444743693Sstephh  * The -v flag is similar, but adds some additonal information such as the
11544743693Sstephh  * resource. The -s flag is also similar but just gives the top line summary.
11644743693Sstephh  * All these options (ie without the -f or -r flags) use the print_catalog()
11744743693Sstephh  * function to do the display.
11844743693Sstephh  *
11944743693Sstephh  * The -f flag changes the output so that it appears sorted on a per-fru basis.
12044743693Sstephh  * The output is somewhat cut down compared to the default output. If -f is
12144743693Sstephh  * used, then print_fru() is used to print the output.
12244743693Sstephh  *
12344743693Sstephh  * -----------------------------------------------------------------------------
12444743693Sstephh  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
12544743693Sstephh  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
12644743693Sstephh  *
12744743693Sstephh  * Description	: A problem was detected for a PCI device.
128654b400cSJoshua M. Clulow  *		Refer to http://illumos.org/msg/PCI-8000-7J
129654b400cSJoshua M. Clulow  *		for more information.
13044743693Sstephh  *
13144743693Sstephh  * Response	: One or more device instances may be disabled
13244743693Sstephh  *
13344743693Sstephh  * Impact	: Possible loss of services provided by the device instances
13444743693Sstephh  *		associated with this fault
13544743693Sstephh  *
13644743693Sstephh  * Action	: Schedule a repair procedure to replace the affected device.
137940f2f58SToomas Soome  *		Use fmdump -v -u <EVENT_ID> to identify the device or contact
13844743693Sstephh  *		Sun for support.
13944743693Sstephh  *
14044743693Sstephh  * The -r flag changes the output so that it appears sorted on a per-asru basis.
14144743693Sstephh  * The output is very much cut down compared to the default output, just giving
14244743693Sstephh  * the asru fmri and state. Here print_asru() is used to print the output.
14344743693Sstephh  *
14444743693Sstephh  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
14544743693Sstephh  *
14644743693Sstephh  * For all fmadm faulty options, the sequence of events is
14744743693Sstephh  *
14844743693Sstephh  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
14944743693Sstephh  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
15044743693Sstephh  * This will extract the data from the nvlist and call catalog_new_record() to
15144743693Sstephh  * save the data away in various linked lists in the catalogue.
15244743693Sstephh  *
15344743693Sstephh  * 2) Once this is done, the data can be supplemented by using
15444743693Sstephh  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
15544743693Sstephh  *
15644743693Sstephh  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
15744743693Sstephh  * appropriate to display the information from the catalogue sorted in the
15844743693Sstephh  * requested way.
15944743693Sstephh  *
16044743693Sstephh  */
16144743693Sstephh 
16244743693Sstephh typedef struct name_list {
16344743693Sstephh 	struct name_list *next;
16444743693Sstephh 	struct name_list *prev;
16544743693Sstephh 	char *name;
16644743693Sstephh 	uint8_t pct;
16744743693Sstephh 	uint8_t max_pct;
16844743693Sstephh 	ushort_t count;
16944743693Sstephh 	int status;
170d4c0a8c5Sstephh 	char *label;
17144743693Sstephh } name_list_t;
17244743693Sstephh 
17344743693Sstephh typedef struct ari_list {
17444743693Sstephh 	char *ari_uuid;
17544743693Sstephh 	struct ari_list *next;
17644743693Sstephh } ari_list_t;
17744743693Sstephh 
17844743693Sstephh typedef struct uurec {
17944743693Sstephh 	struct uurec *next;
18044743693Sstephh 	struct uurec *prev;
18144743693Sstephh 	char *uuid;
18244743693Sstephh 	ari_list_t *ari_uuid_list;
18344743693Sstephh 	name_list_t *asru;
18444743693Sstephh 	uint64_t sec;
18588045cffSRobert Johnston 	nvlist_t *event;
18644743693Sstephh } uurec_t;
18744743693Sstephh 
18844743693Sstephh typedef struct uurec_select {
18944743693Sstephh 	struct uurec_select *next;
19044743693Sstephh 	char *uuid;
19144743693Sstephh } uurec_select_t;
19244743693Sstephh 
19344743693Sstephh typedef struct host_id {
19444743693Sstephh 	char *chassis;
19544743693Sstephh 	char *server;
19644743693Sstephh 	char *platform;
197cbf75e67SStephen Hanson 	char *domain;
1989c94f155SCheng Sean Ye 	char *product_sn;
19944743693Sstephh } hostid_t;
20044743693Sstephh 
20144743693Sstephh typedef struct host_id_list {
20244743693Sstephh 	hostid_t hostid;
20344743693Sstephh 	struct host_id_list *next;
20444743693Sstephh } host_id_list_t;
20544743693Sstephh 
20644743693Sstephh typedef struct status_record {
20744743693Sstephh 	hostid_t *host;
20844743693Sstephh 	int nrecs;
20944743693Sstephh 	uurec_t *uurec;
21044743693Sstephh 	char *severity;			/* in C locale */
21144743693Sstephh 	char *msgid;
21244743693Sstephh 	name_list_t *class;
21344743693Sstephh 	name_list_t *resource;
21444743693Sstephh 	name_list_t *asru;
21544743693Sstephh 	name_list_t *fru;
21644743693Sstephh 	name_list_t *serial;
21744743693Sstephh 	uint8_t not_suppressed;
218540db9a9SStephen Hanson 	uint8_t injected;
21944743693Sstephh } status_record_t;
22044743693Sstephh 
22144743693Sstephh typedef struct sr_list {
22244743693Sstephh 	struct sr_list *next;
22344743693Sstephh 	struct sr_list *prev;
22444743693Sstephh 	struct status_record *status_record;
22544743693Sstephh } sr_list_t;
22644743693Sstephh 
22744743693Sstephh typedef struct resource_list {
22844743693Sstephh 	struct resource_list *next;
22944743693Sstephh 	struct resource_list *prev;
23044743693Sstephh 	sr_list_t *status_rec_list;
23144743693Sstephh 	char *resource;
23244743693Sstephh 	uint8_t not_suppressed;
233540db9a9SStephen Hanson 	uint8_t injected;
23444743693Sstephh 	uint8_t max_pct;
23544743693Sstephh } resource_list_t;
23644743693Sstephh 
23744743693Sstephh sr_list_t *status_rec_list;
23844743693Sstephh resource_list_t *status_fru_list;
23944743693Sstephh resource_list_t *status_asru_list;
24044743693Sstephh 
24144743693Sstephh static int max_display;
24244743693Sstephh static int max_fault = 0;
24344743693Sstephh static topo_hdl_t *topo_handle;
24444743693Sstephh static host_id_list_t *host_list;
24544743693Sstephh static int n_server;
24644743693Sstephh static int opt_g;
247b6955755SRobert Johnston static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */
24844743693Sstephh 
24944743693Sstephh static char *
format_date(char * buf,size_t len,uint64_t sec)25044743693Sstephh format_date(char *buf, size_t len, uint64_t sec)
25144743693Sstephh {
25244743693Sstephh 	if (sec > LONG_MAX) {
25344743693Sstephh 		(void) fprintf(stderr,
25444743693Sstephh 		    "record time is too large for 32-bit utility\n");
25544743693Sstephh 		(void) snprintf(buf, len, "0x%llx", sec);
25644743693Sstephh 	} else {
25744743693Sstephh 		time_t tod = (time_t)sec;
258540db9a9SStephen Hanson 		time_t now = time(NULL);
259540db9a9SStephen Hanson 		if (tod > now+60 ||
260540db9a9SStephen Hanson 		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
261540db9a9SStephen Hanson 			(void) strftime(buf, len, "%b %d %Y    ",
262540db9a9SStephen Hanson 			    localtime(&tod));
263540db9a9SStephen Hanson 		} else {
264540db9a9SStephen Hanson 			(void) strftime(buf, len, "%b %d %T", localtime(&tod));
265540db9a9SStephen Hanson 		}
26644743693Sstephh 	}
26744743693Sstephh 
26844743693Sstephh 	return (buf);
26944743693Sstephh }
27044743693Sstephh 
27144743693Sstephh static hostid_t *
find_hostid_in_list(char * platform,char * chassis,char * server,char * domain,char * product_sn)2729c94f155SCheng Sean Ye find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
2739c94f155SCheng Sean Ye     char *product_sn)
27444743693Sstephh {
27544743693Sstephh 	hostid_t *rt = NULL;
27644743693Sstephh 	host_id_list_t *hostp;
27744743693Sstephh 
27844743693Sstephh 	if (platform == NULL)
27944743693Sstephh 		platform = "-";
28044743693Sstephh 	if (server == NULL)
28144743693Sstephh 		server = "-";
28244743693Sstephh 	hostp = host_list;
28344743693Sstephh 	while (hostp) {
28444743693Sstephh 		if (hostp->hostid.platform &&
28544743693Sstephh 		    strcmp(hostp->hostid.platform, platform) == 0 &&
28644743693Sstephh 		    hostp->hostid.server &&
28744743693Sstephh 		    strcmp(hostp->hostid.server, server) == 0 &&
28844743693Sstephh 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
289cbf75e67SStephen Hanson 		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
2909c94f155SCheng Sean Ye 		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
2919c94f155SCheng Sean Ye 		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
292cbf75e67SStephen Hanson 		    (domain == NULL || hostp->hostid.domain == NULL ||
293cbf75e67SStephen Hanson 		    strcmp(domain, hostp->hostid.domain) == 0)) {
29444743693Sstephh 			rt = &hostp->hostid;
29544743693Sstephh 			break;
29644743693Sstephh 		}
29744743693Sstephh 		hostp = hostp->next;
29844743693Sstephh 	}
29944743693Sstephh 	if (rt == NULL) {
30044743693Sstephh 		hostp = malloc(sizeof (host_id_list_t));
30144743693Sstephh 		hostp->hostid.platform = strdup(platform);
3029c94f155SCheng Sean Ye 		hostp->hostid.product_sn =
3039c94f155SCheng Sean Ye 		    product_sn ? strdup(product_sn) : NULL;
30444743693Sstephh 		hostp->hostid.server = strdup(server);
30544743693Sstephh 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
306cbf75e67SStephen Hanson 		hostp->hostid.domain = domain ? strdup(domain) : NULL;
30744743693Sstephh 		hostp->next = host_list;
30844743693Sstephh 		host_list = hostp;
30944743693Sstephh 		rt = &hostp->hostid;
31044743693Sstephh 		n_server++;
31144743693Sstephh 	}
31244743693Sstephh 	return (rt);
31344743693Sstephh }
31444743693Sstephh 
31544743693Sstephh static hostid_t *
find_hostid(nvlist_t * nvl)31644743693Sstephh find_hostid(nvlist_t *nvl)
31744743693Sstephh {
318cbf75e67SStephen Hanson 	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
3199c94f155SCheng Sean Ye 	char *product_sn = NULL;
32044743693Sstephh 	nvlist_t *auth, *fmri;
32144743693Sstephh 	hostid_t *rt = NULL;
32244743693Sstephh 
32344743693Sstephh 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
32444743693Sstephh 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
32544743693Sstephh 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
32644743693Sstephh 		    &platform);
3279c94f155SCheng Sean Ye 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
3289c94f155SCheng Sean Ye 		    &product_sn);
32944743693Sstephh 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
33044743693Sstephh 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
33144743693Sstephh 		    &chassis);
332cbf75e67SStephen Hanson 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
3339c94f155SCheng Sean Ye 		rt = find_hostid_in_list(platform, chassis, server,
3349c94f155SCheng Sean Ye 		    domain, product_sn);
33544743693Sstephh 	}
33644743693Sstephh 	return (rt);
33744743693Sstephh }
33844743693Sstephh 
33944743693Sstephh static char *
get_nvl2str_topo(nvlist_t * nvl)34044743693Sstephh get_nvl2str_topo(nvlist_t *nvl)
34144743693Sstephh {
34244743693Sstephh 	char *name = NULL;
34344743693Sstephh 	char *tname;
34444743693Sstephh 	int err;
34544743693Sstephh 	char *scheme = NULL;
34644743693Sstephh 	char *mod_name = NULL;
34744743693Sstephh 	char buf[128];
34844743693Sstephh 
34944743693Sstephh 	if (topo_handle == NULL)
350e58a33b6SStephen Hanson 		topo_handle = topo_open(TOPO_VERSION, 0, &err);
35144743693Sstephh 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
35244743693Sstephh 		name = strdup(tname);
35344743693Sstephh 		topo_hdl_strfree(topo_handle, tname);
35444743693Sstephh 	} else {
35544743693Sstephh 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
35644743693Sstephh 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
35744743693Sstephh 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
35844743693Sstephh 		    mod_name) {
35944743693Sstephh 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
36044743693Sstephh 			    scheme, mod_name);
36144743693Sstephh 			name = strdup(buf);
36244743693Sstephh 		}
36344743693Sstephh 	}
36444743693Sstephh 	return (name);
36544743693Sstephh }
36644743693Sstephh 
36744743693Sstephh static int
set_priority(char * s)36844743693Sstephh set_priority(char *s)
36944743693Sstephh {
37044743693Sstephh 	int rt = 0;
37144743693Sstephh 
37244743693Sstephh 	if (s) {
37344743693Sstephh 		if (strcmp(s, "Minor") == 0)
37444743693Sstephh 			rt = 1;
37544743693Sstephh 		else if (strcmp(s, "Major") == 0)
37644743693Sstephh 			rt = 10;
37744743693Sstephh 		else if (strcmp(s, "Critical") == 0)
37844743693Sstephh 			rt = 100;
37944743693Sstephh 	}
38044743693Sstephh 	return (rt);
38144743693Sstephh }
38244743693Sstephh 
38344743693Sstephh static int
cmp_priority(char * s1,char * s2,uint64_t t1,uint64_t t2,uint8_t p1,uint8_t p2)38444743693Sstephh cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
38544743693Sstephh     uint8_t p2)
38644743693Sstephh {
38744743693Sstephh 	int r1, r2;
38844743693Sstephh 	int rt;
38944743693Sstephh 
39044743693Sstephh 	r1 = set_priority(s1);
39144743693Sstephh 	r2 = set_priority(s2);
39244743693Sstephh 	rt = r1 - r2;
39344743693Sstephh 	if (rt == 0) {
39444743693Sstephh 		if (t1 > t2)
39544743693Sstephh 			rt = 1;
39644743693Sstephh 		else if (t1 < t2)
39744743693Sstephh 			rt = -1;
39844743693Sstephh 		else
39944743693Sstephh 			rt = p1 - p2;
40044743693Sstephh 	}
40144743693Sstephh 	return (rt);
40244743693Sstephh }
40344743693Sstephh 
40444743693Sstephh /*
40544743693Sstephh  * merge two lists into one, by comparing enties in new and moving into list if
40644743693Sstephh  * name is not there or free off memory for names which are already there
40744743693Sstephh  * add_pct indicates if pct is the sum or highest pct
40844743693Sstephh  */
40944743693Sstephh static name_list_t *
merge_name_list(name_list_t ** list,name_list_t * new,int add_pct)41044743693Sstephh merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
41144743693Sstephh {
41244743693Sstephh 	name_list_t *lp, *np, *sp, *rt = NULL;
41344743693Sstephh 	int max_pct;
41444743693Sstephh 
41544743693Sstephh 	rt = *list;
41644743693Sstephh 	np = new;
41744743693Sstephh 	while (np) {
41844743693Sstephh 		lp = *list;
41944743693Sstephh 		while (lp) {
42044743693Sstephh 			if (strcmp(lp->name, np->name) == 0)
42144743693Sstephh 				break;
42244743693Sstephh 			lp = lp->next;
42344743693Sstephh 			if (lp == *list)
42444743693Sstephh 				lp = NULL;
42544743693Sstephh 		}
42644743693Sstephh 		if (np->next == new)
42744743693Sstephh 			sp = NULL;
42844743693Sstephh 		else
42944743693Sstephh 			sp = np->next;
43044743693Sstephh 		if (lp) {
43144743693Sstephh 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
43244743693Sstephh 			if (add_pct) {
43344743693Sstephh 				lp->pct += np->pct;
43444743693Sstephh 				lp->count += np->count;
43544743693Sstephh 			} else if (np->pct > lp->pct) {
43644743693Sstephh 				lp->pct = np->pct;
43744743693Sstephh 			}
43844743693Sstephh 			max_pct = np->max_pct;
439d4c0a8c5Sstephh 			if (np->label)
440d4c0a8c5Sstephh 				free(np->label);
44144743693Sstephh 			free(np->name);
44244743693Sstephh 			free(np);
44344743693Sstephh 			np = NULL;
44444743693Sstephh 			if (max_pct > lp->max_pct) {
44544743693Sstephh 				lp->max_pct = max_pct;
44644743693Sstephh 				if (lp->max_pct > lp->prev->max_pct &&
44744743693Sstephh 				    lp != *list) {
44844743693Sstephh 					lp->prev->next = lp->next;
44944743693Sstephh 					lp->next->prev = lp->prev;
45044743693Sstephh 					np = lp;
45144743693Sstephh 				}
45244743693Sstephh 			}
45344743693Sstephh 		}
45444743693Sstephh 		if (np) {
45544743693Sstephh 			lp = *list;
45644743693Sstephh 			if (lp) {
45744743693Sstephh 				if (np->max_pct > lp->max_pct) {
45844743693Sstephh 					np->next = lp;
45944743693Sstephh 					np->prev = lp->prev;
46044743693Sstephh 					lp->prev->next = np;
46144743693Sstephh 					lp->prev = np;
46244743693Sstephh 					*list = np;
46344743693Sstephh 					rt = np;
46444743693Sstephh 				} else {
46544743693Sstephh 					lp = lp->next;
46644743693Sstephh 					while (lp != *list &&
46744743693Sstephh 					    np->max_pct < lp->max_pct) {
46844743693Sstephh 						lp = lp->next;
46944743693Sstephh 					}
47044743693Sstephh 					np->next = lp;
47144743693Sstephh 					np->prev = lp->prev;
47244743693Sstephh 					lp->prev->next = np;
47344743693Sstephh 					lp->prev = np;
47444743693Sstephh 				}
47544743693Sstephh 			} else {
47644743693Sstephh 				*list = np;
47744743693Sstephh 				np->next = np;
47844743693Sstephh 				np->prev = np;
47944743693Sstephh 				rt = np;
48044743693Sstephh 			}
48144743693Sstephh 		}
48244743693Sstephh 		np = sp;
48344743693Sstephh 	}
48444743693Sstephh 	return (rt);
48544743693Sstephh }
48644743693Sstephh 
48744743693Sstephh static name_list_t *
alloc_name_list(char * name,uint8_t pct)48844743693Sstephh alloc_name_list(char *name, uint8_t pct)
48944743693Sstephh {
49044743693Sstephh 	name_list_t *nlp;
49144743693Sstephh 
49244743693Sstephh 	nlp = malloc(sizeof (*nlp));
49344743693Sstephh 	nlp->name = strdup(name);
49444743693Sstephh 	nlp->pct = pct;
49544743693Sstephh 	nlp->max_pct = pct;
49644743693Sstephh 	nlp->count = 1;
49744743693Sstephh 	nlp->next = nlp;
49844743693Sstephh 	nlp->prev = nlp;
49944743693Sstephh 	nlp->status = 0;
500d4c0a8c5Sstephh 	nlp->label = NULL;
50144743693Sstephh 	return (nlp);
50244743693Sstephh }
50344743693Sstephh 
50444743693Sstephh static status_record_t *
new_record_init(uurec_t * uurec_p,char * msgid,name_list_t * class,name_list_t * fru,name_list_t * asru,name_list_t * resource,name_list_t * serial,boolean_t not_suppressed,hostid_t * hostid,boolean_t injected)50544743693Sstephh new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
50644743693Sstephh     name_list_t *fru, name_list_t *asru, name_list_t *resource,
507b6955755SRobert Johnston     name_list_t *serial, boolean_t not_suppressed,
508540db9a9SStephen Hanson     hostid_t *hostid, boolean_t injected)
50944743693Sstephh {
51044743693Sstephh 	status_record_t *status_rec_p;
51144743693Sstephh 
51244743693Sstephh 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
51344743693Sstephh 	status_rec_p->nrecs = 1;
51444743693Sstephh 	status_rec_p->host = hostid;
51544743693Sstephh 	status_rec_p->uurec = uurec_p;
51644743693Sstephh 	uurec_p->next = NULL;
51744743693Sstephh 	uurec_p->prev = NULL;
51844743693Sstephh 	uurec_p->asru = asru;
519b6955755SRobert Johnston 	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
520b6955755SRobert Johnston 	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
521b6955755SRobert Johnston 		status_rec_p->severity = strdup("unknown");
52244743693Sstephh 	status_rec_p->class = class;
52344743693Sstephh 	status_rec_p->fru = fru;
52444743693Sstephh 	status_rec_p->asru = asru;
52544743693Sstephh 	status_rec_p->resource = resource;
52644743693Sstephh 	status_rec_p->serial = serial;
52744743693Sstephh 	status_rec_p->msgid = strdup(msgid);
52844743693Sstephh 	status_rec_p->not_suppressed = not_suppressed;
529540db9a9SStephen Hanson 	status_rec_p->injected = injected;
53044743693Sstephh 	return (status_rec_p);
53144743693Sstephh }
53244743693Sstephh 
53344743693Sstephh /*
53444743693Sstephh  * add record to given list maintaining order higher priority first.
53544743693Sstephh  */
53644743693Sstephh static void
add_rec_list(status_record_t * status_rec_p,sr_list_t ** list_pp)53744743693Sstephh add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
53844743693Sstephh {
53944743693Sstephh 	sr_list_t *tp, *np, *sp;
54044743693Sstephh 	int order;
54144743693Sstephh 	uint64_t sec;
5427c478bd9Sstevel@tonic-gate 
54344743693Sstephh 	np = malloc(sizeof (sr_list_t));
54444743693Sstephh 	np->status_record = status_rec_p;
54544743693Sstephh 	sec = status_rec_p->uurec->sec;
54644743693Sstephh 	if ((sp = *list_pp) == NULL) {
54744743693Sstephh 		*list_pp = np;
54844743693Sstephh 		np->next = np;
54944743693Sstephh 		np->prev = np;
55044743693Sstephh 	} else {
55144743693Sstephh 		/* insert new record in front of lower priority */
55244743693Sstephh 		tp = sp;
55344743693Sstephh 		order = cmp_priority(status_rec_p->severity,
55444743693Sstephh 		    sp->status_record->severity, sec,
55544743693Sstephh 		    tp->status_record->uurec->sec, 0, 0);
55644743693Sstephh 		if (order > 0) {
55744743693Sstephh 			*list_pp = np;
55844743693Sstephh 		} else {
55944743693Sstephh 			tp = sp->next;
56044743693Sstephh 			while (tp != sp &&
56144743693