xref: /illumos-gate/usr/src/cmd/fm/fmadm/common/faulty.c (revision 7adb68a6)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2020 Joyent, Inc.
 */

#include <sys/types.h>
#include <fmadm.h>
#include <errno.h>
#include <limits.h>
#include <strings.h>
#include <stdio.h>
#include <unistd.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <fm/fmd_log.h>
#include <sys/fm/protocol.h>
#include <fm/libtopo.h>
#include <fm/fmd_adm.h>
#include <fm/fmd_msg.h>
#include <dlfcn.h>
#include <sys/systeminfo.h>
#include <sys/utsname.h>
#include <libintl.h>
#include <locale.h>
#include <sys/smbios.h>
#include <libdevinfo.h>
#include <stdlib.h>
#include <stddef.h>

/*
 * Fault records are added to catalog by calling add_fault_record_to_catalog()
 * records are stored in order of importance to the system.
 * If -g flag is set or not_suppressed is not set and the class fru, fault,
 * type are the same then details are merged into an existing record, with uuid
 * records are stored in time order.
 * For each record information is extracted from nvlist and merged into linked
 * list each is checked for identical records for which percentage certainty are
 * added together.
 * print_catalog() is called to print out catalog and release external resources
 *
 *                         /---------------\
 *	status_rec_list -> |               | -|
 *                         \---------------/
 *                                \/
 *                         /---------------\    /-------\    /-------\
 *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
 *            \/           |               | |- |       | <- |       |
 *      /-------------\    |               |    \-------/    \-------/
 *      |             | -> |               |       \/           \/
 *      \-------------/    |               |    /-------\    /-------\
 *            \/           |               | -> | asru  | -> | asru  |
 *            ---          |               |    |       | <- |       |
 *                         |               |    \-------/    \-------/
 *      status_asru_list   |  class        |
 *            \/           |  resource     |    /-------\    /-------\
 *      /-------------\    |  fru          | -> | list  | -> | list  |
 *      |             | -> |  serial       |    |       | <- |       |
 *      \-------------/    |               |    \-------/    \-------/
 *            \/           \---------------/
 *            ---               \/    /\
 *                         /---------------\
 *                         | status_record |
 *                         \---------------/
 *
 * Fmadm faulty takes a number of options which affect the format of the
 * output displayed. By default, the display reports the FRU and ASRU along
 * with other information on per-case basis as in the example below.
 *
 * --------------- ------------------------------------  -------------- -------
 * TIME            EVENT-ID                              MSG-ID         SEVERITY
 * --------------- ------------------------------------  -------------- -------
 * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
 *
 * Fault class	: fault.memory.dimm_sb
 * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
 *		    faulted but still in service
 * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
 *		    faulty
 *
 * Description	: The number of errors associated with this memory module has
 *		exceeded acceptable levels.  Refer to
 *		http://illumos.org/msg/AMD-8000-2F for more information.
 *
 * Response	: Pages of memory associated with this memory module are being
 *		removed from service as errors are reported.
 *
 * Impact	: Total system memory capacity will be reduced as pages are
 *		retired.
 *
 * Action	: Schedule a repair procedure to replace the affected memory
 *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
 *
 * The -v flag is similar, but adds some additonal information such as the
 * resource. The -s flag is also similar but just gives the top line summary.
 * All these options (ie without the -f or -r flags) use the print_catalog()
 * function to do the display.
 *
 * The -f flag changes the output so that it appears sorted on a per-fru basis.
 * The output is somewhat cut down compared to the default output. If -f is
 * used, then print_fru() is used to print the output.
 *
 * -----------------------------------------------------------------------------
 * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
 * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
 *
 * Description	: A problem was detected for a PCI device.
 *		Refer to http://illumos.org/msg/PCI-8000-7J
 *		for more information.
 *
 * Response	: One or more device instances may be disabled
 *
 * Impact	: Possible loss of services provided by the device instances
 *		associated with this fault
 *
 * Action	: Schedule a repair procedure to replace the affected device.
 *		Use fmdump -v -u <EVENT_ID> to identify the device or contact
 *		Sun for support.
 *
 * The -r flag changes the output so that it appears sorted on a per-asru basis.
 * The output is very much cut down compared to the default output, just giving
 * the asru fmri and state. Here print_asru() is used to print the output.
 *
 * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
 *
 * For all fmadm faulty options, the sequence of events is
 *
 * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
 * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
 * This will extract the data from the nvlist and call catalog_new_record() to
 * save the data away in various linked lists in the catalogue.
 *
 * 2) Once this is done, the data can be supplemented by using
 * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
 *
 * 3) Finally print_catalog(), print_fru() or print_asru() are called as
 * appropriate to display the information from the catalogue sorted in the
 * requested way.
 *
 */

typedef struct name_list {
	struct name_list *next;
	struct name_list *prev;
	char *name;
	uint8_t pct;
	uint8_t max_pct;
	ushort_t count;
	int status;
	char *label;
} name_list_t;

typedef struct ari_list {
	char *ari_uuid;
	struct ari_list *next;
} ari_list_t;

typedef struct uurec {
	struct uurec *next;
	struct uurec *prev;
	char *uuid;
	ari_list_t *ari_uuid_list;
	name_list_t *asru;
	uint64_t sec;
	nvlist_t *event;
} uurec_t;

typedef struct uurec_select {
	struct uurec_select *next;
	char *uuid;
} uurec_select_t;

typedef struct host_id {
	char *chassis;
	char *server;
	char *platform;
	char *domain;
	char *product_sn;
} hostid_t;

typedef struct host_id_list {
	hostid_t hostid;
	struct host_id_list *next;
} host_id_list_t;

typedef struct status_record {
	hostid_t *host;
	int nrecs;
	uurec_t *uurec;
	char *severity;			/* in C locale */
	char *msgid;
	name_list_t *class;
	name_list_t *resource;
	name_list_t *asru;
	name_list_t *fru;
	name_list_t *serial;
	uint8_t not_suppressed;
	uint8_t injected;
} status_record_t;

typedef struct sr_list {
	struct sr_list *next;
	struct sr_list *prev;
	struct status_record *status_record;
} sr_list_t;

typedef struct resource_list {
	struct resource_list *next;
	struct resource_list *prev;
	sr_list_t *status_rec_list;
	char *resource;
	uint8_t not_suppressed;
	uint8_t injected;
	uint8_t max_pct;
} resource_list_t;

sr_list_t *status_rec_list;
resource_list_t *status_fru_list;
resource_list_t *status_asru_list;

static int max_display;
static int max_fault = 0;
static topo_hdl_t *topo_handle;
static host_id_list_t *host_list;
static int n_server;
static int opt_g;
static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */

static char *
format_date(char *buf, size_t len, uint64_t sec)
{
	if (sec > LONG_MAX) {
		(void) fprintf(stderr,
		    "record time is too large for 32-bit utility\n");
		(void) snprintf(buf, len, "0x%llx", sec);
	} else {
		time_t tod = (time_t)sec;
		time_t now = time(NULL);
		if (tod > now+60 ||
		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
			(void) strftime(buf, len, "%b %d %Y    ",
			    localtime(&tod));
		} else {
			(void) strftime(buf, len, "%b %d %T", localtime(&tod));
		}
	}

	return (buf);
}

static hostid_t *
find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
    char *product_sn)
{
	hostid_t *rt = NULL;
	host_id_list_t *hostp;

	if (platform == NULL)
		platform = "-";
	if (server == NULL)
		server = "-";
	hostp = host_list;
	while (hostp) {
		if (hostp->hostid.platform &&
		    strcmp(hostp->hostid.platform, platform) == 0 &&
		    hostp->hostid.server &&
		    strcmp(hostp->hostid.server, server) == 0 &&
		    (chassis == NULL || hostp->hostid.chassis == NULL ||
		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
		    (domain == NULL || hostp->hostid.domain == NULL ||
		    strcmp(domain, hostp->hostid.domain) == 0)) {
			rt = &hostp->hostid;
			break;
		}
		hostp = hostp->next;
	}
	if (rt == NULL) {
		hostp = malloc(sizeof (host_id_list_t));
		hostp->hostid.platform = strdup(platform);
		hostp->hostid.product_sn =
		    product_sn ? strdup(product_sn) : NULL;
		hostp->hostid.server = strdup(server);
		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
		hostp->hostid.domain = domain ? strdup(domain) : NULL;
		hostp->next = host_list;
		host_list = hostp;
		rt = &hostp->hostid;
		n_server++;
	}
	return (rt);
}

static hostid_t *
find_hostid(nvlist_t *nvl)
{
	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
	char *product_sn = NULL;
	nvlist_t *auth, *fmri;
	hostid_t *rt = NULL;

	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
		    &platform);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
		    &product_sn);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
		    &chassis);
		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
		rt = find_hostid_in_list(platform, chassis, server,
		    domain, product_sn);
	}
	return (rt);
}

static char *
get_nvl2str_topo(nvlist_t *nvl)
{
	char *name = NULL;
	char *tname;
	int err;
	char *scheme = NULL;
	char *mod_name = NULL;
	char buf[128];

	if (topo_handle == NULL)
		topo_handle = topo_open(TOPO_VERSION, 0, &err);
	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
		name = strdup(tname);
		topo_hdl_strfree(topo_handle, tname);
	} else {
		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
		    mod_name) {
			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
			    scheme, mod_name);
			name = strdup(buf);
		}
	}
	return (name);
}

static int
set_priority(char *s)
{
	int rt = 0;

	if (s) {
		if (strcmp(s, "Minor") == 0)
			rt = 1;
		else if (strcmp(s, "Major") == 0)
			rt = 10;
		else if (strcmp(s, "Critical") == 0)
			rt = 100;
	}
	return (rt);
}

static int
cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
    uint8_t p2)
{
	int r1, r2;
	int rt;

	r1 = set_priority(s1);
	r2 = set_priority(s2);
	rt = r1 - r2;
	if (rt == 0) {
		if (t1 > t2)
			rt = 1;
		else if (t1 < t2)
			rt = -1;
		else
			rt = p1 - p2;
	}
	return (rt);
}

/*
 * merge two lists into one, by comparing enties in new and moving into list if
 * name is not there or free off memory for names which are already there
 * add_pct indicates if pct is the sum or highest pct
 */
static name_list_t *
merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
{
	name_list_t *lp, *np, *sp, *rt = NULL;
	int max_pct;

	rt = *list;
	np = new;
	while (np) {
		lp = *list;
		while (lp) {
			if (strcmp(lp->name, np->name) == 0)
				break;
			lp = lp->next;
			if (lp == *list)
				lp = NULL;
		}
		if (np->next == new)
			sp = NULL;
		else
			sp = np->next;
		if (lp) {
			lp->status |= (np->status & FM_SUSPECT_FAULTY);
			if (add_pct) {
				lp->pct += np->pct;
				lp->count += np->count;
			} else if (np->pct > lp->pct) {
				lp->pct = np->pct;
			}
			max_pct = np->max_pct;
			if (np->label)
				free(np->label);
			free(np->name);
			free(np);
			np = NULL;
			if (max_pct > lp->max_pct) {
				lp->max_pct = max_pct;
				if (lp->max_pct > lp->prev->max_pct &&
				    lp != *list) {
					lp->prev->next = lp->next;
					lp->next->prev = lp->prev;
					np = lp;
				}
			}
		}
		if (np) {
			lp = *list;
			if (lp) {
				if (np->max_pct > lp->max_pct) {
					np->next = lp;
					np->prev = lp->prev;
					lp->prev->next = np;
					lp->prev = np;
					*list = np;
					rt = np;
				} else {
					lp = lp->next;
					while (lp != *list &&
					    np->max_pct < lp->max_pct) {
						lp = lp->next;
					}
					np->next = lp;
					np->prev = lp->prev;
					lp->prev->next = np;
					lp->prev = np;
				}
			} else {
				*list = np;
				np->next = np;
				np->prev = np;
				rt = np;
			}
		}
		np = sp;
	}
	return (rt);
}

static name_list_t *
alloc_name_list(char *name, uint8_t pct)
{
	name_list_t *nlp;

	nlp = malloc(sizeof (*nlp));
	nlp->name = strdup(name);
	nlp->pct = pct;
	nlp->max_pct = pct;
	nlp->count = 1;
	nlp->next = nlp;
	nlp->prev = nlp;
	nlp->status = 0;
	nlp->label = NULL;
	return (nlp);
}

static status_record_t *
new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
    name_list_t *fru, name_list_t *asru, name_list_t *resource,
    name_list_t *serial, boolean_t not_suppressed,
    hostid_t *hostid, boolean_t injected)
{
	status_record_t *status_rec_p;

	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
	status_rec_p->nrecs = 1;
	status_rec_p->host = hostid;
	status_rec_p->uurec = uurec_p;
	uurec_p->next = NULL;
	uurec_p->prev = NULL;
	uurec_p->asru = asru;
	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
		status_rec_p->severity = strdup("unknown");
	status_rec_p->class = class;
	status_rec_p->fru = fru;
	status_rec_p->asru = asru;
	status_rec_p->resource = resource;
	status_rec_p->serial = serial;
	status_rec_p->msgid = strdup(msgid);
	status_rec_p->not_suppressed = not_suppressed;
	status_rec_p->injected = injected;
	return (status_rec_p);
}

/*
 * add record to given list maintaining order higher priority first.
 */
static void
add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
{
	sr_list_t *tp, *np, *sp;
	int order;
	uint64_t sec;

	np = malloc(sizeof (sr_list_t));
	np->status_record = status_rec_p;
	sec = status_rec_p->uurec->sec;
	if ((sp = *list_pp) == NULL) {
		*list_pp = np;
		np->next = np;
		np->prev = np;
	} else {
		/* insert new record in front of lower priority */
		tp = sp;
		order = cmp_priority(status_rec_p->severity,
		    sp->status_record->severity, sec,
		    tp->status_record->uurec->sec, 0, 0);
		if (order > 0) {
			*list_pp = np;
		} else {
			tp = sp->next;
			while (tp != sp &&