xref: /illumos-gate/usr/src/cmd/fm/modules/common/eversholt/fme.c (revision cfc9ef1d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2012 Milan Jurik. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 *
 * fme.c -- fault management exercise module
 *
 * this module provides the simulated fault management exercise.
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <ctype.h>
#include <alloca.h>
#include <libnvpair.h>
#include <sys/fm/protocol.h>
#include <fm/fmd_api.h>
#include <fm/libtopo.h>
#include "alloc.h"
#include "out.h"
#include "stats.h"
#include "stable.h"
#include "literals.h"
#include "lut.h"
#include "tree.h"
#include "ptree.h"
#include "itree.h"
#include "ipath.h"
#include "fme.h"
#include "evnv.h"
#include "eval.h"
#include "config.h"
#include "platform.h"
#include "esclex.h"

struct lut *Istats;
struct lut *SerdEngines;
nvlist_t *Action_nvl;

/* imported from eft.c... */
extern hrtime_t Hesitate;
extern char *Serd_Override;
extern nv_alloc_t Eft_nv_hdl;
extern int Max_fme;
extern fmd_hdl_t *Hdl;

static int Istat_need_save;
static int Serd_need_save;
void istat_save(void);
void serd_save(void);

/* fme under construction is global so we can free it on module abort */
static struct fme *Nfmep;

static int Undiag_reason = UD_VAL_UNKNOWN;

static int Nextid = 0;

static int Open_fme_count = 0;	/* Count of open FMEs */

/* list of fault management exercises underway */
static struct fme {
	struct fme *next;		/* next exercise */
	unsigned long long ull;		/* time when fme was created */
	int id;				/* FME id */
	struct config *config;		/* cooked configuration data */
	struct lut *eventtree;		/* propagation tree for this FME */
	/*
	 * The initial error report that created this FME is kept in
	 * two forms.  e0 points to the instance tree node and is used
	 * by fme_eval() as the starting point for the inference
	 * algorithm.  e0r is the event handle FMD passed to us when
	 * the ereport first arrived and is used when setting timers,
	 * which are always relative to the time of this initial
	 * report.
	 */
	struct event *e0;
	fmd_event_t *e0r;

	id_t    timer;			/* for setting an fmd time-out */

	struct event *ecurrent;		/* ereport under consideration */
	struct event *suspects;		/* current suspect list */
	struct event *psuspects;	/* previous suspect list */
	int nsuspects;			/* count of suspects */
	int posted_suspects;		/* true if we've posted a diagnosis */
	int uniqobs;			/* number of unique events observed */
	int peek;			/* just peeking, don't track suspects */
	int overflow;			/* true if overflow FME */
	enum fme_state {
		FME_NOTHING = 5000,	/* not evaluated yet */
		FME_WAIT,		/* need to wait for more info */
		FME_CREDIBLE,		/* suspect list is credible */
		FME_DISPROVED,		/* no valid suspects found */
		FME_DEFERRED		/* don't know yet (k-count not met) */
	} state;

	unsigned long long pull;	/* time passed since created */
	unsigned long long wull;	/* wait until this time for re-eval */
	struct event *observations;	/* observation list */
	struct lut *globals;		/* values of global variables */
	/* fmd interfacing */
	fmd_hdl_t *hdl;			/* handle for talking with fmd */
	fmd_case_t *fmcase;		/* what fmd 'case' we associate with */
	/* stats */
	struct stats *Rcount;
	struct stats *Hcallcount;
	struct stats *Rcallcount;
	struct stats *Ccallcount;
	struct stats *Ecallcount;
	struct stats *Tcallcount;
	struct stats *Marrowcount;
	struct stats *diags;
} *FMElist, *EFMElist, *ClosedFMEs;

static struct case_list {
	fmd_case_t *fmcase;
	struct case_list *next;
} *Undiagablecaselist;

static void fme_eval(struct fme *fmep, fmd_event_t *ffep);
static enum fme_state hypothesise(struct fme *fmep, struct event *ep,
	unsigned long long at_latest_by, unsigned long long *pdelay);
static struct node *eventprop_lookup(struct event *ep, const char *propname);
static struct node *pathstring2epnamenp(char *path);
static void publish_undiagnosable(fmd_hdl_t *hdl, fmd_event_t *ffep,
	fmd_case_t *fmcase, nvlist_t *detector, char *arg);
static char *undiag_2reason_str(int ud, char *arg);
static const char *undiag_2defect_str(int ud);
static void restore_suspects(struct fme *fmep);
static void save_suspects(struct fme *fmep);
static void destroy_fme(struct fme *f);
static void fme_receive_report(fmd_hdl_t *hdl, fmd_event_t *ffep,
    const char *eventstring, const struct ipath *ipp, nvlist_t *nvl);
static void istat_counter_reset_cb(struct istat_entry *entp,
    struct stats *statp, const struct ipath *ipp);
static void istat_counter_topo_chg_cb(struct istat_entry *entp,
    struct stats *statp, void *unused);
static void serd_reset_cb(struct serd_entry *entp, void *unused,
    const struct ipath *ipp);
static void serd_topo_chg_cb(struct serd_entry *entp, void *unused,
    void *unused2);
static void destroy_fme_bufs(struct fme *fp);

static struct fme *
alloc_fme(void)
{
	struct fme *fmep;

	fmep = MALLOC(sizeof (*fmep));
	bzero(fmep, sizeof (*fmep));
	return (fmep);
}

/*
 * fme_ready -- called when all initialization of the FME (except for
 *	stats) has completed successfully.  Adds the fme to global lists
 *	and establishes its stats.
 */
static struct fme *
fme_ready(struct fme *fmep)
{
	char nbuf[100];

	Nfmep = NULL;	/* don't need to free this on module abort now */

	if (EFMElist) {
		EFMElist->next = fmep;
		EFMElist = fmep;
	} else
		FMElist = EFMElist = fmep;

	(void) sprintf(nbuf, "fme%d.Rcount", fmep->id);
	fmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
	(void) sprintf(nbuf, "fme%d.Hcall", fmep->id);
	fmep->Hcallcount = stats_new_counter(nbuf, "calls to hypothesise()", 1);
	(void) sprintf(nbuf, "fme%d.Rcall", fmep->id);
	fmep->Rcallcount = stats_new_counter(nbuf,
	    "calls to requirements_test()", 1);
	(void) sprintf(nbuf, "fme%d.Ccall", fmep->id);
	fmep->Ccallcount = stats_new_counter(nbuf, "calls to causes_test()", 1);
	(void) sprintf(nbuf, "fme%d.Ecall", fmep->id);
	fmep->Ecallcount =
	    stats_new_counter(nbuf, "calls to effects_test()", 1);
	(void) sprintf(nbuf, "fme%d.Tcall", fmep->id);
	fmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
	(void) sprintf(nbuf, "fme%d.Marrow", fmep->id);
	fmep->Marrowcount = stats_new_counter(nbuf,
	    "arrows marked by mark_arrows()", 1);
	(void) sprintf(nbuf, "fme%d.diags", fmep->id);
	fmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);

	out(O_ALTFP|O_VERB2, "newfme: config snapshot contains...");
	config_print(O_ALTFP|O_VERB2, fmep->config);

	return (fmep);
}

extern void ipath_dummy_lut(struct arrow *);
extern struct lut *itree_create_dummy(const char *, const struct ipath *);

/* ARGSUSED */
static void
set_needed_arrows(struct event *ep, struct event *ep2, struct fme *fmep)
{
	struct bubble *bp;
	struct arrowlist *ap;

	for (bp = itree_next_bubble(ep, NULL); bp;
	    bp = itree_next_bubble(ep, bp)) {
		if (bp->t != B_FROM)
			continue;
		for (ap = itree_next_arrow(bp, NULL); ap;
		    ap = itree_next_arrow(bp, ap)) {
			ap->arrowp->pnode->u.arrow.needed = 1;
			ipath_dummy_lut(ap->arrowp);
		}
	}
}

/* ARGSUSED */
static void
unset_needed_arrows(struct event *ep, struct event *ep2, struct fme *fmep)
{
	struct bubble *bp;
	struct arrowlist *ap;

	for (bp = itree_next_bubble(ep, NULL); bp;
	    bp = itree_next_bubble(ep, bp)) {
		if (bp->t != B_FROM)
			continue;
		for (ap = itree_next_arrow(bp, NULL); ap;
		    ap = itree_next_arrow(bp, ap))
			ap->arrowp->pnode->u.arrow.needed = 0;
	}
}

static void globals_destructor(void *left, void *right, void *arg);
static void clear_arrows(struct event *ep, struct event *ep2, struct fme *fmep);

static boolean_t
prune_propagations(const char *e0class, const struct ipath *e0ipp)
{
	char nbuf[100];
	unsigned long long my_delay = TIMEVAL_EVENTUALLY;
	extern struct lut *Usednames;

	Nfmep = alloc_fme();
	Nfmep->id = Nextid;
	Nfmep->state = FME_NOTHING;
	Nfmep->eventtree = itree_create_dummy(e0class, e0ipp);
	if ((Nfmep->e0 =
	    itree_lookup(Nfmep->eventtree, e0class, e0ipp)) == NULL) {
		itree_free(Nfmep->eventtree);
		FREE(Nfmep);
		Nfmep = NULL;
		return (B_FALSE);
	}
	Nfmep->ecurrent = Nfmep->observations = Nfmep->e0;
	Nfmep->e0->count++;

	(void) sprintf(nbuf, "fme%d.Rcount", Nfmep->id);
	Nfmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
	(void) sprintf(nbuf, "fme%d.Hcall", Nfmep->id);
	Nfmep->Hcallcount =
	    stats_new_counter(nbuf, "calls to hypothesise()", 1);
	(void) sprintf(nbuf, "fme%d.Rcall", Nfmep->id);
	Nfmep->Rcallcount = stats_new_counter(nbuf,
	    "calls to requirements_test()", 1);
	(void) sprintf(nbuf, "fme%d.Ccall", Nfmep->id);
	Nfmep->Ccallcount =
	    stats_new_counter(nbuf, "calls to causes_test()", 1);
	(void) sprintf(nbuf, "fme%d.Ecall", Nfmep->id);
	Nfmep->Ecallcount =
	    stats_new_counter(nbuf, "calls to effects_test()", 1);
	(void) sprintf(nbuf, "fme%d.Tcall", Nfmep->id);
	Nfmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
	(void) sprintf(nbuf, "fme%d.Marrow", Nfmep->id);
	Nfmep->Marrowcount = stats_new_counter(nbuf,
	    "arrows marked by mark_arrows()", 1);
	(void) sprintf(nbuf, "fme%d.diags", Nfmep->id);
	Nfmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);

	Nfmep->peek = 1;
	lut_walk(Nfmep->eventtree, (lut_cb)unset_needed_arrows, (void *)Nfmep);
	lut_free(Usednames, NULL, NULL);
	Usednames = NULL;
	lut_walk(Nfmep->eventtree, (lut_cb)clear_arrows, (void *)Nfmep);
	(void) hypothesise(Nfmep, Nfmep->e0, Nfmep->ull, &my_delay);
	itree_prune(Nfmep->eventtree);
	lut_walk(Nfmep->eventtree, (lut_cb)set_needed_arrows, (void *)Nfmep);

	stats_delete(Nfmep->Rcount);
	stats_delete(Nfmep->Hcallcount);
	stats_delete(Nfmep->Rcallcount);
	stats_delete(Nfmep->Ccallcount);
	stats_delete(Nfmep->Ecallcount);
	stats_delete(Nfmep->Tcallcount);
	stats_delete(Nfmep->Marrowcount);
	stats_delete(Nfmep->diags);
	itree_free(Nfmep->eventtree);
	lut_free(Nfmep->globals, globals_destructor, NULL);
	FREE(Nfmep);
	return (B_TRUE);
}

static struct fme *
newfme(const char *e0class, const struct ipath *e0ipp, fmd_hdl_t *hdl,
    fmd_case_t *fmcase, fmd_event_t *ffep, nvlist_t *nvl)
{
	struct cfgdata *cfgdata;
	int init_size;
	extern int alloc_total();
	nvlist_t *detector = NULL;
	char *pathstr;
	char *arg;

	/*
	 * First check if e0ipp is actually in the topology so we can give a
	 * more useful error message.
	 */
	ipathlastcomp(e0ipp);
	pathstr = ipath2str(NULL, e0ipp);
	cfgdata = config_snapshot();
	platform_unit_translate(0, cfgdata->cooked, TOPO_PROP_RESOURCE,
	    &detector, pathstr);
	FREE(pathstr);
	structconfig_free(cfgdata->cooked);
	config_free(cfgdata);
	if (detector == NULL) {
		/* See if class permits silent discard on unknown component. */
		if (lut_lookup(Ereportenames_discard, (void *)e0class, NULL)) {
			out(O_ALTFP|O_VERB2, "Unable to map \"%s\" ereport "
			    "to component path, but silent discard allowed.",
			    e0class);
			fmd_case_close(hdl, fmcase);
		} else {
			Undiag_reason = UD_VAL_BADEVENTPATH;
			(void) nvlist_lookup_nvlist(nvl, FM_EREPORT_DETECTOR,
			    &detector);
			arg = ipath2str(e0class, e0ipp);
			publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
			FREE(arg);
		}
		return (NULL);
	}

	/*
	 * Next run a quick first pass of the rules with a dummy config. This
	 * allows us to prune those rules which can't possibly cause this
	 * ereport.
	 */
	if (!prune_propagations(e0class, e0ipp)) {
		/*
		 * The fault class must have been in the rules or we would
		 * not have registered for it (and got a "nosub"), and the
		 * pathname must be in the topology or we would have failed the
		 * previous test. So to get here means the combination of
		 * class and pathname in the ereport must be invalid.
		 */
		Undiag_reason = UD_VAL_BADEVENTCLASS;
		arg = ipath2str(e0class, e0ipp);
		publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
		nvlist_free(detector);
		FREE(arg);
		return (NULL);
	}

	/*
	 * Now go ahead and create the real fme using the pruned rules.
	 */
	init_size = alloc_total();
	out(O_ALTFP|O_STAMP, "start config_snapshot using %d bytes", init_size);
	nvlist_free(detector);
	pathstr = ipath2str(NULL, e0ipp);
	cfgdata = config_snapshot();
	platform_unit_translate(0, cfgdata->cooked, TOPO_PROP_RESOURCE,
	    &detector, pathstr);
	FREE(pathstr);
	platform_save_config(hdl, fmcase);
	out(O_ALTFP|O_STAMP, "config_snapshot added %d bytes",
	    alloc_total() - init_size);

	Nfmep = alloc_fme();

	Nfmep->id = Nextid++;
	Nfmep->config = cfgdata->cooked;
	config_free(cfgdata);
	Nfmep->posted_suspects = 0;
	Nfmep->uniqobs = 0;
	Nfmep->state = FME_NOTHING;
	Nfmep->pull = 0ULL;
	Nfmep->overflow = 0;

	Nfmep->fmcase = fmcase;
	Nfmep->hdl = hdl;

	if ((Nfmep->eventtree = itree_create(Nfmep->config)) == NULL) {
		Undiag_reason = UD_VAL_INSTFAIL;
		arg = ipath2str(e0class, e0ipp);
		publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
		nvlist_free(detector);
		FREE(arg);
		structconfig_free(Nfmep->config);
		destroy_fme_bufs(Nfmep);
		FREE(Nfmep);
		Nfmep = NULL;
		return (NULL);
	}

	itree_ptree(O_ALTFP|O_VERB2, Nfmep->eventtree);

	if ((Nfmep->e0 =
	    itree_lookup(Nfmep->eventtree, e0class, e0ipp)) == NULL) {
		Undiag_reason = UD_VAL_BADEVENTI;
		arg = ipath2str(e0class, e0ipp);
		publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
		nvlist_free(detector);
		FREE(arg);
		itree_free(Nfmep->eventtree);
		structconfig_free(Nfmep->config);
		destroy_fme_bufs(Nfmep);
		FREE(Nfmep);
		Nfmep = NULL;
		return (NULL);
	}

	nvlist_free(detector);
	return (fme_ready(Nfmep));
}

void
fme_fini(void)
{
	struct fme *sfp, *fp;
	struct case_list *ucasep, *nextcasep;

	ucasep = Undiagablecaselist;
	while (ucasep != NULL) {
		nextcasep = ucasep->next;
		FREE(ucasep);
		ucasep = nextcasep;
	}
	Undiagablecaselist = NULL;

	/* clean up closed fmes */
	fp = ClosedFMEs;
	while (fp != NULL) {
		sfp = fp->next;
		destroy_fme(fp);
		fp = sfp;
	}
	ClosedFMEs = NULL;

	fp = FMElist;
	while (fp != NULL) {
		sfp = fp->next;
		destroy_fme(fp);
		fp = sfp;
	}
	FMElist = EFMElist = NULL;

	/* if we were in the middle of creating an fme, free it now */
	if (Nfmep) {
		destroy_fme(Nfmep);
		Nfmep = NULL;
	}
}

/*
 * Allocated space for a buffer name.  20 bytes allows for
 * a ridiculous 9,999,999 unique observations.
 */
#define	OBBUFNMSZ 20

/*
 *  serialize_observation
 *
 *  Create a recoverable version of the current observation
 *  (f->ecurrent).  We keep a serialized version of each unique
 *  observation in order that we may resume correctly the fme in the
 *  correct state if eft or fmd crashes and we're restarted.
 */
static void
serialize_observation(struct fme *fp, const char *cls, const struct ipath *ipp)
{
	size_t pkdlen;
	char tmpbuf[OBBUFNMSZ];
	char *pkd = NULL;
	char *estr;

	(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d", fp->uniqobs);
	estr = ipath2str(cls, ipp);
	fmd_buf_create(fp->hdl, fp->fmcase, tmpbuf, strlen(estr) + 1);
	fmd_buf_write(fp->hdl, fp->fmcase, tmpbuf, (void *)estr,
	    strlen(estr) + 1);
	FREE(estr);

	if (fp->ecurrent != NULL && fp->ecurrent->nvp != NULL) {
		(void) snprintf(tmpbuf,
		    OBBUFNMSZ, "observed%d.nvp", fp->uniqobs);