17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
55f25dc2aSgavinm  * Common Development and Distribution License (the "License").
65f25dc2aSgavinm  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
215f25dc2aSgavinm 
227c478bd9Sstevel@tonic-gate /*
23f5961f52SAdrian Frost  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
2433f5ff17SMilan Jurik  * Copyright 2012 Milan Jurik. All rights reserved.
25a3f6a2a4SRob Johnston  * Copyright (c) 2018, Joyent, Inc.
267c478bd9Sstevel@tonic-gate  *
277c478bd9Sstevel@tonic-gate  * fme.c -- fault management exercise module
287c478bd9Sstevel@tonic-gate  *
297c478bd9Sstevel@tonic-gate  * this module provides the simulated fault management exercise.
307c478bd9Sstevel@tonic-gate  */
317c478bd9Sstevel@tonic-gate 
327c478bd9Sstevel@tonic-gate #include <stdio.h>
337c478bd9Sstevel@tonic-gate #include <stdlib.h>
347c478bd9Sstevel@tonic-gate #include <string.h>
357c478bd9Sstevel@tonic-gate #include <strings.h>
367c478bd9Sstevel@tonic-gate #include <ctype.h>
377c478bd9Sstevel@tonic-gate #include <alloca.h>
387c478bd9Sstevel@tonic-gate #include <libnvpair.h>
397c478bd9Sstevel@tonic-gate #include <sys/fm/protocol.h>
407c478bd9Sstevel@tonic-gate #include <fm/fmd_api.h>
41da40b264SAlex Wilson #include <fm/libtopo.h>
427c478bd9Sstevel@tonic-gate #include "alloc.h"
437c478bd9Sstevel@tonic-gate #include "out.h"
447c478bd9Sstevel@tonic-gate #include "stats.h"
457c478bd9Sstevel@tonic-gate #include "stable.h"
467c478bd9Sstevel@tonic-gate #include "literals.h"
477c478bd9Sstevel@tonic-gate #include "lut.h"
487c478bd9Sstevel@tonic-gate #include "tree.h"
497c478bd9Sstevel@tonic-gate #include "ptree.h"
507c478bd9Sstevel@tonic-gate #include "itree.h"
517c478bd9Sstevel@tonic-gate #include "ipath.h"
527c478bd9Sstevel@tonic-gate #include "fme.h"
537c478bd9Sstevel@tonic-gate #include "evnv.h"
547c478bd9Sstevel@tonic-gate #include "eval.h"
557c478bd9Sstevel@tonic-gate #include "config.h"
567c478bd9Sstevel@tonic-gate #include "platform.h"
57e5ba14ffSstephh #include "esclex.h"
587c478bd9Sstevel@tonic-gate 
59*cfc9ef1dSToomas Soome struct lut *Istats;
60*cfc9ef1dSToomas Soome struct lut *SerdEngines;
61*cfc9ef1dSToomas Soome nvlist_t *Action_nvl;
62*cfc9ef1dSToomas Soome 
637c478bd9Sstevel@tonic-gate /* imported from eft.c... */
647c478bd9Sstevel@tonic-gate extern hrtime_t Hesitate;
65e5ba14ffSstephh extern char *Serd_Override;
667c478bd9Sstevel@tonic-gate extern nv_alloc_t Eft_nv_hdl;
670cc1f05eSjrutt extern int Max_fme;
687aec1d6eScindi extern fmd_hdl_t *Hdl;
697aec1d6eScindi 
707aec1d6eScindi static int Istat_need_save;
71b5016cbbSstephh static int Serd_need_save;
7208f6c065Sgavinm void istat_save(void);
73b5016cbbSstephh void serd_save(void);
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate /* fme under construction is global so we can free it on module abort */
767c478bd9Sstevel@tonic-gate static struct fme *Nfmep;
777c478bd9Sstevel@tonic-gate 
7832d4e834STarik Soydan static int Undiag_reason = UD_VAL_UNKNOWN;
797c478bd9Sstevel@tonic-gate 
807c478bd9Sstevel@tonic-gate static int Nextid = 0;
817c478bd9Sstevel@tonic-gate 
820cc1f05eSjrutt static int Open_fme_count = 0;	/* Count of open FMEs */
830cc1f05eSjrutt 
847c478bd9Sstevel@tonic-gate /* list of fault management exercises underway */
857c478bd9Sstevel@tonic-gate static struct fme {
867c478bd9Sstevel@tonic-gate 	struct fme *next;		/* next exercise */
877c478bd9Sstevel@tonic-gate 	unsigned long long ull;		/* time when fme was created */
887c478bd9Sstevel@tonic-gate 	int id;				/* FME id */
89e5ba14ffSstephh 	struct config *config;		/* cooked configuration data */
907c478bd9Sstevel@tonic-gate 	struct lut *eventtree;		/* propagation tree for this FME */
917c478bd9Sstevel@tonic-gate 	/*
927c478bd9Sstevel@tonic-gate 	 * The initial error report that created this FME is kept in
937c478bd9Sstevel@tonic-gate 	 * two forms.  e0 points to the instance tree node and is used
947c478bd9Sstevel@tonic-gate 	 * by fme_eval() as the starting point for the inference
957c478bd9Sstevel@tonic-gate 	 * algorithm.  e0r is the event handle FMD passed to us when
967c478bd9Sstevel@tonic-gate 	 * the ereport first arrived and is used when setting timers,
977c478bd9Sstevel@tonic-gate 	 * which are always relative to the time of this initial
987c478bd9Sstevel@tonic-gate 	 * report.
997c478bd9Sstevel@tonic-gate 	 */
1007c478bd9Sstevel@tonic-gate 	struct event *e0;
1017c478bd9Sstevel@tonic-gate 	fmd_event_t *e0r;
1027c478bd9Sstevel@tonic-gate 
1037c478bd9Sstevel@tonic-gate 	id_t    timer;			/* for setting an fmd time-out */
1047c478bd9Sstevel@tonic-gate 
1057c478bd9Sstevel@tonic-gate 	struct event *ecurrent;		/* ereport under consideration */
1067c478bd9Sstevel@tonic-gate 	struct event *suspects;		/* current suspect list */
1077c478bd9Sstevel@tonic-gate 	struct event *psuspects;	/* previous suspect list */
1087c478bd9Sstevel@tonic-gate 	int nsuspects;			/* count of suspects */
1097c478bd9Sstevel@tonic-gate 	int posted_suspects;		/* true if we've posted a diagnosis */
1107c478bd9Sstevel@tonic-gate 	int uniqobs;			/* number of unique events observed */
1117c478bd9Sstevel@tonic-gate 	int peek;			/* just peeking, don't track suspects */
1120cc1f05eSjrutt 	int overflow;			/* true if overflow FME */
1137c478bd9Sstevel@tonic-gate 	enum fme_state {
1147c478bd9Sstevel@tonic-gate 		FME_NOTHING = 5000,	/* not evaluated yet */
1157c478bd9Sstevel@tonic-gate 		FME_WAIT,		/* need to wait for more info */
1167c478bd9Sstevel@tonic-gate 		FME_CREDIBLE,		/* suspect list is credible */
1177aec1d6eScindi 		FME_DISPROVED,		/* no valid suspects found */
1187aec1d6eScindi 		FME_DEFERRED		/* don't know yet (k-count not met) */
1197c478bd9Sstevel@tonic-gate 	} state;
1207c478bd9Sstevel@tonic-gate 
1217c478bd9Sstevel@tonic-gate 	unsigned long long pull;	/* time passed since created */
1227c478bd9Sstevel@tonic-gate 	unsigned long long wull;	/* wait until this time for re-eval */
1237c478bd9Sstevel@tonic-gate 	struct event *observations;	/* observation list */
1247c478bd9Sstevel@tonic-gate 	struct lut *globals;		/* values of global variables */
1257c478bd9Sstevel@tonic-gate 	/* fmd interfacing */
1267c478bd9Sstevel@tonic-gate 	fmd_hdl_t *hdl;			/* handle for talking with fmd */
1277c478bd9Sstevel@tonic-gate 	fmd_case_t *fmcase;		/* what fmd 'case' we associate with */
1287c478bd9Sstevel@tonic-gate 	/* stats */
1297c478bd9Sstevel@tonic-gate 	struct stats *Rcount;
1307c478bd9Sstevel@tonic-gate 	struct stats *Hcallcount;
1317c478bd9Sstevel@tonic-gate 	struct stats *Rcallcount;
1327c478bd9Sstevel@tonic-gate 	struct stats *Ccallcount;
1337c478bd9Sstevel@tonic-gate 	struct stats *Ecallcount;
1347c478bd9Sstevel@tonic-gate 	struct stats *Tcallcount;
1357c478bd9Sstevel@tonic-gate 	struct stats *Marrowcount;
1367c478bd9Sstevel@tonic-gate 	struct stats *diags;
1377c478bd9Sstevel@tonic-gate } *FMElist, *EFMElist, *ClosedFMEs;
1387c478bd9Sstevel@tonic-gate 
1397c478bd9Sstevel@tonic-gate static struct case_list {
1407c478bd9Sstevel@tonic-gate 	fmd_case_t *fmcase;
1417c478bd9Sstevel@tonic-gate 	struct case_list *next;
1427c478bd9Sstevel@tonic-gate } *Undiagablecaselist;
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate static void fme_eval(struct fme *fmep, fmd_event_t *ffep);
1457c478bd9Sstevel@tonic-gate static enum fme_state hypothesise(struct fme *fmep, struct event *ep,
1467aec1d6eScindi 	unsigned long long at_latest_by, unsigned long long *pdelay);
1477c478bd9Sstevel@tonic-gate static struct node *eventprop_lookup(struct event *ep, const char *propname);
1487c478bd9Sstevel@tonic-gate static struct node *pathstring2epnamenp(char *path);
149b5016cbbSstephh static void publish_undiagnosable(fmd_hdl_t *hdl, fmd_event_t *ffep,
150705e9f42SStephen Hanson 	fmd_case_t *fmcase, nvlist_t *detector, char *arg);
151705e9f42SStephen Hanson static char *undiag_2reason_str(int ud, char *arg);
15232d4e834STarik Soydan static const char *undiag_2defect_str(int ud);
1537c478bd9Sstevel@tonic-gate static void restore_suspects(struct fme *fmep);
1547c478bd9Sstevel@tonic-gate static void save_suspects(struct fme *fmep);
1557c478bd9Sstevel@tonic-gate static void destroy_fme(struct fme *f);
1567c478bd9Sstevel@tonic-gate static void fme_receive_report(fmd_hdl_t *hdl, fmd_event_t *ffep,
1577c478bd9Sstevel@tonic-gate     const char *eventstring, const struct ipath *ipp, nvlist_t *nvl);
15808f6c065Sgavinm static void istat_counter_reset_cb(struct istat_entry *entp,
15908f6c065Sgavinm     struct stats *statp, const struct ipath *ipp);
160e5ba14ffSstephh static void istat_counter_topo_chg_cb(struct istat_entry *entp,
161e5ba14ffSstephh     struct stats *statp, void *unused);
162b5016cbbSstephh static void serd_reset_cb(struct serd_entry *entp, void *unused,
163b5016cbbSstephh     const struct ipath *ipp);
164e5ba14ffSstephh static void serd_topo_chg_cb(struct serd_entry *entp, void *unused,
165e5ba14ffSstephh     void *unused2);
166b5016cbbSstephh static void destroy_fme_bufs(struct fme *fp);
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate static struct fme *
alloc_fme(void)1697c478bd9Sstevel@tonic-gate alloc_fme(void)
1707c478bd9Sstevel@tonic-gate {
1717c478bd9Sstevel@tonic-gate 	struct fme *fmep;
1727c478bd9Sstevel@tonic-gate 
1737c478bd9Sstevel@tonic-gate 	fmep = MALLOC(sizeof (*fmep));
1747c478bd9Sstevel@tonic-gate 	bzero(fmep, sizeof (*fmep));
1757c478bd9Sstevel@tonic-gate 	return (fmep);
1767c478bd9Sstevel@tonic-gate }
1777c478bd9Sstevel@tonic-gate 
1787c478bd9Sstevel@tonic-gate /*
1797c478bd9Sstevel@tonic-gate  * fme_ready -- called when all initialization of the FME (except for
1807c478bd9Sstevel@tonic-gate  *	stats) has completed successfully.  Adds the fme to global lists
1817c478bd9Sstevel@tonic-gate  *	and establishes its stats.
1827c478bd9Sstevel@tonic-gate  */
1837c478bd9Sstevel@tonic-gate static struct fme *
fme_ready(struct fme * fmep)1847c478bd9Sstevel@tonic-gate fme_ready(struct fme *fmep)
1857c478bd9Sstevel@tonic-gate {
1867c478bd9Sstevel@tonic-gate 	char nbuf[100];
1877c478bd9Sstevel@tonic-gate 
1887c478bd9Sstevel@tonic-gate 	Nfmep = NULL;	/* don't need to free this on module abort now */
1897c478bd9Sstevel@tonic-gate 
1907c478bd9Sstevel@tonic-gate 	if (EFMElist) {
1917c478bd9Sstevel@tonic-gate 		EFMElist->next = fmep;
1927c478bd9Sstevel@tonic-gate 		EFMElist = fmep;
1937c478bd9Sstevel@tonic-gate 	} else
1947c478bd9Sstevel@tonic-gate 		FMElist = EFMElist = fmep;
1957c478bd9Sstevel@tonic-gate 
1967c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Rcount", fmep->id);
1977c478bd9Sstevel@tonic-gate 	fmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
1987c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Hcall", fmep->id);
1997c478bd9Sstevel@tonic-gate 	fmep->Hcallcount = stats_new_counter(nbuf, "calls to hypothesise()", 1);
2007c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Rcall", fmep->id);
2017c478bd9Sstevel@tonic-gate 	fmep->Rcallcount = stats_new_counter(nbuf,
2027c478bd9Sstevel@tonic-gate 	    "calls to requirements_test()", 1);
2037c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Ccall", fmep->id);
2047c478bd9Sstevel@tonic-gate 	fmep->Ccallcount = stats_new_counter(nbuf, "calls to causes_test()", 1);
2057c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Ecall", fmep->id);
2067c478bd9Sstevel@tonic-gate 	fmep->Ecallcount =
2077c478bd9Sstevel@tonic-gate 	    stats_new_counter(nbuf, "calls to effects_test()", 1);
2087c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Tcall", fmep->id);
2097c478bd9Sstevel@tonic-gate 	fmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
2107c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Marrow", fmep->id);
2117c478bd9Sstevel@tonic-gate 	fmep->Marrowcount = stats_new_counter(nbuf,
2127c478bd9Sstevel@tonic-gate 	    "arrows marked by mark_arrows()", 1);
2137c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.diags", fmep->id);
2147c478bd9Sstevel@tonic-gate 	fmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);
2157c478bd9Sstevel@tonic-gate 
2167c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB2, "newfme: config snapshot contains...");
217e5ba14ffSstephh 	config_print(O_ALTFP|O_VERB2, fmep->config);
2187c478bd9Sstevel@tonic-gate 
2197c478bd9Sstevel@tonic-gate 	return (fmep);
2207c478bd9Sstevel@tonic-gate }
2217c478bd9Sstevel@tonic-gate 
222b5016cbbSstephh extern void ipath_dummy_lut(struct arrow *);
223b5016cbbSstephh extern struct lut *itree_create_dummy(const char *, const struct ipath *);
224b5016cbbSstephh 
225b5016cbbSstephh /* ARGSUSED */
226b5016cbbSstephh static void
set_needed_arrows(struct event * ep,struct event * ep2,struct fme * fmep)227b5016cbbSstephh set_needed_arrows(struct event *ep, struct event *ep2, struct fme *fmep)
228b5016cbbSstephh {
229b5016cbbSstephh 	struct bubble *bp;
230b5016cbbSstephh 	struct arrowlist *ap;
231b5016cbbSstephh 
232b5016cbbSstephh 	for (bp = itree_next_bubble(ep, NULL); bp;
233b5016cbbSstephh 	    bp = itree_next_bubble(ep, bp)) {
234b5016cbbSstephh 		if (bp->t != B_FROM)
235b5016cbbSstephh 			continue;
236b5016cbbSstephh 		for (ap = itree_next_arrow(bp, NULL); ap;
237b5016cbbSstephh 		    ap = itree_next_arrow(bp, ap)) {
238b5016cbbSstephh 			ap->arrowp->pnode->u.arrow.needed = 1;
239b5016cbbSstephh 			ipath_dummy_lut(ap->arrowp);
240b5016cbbSstephh 		}
241b5016cbbSstephh 	}
242b5016cbbSstephh }
243b5016cbbSstephh 
244b5016cbbSstephh /* ARGSUSED */
245b5016cbbSstephh static void
unset_needed_arrows(struct event * ep,struct event * ep2,struct fme * fmep)246b5016cbbSstephh unset_needed_arrows(struct event *ep, struct event *ep2, struct fme *fmep)
247b5016cbbSstephh {
248b5016cbbSstephh 	struct bubble *bp;
249b5016cbbSstephh 	struct arrowlist *ap;
250b5016cbbSstephh 
251b5016cbbSstephh 	for (bp = itree_next_bubble(ep, NULL); bp;
252b5016cbbSstephh 	    bp = itree_next_bubble(ep, bp)) {
253b5016cbbSstephh 		if (bp->t != B_FROM)
254b5016cbbSstephh 			continue;
255b5016cbbSstephh 		for (ap = itree_next_arrow(bp, NULL); ap;
256b5016cbbSstephh 		    ap = itree_next_arrow(bp, ap))
257b5016cbbSstephh 			ap->arrowp->pnode->u.arrow.needed = 0;
258b5016cbbSstephh 	}
259b5016cbbSstephh }
260b5016cbbSstephh 
261b5016cbbSstephh static void globals_destructor(void *left, void *right, void *arg);
262b5016cbbSstephh static void clear_arrows(struct event *ep, struct event *ep2, struct fme *fmep);
263b5016cbbSstephh 
264705e9f42SStephen Hanson static boolean_t
prune_propagations(const char * e0class,const struct ipath * e0ipp)265b5016cbbSstephh prune_propagations(const char *e0class, const struct ipath *e0ipp)
266b5016cbbSstephh {
267b5016cbbSstephh 	char nbuf[100];
268b5016cbbSstephh 	unsigned long long my_delay = TIMEVAL_EVENTUALLY;
269b5016cbbSstephh 	extern struct lut *Usednames;
270b5016cbbSstephh 
271b5016cbbSstephh 	Nfmep = alloc_fme();
272b5016cbbSstephh 	Nfmep->id = Nextid;
273b5016cbbSstephh 	Nfmep->state = FME_NOTHING;
274b5016cbbSstephh 	Nfmep->eventtree = itree_create_dummy(e0class, e0ipp);
275b5016cbbSstephh 	if ((Nfmep->e0 =
276b5016cbbSstephh 	    itree_lookup(Nfmep->eventtree, e0class, e0ipp)) == NULL) {
277b5016cbbSstephh 		itree_free(Nfmep->eventtree);
278b5016cbbSstephh 		FREE(Nfmep);
279b5016cbbSstephh 		Nfmep = NULL;
280705e9f42SStephen Hanson 		return (B_FALSE);
281b5016cbbSstephh 	}
282b5016cbbSstephh 	Nfmep->ecurrent = Nfmep->observations = Nfmep->e0;
283b5016cbbSstephh 	Nfmep->e0->count++;
284b5016cbbSstephh 
285b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Rcount", Nfmep->id);
286b5016cbbSstephh 	Nfmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
287b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Hcall", Nfmep->id);
288b5016cbbSstephh 	Nfmep->Hcallcount =
289b5016cbbSstephh 	    stats_new_counter(nbuf, "calls to hypothesise()", 1);
290b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Rcall", Nfmep->id);
291b5016cbbSstephh 	Nfmep->Rcallcount = stats_new_counter(nbuf,
292b5016cbbSstephh 	    "calls to requirements_test()", 1);
293b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Ccall", Nfmep->id);
294b5016cbbSstephh 	Nfmep->Ccallcount =
295b5016cbbSstephh 	    stats_new_counter(nbuf, "calls to causes_test()", 1);
296b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Ecall", Nfmep->id);
297b5016cbbSstephh 	Nfmep->Ecallcount =
298b5016cbbSstephh 	    stats_new_counter(nbuf, "calls to effects_test()", 1);
299b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Tcall", Nfmep->id);
300b5016cbbSstephh 	Nfmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
301b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.Marrow", Nfmep->id);
302b5016cbbSstephh 	Nfmep->Marrowcount = stats_new_counter(nbuf,
303b5016cbbSstephh 	    "arrows marked by mark_arrows()", 1);
304b5016cbbSstephh 	(void) sprintf(nbuf, "fme%d.diags", Nfmep->id);
305b5016cbbSstephh 	Nfmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);
306b5016cbbSstephh 
307b5016cbbSstephh 	Nfmep->peek = 1;
308b5016cbbSstephh 	lut_walk(Nfmep->eventtree, (lut_cb)unset_needed_arrows, (void *)Nfmep);
309b5016cbbSstephh 	lut_free(Usednames, NULL, NULL);
310b5016cbbSstephh 	Usednames = NULL;
311b5016cbbSstephh 	lut_walk(Nfmep->eventtree, (lut_cb)clear_arrows, (void *)Nfmep);
312b5016cbbSstephh 	(void) hypothesise(Nfmep, Nfmep->e0, Nfmep->ull, &my_delay);
313b5016cbbSstephh 	itree_prune(Nfmep->eventtree);
314b5016cbbSstephh 	lut_walk(Nfmep->eventtree, (lut_cb)set_needed_arrows, (void *)Nfmep);
315b5016cbbSstephh 
316b5016cbbSstephh 	stats_delete(Nfmep->Rcount);
317b5016cbbSstephh 	stats_delete(Nfmep->Hcallcount);
318b5016cbbSstephh 	stats_delete(Nfmep->Rcallcount);
319b5016cbbSstephh 	stats_delete(Nfmep->Ccallcount);
320b5016cbbSstephh 	stats_delete(Nfmep->Ecallcount);
321b5016cbbSstephh 	stats_delete(Nfmep->Tcallcount);
322b5016cbbSstephh 	stats_delete(Nfmep->Marrowcount);
323b5016cbbSstephh 	stats_delete(Nfmep->diags);
324b5016cbbSstephh 	itree_free(Nfmep->eventtree);
325b5016cbbSstephh 	lut_free(Nfmep->globals, globals_destructor, NULL);
326b5016cbbSstephh 	FREE(Nfmep);
327705e9f42SStephen Hanson 	return (B_TRUE);
328b5016cbbSstephh }
329b5016cbbSstephh 
3307c478bd9Sstevel@tonic-gate static struct fme *
newfme(const char * e0class,const struct ipath * e0ipp,fmd_hdl_t * hdl,fmd_case_t * fmcase,fmd_event_t * ffep,nvlist_t * nvl)331b5016cbbSstephh newfme(const char *e0class, const struct ipath *e0ipp, fmd_hdl_t *hdl,
332a3f6a2a4SRob Johnston     fmd_case_t *fmcase, fmd_event_t *ffep, nvlist_t *nvl)
3337c478bd9Sstevel@tonic-gate {
3347c478bd9Sstevel@tonic-gate 	struct cfgdata *cfgdata;
335b5016cbbSstephh 	int init_size;
336b5016cbbSstephh 	extern int alloc_total();
337705e9f42SStephen Hanson 	nvlist_t *detector = NULL;
338705e9f42SStephen Hanson 	char *pathstr;
339705e9f42SStephen Hanson 	char *arg;
340705e9f42SStephen Hanson 
341705e9f42SStephen Hanson 	/*
342705e9f42SStephen Hanson 	 * First check if e0ipp is actually in the topology so we can give a
343705e9f42SStephen Hanson 	 * more useful error message.
344705e9f42SStephen Hanson 	 */
345705e9f42SStephen Hanson 	ipathlastcomp(e0ipp);
346705e9f42SStephen Hanson 	pathstr = ipath2str(NULL, e0ipp);
347705e9f42SStephen Hanson 	cfgdata = config_snapshot();
348da40b264SAlex Wilson 	platform_unit_translate(0, cfgdata->cooked, TOPO_PROP_RESOURCE,
349705e9f42SStephen Hanson 	    &detector, pathstr);
350705e9f42SStephen Hanson 	FREE(pathstr);
351705e9f42SStephen Hanson 	structconfig_free(cfgdata->cooked);
352705e9f42SStephen Hanson 	config_free(cfgdata);
353705e9f42SStephen Hanson 	if (detector == NULL) {
354f5961f52SAdrian Frost 		/* See if class permits silent discard on unknown component. */
355f5961f52SAdrian Frost 		if (lut_lookup(Ereportenames_discard, (void *)e0class, NULL)) {
356f5961f52SAdrian Frost 			out(O_ALTFP|O_VERB2, "Unable to map \"%s\" ereport "
357f5961f52SAdrian Frost 			    "to component path, but silent discard allowed.",
358f5961f52SAdrian Frost 			    e0class);
359a3f6a2a4SRob Johnston 			fmd_case_close(hdl, fmcase);
360f5961f52SAdrian Frost 		} else {
361f5961f52SAdrian Frost 			Undiag_reason = UD_VAL_BADEVENTPATH;
362f5961f52SAdrian Frost 			(void) nvlist_lookup_nvlist(nvl, FM_EREPORT_DETECTOR,
363f5961f52SAdrian Frost 			    &detector);
364f5961f52SAdrian Frost 			arg = ipath2str(e0class, e0ipp);
365f5961f52SAdrian Frost 			publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
366f5961f52SAdrian Frost 			FREE(arg);
367f5961f52SAdrian Frost 		}
368705e9f42SStephen Hanson 		return (NULL);
369705e9f42SStephen Hanson 	}
370705e9f42SStephen Hanson 
371705e9f42SStephen Hanson 	/*
372705e9f42SStephen Hanson 	 * Next run a quick first pass of the rules with a dummy config. This
373705e9f42SStephen Hanson 	 * allows us to prune those rules which can't possibly cause this
374705e9f42SStephen Hanson 	 * ereport.
375705e9f42SStephen Hanson 	 */
376705e9f42SStephen Hanson 	if (!prune_propagations(e0class, e0ipp)) {
377705e9f42SStephen Hanson 		/*
378705e9f42SStephen Hanson 		 * The fault class must have been in the rules or we would
379705e9f42SStephen Hanson 		 * not have registered for it (and got a "nosub"), and the
380705e9f42SStephen Hanson 		 * pathname must be in the topology or we would have failed the
381705e9f42SStephen Hanson 		 * previous test. So to get here means the combination of
382705e9f42SStephen Hanson 		 * class and pathname in the ereport must be invalid.
383705e9f42SStephen Hanson 		 */
384705e9f42SStephen Hanson 		Undiag_reason = UD_VAL_BADEVENTCLASS;
385705e9f42SStephen Hanson 		arg = ipath2str(e0class, e0ipp);
386705e9f42SStephen Hanson 		publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
387705e9f42SStephen Hanson 		nvlist_free(detector);
388705e9f42SStephen Hanson 		FREE(arg);
389705e9f42SStephen Hanson 		return (NULL);
390705e9f42SStephen Hanson 	}
3917c478bd9Sstevel@tonic-gate 
392705e9f42SStephen Hanson 	/*
393705e9f42SStephen Hanson 	 * Now go ahead and create the real fme using the pruned rules.
394705e9f42SStephen Hanson 	 */
395b5016cbbSstephh 	init_size = alloc_total();
396b5016cbbSstephh 	out(O_ALTFP|O_STAMP, "start config_snapshot using %d bytes", init_size);
39735f59e50SStephen Hanson 	nvlist_free(detector);
39835f59e50SStephen Hanson 	pathstr = ipath2str(NULL, e0ipp);
39932d4e834STarik Soydan 	cfgdata = config_snapshot();
400da40b264SAlex Wilson 	platform_unit_translate(0, cfgdata->cooked, TOPO_PROP_RESOURCE,
40135f59e50SStephen Hanson 	    &detector, pathstr);
40235f59e50SStephen Hanson 	FREE(pathstr);
403b5016cbbSstephh 	platform_save_config(hdl, fmcase);
404b5016cbbSstephh 	out(O_ALTFP|O_STAMP, "config_snapshot added %d bytes",
405b5016cbbSstephh 	    alloc_total() - init_size);
4067c478bd9Sstevel@tonic-gate 
4077c478bd9Sstevel@tonic-gate 	Nfmep = alloc_fme();
4087c478bd9Sstevel@tonic-gate 
4097c478bd9Sstevel@tonic-gate 	Nfmep->id = Nextid++;
410e5ba14ffSstephh 	Nfmep->config = cfgdata->cooked;
411e5ba14ffSstephh 	config_free(cfgdata);
4127c478bd9Sstevel@tonic-gate 	Nfmep->posted_suspects = 0;
4137c478bd9Sstevel@tonic-gate 	Nfmep->uniqobs = 0;
4147c478bd9Sstevel@tonic-gate 	Nfmep->state = FME_NOTHING;
4157c478bd9Sstevel@tonic-gate 	Nfmep->pull = 0ULL;
4160cc1f05eSjrutt 	Nfmep->overflow = 0;
4177c478bd9Sstevel@tonic-gate 
418b5016cbbSstephh 	Nfmep->fmcase = fmcase;
419b5016cbbSstephh 	Nfmep->hdl = hdl;
4207c478bd9Sstevel@tonic-gate 
421e5ba14ffSstephh 	if ((Nfmep->eventtree = itree_create(Nfmep->config)) == NULL) {
42232d4e834STarik Soydan 		Undiag_reason = UD_VAL_INSTFAIL;
423705e9f42SStephen Hanson 		arg = ipath2str(e0class, e0ipp);
424705e9f42SStephen Hanson 		publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
425705e9f42SStephen Hanson 		nvlist_free(detector);
426705e9f42SStephen Hanson 		FREE(arg);
427e5ba14ffSstephh 		structconfig_free(Nfmep->config);
428b5016cbbSstephh 		destroy_fme_bufs(Nfmep);
4297c478bd9Sstevel@tonic-gate 		FREE(Nfmep);
4307c478bd9Sstevel@tonic-gate 		Nfmep = NULL;
4317c478bd9Sstevel@tonic-gate 		return (NULL);
4327c478bd9Sstevel@tonic-gate 	}
4337c478bd9Sstevel@tonic-gate 
4347c478bd9Sstevel@tonic-gate 	itree_ptree(O_ALTFP|O_VERB2, Nfmep->eventtree);
4357c478bd9Sstevel@tonic-gate 
4367c478bd9Sstevel@tonic-gate 	if ((Nfmep->e0 =
4377c478bd9Sstevel@tonic-gate 	    itree_lookup(Nfmep->eventtree, e0class, e0ipp)) == NULL) {
43832d4e834STarik Soydan 		Undiag_reason = UD_VAL_BADEVENTI;
439705e9f42SStephen Hanson 		arg = ipath2str(e0class, e0ipp);
440705e9f42SStephen Hanson 		publish_undiagnosable(hdl, ffep, fmcase, detector, arg);
441705e9f42SStephen Hanson 		nvlist_free(detector);
442705e9f42SStephen Hanson 		FREE(arg);
4437c478bd9Sstevel@tonic-gate 		itree_free(Nfmep->eventtree);
444e5ba14ffSstephh 		structconfig_free(Nfmep->config);
445b5016cbbSstephh 		destroy_fme_bufs(Nfmep);
4467c478bd9Sstevel@tonic-gate 		FREE(Nfmep);
4477c478bd9Sstevel@tonic-gate 		Nfmep = NULL;
4487c478bd9Sstevel@tonic-gate 		return (NULL);
4497c478bd9Sstevel@tonic-gate 	}
4507c478bd9Sstevel@tonic-gate 
451705e9f42SStephen Hanson 	nvlist_free(detector);
4527c478bd9Sstevel@tonic-gate 	return (fme_ready(Nfmep));
4537c478bd9Sstevel@tonic-gate }
4547c478bd9Sstevel@tonic-gate 
4557c478bd9Sstevel@tonic-gate void
fme_fini(void)4567c478bd9Sstevel@tonic-gate fme_fini(void)
4577c478bd9Sstevel@tonic-gate {
4587c478bd9Sstevel@tonic-gate 	struct fme *sfp, *fp;
4597c478bd9Sstevel@tonic-gate 	struct case_list *ucasep, *nextcasep;
4607c478bd9Sstevel@tonic-gate 
4617c478bd9Sstevel@tonic-gate 	ucasep = Undiagablecaselist;
4627c478bd9Sstevel@tonic-gate 	while (ucasep != NULL) {
4637c478bd9Sstevel@tonic-gate 		nextcasep = ucasep->next;
4647c478bd9Sstevel@tonic-gate 		FREE(ucasep);
4657c478bd9Sstevel@tonic-gate 		ucasep = nextcasep;
4667c478bd9Sstevel@tonic-gate 	}
4677c478bd9Sstevel@tonic-gate 	Undiagablecaselist = NULL;
4687c478bd9Sstevel@tonic-gate 
4697c478bd9Sstevel@tonic-gate 	/* clean up closed fmes */
4707c478bd9Sstevel@tonic-gate 	fp = ClosedFMEs;
4717c478bd9Sstevel@tonic-gate 	while (fp != NULL) {
4727c478bd9Sstevel@tonic-gate 		sfp = fp->next;
4737c478bd9Sstevel@tonic-gate 		destroy_fme(fp);
4747c478bd9Sstevel@tonic-gate 		fp = sfp;
4757c478bd9Sstevel@tonic-gate 	}
4767c478bd9Sstevel@tonic-gate 	ClosedFMEs = NULL;
4777c478bd9Sstevel@tonic-gate 
4787c478bd9Sstevel@tonic-gate 	fp = FMElist;
4797c478bd9Sstevel@tonic-gate 	while (fp != NULL) {
4807c478bd9Sstevel@tonic-gate 		sfp = fp->next;
4817c478bd9Sstevel@tonic-gate 		destroy_fme(fp);
4827c478bd9Sstevel@tonic-gate 		fp = sfp;
4837c478bd9Sstevel@tonic-gate 	}
4847c478bd9Sstevel@tonic-gate 	FMElist = EFMElist = NULL;
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate 	/* if we were in the middle of creating an fme, free it now */
4877c478bd9Sstevel@tonic-gate 	if (Nfmep) {
4887c478bd9Sstevel@tonic-gate 		destroy_fme(Nfmep);
4897c478bd9Sstevel@tonic-gate 		Nfmep = NULL;
4907c478bd9Sstevel@tonic-gate 	}
4917c478bd9Sstevel@tonic-gate }
4927c478bd9Sstevel@tonic-gate 
4937c478bd9Sstevel@tonic-gate /*
4947c478bd9Sstevel@tonic-gate  * Allocated space for a buffer name.  20 bytes allows for
4957c478bd9Sstevel@tonic-gate  * a ridiculous 9,999,999 unique observations.
4967c478bd9Sstevel@tonic-gate  */
4977c478bd9Sstevel@tonic-gate #define	OBBUFNMSZ 20
4987c478bd9Sstevel@tonic-gate 
4997c478bd9Sstevel@tonic-gate /*
5007c478bd9Sstevel@tonic-gate  *  serialize_observation
5017c478bd9Sstevel@tonic-gate  *
5027c478bd9Sstevel@tonic-gate  *  Create a recoverable version of the current observation
5037c478bd9Sstevel@tonic-gate  *  (f->ecurrent).  We keep a serialized version of each unique
5047c478bd9Sstevel@tonic-gate  *  observation in order that we may resume correctly the fme in the
5057c478bd9Sstevel@tonic-gate  *  correct state if eft or fmd crashes and we're restarted.
5067c478bd9Sstevel@tonic-gate  */
5077c478bd9Sstevel@tonic-gate static void
serialize_observation(struct fme * fp,const char * cls,const struct ipath * ipp)5087c478bd9Sstevel@tonic-gate serialize_observation(struct fme *fp, const char *cls, const struct ipath *ipp)
5097c478bd9Sstevel@tonic-gate {
5107c478bd9Sstevel@tonic-gate 	size_t pkdlen;
5117c478bd9Sstevel@tonic-gate 	char tmpbuf[OBBUFNMSZ];
5127c478bd9Sstevel@tonic-gate 	char *pkd = NULL;
5137c478bd9Sstevel@tonic-gate 	char *estr;
5147c478bd9Sstevel@tonic-gate 
5157c478bd9Sstevel@tonic-gate 	(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d", fp->uniqobs);
5167c478bd9Sstevel@tonic-gate 	estr = ipath2str(cls, ipp);
5177c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, tmpbuf, strlen(estr) + 1);
5187c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, tmpbuf, (void *)estr,
5197c478bd9Sstevel@tonic-gate 	    strlen(estr) + 1);
5207c478bd9Sstevel@tonic-gate 	FREE(estr);
5217c478bd9Sstevel@tonic-gate 
5227c478bd9Sstevel@tonic-gate 	if (fp->ecurrent != NULL && fp->ecurrent->nvp != NULL) {
5237c478bd9Sstevel@tonic-gate 		(void) snprintf(tmpbuf,
5247c478bd9Sstevel@tonic-gate 		    OBBUFNMSZ, "observed%d.nvp", fp->uniqobs);
5257c478bd9