1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * CDDL HEADER START
3*7c478bd9Sstevel@tonic-gate  *
4*7c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*7c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*7c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*7c478bd9Sstevel@tonic-gate  * with the License.
8*7c478bd9Sstevel@tonic-gate  *
9*7c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*7c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*7c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*7c478bd9Sstevel@tonic-gate  * and limitations under the License.
13*7c478bd9Sstevel@tonic-gate  *
14*7c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*7c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*7c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*7c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*7c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*7c478bd9Sstevel@tonic-gate  *
20*7c478bd9Sstevel@tonic-gate  * CDDL HEADER END
21*7c478bd9Sstevel@tonic-gate  */
22*7c478bd9Sstevel@tonic-gate /*
23*7c478bd9Sstevel@tonic-gate  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
25*7c478bd9Sstevel@tonic-gate  *
26*7c478bd9Sstevel@tonic-gate  * fme.c -- fault management exercise module
27*7c478bd9Sstevel@tonic-gate  *
28*7c478bd9Sstevel@tonic-gate  * this module provides the simulated fault management exercise.
29*7c478bd9Sstevel@tonic-gate  */
30*7c478bd9Sstevel@tonic-gate 
31*7c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
32*7c478bd9Sstevel@tonic-gate 
33*7c478bd9Sstevel@tonic-gate #include <stdio.h>
34*7c478bd9Sstevel@tonic-gate #include <stdlib.h>
35*7c478bd9Sstevel@tonic-gate #include <string.h>
36*7c478bd9Sstevel@tonic-gate #include <strings.h>
37*7c478bd9Sstevel@tonic-gate #include <ctype.h>
38*7c478bd9Sstevel@tonic-gate #include <alloca.h>
39*7c478bd9Sstevel@tonic-gate #include <libnvpair.h>
40*7c478bd9Sstevel@tonic-gate #include <sys/fm/protocol.h>
41*7c478bd9Sstevel@tonic-gate #include <fm/fmd_api.h>
42*7c478bd9Sstevel@tonic-gate #include "alloc.h"
43*7c478bd9Sstevel@tonic-gate #include "out.h"
44*7c478bd9Sstevel@tonic-gate #include "stats.h"
45*7c478bd9Sstevel@tonic-gate #include "stable.h"
46*7c478bd9Sstevel@tonic-gate #include "literals.h"
47*7c478bd9Sstevel@tonic-gate #include "lut.h"
48*7c478bd9Sstevel@tonic-gate #include "tree.h"
49*7c478bd9Sstevel@tonic-gate #include "ptree.h"
50*7c478bd9Sstevel@tonic-gate #include "itree.h"
51*7c478bd9Sstevel@tonic-gate #include "ipath.h"
52*7c478bd9Sstevel@tonic-gate #include "fme.h"
53*7c478bd9Sstevel@tonic-gate #include "evnv.h"
54*7c478bd9Sstevel@tonic-gate #include "eval.h"
55*7c478bd9Sstevel@tonic-gate #include "config.h"
56*7c478bd9Sstevel@tonic-gate #include "platform.h"
57*7c478bd9Sstevel@tonic-gate 
58*7c478bd9Sstevel@tonic-gate /* imported from eft.c... */
59*7c478bd9Sstevel@tonic-gate extern int Autoconvict;
60*7c478bd9Sstevel@tonic-gate extern char *Autoclose;
61*7c478bd9Sstevel@tonic-gate extern hrtime_t Hesitate;
62*7c478bd9Sstevel@tonic-gate extern nv_alloc_t Eft_nv_hdl;
63*7c478bd9Sstevel@tonic-gate 
64*7c478bd9Sstevel@tonic-gate /* fme under construction is global so we can free it on module abort */
65*7c478bd9Sstevel@tonic-gate static struct fme *Nfmep;
66*7c478bd9Sstevel@tonic-gate 
67*7c478bd9Sstevel@tonic-gate static const char *Undiag_reason;
68*7c478bd9Sstevel@tonic-gate 
69*7c478bd9Sstevel@tonic-gate static int Nextid = 0;
70*7c478bd9Sstevel@tonic-gate 
71*7c478bd9Sstevel@tonic-gate /* list of fault management exercises underway */
72*7c478bd9Sstevel@tonic-gate static struct fme {
73*7c478bd9Sstevel@tonic-gate 	struct fme *next;		/* next exercise */
74*7c478bd9Sstevel@tonic-gate 	unsigned long long ull;		/* time when fme was created */
75*7c478bd9Sstevel@tonic-gate 	int id;				/* FME id */
76*7c478bd9Sstevel@tonic-gate 	struct cfgdata *cfgdata;	/* full configuration data */
77*7c478bd9Sstevel@tonic-gate 	struct lut *eventtree;		/* propagation tree for this FME */
78*7c478bd9Sstevel@tonic-gate 	/*
79*7c478bd9Sstevel@tonic-gate 	 * The initial error report that created this FME is kept in
80*7c478bd9Sstevel@tonic-gate 	 * two forms.  e0 points to the instance tree node and is used
81*7c478bd9Sstevel@tonic-gate 	 * by fme_eval() as the starting point for the inference
82*7c478bd9Sstevel@tonic-gate 	 * algorithm.  e0r is the event handle FMD passed to us when
83*7c478bd9Sstevel@tonic-gate 	 * the ereport first arrived and is used when setting timers,
84*7c478bd9Sstevel@tonic-gate 	 * which are always relative to the time of this initial
85*7c478bd9Sstevel@tonic-gate 	 * report.
86*7c478bd9Sstevel@tonic-gate 	 */
87*7c478bd9Sstevel@tonic-gate 	struct event *e0;
88*7c478bd9Sstevel@tonic-gate 	fmd_event_t *e0r;
89*7c478bd9Sstevel@tonic-gate 
90*7c478bd9Sstevel@tonic-gate 	id_t    timer;			/* for setting an fmd time-out */
91*7c478bd9Sstevel@tonic-gate 	id_t	htid;			/* for setting hesitation timer */
92*7c478bd9Sstevel@tonic-gate 
93*7c478bd9Sstevel@tonic-gate 	struct event *ecurrent;		/* ereport under consideration */
94*7c478bd9Sstevel@tonic-gate 	struct event *suspects;		/* current suspect list */
95*7c478bd9Sstevel@tonic-gate 	struct event *psuspects;	/* previous suspect list */
96*7c478bd9Sstevel@tonic-gate 	int nsuspects;			/* count of suspects */
97*7c478bd9Sstevel@tonic-gate 	int nonfault;			/* zero if all suspects T_FAULT */
98*7c478bd9Sstevel@tonic-gate 	int posted_suspects;		/* true if we've posted a diagnosis */
99*7c478bd9Sstevel@tonic-gate 	int hesitated;			/* true if we hesitated */
100*7c478bd9Sstevel@tonic-gate 	int uniqobs;			/* number of unique events observed */
101*7c478bd9Sstevel@tonic-gate 	int peek;			/* just peeking, don't track suspects */
102*7c478bd9Sstevel@tonic-gate 	enum fme_state {
103*7c478bd9Sstevel@tonic-gate 		FME_NOTHING = 5000,	/* not evaluated yet */
104*7c478bd9Sstevel@tonic-gate 		FME_WAIT,		/* need to wait for more info */
105*7c478bd9Sstevel@tonic-gate 		FME_CREDIBLE,		/* suspect list is credible */
106*7c478bd9Sstevel@tonic-gate 		FME_DISPROVED		/* no valid suspects found */
107*7c478bd9Sstevel@tonic-gate 	} state;
108*7c478bd9Sstevel@tonic-gate 
109*7c478bd9Sstevel@tonic-gate 	unsigned long long pull;	/* time passed since created */
110*7c478bd9Sstevel@tonic-gate 	unsigned long long wull;	/* wait until this time for re-eval */
111*7c478bd9Sstevel@tonic-gate 	struct event *observations;	/* observation list */
112*7c478bd9Sstevel@tonic-gate 	struct lut *globals;		/* values of global variables */
113*7c478bd9Sstevel@tonic-gate 	/* fmd interfacing */
114*7c478bd9Sstevel@tonic-gate 	fmd_hdl_t *hdl;			/* handle for talking with fmd */
115*7c478bd9Sstevel@tonic-gate 	fmd_case_t *fmcase;		/* what fmd 'case' we associate with */
116*7c478bd9Sstevel@tonic-gate 	/* stats */
117*7c478bd9Sstevel@tonic-gate 	struct stats *Rcount;
118*7c478bd9Sstevel@tonic-gate 	struct stats *Hcallcount;
119*7c478bd9Sstevel@tonic-gate 	struct stats *Rcallcount;
120*7c478bd9Sstevel@tonic-gate 	struct stats *Ccallcount;
121*7c478bd9Sstevel@tonic-gate 	struct stats *Ecallcount;
122*7c478bd9Sstevel@tonic-gate 	struct stats *Tcallcount;
123*7c478bd9Sstevel@tonic-gate 	struct stats *Marrowcount;
124*7c478bd9Sstevel@tonic-gate 	struct stats *diags;
125*7c478bd9Sstevel@tonic-gate } *FMElist, *EFMElist, *ClosedFMEs;
126*7c478bd9Sstevel@tonic-gate 
127*7c478bd9Sstevel@tonic-gate static struct case_list {
128*7c478bd9Sstevel@tonic-gate 	fmd_case_t *fmcase;
129*7c478bd9Sstevel@tonic-gate 	struct case_list *next;
130*7c478bd9Sstevel@tonic-gate } *Undiagablecaselist;
131*7c478bd9Sstevel@tonic-gate 
132*7c478bd9Sstevel@tonic-gate static void fme_eval(struct fme *fmep, fmd_event_t *ffep);
133*7c478bd9Sstevel@tonic-gate static enum fme_state hypothesise(struct fme *fmep, struct event *ep,
134*7c478bd9Sstevel@tonic-gate 	unsigned long long at_latest_by, unsigned long long *pdelay,
135*7c478bd9Sstevel@tonic-gate 	struct arrow *arrowp);
136*7c478bd9Sstevel@tonic-gate static struct node *eventprop_lookup(struct event *ep, const char *propname);
137*7c478bd9Sstevel@tonic-gate static struct node *pathstring2epnamenp(char *path);
138*7c478bd9Sstevel@tonic-gate static void publish_undiagnosable(fmd_hdl_t *hdl, fmd_event_t *ffep);
139*7c478bd9Sstevel@tonic-gate static void restore_suspects(struct fme *fmep);
140*7c478bd9Sstevel@tonic-gate static void save_suspects(struct fme *fmep);
141*7c478bd9Sstevel@tonic-gate static void destroy_fme(struct fme *f);
142*7c478bd9Sstevel@tonic-gate static void fme_receive_report(fmd_hdl_t *hdl, fmd_event_t *ffep,
143*7c478bd9Sstevel@tonic-gate     const char *eventstring, const struct ipath *ipp, nvlist_t *nvl);
144*7c478bd9Sstevel@tonic-gate 
145*7c478bd9Sstevel@tonic-gate static struct fme *
146*7c478bd9Sstevel@tonic-gate alloc_fme(void)
147*7c478bd9Sstevel@tonic-gate {
148*7c478bd9Sstevel@tonic-gate 	struct fme *fmep;
149*7c478bd9Sstevel@tonic-gate 
150*7c478bd9Sstevel@tonic-gate 	fmep = MALLOC(sizeof (*fmep));
151*7c478bd9Sstevel@tonic-gate 	bzero(fmep, sizeof (*fmep));
152*7c478bd9Sstevel@tonic-gate 	return (fmep);
153*7c478bd9Sstevel@tonic-gate }
154*7c478bd9Sstevel@tonic-gate 
155*7c478bd9Sstevel@tonic-gate /*
156*7c478bd9Sstevel@tonic-gate  * fme_ready -- called when all initialization of the FME (except for
157*7c478bd9Sstevel@tonic-gate  *	stats) has completed successfully.  Adds the fme to global lists
158*7c478bd9Sstevel@tonic-gate  *	and establishes its stats.
159*7c478bd9Sstevel@tonic-gate  */
160*7c478bd9Sstevel@tonic-gate static struct fme *
161*7c478bd9Sstevel@tonic-gate fme_ready(struct fme *fmep)
162*7c478bd9Sstevel@tonic-gate {
163*7c478bd9Sstevel@tonic-gate 	char nbuf[100];
164*7c478bd9Sstevel@tonic-gate 
165*7c478bd9Sstevel@tonic-gate 	Nfmep = NULL;	/* don't need to free this on module abort now */
166*7c478bd9Sstevel@tonic-gate 
167*7c478bd9Sstevel@tonic-gate 	if (EFMElist) {
168*7c478bd9Sstevel@tonic-gate 		EFMElist->next = fmep;
169*7c478bd9Sstevel@tonic-gate 		EFMElist = fmep;
170*7c478bd9Sstevel@tonic-gate 	} else
171*7c478bd9Sstevel@tonic-gate 		FMElist = EFMElist = fmep;
172*7c478bd9Sstevel@tonic-gate 
173*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Rcount", fmep->id);
174*7c478bd9Sstevel@tonic-gate 	fmep->Rcount = stats_new_counter(nbuf, "ereports received", 0);
175*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Hcall", fmep->id);
176*7c478bd9Sstevel@tonic-gate 	fmep->Hcallcount = stats_new_counter(nbuf, "calls to hypothesise()", 1);
177*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Rcall", fmep->id);
178*7c478bd9Sstevel@tonic-gate 	fmep->Rcallcount = stats_new_counter(nbuf,
179*7c478bd9Sstevel@tonic-gate 	    "calls to requirements_test()", 1);
180*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Ccall", fmep->id);
181*7c478bd9Sstevel@tonic-gate 	fmep->Ccallcount = stats_new_counter(nbuf, "calls to causes_test()", 1);
182*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Ecall", fmep->id);
183*7c478bd9Sstevel@tonic-gate 	fmep->Ecallcount =
184*7c478bd9Sstevel@tonic-gate 	    stats_new_counter(nbuf, "calls to effects_test()", 1);
185*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Tcall", fmep->id);
186*7c478bd9Sstevel@tonic-gate 	fmep->Tcallcount = stats_new_counter(nbuf, "calls to triggered()", 1);
187*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.Marrow", fmep->id);
188*7c478bd9Sstevel@tonic-gate 	fmep->Marrowcount = stats_new_counter(nbuf,
189*7c478bd9Sstevel@tonic-gate 	    "arrows marked by mark_arrows()", 1);
190*7c478bd9Sstevel@tonic-gate 	(void) sprintf(nbuf, "fme%d.diags", fmep->id);
191*7c478bd9Sstevel@tonic-gate 	fmep->diags = stats_new_counter(nbuf, "suspect lists diagnosed", 0);
192*7c478bd9Sstevel@tonic-gate 
193*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB2, "newfme: config snapshot contains...");
194*7c478bd9Sstevel@tonic-gate 	config_print(O_ALTFP|O_VERB2, fmep->cfgdata->cooked);
195*7c478bd9Sstevel@tonic-gate 
196*7c478bd9Sstevel@tonic-gate 	return (fmep);
197*7c478bd9Sstevel@tonic-gate }
198*7c478bd9Sstevel@tonic-gate 
199*7c478bd9Sstevel@tonic-gate static struct fme *
200*7c478bd9Sstevel@tonic-gate newfme(const char *e0class, const struct ipath *e0ipp)
201*7c478bd9Sstevel@tonic-gate {
202*7c478bd9Sstevel@tonic-gate 	struct cfgdata *cfgdata;
203*7c478bd9Sstevel@tonic-gate 
204*7c478bd9Sstevel@tonic-gate 	if ((cfgdata = config_snapshot()) == NULL) {
205*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "newfme: NULL configuration");
206*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_NOCONF;
207*7c478bd9Sstevel@tonic-gate 		return (NULL);
208*7c478bd9Sstevel@tonic-gate 	}
209*7c478bd9Sstevel@tonic-gate 
210*7c478bd9Sstevel@tonic-gate 	Nfmep = alloc_fme();
211*7c478bd9Sstevel@tonic-gate 
212*7c478bd9Sstevel@tonic-gate 	Nfmep->id = Nextid++;
213*7c478bd9Sstevel@tonic-gate 	Nfmep->cfgdata = cfgdata;
214*7c478bd9Sstevel@tonic-gate 	Nfmep->posted_suspects = 0;
215*7c478bd9Sstevel@tonic-gate 	Nfmep->uniqobs = 0;
216*7c478bd9Sstevel@tonic-gate 	Nfmep->state = FME_NOTHING;
217*7c478bd9Sstevel@tonic-gate 	Nfmep->pull = 0ULL;
218*7c478bd9Sstevel@tonic-gate 
219*7c478bd9Sstevel@tonic-gate 	Nfmep->fmcase = NULL;
220*7c478bd9Sstevel@tonic-gate 	Nfmep->hdl = NULL;
221*7c478bd9Sstevel@tonic-gate 
222*7c478bd9Sstevel@tonic-gate 	if ((Nfmep->eventtree = itree_create(cfgdata->cooked)) == NULL) {
223*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "newfme: NULL instance tree");
224*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_INSTFAIL;
225*7c478bd9Sstevel@tonic-gate 		config_free(cfgdata);
226*7c478bd9Sstevel@tonic-gate 		FREE(Nfmep);
227*7c478bd9Sstevel@tonic-gate 		Nfmep = NULL;
228*7c478bd9Sstevel@tonic-gate 		return (NULL);
229*7c478bd9Sstevel@tonic-gate 	}
230*7c478bd9Sstevel@tonic-gate 
231*7c478bd9Sstevel@tonic-gate 	itree_ptree(O_ALTFP|O_VERB2, Nfmep->eventtree);
232*7c478bd9Sstevel@tonic-gate 
233*7c478bd9Sstevel@tonic-gate 	if ((Nfmep->e0 =
234*7c478bd9Sstevel@tonic-gate 	    itree_lookup(Nfmep->eventtree, e0class, e0ipp)) == NULL) {
235*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "newfme: e0 not in instance tree");
236*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_BADEVENTI;
237*7c478bd9Sstevel@tonic-gate 		itree_free(Nfmep->eventtree);
238*7c478bd9Sstevel@tonic-gate 		config_free(cfgdata);
239*7c478bd9Sstevel@tonic-gate 		FREE(Nfmep);
240*7c478bd9Sstevel@tonic-gate 		Nfmep = NULL;
241*7c478bd9Sstevel@tonic-gate 		return (NULL);
242*7c478bd9Sstevel@tonic-gate 	}
243*7c478bd9Sstevel@tonic-gate 
244*7c478bd9Sstevel@tonic-gate 	return (fme_ready(Nfmep));
245*7c478bd9Sstevel@tonic-gate }
246*7c478bd9Sstevel@tonic-gate 
247*7c478bd9Sstevel@tonic-gate void
248*7c478bd9Sstevel@tonic-gate fme_fini(void)
249*7c478bd9Sstevel@tonic-gate {
250*7c478bd9Sstevel@tonic-gate 	struct fme *sfp, *fp;
251*7c478bd9Sstevel@tonic-gate 	struct case_list *ucasep, *nextcasep;
252*7c478bd9Sstevel@tonic-gate 
253*7c478bd9Sstevel@tonic-gate 	ucasep = Undiagablecaselist;
254*7c478bd9Sstevel@tonic-gate 	while (ucasep != NULL) {
255*7c478bd9Sstevel@tonic-gate 		nextcasep = ucasep->next;
256*7c478bd9Sstevel@tonic-gate 		FREE(ucasep);
257*7c478bd9Sstevel@tonic-gate 		ucasep = nextcasep;
258*7c478bd9Sstevel@tonic-gate 	}
259*7c478bd9Sstevel@tonic-gate 	Undiagablecaselist = NULL;
260*7c478bd9Sstevel@tonic-gate 
261*7c478bd9Sstevel@tonic-gate 	/* clean up closed fmes */
262*7c478bd9Sstevel@tonic-gate 	fp = ClosedFMEs;
263*7c478bd9Sstevel@tonic-gate 	while (fp != NULL) {
264*7c478bd9Sstevel@tonic-gate 		sfp = fp->next;
265*7c478bd9Sstevel@tonic-gate 		destroy_fme(fp);
266*7c478bd9Sstevel@tonic-gate 		fp = sfp;
267*7c478bd9Sstevel@tonic-gate 	}
268*7c478bd9Sstevel@tonic-gate 	ClosedFMEs = NULL;
269*7c478bd9Sstevel@tonic-gate 
270*7c478bd9Sstevel@tonic-gate 	fp = FMElist;
271*7c478bd9Sstevel@tonic-gate 	while (fp != NULL) {
272*7c478bd9Sstevel@tonic-gate 		sfp = fp->next;
273*7c478bd9Sstevel@tonic-gate 		destroy_fme(fp);
274*7c478bd9Sstevel@tonic-gate 		fp = sfp;
275*7c478bd9Sstevel@tonic-gate 	}
276*7c478bd9Sstevel@tonic-gate 	FMElist = EFMElist = NULL;
277*7c478bd9Sstevel@tonic-gate 
278*7c478bd9Sstevel@tonic-gate 	/* if we were in the middle of creating an fme, free it now */
279*7c478bd9Sstevel@tonic-gate 	if (Nfmep) {
280*7c478bd9Sstevel@tonic-gate 		destroy_fme(Nfmep);
281*7c478bd9Sstevel@tonic-gate 		Nfmep = NULL;
282*7c478bd9Sstevel@tonic-gate 	}
283*7c478bd9Sstevel@tonic-gate }
284*7c478bd9Sstevel@tonic-gate 
285*7c478bd9Sstevel@tonic-gate /*
286*7c478bd9Sstevel@tonic-gate  * Allocated space for a buffer name.  20 bytes allows for
287*7c478bd9Sstevel@tonic-gate  * a ridiculous 9,999,999 unique observations.
288*7c478bd9Sstevel@tonic-gate  */
289*7c478bd9Sstevel@tonic-gate #define	OBBUFNMSZ 20
290*7c478bd9Sstevel@tonic-gate 
291*7c478bd9Sstevel@tonic-gate /*
292*7c478bd9Sstevel@tonic-gate  *  serialize_observation
293*7c478bd9Sstevel@tonic-gate  *
294*7c478bd9Sstevel@tonic-gate  *  Create a recoverable version of the current observation
295*7c478bd9Sstevel@tonic-gate  *  (f->ecurrent).  We keep a serialized version of each unique
296*7c478bd9Sstevel@tonic-gate  *  observation in order that we may resume correctly the fme in the
297*7c478bd9Sstevel@tonic-gate  *  correct state if eft or fmd crashes and we're restarted.
298*7c478bd9Sstevel@tonic-gate  */
299*7c478bd9Sstevel@tonic-gate static void
300*7c478bd9Sstevel@tonic-gate serialize_observation(struct fme *fp, const char *cls, const struct ipath *ipp)
301*7c478bd9Sstevel@tonic-gate {
302*7c478bd9Sstevel@tonic-gate 	size_t pkdlen;
303*7c478bd9Sstevel@tonic-gate 	char tmpbuf[OBBUFNMSZ];
304*7c478bd9Sstevel@tonic-gate 	char *pkd = NULL;
305*7c478bd9Sstevel@tonic-gate 	char *estr;
306*7c478bd9Sstevel@tonic-gate 
307*7c478bd9Sstevel@tonic-gate 	(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d", fp->uniqobs);
308*7c478bd9Sstevel@tonic-gate 	estr = ipath2str(cls, ipp);
309*7c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, tmpbuf, strlen(estr) + 1);
310*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, tmpbuf, (void *)estr,
311*7c478bd9Sstevel@tonic-gate 	    strlen(estr) + 1);
312*7c478bd9Sstevel@tonic-gate 	FREE(estr);
313*7c478bd9Sstevel@tonic-gate 
314*7c478bd9Sstevel@tonic-gate 	if (fp->ecurrent != NULL && fp->ecurrent->nvp != NULL) {
315*7c478bd9Sstevel@tonic-gate 		(void) snprintf(tmpbuf,
316*7c478bd9Sstevel@tonic-gate 		    OBBUFNMSZ, "observed%d.nvp", fp->uniqobs);
317*7c478bd9Sstevel@tonic-gate 		if (nvlist_xpack(fp->ecurrent->nvp,
318*7c478bd9Sstevel@tonic-gate 		    &pkd, &pkdlen, NV_ENCODE_XDR, &Eft_nv_hdl) != 0)
319*7c478bd9Sstevel@tonic-gate 			out(O_DIE|O_SYS, "pack of observed nvl failed");
320*7c478bd9Sstevel@tonic-gate 		fmd_buf_create(fp->hdl, fp->fmcase, tmpbuf, pkdlen);
321*7c478bd9Sstevel@tonic-gate 		fmd_buf_write(fp->hdl, fp->fmcase, tmpbuf, (void *)pkd, pkdlen);
322*7c478bd9Sstevel@tonic-gate 		FREE(pkd);
323*7c478bd9Sstevel@tonic-gate 	}
324*7c478bd9Sstevel@tonic-gate 
325*7c478bd9Sstevel@tonic-gate 	fp->uniqobs++;
326*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_NOBS, (void *)&fp->uniqobs,
327*7c478bd9Sstevel@tonic-gate 	    sizeof (fp->uniqobs));
328*7c478bd9Sstevel@tonic-gate }
329*7c478bd9Sstevel@tonic-gate 
330*7c478bd9Sstevel@tonic-gate /*
331*7c478bd9Sstevel@tonic-gate  *  init_fme_bufs -- We keep several bits of state about an fme for
332*7c478bd9Sstevel@tonic-gate  *	use if eft or fmd crashes and we're restarted.
333*7c478bd9Sstevel@tonic-gate  */
334*7c478bd9Sstevel@tonic-gate static void
335*7c478bd9Sstevel@tonic-gate init_fme_bufs(struct fme *fp)
336*7c478bd9Sstevel@tonic-gate {
337*7c478bd9Sstevel@tonic-gate 	size_t cfglen = fp->cfgdata->nextfree - fp->cfgdata->begin;
338*7c478bd9Sstevel@tonic-gate 
339*7c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, WOBUF_CFGLEN, sizeof (cfglen));
340*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_CFGLEN, (void *)&cfglen,
341*7c478bd9Sstevel@tonic-gate 	    sizeof (cfglen));
342*7c478bd9Sstevel@tonic-gate 	if (cfglen != 0) {
343*7c478bd9Sstevel@tonic-gate 		fmd_buf_create(fp->hdl, fp->fmcase, WOBUF_CFG, cfglen);
344*7c478bd9Sstevel@tonic-gate 		fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_CFG,
345*7c478bd9Sstevel@tonic-gate 		    fp->cfgdata->begin, cfglen);
346*7c478bd9Sstevel@tonic-gate 	}
347*7c478bd9Sstevel@tonic-gate 
348*7c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, WOBUF_PULL, sizeof (fp->pull));
349*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_PULL, (void *)&fp->pull,
350*7c478bd9Sstevel@tonic-gate 	    sizeof (fp->pull));
351*7c478bd9Sstevel@tonic-gate 
352*7c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, WOBUF_ID, sizeof (fp->id));
353*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_ID, (void *)&fp->id,
354*7c478bd9Sstevel@tonic-gate 	    sizeof (fp->id));
355*7c478bd9Sstevel@tonic-gate 
356*7c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, WOBUF_NOBS, sizeof (fp->uniqobs));
357*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_NOBS, (void *)&fp->uniqobs,
358*7c478bd9Sstevel@tonic-gate 	    sizeof (fp->uniqobs));
359*7c478bd9Sstevel@tonic-gate 
360*7c478bd9Sstevel@tonic-gate 	fmd_buf_create(fp->hdl, fp->fmcase, WOBUF_POSTD,
361*7c478bd9Sstevel@tonic-gate 	    sizeof (fp->posted_suspects));
362*7c478bd9Sstevel@tonic-gate 	fmd_buf_write(fp->hdl, fp->fmcase, WOBUF_POSTD,
363*7c478bd9Sstevel@tonic-gate 	    (void *)&fp->posted_suspects, sizeof (fp->posted_suspects));
364*7c478bd9Sstevel@tonic-gate }
365*7c478bd9Sstevel@tonic-gate 
366*7c478bd9Sstevel@tonic-gate static void
367*7c478bd9Sstevel@tonic-gate destroy_fme_bufs(struct fme *fp)
368*7c478bd9Sstevel@tonic-gate {
369*7c478bd9Sstevel@tonic-gate 	char tmpbuf[OBBUFNMSZ];
370*7c478bd9Sstevel@tonic-gate 	int o;
371*7c478bd9Sstevel@tonic-gate 
372*7c478bd9Sstevel@tonic-gate 	fmd_buf_destroy(fp->hdl, fp->fmcase, WOBUF_CFGLEN);
373*7c478bd9Sstevel@tonic-gate 	fmd_buf_destroy(fp->hdl, fp->fmcase, WOBUF_CFG);
374*7c478bd9Sstevel@tonic-gate 	fmd_buf_destroy(fp->hdl, fp->fmcase, WOBUF_PULL);
375*7c478bd9Sstevel@tonic-gate 	fmd_buf_destroy(fp->hdl, fp->fmcase, WOBUF_ID);
376*7c478bd9Sstevel@tonic-gate 	fmd_buf_destroy(fp->hdl, fp->fmcase, WOBUF_POSTD);
377*7c478bd9Sstevel@tonic-gate 	fmd_buf_destroy(fp->hdl, fp->fmcase, WOBUF_NOBS);
378*7c478bd9Sstevel@tonic-gate 
379*7c478bd9Sstevel@tonic-gate 	for (o = 0; o < fp->uniqobs; o++) {
380*7c478bd9Sstevel@tonic-gate 		(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d", o);
381*7c478bd9Sstevel@tonic-gate 		fmd_buf_destroy(fp->hdl, fp->fmcase, tmpbuf);
382*7c478bd9Sstevel@tonic-gate 		(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d.nvp", o);
383*7c478bd9Sstevel@tonic-gate 		fmd_buf_destroy(fp->hdl, fp->fmcase, tmpbuf);
384*7c478bd9Sstevel@tonic-gate 	}
385*7c478bd9Sstevel@tonic-gate }
386*7c478bd9Sstevel@tonic-gate 
387*7c478bd9Sstevel@tonic-gate /*
388*7c478bd9Sstevel@tonic-gate  * reconstitute_observations -- convert a case's serialized observations
389*7c478bd9Sstevel@tonic-gate  *	back into struct events.  Returns zero if all observations are
390*7c478bd9Sstevel@tonic-gate  *	successfully reconstituted.
391*7c478bd9Sstevel@tonic-gate  */
392*7c478bd9Sstevel@tonic-gate static int
393*7c478bd9Sstevel@tonic-gate reconstitute_observations(struct fme *fmep)
394*7c478bd9Sstevel@tonic-gate {
395*7c478bd9Sstevel@tonic-gate 	struct event *ep;
396*7c478bd9Sstevel@tonic-gate 	struct node *epnamenp = NULL;
397*7c478bd9Sstevel@tonic-gate 	size_t pkdlen;
398*7c478bd9Sstevel@tonic-gate 	char *pkd = NULL;
399*7c478bd9Sstevel@tonic-gate 	char *tmpbuf = alloca(OBBUFNMSZ);
400*7c478bd9Sstevel@tonic-gate 	char *sepptr;
401*7c478bd9Sstevel@tonic-gate 	char *estr;
402*7c478bd9Sstevel@tonic-gate 	int ocnt;
403*7c478bd9Sstevel@tonic-gate 	int elen;
404*7c478bd9Sstevel@tonic-gate 
405*7c478bd9Sstevel@tonic-gate 	for (ocnt = 0; ocnt < fmep->uniqobs; ocnt++) {
406*7c478bd9Sstevel@tonic-gate 		(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d", ocnt);
407*7c478bd9Sstevel@tonic-gate 		elen = fmd_buf_size(fmep->hdl, fmep->fmcase, tmpbuf);
408*7c478bd9Sstevel@tonic-gate 		if (elen == 0) {
409*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP,
410*7c478bd9Sstevel@tonic-gate 			    "reconstitute_observation: no %s buffer found.",
411*7c478bd9Sstevel@tonic-gate 			    tmpbuf);
412*7c478bd9Sstevel@tonic-gate 			Undiag_reason = UD_MISSINGOBS;
413*7c478bd9Sstevel@tonic-gate 			break;
414*7c478bd9Sstevel@tonic-gate 		}
415*7c478bd9Sstevel@tonic-gate 
416*7c478bd9Sstevel@tonic-gate 		estr = MALLOC(elen);
417*7c478bd9Sstevel@tonic-gate 		fmd_buf_read(fmep->hdl, fmep->fmcase, tmpbuf, estr, elen);
418*7c478bd9Sstevel@tonic-gate 		sepptr = strchr(estr, '@');
419*7c478bd9Sstevel@tonic-gate 		if (sepptr == NULL) {
420*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP,
421*7c478bd9Sstevel@tonic-gate 			    "reconstitute_observation: %s: "
422*7c478bd9Sstevel@tonic-gate 			    "missing @ separator in %s.",
423*7c478bd9Sstevel@tonic-gate 			    tmpbuf, estr);
424*7c478bd9Sstevel@tonic-gate 			Undiag_reason = UD_MISSINGPATH;
425*7c478bd9Sstevel@tonic-gate 			FREE(estr);
426*7c478bd9Sstevel@tonic-gate 			break;
427*7c478bd9Sstevel@tonic-gate 		}
428*7c478bd9Sstevel@tonic-gate 
429*7c478bd9Sstevel@tonic-gate 		*sepptr = '\0';
430*7c478bd9Sstevel@tonic-gate 		if ((epnamenp = pathstring2epnamenp(sepptr + 1)) == NULL) {
431*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP,
432*7c478bd9Sstevel@tonic-gate 			    "reconstitute_observation: %s: "
433*7c478bd9Sstevel@tonic-gate 			    "trouble converting path string \"%s\" "
434*7c478bd9Sstevel@tonic-gate 			    "to internal representation.",
435*7c478bd9Sstevel@tonic-gate 			    tmpbuf, sepptr + 1);
436*7c478bd9Sstevel@tonic-gate 			Undiag_reason = UD_MISSINGPATH;
437*7c478bd9Sstevel@tonic-gate 			FREE(estr);
438*7c478bd9Sstevel@tonic-gate 			break;
439*7c478bd9Sstevel@tonic-gate 		}
440*7c478bd9Sstevel@tonic-gate 
441*7c478bd9Sstevel@tonic-gate 		/* construct the event */
442*7c478bd9Sstevel@tonic-gate 		ep = itree_lookup(fmep->eventtree,
443*7c478bd9Sstevel@tonic-gate 		    stable(estr), ipath(epnamenp));
444*7c478bd9Sstevel@tonic-gate 		if (ep == NULL) {
445*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP,
446*7c478bd9Sstevel@tonic-gate 			    "reconstitute_observation: %s: "
447*7c478bd9Sstevel@tonic-gate 			    "lookup of  \"%s\" in itree failed.",
448*7c478bd9Sstevel@tonic-gate 			    tmpbuf, ipath2str(estr, ipath(epnamenp)));
449*7c478bd9Sstevel@tonic-gate 			Undiag_reason = UD_BADOBS;
450*7c478bd9Sstevel@tonic-gate 			tree_free(epnamenp);
451*7c478bd9Sstevel@tonic-gate 			FREE(estr);
452*7c478bd9Sstevel@tonic-gate 			break;
453*7c478bd9Sstevel@tonic-gate 		}
454*7c478bd9Sstevel@tonic-gate 		tree_free(epnamenp);
455*7c478bd9Sstevel@tonic-gate 
456*7c478bd9Sstevel@tonic-gate 		/*
457*7c478bd9Sstevel@tonic-gate 		 * We may or may not have a saved nvlist for the observation
458*7c478bd9Sstevel@tonic-gate 		 */
459*7c478bd9Sstevel@tonic-gate 		(void) snprintf(tmpbuf, OBBUFNMSZ, "observed%d.nvp", ocnt);
460*7c478bd9Sstevel@tonic-gate 		pkdlen = fmd_buf_size(fmep->hdl, fmep->fmcase, tmpbuf);
461*7c478bd9Sstevel@tonic-gate 		if (pkdlen != 0) {
462*7c478bd9Sstevel@tonic-gate 			pkd = MALLOC(pkdlen);
463*7c478bd9Sstevel@tonic-gate 			fmd_buf_read(fmep->hdl,
464*7c478bd9Sstevel@tonic-gate 			    fmep->fmcase, tmpbuf, pkd, pkdlen);
465*7c478bd9Sstevel@tonic-gate 			if (nvlist_xunpack(pkd,
466*7c478bd9Sstevel@tonic-gate 			    pkdlen, &ep->nvp, &Eft_nv_hdl) != 0)
467*7c478bd9Sstevel@tonic-gate 				out(O_DIE|O_SYS, "pack of observed nvl failed");
468*7c478bd9Sstevel@tonic-gate 			FREE(pkd);
469*7c478bd9Sstevel@tonic-gate 		}
470*7c478bd9Sstevel@tonic-gate 
471*7c478bd9Sstevel@tonic-gate 		if (ocnt == 0)
472*7c478bd9Sstevel@tonic-gate 			fmep->e0 = ep;
473*7c478bd9Sstevel@tonic-gate 
474*7c478bd9Sstevel@tonic-gate 		FREE(estr);
475*7c478bd9Sstevel@tonic-gate 		fmep->ecurrent = ep;
476*7c478bd9Sstevel@tonic-gate 		ep->count++;
477*7c478bd9Sstevel@tonic-gate 
478*7c478bd9Sstevel@tonic-gate 		/* link it into list of observations seen */
479*7c478bd9Sstevel@tonic-gate 		ep->observations = fmep->observations;
480*7c478bd9Sstevel@tonic-gate 		fmep->observations = ep;
481*7c478bd9Sstevel@tonic-gate 	}
482*7c478bd9Sstevel@tonic-gate 
483*7c478bd9Sstevel@tonic-gate 	if (ocnt == fmep->uniqobs) {
484*7c478bd9Sstevel@tonic-gate 		(void) fme_ready(fmep);
485*7c478bd9Sstevel@tonic-gate 		return (0);
486*7c478bd9Sstevel@tonic-gate 	}
487*7c478bd9Sstevel@tonic-gate 
488*7c478bd9Sstevel@tonic-gate 	return (1);
489*7c478bd9Sstevel@tonic-gate }
490*7c478bd9Sstevel@tonic-gate 
491*7c478bd9Sstevel@tonic-gate /*
492*7c478bd9Sstevel@tonic-gate  * restart_fme -- called during eft initialization.  Reconstitutes
493*7c478bd9Sstevel@tonic-gate  *	an in-progress fme.
494*7c478bd9Sstevel@tonic-gate  */
495*7c478bd9Sstevel@tonic-gate void
496*7c478bd9Sstevel@tonic-gate fme_restart(fmd_hdl_t *hdl, fmd_case_t *inprogress)
497*7c478bd9Sstevel@tonic-gate {
498*7c478bd9Sstevel@tonic-gate 	nvlist_t *defect;
499*7c478bd9Sstevel@tonic-gate 	struct case_list *bad;
500*7c478bd9Sstevel@tonic-gate 	struct fme *fmep;
501*7c478bd9Sstevel@tonic-gate 	struct cfgdata *cfgdata = NULL;
502*7c478bd9Sstevel@tonic-gate 	size_t rawsz;
503*7c478bd9Sstevel@tonic-gate 
504*7c478bd9Sstevel@tonic-gate 	fmep = alloc_fme();
505*7c478bd9Sstevel@tonic-gate 	fmep->fmcase = inprogress;
506*7c478bd9Sstevel@tonic-gate 	fmep->hdl = hdl;
507*7c478bd9Sstevel@tonic-gate 
508*7c478bd9Sstevel@tonic-gate 	if (fmd_buf_size(hdl, inprogress, WOBUF_CFGLEN) != sizeof (size_t)) {
509*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: No config data");
510*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_MISSINGINFO;
511*7c478bd9Sstevel@tonic-gate 		goto badcase;
512*7c478bd9Sstevel@tonic-gate 	}
513*7c478bd9Sstevel@tonic-gate 	fmd_buf_read(hdl, inprogress, WOBUF_CFGLEN, (void *)&rawsz,
514*7c478bd9Sstevel@tonic-gate 	    sizeof (size_t));
515*7c478bd9Sstevel@tonic-gate 
516*7c478bd9Sstevel@tonic-gate 	if ((fmep->e0r = fmd_case_getprincipal(hdl, inprogress)) == NULL) {
517*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: No event zero");
518*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_MISSINGZERO;
519*7c478bd9Sstevel@tonic-gate 		goto badcase;
520*7c478bd9Sstevel@tonic-gate 	}
521*7c478bd9Sstevel@tonic-gate 
522*7c478bd9Sstevel@tonic-gate 	cfgdata = MALLOC(sizeof (struct cfgdata));
523*7c478bd9Sstevel@tonic-gate 	cfgdata->cooked = NULL;
524*7c478bd9Sstevel@tonic-gate 	cfgdata->devcache = NULL;
525*7c478bd9Sstevel@tonic-gate 	cfgdata->cpucache = NULL;
526*7c478bd9Sstevel@tonic-gate 	cfgdata->refcnt = 1;
527*7c478bd9Sstevel@tonic-gate 
528*7c478bd9Sstevel@tonic-gate 	if (rawsz > 0) {
529*7c478bd9Sstevel@tonic-gate 		if (fmd_buf_size(hdl, inprogress, WOBUF_CFG) != rawsz) {
530*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP, "restart_fme: Config data size mismatch");
531*7c478bd9Sstevel@tonic-gate 			Undiag_reason = UD_CFGMISMATCH;
532*7c478bd9Sstevel@tonic-gate 			goto badcase;
533*7c478bd9Sstevel@tonic-gate 		}
534*7c478bd9Sstevel@tonic-gate 		cfgdata->begin = MALLOC(rawsz);
535*7c478bd9Sstevel@tonic-gate 		cfgdata->end = cfgdata->nextfree = cfgdata->begin + rawsz;
536*7c478bd9Sstevel@tonic-gate 		fmd_buf_read(hdl,
537*7c478bd9Sstevel@tonic-gate 		    inprogress, WOBUF_CFG, cfgdata->begin, rawsz);
538*7c478bd9Sstevel@tonic-gate 	} else {
539*7c478bd9Sstevel@tonic-gate 		cfgdata->begin = cfgdata->end = cfgdata->nextfree = NULL;
540*7c478bd9Sstevel@tonic-gate 	}
541*7c478bd9Sstevel@tonic-gate 	fmep->cfgdata = cfgdata;
542*7c478bd9Sstevel@tonic-gate 
543*7c478bd9Sstevel@tonic-gate 	config_cook(cfgdata);
544*7c478bd9Sstevel@tonic-gate 	if ((fmep->eventtree = itree_create(cfgdata->cooked)) == NULL) {
545*7c478bd9Sstevel@tonic-gate 		/* case not properly saved or irretrievable */
546*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: NULL instance tree");
547*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_INSTFAIL;
548*7c478bd9Sstevel@tonic-gate 		goto badcase;
549*7c478bd9Sstevel@tonic-gate 	}
550*7c478bd9Sstevel@tonic-gate 
551*7c478bd9Sstevel@tonic-gate 	itree_ptree(O_ALTFP|O_VERB2, fmep->eventtree);
552*7c478bd9Sstevel@tonic-gate 
553*7c478bd9Sstevel@tonic-gate 	if (fmd_buf_size(hdl, inprogress, WOBUF_PULL) == 0) {
554*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: no saved wait time");
555*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_MISSINGINFO;
556*7c478bd9Sstevel@tonic-gate 		goto badcase;
557*7c478bd9Sstevel@tonic-gate 	} else {
558*7c478bd9Sstevel@tonic-gate 		fmd_buf_read(hdl, inprogress, WOBUF_PULL, (void *)&fmep->pull,
559*7c478bd9Sstevel@tonic-gate 		    sizeof (fmep->pull));
560*7c478bd9Sstevel@tonic-gate 	}
561*7c478bd9Sstevel@tonic-gate 
562*7c478bd9Sstevel@tonic-gate 	if (fmd_buf_size(hdl, inprogress, WOBUF_POSTD) == 0) {
563*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: no saved posted status");
564*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_MISSINGINFO;
565*7c478bd9Sstevel@tonic-gate 		goto badcase;
566*7c478bd9Sstevel@tonic-gate 	} else {
567*7c478bd9Sstevel@tonic-gate 		fmd_buf_read(hdl, inprogress, WOBUF_POSTD,
568*7c478bd9Sstevel@tonic-gate 		    (void *)&fmep->posted_suspects,
569*7c478bd9Sstevel@tonic-gate 		    sizeof (fmep->posted_suspects));
570*7c478bd9Sstevel@tonic-gate 	}
571*7c478bd9Sstevel@tonic-gate 
572*7c478bd9Sstevel@tonic-gate 	if (fmd_buf_size(hdl, inprogress, WOBUF_ID) == 0) {
573*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: no saved id");
574*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_MISSINGINFO;
575*7c478bd9Sstevel@tonic-gate 		goto badcase;
576*7c478bd9Sstevel@tonic-gate 	} else {
577*7c478bd9Sstevel@tonic-gate 		fmd_buf_read(hdl, inprogress, WOBUF_ID, (void *)&fmep->id,
578*7c478bd9Sstevel@tonic-gate 		    sizeof (fmep->id));
579*7c478bd9Sstevel@tonic-gate 	}
580*7c478bd9Sstevel@tonic-gate 	if (Nextid <= fmep->id)
581*7c478bd9Sstevel@tonic-gate 		Nextid = fmep->id + 1;
582*7c478bd9Sstevel@tonic-gate 
583*7c478bd9Sstevel@tonic-gate 	if (fmd_buf_size(hdl, inprogress, WOBUF_NOBS) == 0) {
584*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "restart_fme: no count of observations");
585*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_MISSINGINFO;
586*7c478bd9Sstevel@tonic-gate 		goto badcase;
587*7c478bd9Sstevel@tonic-gate 	} else {
588*7c478bd9Sstevel@tonic-gate 		fmd_buf_read(hdl, inprogress, WOBUF_NOBS,
589*7c478bd9Sstevel@tonic-gate 		    (void *)&fmep->uniqobs, sizeof (fmep->uniqobs));
590*7c478bd9Sstevel@tonic-gate 	}
591*7c478bd9Sstevel@tonic-gate 
592*7c478bd9Sstevel@tonic-gate 	if (reconstitute_observations(fmep) != 0)
593*7c478bd9Sstevel@tonic-gate 		goto badcase;
594*7c478bd9Sstevel@tonic-gate 
595*7c478bd9Sstevel@tonic-gate 	/* give the diagnosis algorithm a shot at the new FME state */
596*7c478bd9Sstevel@tonic-gate 	fme_eval(fmep, NULL);
597*7c478bd9Sstevel@tonic-gate 	return;
598*7c478bd9Sstevel@tonic-gate 
599*7c478bd9Sstevel@tonic-gate badcase:
600*7c478bd9Sstevel@tonic-gate 	if (fmep->eventtree != NULL)
601*7c478bd9Sstevel@tonic-gate 		itree_free(fmep->eventtree);
602*7c478bd9Sstevel@tonic-gate 	config_free(cfgdata);
603*7c478bd9Sstevel@tonic-gate 	destroy_fme_bufs(fmep);
604*7c478bd9Sstevel@tonic-gate 	FREE(fmep);
605*7c478bd9Sstevel@tonic-gate 
606*7c478bd9Sstevel@tonic-gate 	/*
607*7c478bd9Sstevel@tonic-gate 	 * Since we're unable to restart the case, add it to the undiagable
608*7c478bd9Sstevel@tonic-gate 	 * list and solve and close it as appropriate.
609*7c478bd9Sstevel@tonic-gate 	 */
610*7c478bd9Sstevel@tonic-gate 	bad = MALLOC(sizeof (struct case_list));
611*7c478bd9Sstevel@tonic-gate 	bad->next = NULL;
612*7c478bd9Sstevel@tonic-gate 
613*7c478bd9Sstevel@tonic-gate 	if (Undiagablecaselist != NULL)
614*7c478bd9Sstevel@tonic-gate 		bad->next = Undiagablecaselist;
615*7c478bd9Sstevel@tonic-gate 	Undiagablecaselist = bad;
616*7c478bd9Sstevel@tonic-gate 	bad->fmcase = inprogress;
617*7c478bd9Sstevel@tonic-gate 
618*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP, "[case %s (unable to restart), ",
619*7c478bd9Sstevel@tonic-gate 	    fmd_case_uuid(hdl, bad->fmcase));
620*7c478bd9Sstevel@tonic-gate 
621*7c478bd9Sstevel@tonic-gate 	if (fmd_case_solved(hdl, bad->fmcase)) {
622*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "already solved, ");
623*7c478bd9Sstevel@tonic-gate 	} else {
624*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "solving, ");
625*7c478bd9Sstevel@tonic-gate 		defect = fmd_nvl_create_fault(hdl, UNDIAGNOSABLE_DEFECT, 100,
626*7c478bd9Sstevel@tonic-gate 		    NULL, NULL, NULL);
627*7c478bd9Sstevel@tonic-gate 		if (Undiag_reason != NULL)
628*7c478bd9Sstevel@tonic-gate 			(void) nvlist_add_string(defect,
629*7c478bd9Sstevel@tonic-gate 			    UNDIAG_REASON, Undiag_reason);
630*7c478bd9Sstevel@tonic-gate 		fmd_case_add_suspect(hdl, bad->fmcase, defect);
631*7c478bd9Sstevel@tonic-gate 		fmd_case_solve(hdl, bad->fmcase);
632*7c478bd9Sstevel@tonic-gate 	}
633*7c478bd9Sstevel@tonic-gate 
634*7c478bd9Sstevel@tonic-gate 	if (fmd_case_closed(hdl, bad->fmcase)) {
635*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "already closed ]");
636*7c478bd9Sstevel@tonic-gate 	} else {
637*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "closing ]");
638*7c478bd9Sstevel@tonic-gate 		fmd_case_close(hdl, bad->fmcase);
639*7c478bd9Sstevel@tonic-gate 	}
640*7c478bd9Sstevel@tonic-gate }
641*7c478bd9Sstevel@tonic-gate 
642*7c478bd9Sstevel@tonic-gate void
643*7c478bd9Sstevel@tonic-gate destroy_fme(struct fme *f)
644*7c478bd9Sstevel@tonic-gate {
645*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Rcount);
646*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Hcallcount);
647*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Rcallcount);
648*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Ccallcount);
649*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Ecallcount);
650*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Tcallcount);
651*7c478bd9Sstevel@tonic-gate 	stats_delete(f->Marrowcount);
652*7c478bd9Sstevel@tonic-gate 	stats_delete(f->diags);
653*7c478bd9Sstevel@tonic-gate 
654*7c478bd9Sstevel@tonic-gate 	itree_free(f->eventtree);
655*7c478bd9Sstevel@tonic-gate 	config_free(f->cfgdata);
656*7c478bd9Sstevel@tonic-gate 	FREE(f);
657*7c478bd9Sstevel@tonic-gate }
658*7c478bd9Sstevel@tonic-gate 
659*7c478bd9Sstevel@tonic-gate static const char *
660*7c478bd9Sstevel@tonic-gate fme_state2str(enum fme_state s)
661*7c478bd9Sstevel@tonic-gate {
662*7c478bd9Sstevel@tonic-gate 	switch (s) {
663*7c478bd9Sstevel@tonic-gate 	case FME_NOTHING:	return ("NOTHING");
664*7c478bd9Sstevel@tonic-gate 	case FME_WAIT:		return ("WAIT");
665*7c478bd9Sstevel@tonic-gate 	case FME_CREDIBLE:	return ("CREDIBLE");
666*7c478bd9Sstevel@tonic-gate 	case FME_DISPROVED:	return ("DISPROVED");
667*7c478bd9Sstevel@tonic-gate 	default:		return ("UNKNOWN");
668*7c478bd9Sstevel@tonic-gate 	}
669*7c478bd9Sstevel@tonic-gate }
670*7c478bd9Sstevel@tonic-gate 
671*7c478bd9Sstevel@tonic-gate static int
672*7c478bd9Sstevel@tonic-gate is_problem(enum nametype t)
673*7c478bd9Sstevel@tonic-gate {
674*7c478bd9Sstevel@tonic-gate 	return (t == N_FAULT || t == N_DEFECT || t == N_UPSET);
675*7c478bd9Sstevel@tonic-gate }
676*7c478bd9Sstevel@tonic-gate 
677*7c478bd9Sstevel@tonic-gate static int
678*7c478bd9Sstevel@tonic-gate is_fault(enum nametype t)
679*7c478bd9Sstevel@tonic-gate {
680*7c478bd9Sstevel@tonic-gate 	return (t == N_FAULT);
681*7c478bd9Sstevel@tonic-gate }
682*7c478bd9Sstevel@tonic-gate 
683*7c478bd9Sstevel@tonic-gate static int
684*7c478bd9Sstevel@tonic-gate is_defect(enum nametype t)
685*7c478bd9Sstevel@tonic-gate {
686*7c478bd9Sstevel@tonic-gate 	return (t == N_DEFECT);
687*7c478bd9Sstevel@tonic-gate }
688*7c478bd9Sstevel@tonic-gate 
689*7c478bd9Sstevel@tonic-gate static int
690*7c478bd9Sstevel@tonic-gate is_upset(enum nametype t)
691*7c478bd9Sstevel@tonic-gate {
692*7c478bd9Sstevel@tonic-gate 	return (t == N_UPSET);
693*7c478bd9Sstevel@tonic-gate }
694*7c478bd9Sstevel@tonic-gate 
695*7c478bd9Sstevel@tonic-gate /*ARGSUSED*/
696*7c478bd9Sstevel@tonic-gate static void
697*7c478bd9Sstevel@tonic-gate clear_causes_tested(struct event *lhs, struct event *ep, void *arg)
698*7c478bd9Sstevel@tonic-gate {
699*7c478bd9Sstevel@tonic-gate 	struct bubble *bp;
700*7c478bd9Sstevel@tonic-gate 	struct arrowlist *ap;
701*7c478bd9Sstevel@tonic-gate 
702*7c478bd9Sstevel@tonic-gate 	for (bp = itree_next_bubble(ep, NULL); bp;
703*7c478bd9Sstevel@tonic-gate 	    bp = itree_next_bubble(ep, bp)) {
704*7c478bd9Sstevel@tonic-gate 		if (bp->t != B_FROM)
705*7c478bd9Sstevel@tonic-gate 			continue;
706*7c478bd9Sstevel@tonic-gate 		for (ap = itree_next_arrow(bp, NULL); ap;
707*7c478bd9Sstevel@tonic-gate 		    ap = itree_next_arrow(bp, ap))
708*7c478bd9Sstevel@tonic-gate 			ap->arrowp->causes_tested = 0;
709*7c478bd9Sstevel@tonic-gate 	}
710*7c478bd9Sstevel@tonic-gate }
711*7c478bd9Sstevel@tonic-gate 
712*7c478bd9Sstevel@tonic-gate /*
713*7c478bd9Sstevel@tonic-gate  * call this function with initcode set to 0 to initialize cycle tracking
714*7c478bd9Sstevel@tonic-gate  */
715*7c478bd9Sstevel@tonic-gate static void
716*7c478bd9Sstevel@tonic-gate initialize_cycles(struct fme *fmep)
717*7c478bd9Sstevel@tonic-gate {
718*7c478bd9Sstevel@tonic-gate 	lut_walk(fmep->eventtree, (lut_cb)clear_causes_tested, NULL);
719*7c478bd9Sstevel@tonic-gate }
720*7c478bd9Sstevel@tonic-gate 
721*7c478bd9Sstevel@tonic-gate static void
722*7c478bd9Sstevel@tonic-gate fme_print(int flags, struct fme *fmep)
723*7c478bd9Sstevel@tonic-gate {
724*7c478bd9Sstevel@tonic-gate 	struct event *ep;
725*7c478bd9Sstevel@tonic-gate 
726*7c478bd9Sstevel@tonic-gate 	out(flags, "Fault Management Exercise %d", fmep->id);
727*7c478bd9Sstevel@tonic-gate 	out(flags, "\t       State: %s", fme_state2str(fmep->state));
728*7c478bd9Sstevel@tonic-gate 	out(flags|O_NONL, "\t  Start time: ");
729*7c478bd9Sstevel@tonic-gate 	ptree_timeval(flags|O_NONL, &fmep->ull);
730*7c478bd9Sstevel@tonic-gate 	out(flags, NULL);
731*7c478bd9Sstevel@tonic-gate 	if (fmep->wull) {
732*7c478bd9Sstevel@tonic-gate 		out(flags|O_NONL, "\t   Wait time: ");
733*7c478bd9Sstevel@tonic-gate 		ptree_timeval(flags|O_NONL, &fmep->wull);
734*7c478bd9Sstevel@tonic-gate 		out(flags, NULL);
735*7c478bd9Sstevel@tonic-gate 	}
736*7c478bd9Sstevel@tonic-gate 	out(flags|O_NONL, "\t          E0: ");
737*7c478bd9Sstevel@tonic-gate 	if (fmep->e0)
738*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(flags|O_NONL, fmep->e0);
739*7c478bd9Sstevel@tonic-gate 	else
740*7c478bd9Sstevel@tonic-gate 		out(flags|O_NONL, "NULL");
741*7c478bd9Sstevel@tonic-gate 	out(flags, NULL);
742*7c478bd9Sstevel@tonic-gate 	out(flags|O_NONL, "\tObservations:");
743*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->observations; ep; ep = ep->observations) {
744*7c478bd9Sstevel@tonic-gate 		out(flags|O_NONL, " ");
745*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(flags|O_NONL, ep);
746*7c478bd9Sstevel@tonic-gate 	}
747*7c478bd9Sstevel@tonic-gate 	out(flags, NULL);
748*7c478bd9Sstevel@tonic-gate 	out(flags|O_NONL, "\tSuspect list:");
749*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->suspects; ep; ep = ep->suspects) {
750*7c478bd9Sstevel@tonic-gate 		out(flags|O_NONL, " ");
751*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(flags|O_NONL, ep);
752*7c478bd9Sstevel@tonic-gate 	}
753*7c478bd9Sstevel@tonic-gate 	out(flags, NULL);
754*7c478bd9Sstevel@tonic-gate 	out(flags|O_VERB2, "\t        Tree:");
755*7c478bd9Sstevel@tonic-gate 	itree_ptree(flags|O_VERB2, fmep->eventtree);
756*7c478bd9Sstevel@tonic-gate }
757*7c478bd9Sstevel@tonic-gate 
758*7c478bd9Sstevel@tonic-gate static struct node *
759*7c478bd9Sstevel@tonic-gate pathstring2epnamenp(char *path)
760*7c478bd9Sstevel@tonic-gate {
761*7c478bd9Sstevel@tonic-gate 	char *sep = "/";
762*7c478bd9Sstevel@tonic-gate 	struct node *ret;
763*7c478bd9Sstevel@tonic-gate 	char *ptr;
764*7c478bd9Sstevel@tonic-gate 
765*7c478bd9Sstevel@tonic-gate 	if ((ptr = strtok(path, sep)) == NULL)
766*7c478bd9Sstevel@tonic-gate 		out(O_DIE, "pathstring2epnamenp: invalid empty class");
767*7c478bd9Sstevel@tonic-gate 
768*7c478bd9Sstevel@tonic-gate 	ret = tree_iname(stable(ptr), NULL, 0);
769*7c478bd9Sstevel@tonic-gate 
770*7c478bd9Sstevel@tonic-gate 	while ((ptr = strtok(NULL, sep)) != NULL)
771*7c478bd9Sstevel@tonic-gate 		ret = tree_name_append(ret,
772*7c478bd9Sstevel@tonic-gate 		    tree_iname(stable(ptr), NULL, 0));
773*7c478bd9Sstevel@tonic-gate 
774*7c478bd9Sstevel@tonic-gate 	return (ret);
775*7c478bd9Sstevel@tonic-gate }
776*7c478bd9Sstevel@tonic-gate 
777*7c478bd9Sstevel@tonic-gate /*
778*7c478bd9Sstevel@tonic-gate  * for a given upset sp, increment the corresponding SERD engine.  if the
779*7c478bd9Sstevel@tonic-gate  * SERD engine trips, return the ename and ipp of the resulting ereport.
780*7c478bd9Sstevel@tonic-gate  * returns true if engine tripped and *enamep and *ippp were filled in.
781*7c478bd9Sstevel@tonic-gate  */
782*7c478bd9Sstevel@tonic-gate static int
783*7c478bd9Sstevel@tonic-gate serd_eval(fmd_hdl_t *hdl, fmd_event_t *ffep, struct event *sp,
784*7c478bd9Sstevel@tonic-gate     const char **enamep, const struct ipath **ippp)
785*7c478bd9Sstevel@tonic-gate {
786*7c478bd9Sstevel@tonic-gate 	struct node *serdinst;
787*7c478bd9Sstevel@tonic-gate 	char *serdname;
788*7c478bd9Sstevel@tonic-gate 
789*7c478bd9Sstevel@tonic-gate 	ASSERT(sp->t == N_UPSET);
790*7c478bd9Sstevel@tonic-gate 	ASSERT(ffep != NULL);
791*7c478bd9Sstevel@tonic-gate 
792*7c478bd9Sstevel@tonic-gate 	/*
793*7c478bd9Sstevel@tonic-gate 	 * obtain instanced SERD engine from the upset sp.  from this
794*7c478bd9Sstevel@tonic-gate 	 * derive serdname, the string used to identify the SERD engine.
795*7c478bd9Sstevel@tonic-gate 	 */
796*7c478bd9Sstevel@tonic-gate 	serdinst = eventprop_lookup(sp, L_engine);
797*7c478bd9Sstevel@tonic-gate 
798*7c478bd9Sstevel@tonic-gate 	if (serdinst == NULL)
799*7c478bd9Sstevel@tonic-gate 		return (NULL);
800*7c478bd9Sstevel@tonic-gate 
801*7c478bd9Sstevel@tonic-gate 	serdname = ipath2str(serdinst->u.stmt.np->u.event.ename->u.name.s,
802*7c478bd9Sstevel@tonic-gate 	    ipath(serdinst->u.stmt.np->u.event.epname));
803*7c478bd9Sstevel@tonic-gate 
804*7c478bd9Sstevel@tonic-gate 	if (!fmd_serd_exists(hdl, serdname)) {
805*7c478bd9Sstevel@tonic-gate 		struct node *nN, *nT;
806*7c478bd9Sstevel@tonic-gate 
807*7c478bd9Sstevel@tonic-gate 		/* no SERD engine yet, so create it */
808*7c478bd9Sstevel@tonic-gate 		nN = lut_lookup(serdinst->u.stmt.lutp, (void *)L_N, NULL);
809*7c478bd9Sstevel@tonic-gate 		nT = lut_lookup(serdinst->u.stmt.lutp, (void *)L_T, NULL);
810*7c478bd9Sstevel@tonic-gate 
811*7c478bd9Sstevel@tonic-gate 		ASSERT(nN->t == T_NUM);
812*7c478bd9Sstevel@tonic-gate 		ASSERT(nT->t == T_TIMEVAL);
813*7c478bd9Sstevel@tonic-gate 
814*7c478bd9Sstevel@tonic-gate 		fmd_serd_create(hdl, serdname, (uint_t)nN->u.ull,
815*7c478bd9Sstevel@tonic-gate 		    (hrtime_t)nT->u.ull);
816*7c478bd9Sstevel@tonic-gate 	}
817*7c478bd9Sstevel@tonic-gate 
818*7c478bd9Sstevel@tonic-gate 
819*7c478bd9Sstevel@tonic-gate 	/*
820*7c478bd9Sstevel@tonic-gate 	 * increment SERD engine.  if engine fires, reset serd
821*7c478bd9Sstevel@tonic-gate 	 * engine and return trip_strcode
822*7c478bd9Sstevel@tonic-gate 	 */
823*7c478bd9Sstevel@tonic-gate 	if (fmd_serd_record(hdl, serdname, ffep)) {
824*7c478bd9Sstevel@tonic-gate 		struct node *tripinst = lut_lookup(serdinst->u.stmt.lutp,
825*7c478bd9Sstevel@tonic-gate 		    (void *)L_trip, NULL);
826*7c478bd9Sstevel@tonic-gate 
827*7c478bd9Sstevel@tonic-gate 		ASSERT(tripinst != NULL);
828*7c478bd9Sstevel@tonic-gate 
829*7c478bd9Sstevel@tonic-gate 		*enamep = tripinst->u.event.ename->u.name.s;
830*7c478bd9Sstevel@tonic-gate 		*ippp = ipath(tripinst->u.event.epname);
831*7c478bd9Sstevel@tonic-gate 
832*7c478bd9Sstevel@tonic-gate 		fmd_serd_reset(hdl, serdname);
833*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "[engine fired: %s, sending: ", serdname);
834*7c478bd9Sstevel@tonic-gate 		ipath_print(O_ALTFP|O_NONL, *enamep, *ippp);
835*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "]");
836*7c478bd9Sstevel@tonic-gate 
837*7c478bd9Sstevel@tonic-gate 		FREE(serdname);
838*7c478bd9Sstevel@tonic-gate 		return (1);
839*7c478bd9Sstevel@tonic-gate 	}
840*7c478bd9Sstevel@tonic-gate 
841*7c478bd9Sstevel@tonic-gate 	FREE(serdname);
842*7c478bd9Sstevel@tonic-gate 	return (0);
843*7c478bd9Sstevel@tonic-gate }
844*7c478bd9Sstevel@tonic-gate 
845*7c478bd9Sstevel@tonic-gate /*
846*7c478bd9Sstevel@tonic-gate  * search a suspect list for upsets.  feed each upset to serd_eval() and
847*7c478bd9Sstevel@tonic-gate  * build up tripped[], an array of ereports produced by the firing of
848*7c478bd9Sstevel@tonic-gate  * any SERD engines.  then feed each ereport back into
849*7c478bd9Sstevel@tonic-gate  * fme_receive_report().
850*7c478bd9Sstevel@tonic-gate  *
851*7c478bd9Sstevel@tonic-gate  * returns ntrip, the number of these ereports produced.
852*7c478bd9Sstevel@tonic-gate  */
853*7c478bd9Sstevel@tonic-gate static int
854*7c478bd9Sstevel@tonic-gate upsets_eval(struct fme *fmep, fmd_event_t *ffep)
855*7c478bd9Sstevel@tonic-gate {
856*7c478bd9Sstevel@tonic-gate 	/* we build an array of tripped ereports that we send ourselves */
857*7c478bd9Sstevel@tonic-gate 	struct {
858*7c478bd9Sstevel@tonic-gate 		const char *ename;
859*7c478bd9Sstevel@tonic-gate 		const struct ipath *ipp;
860*7c478bd9Sstevel@tonic-gate 	} *tripped;
861*7c478bd9Sstevel@tonic-gate 	struct event *sp;
862*7c478bd9Sstevel@tonic-gate 	int ntrip, nupset, i;
863*7c478bd9Sstevel@tonic-gate 
864*7c478bd9Sstevel@tonic-gate 	/*
865*7c478bd9Sstevel@tonic-gate 	 * we avoid recursion by calling fme_receive_report() at the end of
866*7c478bd9Sstevel@tonic-gate 	 * this function with a NULL ffep
867*7c478bd9Sstevel@tonic-gate 	 */
868*7c478bd9Sstevel@tonic-gate 	if (ffep == NULL)
869*7c478bd9Sstevel@tonic-gate 		return (0);
870*7c478bd9Sstevel@tonic-gate 
871*7c478bd9Sstevel@tonic-gate 	/*
872*7c478bd9Sstevel@tonic-gate 	 * count the number of upsets to determine the upper limit on
873*7c478bd9Sstevel@tonic-gate 	 * expected trip ereport strings.  remember that one upset can
874*7c478bd9Sstevel@tonic-gate 	 * lead to at most one ereport.
875*7c478bd9Sstevel@tonic-gate 	 */
876*7c478bd9Sstevel@tonic-gate 	nupset = 0;
877*7c478bd9Sstevel@tonic-gate 	for (sp = fmep->suspects; sp; sp = sp->suspects) {
878*7c478bd9Sstevel@tonic-gate 		if (sp->t == N_UPSET)
879*7c478bd9Sstevel@tonic-gate 			nupset++;
880*7c478bd9Sstevel@tonic-gate 	}
881*7c478bd9Sstevel@tonic-gate 
882*7c478bd9Sstevel@tonic-gate 	if (nupset == 0)
883*7c478bd9Sstevel@tonic-gate 		return (0);
884*7c478bd9Sstevel@tonic-gate 
885*7c478bd9Sstevel@tonic-gate 	/*
886*7c478bd9Sstevel@tonic-gate 	 * get to this point if we have upsets and expect some trip
887*7c478bd9Sstevel@tonic-gate 	 * ereports
888*7c478bd9Sstevel@tonic-gate 	 */
889*7c478bd9Sstevel@tonic-gate 	tripped = alloca(sizeof (*tripped) * nupset);
890*7c478bd9Sstevel@tonic-gate 	bzero((void *)tripped, sizeof (*tripped) * nupset);
891*7c478bd9Sstevel@tonic-gate 
892*7c478bd9Sstevel@tonic-gate 	ntrip = 0;
893*7c478bd9Sstevel@tonic-gate 	for (sp = fmep->suspects; sp; sp = sp->suspects)
894*7c478bd9Sstevel@tonic-gate 		if (sp->t == N_UPSET && serd_eval(fmep->hdl, ffep, sp,
895*7c478bd9Sstevel@tonic-gate 		    &tripped[ntrip].ename, &tripped[ntrip].ipp))
896*7c478bd9Sstevel@tonic-gate 			ntrip++;
897*7c478bd9Sstevel@tonic-gate 
898*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < ntrip; i++)
899*7c478bd9Sstevel@tonic-gate 		fme_receive_report(fmep->hdl, NULL,
900*7c478bd9Sstevel@tonic-gate 		    tripped[i].ename, tripped[i].ipp, NULL);
901*7c478bd9Sstevel@tonic-gate 
902*7c478bd9Sstevel@tonic-gate 	return (ntrip);
903*7c478bd9Sstevel@tonic-gate }
904*7c478bd9Sstevel@tonic-gate 
905*7c478bd9Sstevel@tonic-gate /*
906*7c478bd9Sstevel@tonic-gate  * fme_receive_external_report -- call when an external ereport comes in
907*7c478bd9Sstevel@tonic-gate  *
908*7c478bd9Sstevel@tonic-gate  * this routine just converts the relevant information from the ereport
909*7c478bd9Sstevel@tonic-gate  * into a format used internally and passes it on to fme_receive_report().
910*7c478bd9Sstevel@tonic-gate  */
911*7c478bd9Sstevel@tonic-gate void
912*7c478bd9Sstevel@tonic-gate fme_receive_external_report(fmd_hdl_t *hdl, fmd_event_t *ffep, nvlist_t *nvl,
913*7c478bd9Sstevel@tonic-gate     const char *eventstring)
914*7c478bd9Sstevel@tonic-gate {
915*7c478bd9Sstevel@tonic-gate 	struct node *epnamenp = platform_getpath(nvl);
916*7c478bd9Sstevel@tonic-gate 	const struct ipath *ipp;
917*7c478bd9Sstevel@tonic-gate 
918*7c478bd9Sstevel@tonic-gate 	/*
919*7c478bd9Sstevel@tonic-gate 	 * XFILE: If we ended up without a path, it's an X-file.
920*7c478bd9Sstevel@tonic-gate 	 * For now, use our undiagnosable interface.
921*7c478bd9Sstevel@tonic-gate 	 */
922*7c478bd9Sstevel@tonic-gate 	if (epnamenp == NULL) {
923*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "XFILE: Unable to get path from ereport");
924*7c478bd9Sstevel@tonic-gate 		Undiag_reason = UD_NOPATH;
925*7c478bd9Sstevel@tonic-gate 		publish_undiagnosable(hdl, ffep);
926*7c478bd9Sstevel@tonic-gate 		return;
927*7c478bd9Sstevel@tonic-gate 	}
928*7c478bd9Sstevel@tonic-gate 
929*7c478bd9Sstevel@tonic-gate 	ipp = ipath(epnamenp);
930*7c478bd9Sstevel@tonic-gate 	tree_free(epnamenp);
931*7c478bd9Sstevel@tonic-gate 	fme_receive_report(hdl, ffep, stable(eventstring), ipp, nvl);
932*7c478bd9Sstevel@tonic-gate }
933*7c478bd9Sstevel@tonic-gate 
934*7c478bd9Sstevel@tonic-gate static void
935*7c478bd9Sstevel@tonic-gate fme_receive_report(fmd_hdl_t *hdl, fmd_event_t *ffep,
936*7c478bd9Sstevel@tonic-gate     const char *eventstring, const struct ipath *ipp, nvlist_t *nvl)
937*7c478bd9Sstevel@tonic-gate {
938*7c478bd9Sstevel@tonic-gate 	struct event *ep;
939*7c478bd9Sstevel@tonic-gate 	struct fme *fmep = NULL;
940*7c478bd9Sstevel@tonic-gate 	struct fme *ofmep, *svfmep;
941*7c478bd9Sstevel@tonic-gate 	int matched = 0;
942*7c478bd9Sstevel@tonic-gate 
943*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_NONL, "fme_receive_report: ");
944*7c478bd9Sstevel@tonic-gate 	ipath_print(O_ALTFP|O_NONL, eventstring, ipp);
945*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_STAMP, NULL);
946*7c478bd9Sstevel@tonic-gate 
947*7c478bd9Sstevel@tonic-gate 	/* decide which FME it goes to */
948*7c478bd9Sstevel@tonic-gate 	for (fmep = FMElist; fmep; fmep = fmep->next) {
949*7c478bd9Sstevel@tonic-gate 		int prev_verbose;
950*7c478bd9Sstevel@tonic-gate 		unsigned long long my_delay = TIMEVAL_EVENTUALLY;
951*7c478bd9Sstevel@tonic-gate 		enum fme_state state;
952*7c478bd9Sstevel@tonic-gate 
953*7c478bd9Sstevel@tonic-gate 		/* look up event in event tree for this FME */
954*7c478bd9Sstevel@tonic-gate 		if ((ep = itree_lookup(fmep->eventtree,
955*7c478bd9Sstevel@tonic-gate 		    eventstring, ipp)) == NULL)
956*7c478bd9Sstevel@tonic-gate 			continue;
957*7c478bd9Sstevel@tonic-gate 
958*7c478bd9Sstevel@tonic-gate 		/* note observation */
959*7c478bd9Sstevel@tonic-gate 		fmep->ecurrent = ep;
960*7c478bd9Sstevel@tonic-gate 		if (ep->count++ == 0) {
961*7c478bd9Sstevel@tonic-gate 			/* link it into list of observations seen */
962*7c478bd9Sstevel@tonic-gate 			ep->observations = fmep->observations;
963*7c478bd9Sstevel@tonic-gate 			fmep->observations = ep;
964*7c478bd9Sstevel@tonic-gate 			ep->nvp = evnv_dupnvl(nvl);
965*7c478bd9Sstevel@tonic-gate 		}
966*7c478bd9Sstevel@tonic-gate 
967*7c478bd9Sstevel@tonic-gate 		/* tell hypothesise() not to mess with suspect list */
968*7c478bd9Sstevel@tonic-gate 		fmep->peek = 1;
969*7c478bd9Sstevel@tonic-gate 
970*7c478bd9Sstevel@tonic-gate 		/* don't want this to be verbose (unless Debug is set) */
971*7c478bd9Sstevel@tonic-gate 		prev_verbose = Verbose;
972*7c478bd9Sstevel@tonic-gate 		if (Debug == 0)
973*7c478bd9Sstevel@tonic-gate 			Verbose = 0;
974*7c478bd9Sstevel@tonic-gate 
975*7c478bd9Sstevel@tonic-gate 		initialize_cycles(fmep);
976*7c478bd9Sstevel@tonic-gate 		state = hypothesise(fmep, fmep->e0, fmep->ull, &my_delay, NULL);
977*7c478bd9Sstevel@tonic-gate 
978*7c478bd9Sstevel@tonic-gate 		fmep->peek = 0;
979*7c478bd9Sstevel@tonic-gate 
980*7c478bd9Sstevel@tonic-gate 		/* put verbose flag back */
981*7c478bd9Sstevel@tonic-gate 		Verbose = prev_verbose;
982*7c478bd9Sstevel@tonic-gate 
983*7c478bd9Sstevel@tonic-gate 		if (state != FME_DISPROVED) {
984*7c478bd9Sstevel@tonic-gate 			/* found an FME that explains the ereport */
985*7c478bd9Sstevel@tonic-gate 			matched++;
986*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_NONL, "[");
987*7c478bd9Sstevel@tonic-gate 			ipath_print(O_ALTFP|O_NONL, eventstring, ipp);
988*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP, " explained by FME%d]", fmep->id);
989*7c478bd9Sstevel@tonic-gate 
990*7c478bd9Sstevel@tonic-gate 			if (ep->count == 1)
991*7c478bd9Sstevel@tonic-gate 				serialize_observation(fmep, eventstring, ipp);
992*7c478bd9Sstevel@tonic-gate 
993*7c478bd9Sstevel@tonic-gate 			if (ffep)
994*7c478bd9Sstevel@tonic-gate 				fmd_case_add_ereport(hdl, fmep->fmcase, ffep);
995*7c478bd9Sstevel@tonic-gate 
996*7c478bd9Sstevel@tonic-gate 			stats_counter_bump(fmep->Rcount);
997*7c478bd9Sstevel@tonic-gate 
998*7c478bd9Sstevel@tonic-gate 			/* re-eval FME */
999*7c478bd9Sstevel@tonic-gate 			fme_eval(fmep, ffep);
1000*7c478bd9Sstevel@tonic-gate 		} else {
1001*7c478bd9Sstevel@tonic-gate 
1002*7c478bd9Sstevel@tonic-gate 			/* not a match, undo noting of observation */
1003*7c478bd9Sstevel@tonic-gate 			fmep->ecurrent = NULL;
1004*7c478bd9Sstevel@tonic-gate 			if (--ep->count == 0) {
1005*7c478bd9Sstevel@tonic-gate 				/* unlink it from observations */
1006*7c478bd9Sstevel@tonic-gate 				fmep->observations = ep->observations;
1007*7c478bd9Sstevel@tonic-gate 				ep->observations = NULL;
1008*7c478bd9Sstevel@tonic-gate 				nvlist_free(ep->nvp);
1009*7c478bd9Sstevel@tonic-gate 				ep->nvp = NULL;
1010*7c478bd9Sstevel@tonic-gate 			}
1011*7c478bd9Sstevel@tonic-gate 		}
1012*7c478bd9Sstevel@tonic-gate 	}
1013*7c478bd9Sstevel@tonic-gate 
1014*7c478bd9Sstevel@tonic-gate 	if (matched)
1015*7c478bd9Sstevel@tonic-gate 		return;	/* explained by at least one existing FME */
1016*7c478bd9Sstevel@tonic-gate 
1017*7c478bd9Sstevel@tonic-gate 	/* clean up closed fmes */
1018*7c478bd9Sstevel@tonic-gate 	ofmep = ClosedFMEs;
1019*7c478bd9Sstevel@tonic-gate 	while (ofmep != NULL) {
1020*7c478bd9Sstevel@tonic-gate 		svfmep = ofmep->next;
1021*7c478bd9Sstevel@tonic-gate 		destroy_fme(ofmep);
1022*7c478bd9Sstevel@tonic-gate 		ofmep = svfmep;
1023*7c478bd9Sstevel@tonic-gate 	}
1024*7c478bd9Sstevel@tonic-gate 	ClosedFMEs = NULL;
1025*7c478bd9Sstevel@tonic-gate 
1026*7c478bd9Sstevel@tonic-gate 	/* start a new FME */
1027*7c478bd9Sstevel@tonic-gate 	if ((fmep = newfme(eventstring, ipp)) == NULL) {
1028*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "[");
1029*7c478bd9Sstevel@tonic-gate 		ipath_print(O_ALTFP|O_NONL, eventstring, ipp);
1030*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, " CANNOT DIAGNOSE]");
1031*7c478bd9Sstevel@tonic-gate 		publish_undiagnosable(hdl, ffep);
1032*7c478bd9Sstevel@tonic-gate 		return;
1033*7c478bd9Sstevel@tonic-gate 	}
1034*7c478bd9Sstevel@tonic-gate 
1035*7c478bd9Sstevel@tonic-gate 	/* open a case */
1036*7c478bd9Sstevel@tonic-gate 	fmep->fmcase = fmd_case_open(hdl, NULL);
1037*7c478bd9Sstevel@tonic-gate 	fmep->hdl = hdl;
1038*7c478bd9Sstevel@tonic-gate 	init_fme_bufs(fmep);
1039*7c478bd9Sstevel@tonic-gate 
1040*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_NONL, "[");
1041*7c478bd9Sstevel@tonic-gate 	ipath_print(O_ALTFP|O_NONL, eventstring, ipp);
1042*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP, " created FME%d, case %s]", fmep->id,
1043*7c478bd9Sstevel@tonic-gate 	    fmd_case_uuid(hdl, fmep->fmcase));
1044*7c478bd9Sstevel@tonic-gate 
1045*7c478bd9Sstevel@tonic-gate 	ep = fmep->e0;
1046*7c478bd9Sstevel@tonic-gate 	ASSERT(ep != NULL);
1047*7c478bd9Sstevel@tonic-gate 
1048*7c478bd9Sstevel@tonic-gate 	/* note observation */
1049*7c478bd9Sstevel@tonic-gate 	fmep->ecurrent = ep;
1050*7c478bd9Sstevel@tonic-gate 	if (ep->count++ == 0) {
1051*7c478bd9Sstevel@tonic-gate 		/* link it into list of observations seen */
1052*7c478bd9Sstevel@tonic-gate 		ep->observations = fmep->observations;
1053*7c478bd9Sstevel@tonic-gate 		fmep->observations = ep;
1054*7c478bd9Sstevel@tonic-gate 		ep->nvp = evnv_dupnvl(nvl);
1055*7c478bd9Sstevel@tonic-gate 		serialize_observation(fmep, eventstring, ipp);
1056*7c478bd9Sstevel@tonic-gate 	}
1057*7c478bd9Sstevel@tonic-gate 
1058*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->Rcount);
1059*7c478bd9Sstevel@tonic-gate 
1060*7c478bd9Sstevel@tonic-gate 	if (ffep) {
1061*7c478bd9Sstevel@tonic-gate 		fmd_case_add_ereport(hdl, fmep->fmcase, ffep);
1062*7c478bd9Sstevel@tonic-gate 		fmd_case_setprincipal(hdl, fmep->fmcase, ffep);
1063*7c478bd9Sstevel@tonic-gate 		fmep->e0r = ffep;
1064*7c478bd9Sstevel@tonic-gate 	}
1065*7c478bd9Sstevel@tonic-gate 
1066*7c478bd9Sstevel@tonic-gate 	/* give the diagnosis algorithm a shot at the new FME state */
1067*7c478bd9Sstevel@tonic-gate 	fme_eval(fmep, ffep);
1068*7c478bd9Sstevel@tonic-gate }
1069*7c478bd9Sstevel@tonic-gate 
1070*7c478bd9Sstevel@tonic-gate void
1071*7c478bd9Sstevel@tonic-gate fme_status(int flags)
1072*7c478bd9Sstevel@tonic-gate {
1073*7c478bd9Sstevel@tonic-gate 	struct fme *fmep;
1074*7c478bd9Sstevel@tonic-gate 
1075*7c478bd9Sstevel@tonic-gate 	if (FMElist == NULL) {
1076*7c478bd9Sstevel@tonic-gate 		out(flags, "No fault management exercises underway.");
1077*7c478bd9Sstevel@tonic-gate 		return;
1078*7c478bd9Sstevel@tonic-gate 	}
1079*7c478bd9Sstevel@tonic-gate 
1080*7c478bd9Sstevel@tonic-gate 	for (fmep = FMElist; fmep; fmep = fmep->next)
1081*7c478bd9Sstevel@tonic-gate 		fme_print(flags, fmep);
1082*7c478bd9Sstevel@tonic-gate }
1083*7c478bd9Sstevel@tonic-gate 
1084*7c478bd9Sstevel@tonic-gate /*
1085*7c478bd9Sstevel@tonic-gate  * "indent" routines used mostly for nicely formatted debug output, but also
1086*7c478bd9Sstevel@tonic-gate  * for sanity checking for infinite recursion bugs.
1087*7c478bd9Sstevel@tonic-gate  */
1088*7c478bd9Sstevel@tonic-gate 
1089*7c478bd9Sstevel@tonic-gate #define	MAX_INDENT 1024
1090*7c478bd9Sstevel@tonic-gate static const char *indent_s[MAX_INDENT];
1091*7c478bd9Sstevel@tonic-gate static int current_indent;
1092*7c478bd9Sstevel@tonic-gate 
1093*7c478bd9Sstevel@tonic-gate static void
1094*7c478bd9Sstevel@tonic-gate indent_push(const char *s)
1095*7c478bd9Sstevel@tonic-gate {
1096*7c478bd9Sstevel@tonic-gate 	if (current_indent < MAX_INDENT)
1097*7c478bd9Sstevel@tonic-gate 		indent_s[current_indent++] = s;
1098*7c478bd9Sstevel@tonic-gate 	else
1099*7c478bd9Sstevel@tonic-gate 		out(O_DIE, "unexpected recursion depth (%d)", current_indent);
1100*7c478bd9Sstevel@tonic-gate }
1101*7c478bd9Sstevel@tonic-gate 
1102*7c478bd9Sstevel@tonic-gate static void
1103*7c478bd9Sstevel@tonic-gate indent_set(const char *s)
1104*7c478bd9Sstevel@tonic-gate {
1105*7c478bd9Sstevel@tonic-gate 	current_indent = 0;
1106*7c478bd9Sstevel@tonic-gate 	indent_push(s);
1107*7c478bd9Sstevel@tonic-gate }
1108*7c478bd9Sstevel@tonic-gate 
1109*7c478bd9Sstevel@tonic-gate static void
1110*7c478bd9Sstevel@tonic-gate indent_pop(void)
1111*7c478bd9Sstevel@tonic-gate {
1112*7c478bd9Sstevel@tonic-gate 	if (current_indent > 0)
1113*7c478bd9Sstevel@tonic-gate 		current_indent--;
1114*7c478bd9Sstevel@tonic-gate 	else
1115*7c478bd9Sstevel@tonic-gate 		out(O_DIE, "recursion underflow");
1116*7c478bd9Sstevel@tonic-gate }
1117*7c478bd9Sstevel@tonic-gate 
1118*7c478bd9Sstevel@tonic-gate static void
1119*7c478bd9Sstevel@tonic-gate indent(void)
1120*7c478bd9Sstevel@tonic-gate {
1121*7c478bd9Sstevel@tonic-gate 	int i;
1122*7c478bd9Sstevel@tonic-gate 	if (!Verbose)
1123*7c478bd9Sstevel@tonic-gate 		return;
1124*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < current_indent; i++)
1125*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, indent_s[i]);
1126*7c478bd9Sstevel@tonic-gate }
1127*7c478bd9Sstevel@tonic-gate 
1128*7c478bd9Sstevel@tonic-gate static int
1129*7c478bd9Sstevel@tonic-gate suspects_changed(struct fme *fmep)
1130*7c478bd9Sstevel@tonic-gate {
1131*7c478bd9Sstevel@tonic-gate 	struct event *suspects = fmep->suspects;
1132*7c478bd9Sstevel@tonic-gate 	struct event *psuspects = fmep->psuspects;
1133*7c478bd9Sstevel@tonic-gate 
1134*7c478bd9Sstevel@tonic-gate 	while (suspects != NULL && psuspects != NULL) {
1135*7c478bd9Sstevel@tonic-gate 		if (suspects != psuspects)
1136*7c478bd9Sstevel@tonic-gate 			return (1);
1137*7c478bd9Sstevel@tonic-gate 		suspects = suspects->suspects;
1138*7c478bd9Sstevel@tonic-gate 		psuspects = psuspects->psuspects;
1139*7c478bd9Sstevel@tonic-gate 	}
1140*7c478bd9Sstevel@tonic-gate 
1141*7c478bd9Sstevel@tonic-gate 	return (suspects != psuspects);
1142*7c478bd9Sstevel@tonic-gate }
1143*7c478bd9Sstevel@tonic-gate 
1144*7c478bd9Sstevel@tonic-gate #define	SLNEW		1
1145*7c478bd9Sstevel@tonic-gate #define	SLCHANGED	2
1146*7c478bd9Sstevel@tonic-gate #define	SLWAIT		3
1147*7c478bd9Sstevel@tonic-gate #define	SLDISPROVED	4
1148*7c478bd9Sstevel@tonic-gate 
1149*7c478bd9Sstevel@tonic-gate static void
1150*7c478bd9Sstevel@tonic-gate print_suspects(int circumstance, struct fme *fmep)
1151*7c478bd9Sstevel@tonic-gate {
1152*7c478bd9Sstevel@tonic-gate 	struct event *ep;
1153*7c478bd9Sstevel@tonic-gate 
1154*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_NONL, "[");
1155*7c478bd9Sstevel@tonic-gate 	if (circumstance == SLCHANGED) {
1156*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "FME%d diagnosis changed. state: %s, "
1157*7c478bd9Sstevel@tonic-gate 		    "suspect list:", fmep->id, fme_state2str(fmep->state));
1158*7c478bd9Sstevel@tonic-gate 	} else if (circumstance == SLWAIT) {
1159*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "FME%d set wait timer ", fmep->id);
1160*7c478bd9Sstevel@tonic-gate 		ptree_timeval(O_ALTFP|O_NONL, &fmep->wull);
1161*7c478bd9Sstevel@tonic-gate 	} else if (circumstance == SLDISPROVED) {
1162*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "FME%d DIAGNOSIS UNKNOWN", fmep->id);
1163*7c478bd9Sstevel@tonic-gate 	} else {
1164*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "FME%d DIAGNOSIS PRODUCED:", fmep->id);
1165*7c478bd9Sstevel@tonic-gate 	}
1166*7c478bd9Sstevel@tonic-gate 
1167*7c478bd9Sstevel@tonic-gate 	if (circumstance == SLWAIT || circumstance == SLDISPROVED) {
1168*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "]");
1169*7c478bd9Sstevel@tonic-gate 		return;
1170*7c478bd9Sstevel@tonic-gate 	}
1171*7c478bd9Sstevel@tonic-gate 
1172*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->suspects; ep; ep = ep->suspects) {
1173*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, " ");
1174*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_NONL, ep);
1175*7c478bd9Sstevel@tonic-gate 	}
1176*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP, "]");
1177*7c478bd9Sstevel@tonic-gate }
1178*7c478bd9Sstevel@tonic-gate 
1179*7c478bd9Sstevel@tonic-gate static struct node *
1180*7c478bd9Sstevel@tonic-gate eventprop_lookup(struct event *ep, const char *propname)
1181*7c478bd9Sstevel@tonic-gate {
1182*7c478bd9Sstevel@tonic-gate 	return (lut_lookup(ep->props, (void *)propname, NULL));
1183*7c478bd9Sstevel@tonic-gate }
1184*7c478bd9Sstevel@tonic-gate 
1185*7c478bd9Sstevel@tonic-gate #define	MAXDIGITIDX	23
1186*7c478bd9Sstevel@tonic-gate static char numbuf[MAXDIGITIDX + 1];
1187*7c478bd9Sstevel@tonic-gate 
1188*7c478bd9Sstevel@tonic-gate static int
1189*7c478bd9Sstevel@tonic-gate node2uint(struct node *n, uint_t *valp)
1190*7c478bd9Sstevel@tonic-gate {
1191*7c478bd9Sstevel@tonic-gate 	struct evalue value;
1192*7c478bd9Sstevel@tonic-gate 	struct lut *globals = NULL;
1193*7c478bd9Sstevel@tonic-gate 
1194*7c478bd9Sstevel@tonic-gate 	if (n == NULL)
1195*7c478bd9Sstevel@tonic-gate 		return (1);
1196*7c478bd9Sstevel@tonic-gate 
1197*7c478bd9Sstevel@tonic-gate 	/*
1198*7c478bd9Sstevel@tonic-gate 	 * check value.v since we are being asked to convert an unsigned
1199*7c478bd9Sstevel@tonic-gate 	 * long long int to an unsigned int
1200*7c478bd9Sstevel@tonic-gate 	 */
1201*7c478bd9Sstevel@tonic-gate 	if (! eval_expr(n, NULL, NULL, &globals, NULL, NULL, 0, &value) ||
1202*7c478bd9Sstevel@tonic-gate 	    value.t != UINT64 || value.v > (1ULL << 32))
1203*7c478bd9Sstevel@tonic-gate 		return (1);
1204*7c478bd9Sstevel@tonic-gate 
1205*7c478bd9Sstevel@tonic-gate 	*valp = (uint_t)value.v;
1206*7c478bd9Sstevel@tonic-gate 
1207*7c478bd9Sstevel@tonic-gate 	return (0);
1208*7c478bd9Sstevel@tonic-gate }
1209*7c478bd9Sstevel@tonic-gate 
1210*7c478bd9Sstevel@tonic-gate static nvlist_t *
1211*7c478bd9Sstevel@tonic-gate node2fmri(struct node *n)
1212*7c478bd9Sstevel@tonic-gate {
1213*7c478bd9Sstevel@tonic-gate 	nvlist_t **pa, *f, *p;
1214*7c478bd9Sstevel@tonic-gate 	struct node *nc;
1215*7c478bd9Sstevel@tonic-gate 	uint_t depth = 0;
1216*7c478bd9Sstevel@tonic-gate 	char *numstr, *nullbyte;
1217*7c478bd9Sstevel@tonic-gate 	char *failure;
1218*7c478bd9Sstevel@tonic-gate 	int err, i;
1219*7c478bd9Sstevel@tonic-gate 
1220*7c478bd9Sstevel@tonic-gate 	/* XXX do we need to be able to handle a non-T_NAME node? */
1221*7c478bd9Sstevel@tonic-gate 	if (n == NULL || n->t != T_NAME)
1222*7c478bd9Sstevel@tonic-gate 		return (NULL);
1223*7c478bd9Sstevel@tonic-gate 
1224*7c478bd9Sstevel@tonic-gate 	for (nc = n; nc != NULL; nc = nc->u.name.next) {
1225*7c478bd9Sstevel@tonic-gate 		if (nc->u.name.child == NULL || nc->u.name.child->t != T_NUM)
1226*7c478bd9Sstevel@tonic-gate 			break;
1227*7c478bd9Sstevel@tonic-gate 		depth++;
1228*7c478bd9Sstevel@tonic-gate 	}
1229*7c478bd9Sstevel@tonic-gate 
1230*7c478bd9Sstevel@tonic-gate 	if (nc != NULL) {
1231*7c478bd9Sstevel@tonic-gate 		/* We bailed early, something went wrong */
1232*7c478bd9Sstevel@tonic-gate 		return (NULL);
1233*7c478bd9Sstevel@tonic-gate 	}
1234*7c478bd9Sstevel@tonic-gate 
1235*7c478bd9Sstevel@tonic-gate 	if ((err = nvlist_xalloc(&f, NV_UNIQUE_NAME, &Eft_nv_hdl)) != 0)
1236*7c478bd9Sstevel@tonic-gate 		out(O_DIE|O_SYS, "alloc of fmri nvl failed");
1237*7c478bd9Sstevel@tonic-gate 	pa = alloca(depth * sizeof (nvlist_t *));
1238*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < depth; i++)
1239*7c478bd9Sstevel@tonic-gate 		pa[i] = NULL;
1240*7c478bd9Sstevel@tonic-gate 
1241*7c478bd9Sstevel@tonic-gate 	err = nvlist_add_string(f, FM_FMRI_SCHEME, FM_FMRI_SCHEME_HC);
1242*7c478bd9Sstevel@tonic-gate 	err |= nvlist_add_uint8(f, FM_VERSION, FM_HC_SCHEME_VERSION);
1243*7c478bd9Sstevel@tonic-gate 	err |= nvlist_add_string(f, FM_FMRI_HC_ROOT, "");
1244*7c478bd9Sstevel@tonic-gate 	err |= nvlist_add_uint32(f, FM_FMRI_HC_LIST_SZ, depth);
1245*7c478bd9Sstevel@tonic-gate 	if (err != 0) {
1246*7c478bd9Sstevel@tonic-gate 		failure = "basic construction of FMRI failed";
1247*7c478bd9Sstevel@tonic-gate 		goto boom;
1248*7c478bd9Sstevel@tonic-gate 	}
1249*7c478bd9Sstevel@tonic-gate 
1250*7c478bd9Sstevel@tonic-gate 	numbuf[MAXDIGITIDX] = '\0';
1251*7c478bd9Sstevel@tonic-gate 	nullbyte = &numbuf[MAXDIGITIDX];
1252*7c478bd9Sstevel@tonic-gate 	i = 0;
1253*7c478bd9Sstevel@tonic-gate 
1254*7c478bd9Sstevel@tonic-gate 	for (nc = n; nc != NULL; nc = nc->u.name.next) {
1255*7c478bd9Sstevel@tonic-gate 		err = nvlist_xalloc(&p, NV_UNIQUE_NAME, &Eft_nv_hdl);
1256*7c478bd9Sstevel@tonic-gate 		if (err != 0) {
1257*7c478bd9Sstevel@tonic-gate 			failure = "alloc of an hc-pair failed";
1258*7c478bd9Sstevel@tonic-gate 			goto boom;
1259*7c478bd9Sstevel@tonic-gate 		}
1260*7c478bd9Sstevel@tonic-gate 		err = nvlist_add_string(p, FM_FMRI_HC_NAME, nc->u.name.s);
1261*7c478bd9Sstevel@tonic-gate 		numstr = ulltostr(nc->u.name.child->u.ull, nullbyte);
1262*7c478bd9Sstevel@tonic-gate 		err |= nvlist_add_string(p, FM_FMRI_HC_ID, numstr);
1263*7c478bd9Sstevel@tonic-gate 		if (err != 0) {
1264*7c478bd9Sstevel@tonic-gate 			failure = "construction of an hc-pair failed";
1265*7c478bd9Sstevel@tonic-gate 			goto boom;
1266*7c478bd9Sstevel@tonic-gate 		}
1267*7c478bd9Sstevel@tonic-gate 		pa[i++] = p;
1268*7c478bd9Sstevel@tonic-gate 	}
1269*7c478bd9Sstevel@tonic-gate 
1270*7c478bd9Sstevel@tonic-gate 	err = nvlist_add_nvlist_array(f, FM_FMRI_HC_LIST, pa, depth);
1271*7c478bd9Sstevel@tonic-gate 	if (err == 0) {
1272*7c478bd9Sstevel@tonic-gate 		for (i = 0; i < depth; i++)
1273*7c478bd9Sstevel@tonic-gate 			if (pa[i] != NULL)
1274*7c478bd9Sstevel@tonic-gate 				nvlist_free(pa[i]);
1275*7c478bd9Sstevel@tonic-gate 		return (f);
1276*7c478bd9Sstevel@tonic-gate 	}
1277*7c478bd9Sstevel@tonic-gate 	failure = "addition of hc-pair array to FMRI failed";
1278*7c478bd9Sstevel@tonic-gate 
1279*7c478bd9Sstevel@tonic-gate boom:
1280*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < depth; i++)
1281*7c478bd9Sstevel@tonic-gate 		if (pa[i] != NULL)
1282*7c478bd9Sstevel@tonic-gate 			nvlist_free(pa[i]);
1283*7c478bd9Sstevel@tonic-gate 	nvlist_free(f);
1284*7c478bd9Sstevel@tonic-gate 	out(O_DIE, "%s", failure);
1285*7c478bd9Sstevel@tonic-gate 	/*NOTREACHED*/
1286*7c478bd9Sstevel@tonic-gate }
1287*7c478bd9Sstevel@tonic-gate 
1288*7c478bd9Sstevel@tonic-gate static uint_t
1289*7c478bd9Sstevel@tonic-gate avg(uint_t sum, uint_t cnt)
1290*7c478bd9Sstevel@tonic-gate {
1291*7c478bd9Sstevel@tonic-gate 	unsigned long long s = sum * 10;
1292*7c478bd9Sstevel@tonic-gate 
1293*7c478bd9Sstevel@tonic-gate 	return ((s / cnt / 10) + (((s / cnt % 10) >= 5) ? 1 : 0));
1294*7c478bd9Sstevel@tonic-gate }
1295*7c478bd9Sstevel@tonic-gate 
1296*7c478bd9Sstevel@tonic-gate static uint8_t
1297*7c478bd9Sstevel@tonic-gate percentof(uint_t part, uint_t whole)
1298*7c478bd9Sstevel@tonic-gate {
1299*7c478bd9Sstevel@tonic-gate 	unsigned long long p = part * 1000;
1300*7c478bd9Sstevel@tonic-gate 
1301*7c478bd9Sstevel@tonic-gate 	return ((p / whole / 10) + (((p / whole % 10) >= 5) ? 1 : 0));
1302*7c478bd9Sstevel@tonic-gate }
1303*7c478bd9Sstevel@tonic-gate 
1304*7c478bd9Sstevel@tonic-gate static struct rsl {
1305*7c478bd9Sstevel@tonic-gate 	struct event *suspect;
1306*7c478bd9Sstevel@tonic-gate 	nvlist_t *asru;
1307*7c478bd9Sstevel@tonic-gate 	nvlist_t *fru;
1308*7c478bd9Sstevel@tonic-gate 	nvlist_t *rsrc;
1309*7c478bd9Sstevel@tonic-gate };
1310*7c478bd9Sstevel@tonic-gate 
1311*7c478bd9Sstevel@tonic-gate /*
1312*7c478bd9Sstevel@tonic-gate  *  rslfree -- free internal members of struct rsl not expected to be
1313*7c478bd9Sstevel@tonic-gate  *	freed elsewhere.
1314*7c478bd9Sstevel@tonic-gate  */
1315*7c478bd9Sstevel@tonic-gate static void
1316*7c478bd9Sstevel@tonic-gate rslfree(struct rsl *freeme)
1317*7c478bd9Sstevel@tonic-gate {
1318*7c478bd9Sstevel@tonic-gate 	if (freeme->asru != NULL)
1319*7c478bd9Sstevel@tonic-gate 		nvlist_free(freeme->asru);
1320*7c478bd9Sstevel@tonic-gate 	if (freeme->fru != NULL)
1321*7c478bd9Sstevel@tonic-gate 		nvlist_free(freeme->fru);
1322*7c478bd9Sstevel@tonic-gate 	if (freeme->rsrc != NULL && freeme->rsrc != freeme->asru)
1323*7c478bd9Sstevel@tonic-gate 		nvlist_free(freeme->rsrc);
1324*7c478bd9Sstevel@tonic-gate }
1325*7c478bd9Sstevel@tonic-gate 
1326*7c478bd9Sstevel@tonic-gate /*
1327*7c478bd9Sstevel@tonic-gate  *  rslcmp -- compare two rsl structures.  Use the following
1328*7c478bd9Sstevel@tonic-gate  *	comparisons to establish cardinality:
1329*7c478bd9Sstevel@tonic-gate  *
1330*7c478bd9Sstevel@tonic-gate  *	1. Name of the suspect's class. (simple strcmp)
1331*7c478bd9Sstevel@tonic-gate  *	2. Name of the suspect's ASRU. (trickier, since nvlist)
1332*7c478bd9Sstevel@tonic-gate  *
1333*7c478bd9Sstevel@tonic-gate  */
1334*7c478bd9Sstevel@tonic-gate static int
1335*7c478bd9Sstevel@tonic-gate rslcmp(const void *a, const void *b)
1336*7c478bd9Sstevel@tonic-gate {
1337*7c478bd9Sstevel@tonic-gate 	struct rsl *r1 = (struct rsl *)a;
1338*7c478bd9Sstevel@tonic-gate 	struct rsl *r2 = (struct rsl *)b;
1339*7c478bd9Sstevel@tonic-gate 	int rv;
1340*7c478bd9Sstevel@tonic-gate 
1341*7c478bd9Sstevel@tonic-gate 	rv = strcmp(r1->suspect->enode->u.event.ename->u.name.s,
1342*7c478bd9Sstevel@tonic-gate 	    r2->suspect->enode->u.event.ename->u.name.s);
1343*7c478bd9Sstevel@tonic-gate 	if (rv != 0)
1344*7c478bd9Sstevel@tonic-gate 		return (rv);
1345*7c478bd9Sstevel@tonic-gate 
1346*7c478bd9Sstevel@tonic-gate 	if (r1->asru == NULL && r2->asru == NULL)
1347*7c478bd9Sstevel@tonic-gate 		return (0);
1348*7c478bd9Sstevel@tonic-gate 	if (r1->asru == NULL)
1349*7c478bd9Sstevel@tonic-gate 		return (-1);
1350*7c478bd9Sstevel@tonic-gate 	if (r2->asru == NULL)
1351*7c478bd9Sstevel@tonic-gate 		return (1);
1352*7c478bd9Sstevel@tonic-gate 	return (evnv_cmpnvl(r1->asru, r2->asru, 0));
1353*7c478bd9Sstevel@tonic-gate }
1354*7c478bd9Sstevel@tonic-gate 
1355*7c478bd9Sstevel@tonic-gate /*
1356*7c478bd9Sstevel@tonic-gate  *  rsluniq -- given an array of rsl structures, seek out and "remove"
1357*7c478bd9Sstevel@tonic-gate  *	any duplicates.  Dups are "remove"d by NULLing the suspect pointer
1358*7c478bd9Sstevel@tonic-gate  *	of the array element.  Removal also means updating the number of
1359*7c478bd9Sstevel@tonic-gate  *	problems and the number of problems which are not faults.  User
1360*7c478bd9Sstevel@tonic-gate  *	provides the first and last element pointers.
1361*7c478bd9Sstevel@tonic-gate  */
1362*7c478bd9Sstevel@tonic-gate static void
1363*7c478bd9Sstevel@tonic-gate rsluniq(struct rsl *first, struct rsl *last, int *nprobs, int *nnonf)
1364*7c478bd9Sstevel@tonic-gate {
1365*7c478bd9Sstevel@tonic-gate 	struct rsl *cr;
1366*7c478bd9Sstevel@tonic-gate 
1367*7c478bd9Sstevel@tonic-gate 	if (*nprobs == 1)
1368*7c478bd9Sstevel@tonic-gate 		return;
1369*7c478bd9Sstevel@tonic-gate 
1370*7c478bd9Sstevel@tonic-gate 	/*
1371*7c478bd9Sstevel@tonic-gate 	 *  At this point, we only expect duplicate defects.
1372*7c478bd9Sstevel@tonic-gate 	 *  Eversholt's diagnosis algorithm prevents duplicate
1373*7c478bd9Sstevel@tonic-gate 	 *  suspects, but we rewrite defects in the platform code after
1374*7c478bd9Sstevel@tonic-gate 	 *  the diagnosis is made, and that can introduce new
1375*7c478bd9Sstevel@tonic-gate 	 *  duplicates.
1376*7c478bd9Sstevel@tonic-gate 	 */
1377*7c478bd9Sstevel@tonic-gate 	while (first <= last) {
1378*7c478bd9Sstevel@tonic-gate 		if (first->suspect == NULL || !is_defect(first->suspect->t)) {
1379*7c478bd9Sstevel@tonic-gate 			first++;
1380*7c478bd9Sstevel@tonic-gate 			continue;
1381*7c478bd9Sstevel@tonic-gate 		}
1382*7c478bd9Sstevel@tonic-gate 		cr = first + 1;
1383*7c478bd9Sstevel@tonic-gate 		while (cr <= last) {
1384*7c478bd9Sstevel@tonic-gate 			if (is_defect(first->suspect->t)) {
1385*7c478bd9Sstevel@tonic-gate 				if (rslcmp(first, cr) == 0) {
1386*7c478bd9Sstevel@tonic-gate 					cr->suspect = NULL;
1387*7c478bd9Sstevel@tonic-gate 					rslfree(cr);
1388*7c478bd9Sstevel@tonic-gate 					(*nprobs)--;
1389*7c478bd9Sstevel@tonic-gate 					(*nnonf)--;
1390*7c478bd9Sstevel@tonic-gate 				}
1391*7c478bd9Sstevel@tonic-gate 			}
1392*7c478bd9Sstevel@tonic-gate 			/*
1393*7c478bd9Sstevel@tonic-gate 			 * assume all defects are in order after our
1394*7c478bd9Sstevel@tonic-gate 			 * sort and short circuit here with "else break" ?
1395*7c478bd9Sstevel@tonic-gate 			 */
1396*7c478bd9Sstevel@tonic-gate 			cr++;
1397*7c478bd9Sstevel@tonic-gate 		}
1398*7c478bd9Sstevel@tonic-gate 		first++;
1399*7c478bd9Sstevel@tonic-gate 	}
1400*7c478bd9Sstevel@tonic-gate }
1401*7c478bd9Sstevel@tonic-gate 
1402*7c478bd9Sstevel@tonic-gate /*
1403*7c478bd9Sstevel@tonic-gate  * get_resources -- for a given suspect, determine what ASRU, FRU and
1404*7c478bd9Sstevel@tonic-gate  *     RSRC nvlists should be advertised in the final suspect list.
1405*7c478bd9Sstevel@tonic-gate  */
1406*7c478bd9Sstevel@tonic-gate void
1407*7c478bd9Sstevel@tonic-gate get_resources(struct event *sp, struct rsl *rsrcs, struct config *croot)
1408*7c478bd9Sstevel@tonic-gate {
1409*7c478bd9Sstevel@tonic-gate 	struct node *asrudef, *frudef;
1410*7c478bd9Sstevel@tonic-gate 	nvlist_t *asru, *fru;
1411*7c478bd9Sstevel@tonic-gate 	nvlist_t *rsrc = NULL;
1412*7c478bd9Sstevel@tonic-gate 	char *pathstr;
1413*7c478bd9Sstevel@tonic-gate 
1414*7c478bd9Sstevel@tonic-gate 	/*
1415*7c478bd9Sstevel@tonic-gate 	 * First find any ASRU and/or FRU defined in the
1416*7c478bd9Sstevel@tonic-gate 	 * initial fault tree.
1417*7c478bd9Sstevel@tonic-gate 	 */
1418*7c478bd9Sstevel@tonic-gate 	asrudef = eventprop_lookup(sp, L_ASRU);
1419*7c478bd9Sstevel@tonic-gate 	frudef = eventprop_lookup(sp, L_FRU);
1420*7c478bd9Sstevel@tonic-gate 
1421*7c478bd9Sstevel@tonic-gate 	/*
1422*7c478bd9Sstevel@tonic-gate 	 * Create FMRIs based on those definitions
1423*7c478bd9Sstevel@tonic-gate 	 */
1424*7c478bd9Sstevel@tonic-gate 	asru = node2fmri(asrudef);
1425*7c478bd9Sstevel@tonic-gate 	fru = node2fmri(frudef);
1426*7c478bd9Sstevel@tonic-gate 	pathstr = ipath2str(NULL, sp->ipp);
1427*7c478bd9Sstevel@tonic-gate 
1428*7c478bd9Sstevel@tonic-gate 	/*
1429*7c478bd9Sstevel@tonic-gate 	 * Allow for platform translations of the FMRIs
1430*7c478bd9Sstevel@tonic-gate 	 */
1431*7c478bd9Sstevel@tonic-gate 	platform_units_translate(is_defect(sp->t), croot, &asru, &fru, &rsrc,
1432*7c478bd9Sstevel@tonic-gate 	    pathstr);
1433*7c478bd9Sstevel@tonic-gate 
1434*7c478bd9Sstevel@tonic-gate 	FREE(pathstr);
1435*7c478bd9Sstevel@tonic-gate 	rsrcs->suspect = sp;
1436*7c478bd9Sstevel@tonic-gate 	rsrcs->asru = asru;
1437*7c478bd9Sstevel@tonic-gate 	rsrcs->fru = fru;
1438*7c478bd9Sstevel@tonic-gate 	rsrcs->rsrc = rsrc;
1439*7c478bd9Sstevel@tonic-gate }
1440*7c478bd9Sstevel@tonic-gate 
1441*7c478bd9Sstevel@tonic-gate /*
1442*7c478bd9Sstevel@tonic-gate  * trim_suspects -- prior to publishing, we may need to remove some
1443*7c478bd9Sstevel@tonic-gate  *    suspects from the list.  If we're auto-closing upsets, we don't
1444*7c478bd9Sstevel@tonic-gate  *    want any of those in the published list.  If the ASRUs for multiple
1445*7c478bd9Sstevel@tonic-gate  *    defects resolve to the same ASRU (driver) we only want to publish
1446*7c478bd9Sstevel@tonic-gate  *    that as a single suspect.
1447*7c478bd9Sstevel@tonic-gate  */
1448*7c478bd9Sstevel@tonic-gate static void
1449*7c478bd9Sstevel@tonic-gate trim_suspects(struct fme *fmep, boolean_t no_upsets, struct rsl **begin,
1450*7c478bd9Sstevel@tonic-gate     struct rsl **end)
1451*7c478bd9Sstevel@tonic-gate {
1452*7c478bd9Sstevel@tonic-gate 	struct event *ep;
1453*7c478bd9Sstevel@tonic-gate 	struct rsl *rp;
1454*7c478bd9Sstevel@tonic-gate 	int rpcnt;
1455*7c478bd9Sstevel@tonic-gate 
1456*7c478bd9Sstevel@tonic-gate 	/*
1457*7c478bd9Sstevel@tonic-gate 	 * First save the suspects in the psuspects, then copy back
1458*7c478bd9Sstevel@tonic-gate 	 * only the ones we wish to retain.  This resets nsuspects to
1459*7c478bd9Sstevel@tonic-gate 	 * zero.
1460*7c478bd9Sstevel@tonic-gate 	 */
1461*7c478bd9Sstevel@tonic-gate 	rpcnt = fmep->nsuspects;
1462*7c478bd9Sstevel@tonic-gate 	save_suspects(fmep);
1463*7c478bd9Sstevel@tonic-gate 
1464*7c478bd9Sstevel@tonic-gate 	/*
1465*7c478bd9Sstevel@tonic-gate 	 * allocate an array of resource pointers for the suspects.
1466*7c478bd9Sstevel@tonic-gate 	 * We may end up using less than the full allocation, but this
1467*7c478bd9Sstevel@tonic-gate 	 * is a very short-lived array.  publish_suspects() will free
1468*7c478bd9Sstevel@tonic-gate 	 * this array when it's done using it.
1469*7c478bd9Sstevel@tonic-gate 	 */
1470*7c478bd9Sstevel@tonic-gate 	rp = *begin = MALLOC(rpcnt * sizeof (struct rsl));
1471*7c478bd9Sstevel@tonic-gate 	bzero(rp, rpcnt * sizeof (struct rsl));
1472*7c478bd9Sstevel@tonic-gate 
1473*7c478bd9Sstevel@tonic-gate 	/* first pass, remove any unwanted upsets and populate our array */
1474*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->psuspects; ep; ep = ep->psuspects) {
1475*7c478bd9Sstevel@tonic-gate 		if (no_upsets && is_upset(ep->t))
1476*7c478bd9Sstevel@tonic-gate 			continue;
1477*7c478bd9Sstevel@tonic-gate 		get_resources(ep, rp, fmep->cfgdata->cooked);
1478*7c478bd9Sstevel@tonic-gate 		rp++;
1479*7c478bd9Sstevel@tonic-gate 		fmep->nsuspects++;
1480*7c478bd9Sstevel@tonic-gate 		if (!is_fault(ep->t))
1481*7c478bd9Sstevel@tonic-gate 			fmep->nonfault++;
1482*7c478bd9Sstevel@tonic-gate 	}
1483*7c478bd9Sstevel@tonic-gate 
1484*7c478bd9Sstevel@tonic-gate 	/* if all we had was unwanted upsets, we're done */
1485*7c478bd9Sstevel@tonic-gate 	if (fmep->nsuspects == 0)
1486*7c478bd9Sstevel@tonic-gate 		return;
1487*7c478bd9Sstevel@tonic-gate 
1488*7c478bd9Sstevel@tonic-gate 	*end = rp - 1;
1489*7c478bd9Sstevel@tonic-gate 
1490*7c478bd9Sstevel@tonic-gate 	/* sort the array */
1491*7c478bd9Sstevel@tonic-gate 	qsort(*begin, fmep->nsuspects, sizeof (struct rsl), rslcmp);
1492*7c478bd9Sstevel@tonic-gate 	rsluniq(*begin, *end, &fmep->nsuspects, &fmep->nonfault);
1493*7c478bd9Sstevel@tonic-gate }
1494*7c478bd9Sstevel@tonic-gate 
1495*7c478bd9Sstevel@tonic-gate static void
1496*7c478bd9Sstevel@tonic-gate publish_suspects(struct fme *fmep)
1497*7c478bd9Sstevel@tonic-gate {
1498*7c478bd9Sstevel@tonic-gate 	struct event *ep;
1499*7c478bd9Sstevel@tonic-gate 	struct rsl *srl = NULL;
1500*7c478bd9Sstevel@tonic-gate 	struct rsl *erl;
1501*7c478bd9Sstevel@tonic-gate 	struct rsl *rp;
1502*7c478bd9Sstevel@tonic-gate 	nvlist_t *fault;
1503*7c478bd9Sstevel@tonic-gate 	uint8_t cert;
1504*7c478bd9Sstevel@tonic-gate 	uint_t *frs;
1505*7c478bd9Sstevel@tonic-gate 	uint_t fravg, frsum, fr;
1506*7c478bd9Sstevel@tonic-gate 	int frcnt, fridx;
1507*7c478bd9Sstevel@tonic-gate 	boolean_t no_upsets = B_FALSE;
1508*7c478bd9Sstevel@tonic-gate 
1509*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->diags);
1510*7c478bd9Sstevel@tonic-gate 
1511*7c478bd9Sstevel@tonic-gate 	/*
1512*7c478bd9Sstevel@tonic-gate 	 * The current fmd interfaces don't allow us to solve a case
1513*7c478bd9Sstevel@tonic-gate 	 * that's already solved.  If we make a new case, what of the
1514*7c478bd9Sstevel@tonic-gate 	 * ereports?  We don't appear to have an interface that allows
1515*7c478bd9Sstevel@tonic-gate 	 * us to access the ereports attached to a case (if we wanted
1516*7c478bd9Sstevel@tonic-gate 	 * to copy the original case's ereport attachments to the new
1517*7c478bd9Sstevel@tonic-gate 	 * case) and it's also a bit unclear if there would be any
1518*7c478bd9Sstevel@tonic-gate 	 * problems with having ereports attached to multiple cases
1519*7c478bd9Sstevel@tonic-gate 	 * and/or attaching DIAGNOSED ereports to a case.  For now,
1520*7c478bd9Sstevel@tonic-gate 	 * we'll just output a message.
1521*7c478bd9Sstevel@tonic-gate 	 */
1522*7c478bd9Sstevel@tonic-gate 	if (fmep->posted_suspects ||
1523*7c478bd9Sstevel@tonic-gate 	    fmd_case_solved(fmep->hdl, fmep->fmcase)) {
1524*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_NONL, "Revised diagnosis for case %s: ",
1525*7c478bd9Sstevel@tonic-gate 		    fmd_case_uuid(fmep->hdl, fmep->fmcase));
1526*7c478bd9Sstevel@tonic-gate 		for (ep = fmep->suspects; ep; ep = ep->suspects) {
1527*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_NONL, " ");
1528*7c478bd9Sstevel@tonic-gate 			itree_pevent_brief(O_ALTFP|O_NONL, ep);
1529*7c478bd9Sstevel@tonic-gate 		}
1530*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, NULL);
1531*7c478bd9Sstevel@tonic-gate 		return;
1532*7c478bd9Sstevel@tonic-gate 	}
1533*7c478bd9Sstevel@tonic-gate 
1534*7c478bd9Sstevel@tonic-gate 	/*
1535*7c478bd9Sstevel@tonic-gate 	 * If we're auto-closing upsets, we don't want to include them
1536*7c478bd9Sstevel@tonic-gate 	 * in any produced suspect lists or certainty accounting.
1537*7c478bd9Sstevel@tonic-gate 	 */
1538*7c478bd9Sstevel@tonic-gate 	if (Autoclose != NULL)
1539*7c478bd9Sstevel@tonic-gate 		if (strcmp(Autoclose, "true") == 0 ||
1540*7c478bd9Sstevel@tonic-gate 		    strcmp(Autoclose, "all") == 0 ||
1541*7c478bd9Sstevel@tonic-gate 		    strcmp(Autoclose, "upsets") == 0)
1542*7c478bd9Sstevel@tonic-gate 			no_upsets = B_TRUE;
1543*7c478bd9Sstevel@tonic-gate 
1544*7c478bd9Sstevel@tonic-gate 	trim_suspects(fmep, no_upsets, &srl, &erl);
1545*7c478bd9Sstevel@tonic-gate 
1546*7c478bd9Sstevel@tonic-gate 	/*
1547*7c478bd9Sstevel@tonic-gate 	 * If the resulting suspect list has no members, we're
1548*7c478bd9Sstevel@tonic-gate 	 * done.  Returning here will simply close the case.
1549*7c478bd9Sstevel@tonic-gate 	 */
1550*7c478bd9Sstevel@tonic-gate 	if (fmep->nsuspects == 0) {
1551*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP,
1552*7c478bd9Sstevel@tonic-gate 		    "[FME%d, case %s (all suspects are upsets)]",
1553*7c478bd9Sstevel@tonic-gate 		    fmep->id, fmd_case_uuid(fmep->hdl, fmep->fmcase));
1554*7c478bd9Sstevel@tonic-gate 		FREE(srl);
1555*7c478bd9Sstevel@tonic-gate 		restore_suspects(fmep);
1556*7c478bd9Sstevel@tonic-gate 		return;
1557*7c478bd9Sstevel@tonic-gate 	}
1558*7c478bd9Sstevel@tonic-gate 
1559*7c478bd9Sstevel@tonic-gate 	/*
1560*7c478bd9Sstevel@tonic-gate 	 * If the suspect list is all faults, then for a given fault,
1561*7c478bd9Sstevel@tonic-gate 	 * say X of N, X's certainty is computed via:
1562*7c478bd9Sstevel@tonic-gate 	 *
1563*7c478bd9Sstevel@tonic-gate 	 * fitrate(X) / (fitrate(1) + ... + fitrate(N)) * 100
1564*7c478bd9Sstevel@tonic-gate 	 *
1565*7c478bd9Sstevel@tonic-gate 	 * If none of the suspects are faults, and there are N suspects,
1566*7c478bd9Sstevel@tonic-gate 	 * the certainty of a given suspect is 100/N.
1567*7c478bd9Sstevel@tonic-gate 	 *
1568*7c478bd9Sstevel@tonic-gate 	 * If there are are a mixture of faults and other problems in
1569*7c478bd9Sstevel@tonic-gate 	 * the suspect list, we take an average of the faults'
1570*7c478bd9Sstevel@tonic-gate 	 * FITrates and treat this average as the FITrate for any
1571*7c478bd9Sstevel@tonic-gate 	 * non-faults.  The fitrate of any given suspect is then
1572*7c478bd9Sstevel@tonic-gate 	 * computed per the first formula above.
1573*7c478bd9Sstevel@tonic-gate 	 */
1574*7c478bd9Sstevel@tonic-gate 	if (fmep->nonfault == fmep->nsuspects) {
1575*7c478bd9Sstevel@tonic-gate 		/* NO faults in the suspect list */
1576*7c478bd9Sstevel@tonic-gate 		cert = percentof(1, fmep->nsuspects);
1577*7c478bd9Sstevel@tonic-gate 	} else {
1578*7c478bd9Sstevel@tonic-gate 		/* sum the fitrates */
1579*7c478bd9Sstevel@tonic-gate 		frs = alloca(fmep->nsuspects * sizeof (uint_t));
1580*7c478bd9Sstevel@tonic-gate 		fridx = frcnt = frsum = 0;
1581*7c478bd9Sstevel@tonic-gate 
1582*7c478bd9Sstevel@tonic-gate 		for (rp = srl; rp <= erl; rp++) {
1583*7c478bd9Sstevel@tonic-gate 			struct node *n;
1584*7c478bd9Sstevel@tonic-gate 
1585*7c478bd9Sstevel@tonic-gate 			if (rp->suspect == NULL)
1586*7c478bd9Sstevel@tonic-gate 				continue;
1587*7c478bd9Sstevel@tonic-gate 			if (!is_fault(rp->suspect->t)) {
1588*7c478bd9Sstevel@tonic-gate 				frs[fridx++] = 0;
1589*7c478bd9Sstevel@tonic-gate 				continue;
1590*7c478bd9Sstevel@tonic-gate 			}
1591*7c478bd9Sstevel@tonic-gate 			n = eventprop_lookup(rp->suspect, L_FITrate);
1592*7c478bd9Sstevel@tonic-gate 			if (node2uint(n, &fr) != 0) {
1593*7c478bd9Sstevel@tonic-gate 				out(O_DEBUG|O_NONL, "event ");
1594*7c478bd9Sstevel@tonic-gate 				ipath_print(O_DEBUG|O_NONL,
1595*7c478bd9Sstevel@tonic-gate 				    ep->enode->u.event.ename->u.name.s,
1596*7c478bd9Sstevel@tonic-gate 				    ep->ipp);
1597*7c478bd9Sstevel@tonic-gate 				out(O_DEBUG, " has no FITrate (using 1)");
1598*7c478bd9Sstevel@tonic-gate 				fr = 1;
1599*7c478bd9Sstevel@tonic-gate 			} else if (fr == 0) {
1600*7c478bd9Sstevel@tonic-gate 				out(O_DEBUG|O_NONL, "event ");
1601*7c478bd9Sstevel@tonic-gate 				ipath_print(O_DEBUG|O_NONL,
1602*7c478bd9Sstevel@tonic-gate 				    ep->enode->u.event.ename->u.name.s,
1603*7c478bd9Sstevel@tonic-gate 				    ep->ipp);
1604*7c478bd9Sstevel@tonic-gate 				out(O_DEBUG, " has zero FITrate (using 1)");
1605*7c478bd9Sstevel@tonic-gate 				fr = 1;
1606*7c478bd9Sstevel@tonic-gate 			}
1607*7c478bd9Sstevel@tonic-gate 
1608*7c478bd9Sstevel@tonic-gate 			frs[fridx++] = fr;
1609*7c478bd9Sstevel@tonic-gate 			frsum += fr;
1610*7c478bd9Sstevel@tonic-gate 			frcnt++;
1611*7c478bd9Sstevel@tonic-gate 		}
1612*7c478bd9Sstevel@tonic-gate 		fravg = avg(frsum, frcnt);
1613*7c478bd9Sstevel@tonic-gate 		for (fridx = 0; fridx < fmep->nsuspects; fridx++)
1614*7c478bd9Sstevel@tonic-gate 			if (frs[fridx] == 0) {
1615*7c478bd9Sstevel@tonic-gate 				frs[fridx] = fravg;
1616*7c478bd9Sstevel@tonic-gate 				frsum += fravg;
1617*7c478bd9Sstevel@tonic-gate 			}
1618*7c478bd9Sstevel@tonic-gate 	}
1619*7c478bd9Sstevel@tonic-gate 
1620*7c478bd9Sstevel@tonic-gate 	/* Add them in reverse order of our sort, as fmd reverses order */
1621*7c478bd9Sstevel@tonic-gate 	for (rp = erl; rp >= srl; rp--) {
1622*7c478bd9Sstevel@tonic-gate 		if (rp->suspect == NULL)
1623*7c478bd9Sstevel@tonic-gate 			continue;
1624*7c478bd9Sstevel@tonic-gate 		if (fmep->nonfault != fmep->nsuspects)
1625*7c478bd9Sstevel@tonic-gate 			cert = percentof(frs[--fridx], frsum);
1626*7c478bd9Sstevel@tonic-gate 		fault = fmd_nvl_create_fault(fmep->hdl,
1627*7c478bd9Sstevel@tonic-gate 		    rp->suspect->enode->u.event.ename->u.name.s,
1628*7c478bd9Sstevel@tonic-gate 		    cert,
1629*7c478bd9Sstevel@tonic-gate 		    rp->asru,
1630*7c478bd9Sstevel@tonic-gate 		    rp->fru,
1631*7c478bd9Sstevel@tonic-gate 		    rp->rsrc);
1632*7c478bd9Sstevel@tonic-gate 		if (fault == NULL)
1633*7c478bd9Sstevel@tonic-gate 			out(O_DIE, "fault creation failed");
1634*7c478bd9Sstevel@tonic-gate 		fmd_case_add_suspect(fmep->hdl, fmep->fmcase, fault);
1635*7c478bd9Sstevel@tonic-gate 		rp->suspect->fault = fault;
1636*7c478bd9Sstevel@tonic-gate 		rslfree(rp);
1637*7c478bd9Sstevel@tonic-gate 	}
1638*7c478bd9Sstevel@tonic-gate 	fmd_case_solve(fmep->hdl, fmep->fmcase);
1639*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP, "[solving FME%d, case %s]", fmep->id,
1640*7c478bd9Sstevel@tonic-gate 	    fmd_case_uuid(fmep->hdl, fmep->fmcase));
1641*7c478bd9Sstevel@tonic-gate 
1642*7c478bd9Sstevel@tonic-gate 	if (Autoconvict) {
1643*7c478bd9Sstevel@tonic-gate 		for (rp = srl; rp <= erl; rp++) {
1644*7c478bd9Sstevel@tonic-gate 			if (rp->suspect == NULL)
1645*7c478bd9Sstevel@tonic-gate 				continue;
1646*7c478bd9Sstevel@tonic-gate 			fmd_case_convict(fmep->hdl,
1647*7c478bd9Sstevel@tonic-gate 			    fmep->fmcase, rp->suspect->fault);
1648*7c478bd9Sstevel@tonic-gate 		}
1649*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP, "[convicting FME%d, case %s]", fmep->id,
1650*7c478bd9Sstevel@tonic-gate 		    fmd_case_uuid(fmep->hdl, fmep->fmcase));
1651*7c478bd9Sstevel@tonic-gate 	}
1652*7c478bd9Sstevel@tonic-gate 
1653*7c478bd9Sstevel@tonic-gate 	/*
1654*7c478bd9Sstevel@tonic-gate 	 * revert to the original suspect list
1655*7c478bd9Sstevel@tonic-gate 	 */
1656*7c478bd9Sstevel@tonic-gate 	FREE(srl);
1657*7c478bd9Sstevel@tonic-gate 	restore_suspects(fmep);
1658*7c478bd9Sstevel@tonic-gate }
1659*7c478bd9Sstevel@tonic-gate 
1660*7c478bd9Sstevel@tonic-gate static void
1661*7c478bd9Sstevel@tonic-gate publish_undiagnosable(fmd_hdl_t *hdl, fmd_event_t *ffep)
1662*7c478bd9Sstevel@tonic-gate {
1663*7c478bd9Sstevel@tonic-gate 	struct case_list *newcase;
1664*7c478bd9Sstevel@tonic-gate 	nvlist_t *defect;
1665*7c478bd9Sstevel@tonic-gate 
1666*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP,
1667*7c478bd9Sstevel@tonic-gate 	    "[undiagnosable ereport received, "
1668*7c478bd9Sstevel@tonic-gate 	    "creating and closing a new case (%s)]",
1669*7c478bd9Sstevel@tonic-gate 	    Undiag_reason ? Undiag_reason : "reason not provided");
1670*7c478bd9Sstevel@tonic-gate 
1671*7c478bd9Sstevel@tonic-gate 	newcase = MALLOC(sizeof (struct case_list));
1672*7c478bd9Sstevel@tonic-gate 	newcase->next = NULL;
1673*7c478bd9Sstevel@tonic-gate 
1674*7c478bd9Sstevel@tonic-gate 	newcase->fmcase = fmd_case_open(hdl, NULL);
1675*7c478bd9Sstevel@tonic-gate 	if (Undiagablecaselist != NULL)
1676*7c478bd9Sstevel@tonic-gate 		newcase->next = Undiagablecaselist;
1677*7c478bd9Sstevel@tonic-gate 	Undiagablecaselist = newcase;
1678*7c478bd9Sstevel@tonic-gate 
1679*7c478bd9Sstevel@tonic-gate 	if (ffep != NULL)
1680*7c478bd9Sstevel@tonic-gate 		fmd_case_add_ereport(hdl, newcase->fmcase, ffep);
1681*7c478bd9Sstevel@tonic-gate 
1682*7c478bd9Sstevel@tonic-gate 	defect = fmd_nvl_create_fault(hdl, UNDIAGNOSABLE_DEFECT, 100,
1683*7c478bd9Sstevel@tonic-gate 	    NULL, NULL, NULL);
1684*7c478bd9Sstevel@tonic-gate 	if (Undiag_reason != NULL)
1685*7c478bd9Sstevel@tonic-gate 		(void) nvlist_add_string(defect, UNDIAG_REASON, Undiag_reason);
1686*7c478bd9Sstevel@tonic-gate 	fmd_case_add_suspect(hdl, newcase->fmcase, defect);
1687*7c478bd9Sstevel@tonic-gate 
1688*7c478bd9Sstevel@tonic-gate 	fmd_case_solve(hdl, newcase->fmcase);
1689*7c478bd9Sstevel@tonic-gate 	fmd_case_close(hdl, newcase->fmcase);
1690*7c478bd9Sstevel@tonic-gate }
1691*7c478bd9Sstevel@tonic-gate 
1692*7c478bd9Sstevel@tonic-gate static void
1693*7c478bd9Sstevel@tonic-gate fme_undiagnosable(struct fme *f)
1694*7c478bd9Sstevel@tonic-gate {
1695*7c478bd9Sstevel@tonic-gate 	nvlist_t *defect;
1696*7c478bd9Sstevel@tonic-gate 
1697*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP, "[solving/closing FME%d, case %s (%s)]",
1698*7c478bd9Sstevel@tonic-gate 	    f->id, fmd_case_uuid(f->hdl, f->fmcase),
1699*7c478bd9Sstevel@tonic-gate 	    Undiag_reason ? Undiag_reason : "undiagnosable");
1700*7c478bd9Sstevel@tonic-gate 
1701*7c478bd9Sstevel@tonic-gate 	defect = fmd_nvl_create_fault(f->hdl, UNDIAGNOSABLE_DEFECT, 100,
1702*7c478bd9Sstevel@tonic-gate 	    NULL, NULL, NULL);
1703*7c478bd9Sstevel@tonic-gate 	if (Undiag_reason != NULL)
1704*7c478bd9Sstevel@tonic-gate 		(void) nvlist_add_string(defect, UNDIAG_REASON, Undiag_reason);
1705*7c478bd9Sstevel@tonic-gate 	fmd_case_add_suspect(f->hdl, f->fmcase, defect);
1706*7c478bd9Sstevel@tonic-gate 	fmd_case_solve(f->hdl, f->fmcase);
1707*7c478bd9Sstevel@tonic-gate 	destroy_fme_bufs(f);
1708*7c478bd9Sstevel@tonic-gate 	fmd_case_close(f->hdl, f->fmcase);
1709*7c478bd9Sstevel@tonic-gate }
1710*7c478bd9Sstevel@tonic-gate 
1711*7c478bd9Sstevel@tonic-gate /*
1712*7c478bd9Sstevel@tonic-gate  * fme_close_case
1713*7c478bd9Sstevel@tonic-gate  *
1714*7c478bd9Sstevel@tonic-gate  *	Find the requested case amongst our fmes and close it.  Free up
1715*7c478bd9Sstevel@tonic-gate  *	the related fme.
1716*7c478bd9Sstevel@tonic-gate  */
1717*7c478bd9Sstevel@tonic-gate void
1718*7c478bd9Sstevel@tonic-gate fme_close_case(fmd_hdl_t *hdl, fmd_case_t *fmcase)
1719*7c478bd9Sstevel@tonic-gate {
1720*7c478bd9Sstevel@tonic-gate 	struct case_list *ucasep, *prevcasep = NULL;
1721*7c478bd9Sstevel@tonic-gate 	struct fme *prev = NULL;
1722*7c478bd9Sstevel@tonic-gate 	struct fme *fmep;
1723*7c478bd9Sstevel@tonic-gate 
1724*7c478bd9Sstevel@tonic-gate 	for (ucasep = Undiagablecaselist; ucasep; ucasep = ucasep->next) {
1725*7c478bd9Sstevel@tonic-gate 		if (fmcase != ucasep->fmcase) {
1726*7c478bd9Sstevel@tonic-gate 			prevcasep = ucasep;
1727*7c478bd9Sstevel@tonic-gate 			continue;
1728*7c478bd9Sstevel@tonic-gate 		}
1729*7c478bd9Sstevel@tonic-gate 
1730*7c478bd9Sstevel@tonic-gate 		if (prevcasep == NULL)
1731*7c478bd9Sstevel@tonic-gate 			Undiagablecaselist = Undiagablecaselist->next;
1732*7c478bd9Sstevel@tonic-gate 		else
1733*7c478bd9Sstevel@tonic-gate 			prevcasep->next = ucasep->next;
1734*7c478bd9Sstevel@tonic-gate 
1735*7c478bd9Sstevel@tonic-gate 		FREE(ucasep);
1736*7c478bd9Sstevel@tonic-gate 		return;
1737*7c478bd9Sstevel@tonic-gate 	}
1738*7c478bd9Sstevel@tonic-gate 
1739*7c478bd9Sstevel@tonic-gate 	for (fmep = FMElist; fmep; fmep = fmep->next) {
1740*7c478bd9Sstevel@tonic-gate 		if (fmep->hdl == hdl && fmep->fmcase == fmcase)
1741*7c478bd9Sstevel@tonic-gate 			break;
1742*7c478bd9Sstevel@tonic-gate 		prev = fmep;
1743*7c478bd9Sstevel@tonic-gate 	}
1744*7c478bd9Sstevel@tonic-gate 
1745*7c478bd9Sstevel@tonic-gate 	if (fmep == NULL) {
1746*7c478bd9Sstevel@tonic-gate 		out(O_WARN, "Eft asked to close unrecognized case [%s].",
1747*7c478bd9Sstevel@tonic-gate 		    fmd_case_uuid(hdl, fmcase));
1748*7c478bd9Sstevel@tonic-gate 		return;
1749*7c478bd9Sstevel@tonic-gate 	}
1750*7c478bd9Sstevel@tonic-gate 
1751*7c478bd9Sstevel@tonic-gate 	if (EFMElist == fmep)
1752*7c478bd9Sstevel@tonic-gate 		EFMElist = prev;
1753*7c478bd9Sstevel@tonic-gate 
1754*7c478bd9Sstevel@tonic-gate 	if (prev == NULL)
1755*7c478bd9Sstevel@tonic-gate 		FMElist = FMElist->next;
1756*7c478bd9Sstevel@tonic-gate 	else
1757*7c478bd9Sstevel@tonic-gate 		prev->next = fmep->next;
1758*7c478bd9Sstevel@tonic-gate 
1759*7c478bd9Sstevel@tonic-gate 	fmep->next = NULL;
1760*7c478bd9Sstevel@tonic-gate 
1761*7c478bd9Sstevel@tonic-gate 	/* Get rid of any timer this fme has set */
1762*7c478bd9Sstevel@tonic-gate 	if (fmep->wull != 0)
1763*7c478bd9Sstevel@tonic-gate 		fmd_timer_remove(fmep->hdl, fmep->timer);
1764*7c478bd9Sstevel@tonic-gate 
1765*7c478bd9Sstevel@tonic-gate 	if (ClosedFMEs == NULL) {
1766*7c478bd9Sstevel@tonic-gate 		ClosedFMEs = fmep;
1767*7c478bd9Sstevel@tonic-gate 	} else {
1768*7c478bd9Sstevel@tonic-gate 		fmep->next = ClosedFMEs;
1769*7c478bd9Sstevel@tonic-gate 		ClosedFMEs = fmep;
1770*7c478bd9Sstevel@tonic-gate 	}
1771*7c478bd9Sstevel@tonic-gate }
1772*7c478bd9Sstevel@tonic-gate 
1773*7c478bd9Sstevel@tonic-gate /*
1774*7c478bd9Sstevel@tonic-gate  * fme_set_timer()
1775*7c478bd9Sstevel@tonic-gate  *	If the time we need to wait for the given FME is less than the
1776*7c478bd9Sstevel@tonic-gate  *	current timer, kick that old timer out and establish a new one.
1777*7c478bd9Sstevel@tonic-gate  */
1778*7c478bd9Sstevel@tonic-gate static void
1779*7c478bd9Sstevel@tonic-gate fme_set_timer(struct fme *fmep, unsigned long long wull)
1780*7c478bd9Sstevel@tonic-gate {
1781*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, " fme_set_timer: request to wait ");
1782*7c478bd9Sstevel@tonic-gate 	ptree_timeval(O_ALTFP|O_VERB, &wull);
1783*7c478bd9Sstevel@tonic-gate 
1784*7c478bd9Sstevel@tonic-gate 	if (wull <= fmep->pull) {
1785*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "already have waited at least ");
1786*7c478bd9Sstevel@tonic-gate 		ptree_timeval(O_ALTFP|O_VERB, &fmep->pull);
1787*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
1788*7c478bd9Sstevel@tonic-gate 		/* we've waited at least wull already, don't need timer */
1789*7c478bd9Sstevel@tonic-gate 		return;
1790*7c478bd9Sstevel@tonic-gate 	}
1791*7c478bd9Sstevel@tonic-gate 
1792*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, " currently ");
1793*7c478bd9Sstevel@tonic-gate 	if (fmep->wull != 0) {
1794*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "waiting ");
1795*7c478bd9Sstevel@tonic-gate 		ptree_timeval(O_ALTFP|O_VERB, &fmep->wull);
1796*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
1797*7c478bd9Sstevel@tonic-gate 	} else {
1798*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "not waiting");
1799*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
1800*7c478bd9Sstevel@tonic-gate 	}
1801*7c478bd9Sstevel@tonic-gate 
1802*7c478bd9Sstevel@tonic-gate 	if (fmep->wull != 0)
1803*7c478bd9Sstevel@tonic-gate 		if (wull >= fmep->wull)
1804*7c478bd9Sstevel@tonic-gate 			/* New timer would fire later than established timer */
1805*7c478bd9Sstevel@tonic-gate 			return;
1806*7c478bd9Sstevel@tonic-gate 
1807*7c478bd9Sstevel@tonic-gate 	if (fmep->wull != 0)
1808*7c478bd9Sstevel@tonic-gate 		fmd_timer_remove(fmep->hdl, fmep->timer);
1809*7c478bd9Sstevel@tonic-gate 
1810*7c478bd9Sstevel@tonic-gate 	fmep->timer = fmd_timer_install(fmep->hdl, (void *)fmep,
1811*7c478bd9Sstevel@tonic-gate 	    fmep->e0r, wull);
1812*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, "timer set, id is %ld", fmep->timer);
1813*7c478bd9Sstevel@tonic-gate 	fmep->wull = wull;
1814*7c478bd9Sstevel@tonic-gate }
1815*7c478bd9Sstevel@tonic-gate 
1816*7c478bd9Sstevel@tonic-gate void
1817*7c478bd9Sstevel@tonic-gate fme_timer_fired(struct fme *fmep, id_t tid)
1818*7c478bd9Sstevel@tonic-gate {
1819*7c478bd9Sstevel@tonic-gate 	struct fme *ffmep = NULL;
1820*7c478bd9Sstevel@tonic-gate 
1821*7c478bd9Sstevel@tonic-gate 	for (ffmep = FMElist; ffmep; ffmep = ffmep->next)
1822*7c478bd9Sstevel@tonic-gate 		if (ffmep == fmep)
1823*7c478bd9Sstevel@tonic-gate 			break;
1824*7c478bd9Sstevel@tonic-gate 
1825*7c478bd9Sstevel@tonic-gate 	if (ffmep == NULL) {
1826*7c478bd9Sstevel@tonic-gate 		out(O_WARN, "Timer fired for an FME (%p) not in FMEs list.",
1827*7c478bd9Sstevel@tonic-gate 		    (void *)fmep);
1828*7c478bd9Sstevel@tonic-gate 		return;
1829*7c478bd9Sstevel@tonic-gate 	}
1830*7c478bd9Sstevel@tonic-gate 
1831*7c478bd9Sstevel@tonic-gate 	if (tid != fmep->htid) {
1832*7c478bd9Sstevel@tonic-gate 		/*
1833*7c478bd9Sstevel@tonic-gate 		 * normal timer (not the hesitation timer
1834*7c478bd9Sstevel@tonic-gate 		 */
1835*7c478bd9Sstevel@tonic-gate 		fmep->pull = fmep->wull;
1836*7c478bd9Sstevel@tonic-gate 		fmep->wull = 0;
1837*7c478bd9Sstevel@tonic-gate 		fmd_buf_write(fmep->hdl, fmep->fmcase,
1838*7c478bd9Sstevel@tonic-gate 		    WOBUF_PULL, (void *)&fmep->pull, sizeof (fmep->pull));
1839*7c478bd9Sstevel@tonic-gate 	} else {
1840*7c478bd9Sstevel@tonic-gate 		fmep->hesitated = 1;
1841*7c478bd9Sstevel@tonic-gate 	}
1842*7c478bd9Sstevel@tonic-gate 	fme_eval(fmep, NULL);
1843*7c478bd9Sstevel@tonic-gate }
1844*7c478bd9Sstevel@tonic-gate 
1845*7c478bd9Sstevel@tonic-gate /*
1846*7c478bd9Sstevel@tonic-gate  * Preserve the fme's suspect list in its psuspects list, NULLing the
1847*7c478bd9Sstevel@tonic-gate  * suspects list in the meantime.
1848*7c478bd9Sstevel@tonic-gate  */
1849*7c478bd9Sstevel@tonic-gate static void
1850*7c478bd9Sstevel@tonic-gate save_suspects(struct fme *fmep)
1851*7c478bd9Sstevel@tonic-gate {
1852*7c478bd9Sstevel@tonic-gate 	struct event *ep;
1853*7c478bd9Sstevel@tonic-gate 	struct event *nextep;
1854*7c478bd9Sstevel@tonic-gate 
1855*7c478bd9Sstevel@tonic-gate 	/* zero out the previous suspect list */
1856*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->psuspects; ep; ep = nextep) {
1857*7c478bd9Sstevel@tonic-gate 		nextep = ep->psuspects;
1858*7c478bd9Sstevel@tonic-gate 		ep->psuspects = NULL;
1859*7c478bd9Sstevel@tonic-gate 	}
1860*7c478bd9Sstevel@tonic-gate 	fmep->psuspects = NULL;
1861*7c478bd9Sstevel@tonic-gate 
1862*7c478bd9Sstevel@tonic-gate 	/* zero out the suspect list, copying it to previous suspect list */
1863*7c478bd9Sstevel@tonic-gate 	fmep->psuspects = fmep->suspects;
1864*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->suspects; ep; ep = nextep) {
1865*7c478bd9Sstevel@tonic-gate 		nextep = ep->suspects;
1866*7c478bd9Sstevel@tonic-gate 		ep->psuspects = ep->suspects;
1867*7c478bd9Sstevel@tonic-gate 		ep->suspects = NULL;
1868*7c478bd9Sstevel@tonic-gate 		ep->is_suspect = 0;
1869*7c478bd9Sstevel@tonic-gate 	}
1870*7c478bd9Sstevel@tonic-gate 	fmep->suspects = NULL;
1871*7c478bd9Sstevel@tonic-gate 	fmep->nsuspects = 0;
1872*7c478bd9Sstevel@tonic-gate 	fmep->nonfault = 0;
1873*7c478bd9Sstevel@tonic-gate }
1874*7c478bd9Sstevel@tonic-gate 
1875*7c478bd9Sstevel@tonic-gate /*
1876*7c478bd9Sstevel@tonic-gate  * Retrieve the fme's suspect list from its psuspects list.
1877*7c478bd9Sstevel@tonic-gate  */
1878*7c478bd9Sstevel@tonic-gate static void
1879*7c478bd9Sstevel@tonic-gate restore_suspects(struct fme *fmep)
1880*7c478bd9Sstevel@tonic-gate {
1881*7c478bd9Sstevel@tonic-gate 	struct event *ep;
1882*7c478bd9Sstevel@tonic-gate 	struct event *nextep;
1883*7c478bd9Sstevel@tonic-gate 
1884*7c478bd9Sstevel@tonic-gate 	fmep->nsuspects = fmep->nonfault = 0;
1885*7c478bd9Sstevel@tonic-gate 	fmep->suspects = fmep->psuspects;
1886*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->psuspects; ep; ep = nextep) {
1887*7c478bd9Sstevel@tonic-gate 		fmep->nsuspects++;
1888*7c478bd9Sstevel@tonic-gate 		if (!is_fault(ep->t))
1889*7c478bd9Sstevel@tonic-gate 			fmep->nonfault++;
1890*7c478bd9Sstevel@tonic-gate 		nextep = ep->psuspects;
1891*7c478bd9Sstevel@tonic-gate 		ep->suspects = ep->psuspects;
1892*7c478bd9Sstevel@tonic-gate 	}
1893*7c478bd9Sstevel@tonic-gate }
1894*7c478bd9Sstevel@tonic-gate 
1895*7c478bd9Sstevel@tonic-gate /*
1896*7c478bd9Sstevel@tonic-gate  * this is what we use to call the Emrys prototype code instead of main()
1897*7c478bd9Sstevel@tonic-gate  */
1898*7c478bd9Sstevel@tonic-gate static void
1899*7c478bd9Sstevel@tonic-gate fme_eval(struct fme *fmep, fmd_event_t *ffep)
1900*7c478bd9Sstevel@tonic-gate {
1901*7c478bd9Sstevel@tonic-gate 	struct event *ep;
1902*7c478bd9Sstevel@tonic-gate 	unsigned long long my_delay = TIMEVAL_EVENTUALLY;
1903*7c478bd9Sstevel@tonic-gate 
1904*7c478bd9Sstevel@tonic-gate 	save_suspects(fmep);
1905*7c478bd9Sstevel@tonic-gate 
1906*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, "Evaluate FME %d", fmep->id);
1907*7c478bd9Sstevel@tonic-gate 	indent_set("  ");
1908*7c478bd9Sstevel@tonic-gate 
1909*7c478bd9Sstevel@tonic-gate 	initialize_cycles(fmep);
1910*7c478bd9Sstevel@tonic-gate 	fmep->state = hypothesise(fmep, fmep->e0, fmep->ull, &my_delay, NULL);
1911*7c478bd9Sstevel@tonic-gate 
1912*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "FME%d state: %s, suspect list:", fmep->id,
1913*7c478bd9Sstevel@tonic-gate 	    fme_state2str(fmep->state));
1914*7c478bd9Sstevel@tonic-gate 	for (ep = fmep->suspects; ep; ep = ep->suspects) {
1915*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, " ");
1916*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
1917*7c478bd9Sstevel@tonic-gate 	}
1918*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
1919*7c478bd9Sstevel@tonic-gate 
1920*7c478bd9Sstevel@tonic-gate 	if (fmep->posted_suspects) {
1921*7c478bd9Sstevel@tonic-gate 		/*
1922*7c478bd9Sstevel@tonic-gate 		 * this FME has already posted a diagnosis, so see if
1923*7c478bd9Sstevel@tonic-gate 		 * the event changed the diagnosis and print a warning
1924*7c478bd9Sstevel@tonic-gate 		 * if it did.
1925*7c478bd9Sstevel@tonic-gate 		 *
1926*7c478bd9Sstevel@tonic-gate 		 */
1927*7c478bd9Sstevel@tonic-gate 		if (suspects_changed(fmep)) {
1928*7c478bd9Sstevel@tonic-gate 			print_suspects(SLCHANGED, fmep);
1929*7c478bd9Sstevel@tonic-gate 			publish_suspects(fmep);
1930*7c478bd9Sstevel@tonic-gate 		}
1931*7c478bd9Sstevel@tonic-gate 	} else {
1932*7c478bd9Sstevel@tonic-gate 		switch (fmep->state) {
1933*7c478bd9Sstevel@tonic-gate 		case FME_CREDIBLE:
1934*7c478bd9Sstevel@tonic-gate 			/*
1935*7c478bd9Sstevel@tonic-gate 			 * if the suspect list contains any upsets, we
1936*7c478bd9Sstevel@tonic-gate 			 * turn off the hesitation logic (by setting
1937*7c478bd9Sstevel@tonic-gate 			 * the hesitate flag which normally indicates
1938*7c478bd9Sstevel@tonic-gate 			 * we've already done the hesitate logic).
1939*7c478bd9Sstevel@tonic-gate 			 * this is done because hesitating with upsets
1940*7c478bd9Sstevel@tonic-gate 			 * causes us to explain away additional soft errors
1941*7c478bd9Sstevel@tonic-gate 			 * while the upset FME stays open.
1942*7c478bd9Sstevel@tonic-gate 			 */
1943*7c478bd9Sstevel@tonic-gate 			if (fmep->hesitated == 0) {
1944*7c478bd9Sstevel@tonic-gate 				struct event *s;
1945*7c478bd9Sstevel@tonic-gate 
1946*7c478bd9Sstevel@tonic-gate 				for (s = fmep->suspects; s; s = s->suspects) {
1947*7c478bd9Sstevel@tonic-gate 					if (s->t == N_UPSET) {
1948*7c478bd9Sstevel@tonic-gate 						fmep->hesitated = 1;
1949*7c478bd9Sstevel@tonic-gate 						break;
1950*7c478bd9Sstevel@tonic-gate 					}
1951*7c478bd9Sstevel@tonic-gate 				}
1952*7c478bd9Sstevel@tonic-gate 			}
1953*7c478bd9Sstevel@tonic-gate 
1954*7c478bd9Sstevel@tonic-gate 			if (Hesitate &&
1955*7c478bd9Sstevel@tonic-gate 			    fmep->suspects != NULL &&
1956*7c478bd9Sstevel@tonic-gate 			    fmep->suspects->suspects != NULL &&
1957*7c478bd9Sstevel@tonic-gate 			    fmep->hesitated == 0) {
1958*7c478bd9Sstevel@tonic-gate 				/*
1959*7c478bd9Sstevel@tonic-gate 				 * about to publish multi-entry suspect list,
1960*7c478bd9Sstevel@tonic-gate 				 * set the hesitation timer if not already set.
1961*7c478bd9Sstevel@tonic-gate 				 */
1962*7c478bd9Sstevel@tonic-gate 				if (fmep->htid == 0) {
1963*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_NONL,
1964*7c478bd9Sstevel@tonic-gate 					    "[hesitate FME%d, case %s ",
1965*7c478bd9Sstevel@tonic-gate 					    fmep->id,
1966*7c478bd9Sstevel@tonic-gate 					    fmd_case_uuid(fmep->hdl,
1967*7c478bd9Sstevel@tonic-gate 					    fmep->fmcase));
1968*7c478bd9Sstevel@tonic-gate 					ptree_timeval(O_ALTFP|O_NONL,
1969*7c478bd9Sstevel@tonic-gate 					    (unsigned long long *)&Hesitate);
1970*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP, "]");
1971*7c478bd9Sstevel@tonic-gate 					fme_set_timer(fmep, my_delay);
1972*7c478bd9Sstevel@tonic-gate 					fmep->htid =
1973*7c478bd9Sstevel@tonic-gate 					    fmd_timer_install(fmep->hdl,
1974*7c478bd9Sstevel@tonic-gate 					    (void *)fmep, NULL, Hesitate);
1975*7c478bd9Sstevel@tonic-gate 				} else {
1976*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP,
1977*7c478bd9Sstevel@tonic-gate 					    "[still hesitating FME%d, case %s]",
1978*7c478bd9Sstevel@tonic-gate 					    fmep->id,
1979*7c478bd9Sstevel@tonic-gate 					    fmd_case_uuid(fmep->hdl,
1980*7c478bd9Sstevel@tonic-gate 					    fmep->fmcase));
1981*7c478bd9Sstevel@tonic-gate 				}
1982*7c478bd9Sstevel@tonic-gate 			} else {
1983*7c478bd9Sstevel@tonic-gate 				print_suspects(SLNEW, fmep);
1984*7c478bd9Sstevel@tonic-gate 				(void) upsets_eval(fmep, ffep);
1985*7c478bd9Sstevel@tonic-gate 				publish_suspects(fmep);
1986*7c478bd9Sstevel@tonic-gate 				fmep->posted_suspects = 1;
1987*7c478bd9Sstevel@tonic-gate 				fmd_buf_write(fmep->hdl, fmep->fmcase,
1988*7c478bd9Sstevel@tonic-gate 				    WOBUF_POSTD,
1989*7c478bd9Sstevel@tonic-gate 				    (void *)&fmep->posted_suspects,
1990*7c478bd9Sstevel@tonic-gate 				    sizeof (fmep->posted_suspects));
1991*7c478bd9Sstevel@tonic-gate 			}
1992*7c478bd9Sstevel@tonic-gate 			break;
1993*7c478bd9Sstevel@tonic-gate 
1994*7c478bd9Sstevel@tonic-gate 		case FME_WAIT:
1995*7c478bd9Sstevel@tonic-gate 			/*
1996*7c478bd9Sstevel@tonic-gate 			 * singleton suspect list implies
1997*7c478bd9Sstevel@tonic-gate 			 * no point in waiting
1998*7c478bd9Sstevel@tonic-gate 			 */
1999*7c478bd9Sstevel@tonic-gate 			if (fmep->suspects &&
2000*7c478bd9Sstevel@tonic-gate 			    fmep->suspects->suspects == NULL) {
2001*7c478bd9Sstevel@tonic-gate 				print_suspects(SLNEW, fmep);
2002*7c478bd9Sstevel@tonic-gate 				(void) upsets_eval(fmep, ffep);
2003*7c478bd9Sstevel@tonic-gate 				publish_suspects(fmep);
2004*7c478bd9Sstevel@tonic-gate 				fmep->posted_suspects = 1;
2005*7c478bd9Sstevel@tonic-gate 				fmd_buf_write(fmep->hdl, fmep->fmcase,
2006*7c478bd9Sstevel@tonic-gate 				    WOBUF_POSTD,
2007*7c478bd9Sstevel@tonic-gate 				    (void *)&fmep->posted_suspects,
2008*7c478bd9Sstevel@tonic-gate 				    sizeof (fmep->posted_suspects));
2009*7c478bd9Sstevel@tonic-gate 				fmep->state = FME_CREDIBLE;
2010*7c478bd9Sstevel@tonic-gate 			} else {
2011*7c478bd9Sstevel@tonic-gate 				ASSERT(my_delay > fmep->ull);
2012*7c478bd9Sstevel@tonic-gate 				fme_set_timer(fmep, my_delay);
2013*7c478bd9Sstevel@tonic-gate 				print_suspects(SLWAIT, fmep);
2014*7c478bd9Sstevel@tonic-gate 			}
2015*7c478bd9Sstevel@tonic-gate 			break;
2016*7c478bd9Sstevel@tonic-gate 
2017*7c478bd9Sstevel@tonic-gate 		case FME_DISPROVED:
2018*7c478bd9Sstevel@tonic-gate 			print_suspects(SLDISPROVED, fmep);
2019*7c478bd9Sstevel@tonic-gate 			Undiag_reason = UD_UNSOLVD;
2020*7c478bd9Sstevel@tonic-gate 			fme_undiagnosable(fmep);
2021*7c478bd9Sstevel@tonic-gate 			break;
2022*7c478bd9Sstevel@tonic-gate 		}
2023*7c478bd9Sstevel@tonic-gate 	}
2024*7c478bd9Sstevel@tonic-gate 
2025*7c478bd9Sstevel@tonic-gate 	if (fmep->posted_suspects == 1 && Autoclose != NULL) {
2026*7c478bd9Sstevel@tonic-gate 		int doclose = 0;
2027*7c478bd9Sstevel@tonic-gate 
2028*7c478bd9Sstevel@tonic-gate 		if (strcmp(Autoclose, "true") == 0 ||
2029*7c478bd9Sstevel@tonic-gate 		    strcmp(Autoclose, "all") == 0)
2030*7c478bd9Sstevel@tonic-gate 			doclose = 1;
2031*7c478bd9Sstevel@tonic-gate 
2032*7c478bd9Sstevel@tonic-gate 		if (strcmp(Autoclose, "upsets") == 0) {
2033*7c478bd9Sstevel@tonic-gate 			doclose = 1;
2034*7c478bd9Sstevel@tonic-gate 			for (ep = fmep->suspects; ep; ep = ep->suspects) {
2035*7c478bd9Sstevel@tonic-gate 				if (ep->t != N_UPSET) {
2036*7c478bd9Sstevel@tonic-gate 					doclose = 0;
2037*7c478bd9Sstevel@tonic-gate 					break;
2038*7c478bd9Sstevel@tonic-gate 				}
2039*7c478bd9Sstevel@tonic-gate 			}
2040*7c478bd9Sstevel@tonic-gate 		}
2041*7c478bd9Sstevel@tonic-gate 
2042*7c478bd9Sstevel@tonic-gate 		if (doclose) {
2043*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP, "[closing FME%d, case %s (autoclose)]",
2044*7c478bd9Sstevel@tonic-gate 			    fmep->id, fmd_case_uuid(fmep->hdl, fmep->fmcase));
2045*7c478bd9Sstevel@tonic-gate 
2046*7c478bd9Sstevel@tonic-gate 			destroy_fme_bufs(fmep);
2047*7c478bd9Sstevel@tonic-gate 			fmd_case_close(fmep->hdl, fmep->fmcase);
2048*7c478bd9Sstevel@tonic-gate 		}
2049*7c478bd9Sstevel@tonic-gate 	}
2050*7c478bd9Sstevel@tonic-gate }
2051*7c478bd9Sstevel@tonic-gate 
2052*7c478bd9Sstevel@tonic-gate /*
2053*7c478bd9Sstevel@tonic-gate  * below here is the code derived from the Emrys prototype
2054*7c478bd9Sstevel@tonic-gate  */
2055*7c478bd9Sstevel@tonic-gate 
2056*7c478bd9Sstevel@tonic-gate static void indent(void);
2057*7c478bd9Sstevel@tonic-gate static int triggered(struct fme *fmep, struct event *ep, int mark);
2058*7c478bd9Sstevel@tonic-gate static void mark_arrows(struct fme *fmep, struct event *ep, int mark);
2059*7c478bd9Sstevel@tonic-gate static enum fme_state effects_test(struct fme *fmep,
2060*7c478bd9Sstevel@tonic-gate     struct event *fault_event);
2061*7c478bd9Sstevel@tonic-gate static enum fme_state requirements_test(struct fme *fmep, struct event *ep,
2062*7c478bd9Sstevel@tonic-gate     unsigned long long at_latest_by, unsigned long long *pdelay,
2063*7c478bd9Sstevel@tonic-gate     struct arrow *arrowp);
2064*7c478bd9Sstevel@tonic-gate static enum fme_state causes_test(struct fme *fmep, struct event *ep,
2065*7c478bd9Sstevel@tonic-gate     unsigned long long at_latest_by, unsigned long long *pdelay);
2066*7c478bd9Sstevel@tonic-gate 
2067*7c478bd9Sstevel@tonic-gate static int
2068*7c478bd9Sstevel@tonic-gate triggered(struct fme *fmep, struct event *ep, int mark)
2069*7c478bd9Sstevel@tonic-gate {
2070*7c478bd9Sstevel@tonic-gate 	struct bubble *bp;
2071*7c478bd9Sstevel@tonic-gate 	struct arrowlist *ap;
2072*7c478bd9Sstevel@tonic-gate 	int count = 0;
2073*7c478bd9Sstevel@tonic-gate 
2074*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->Tcallcount);
2075*7c478bd9Sstevel@tonic-gate 	for (bp = itree_next_bubble(ep, NULL); bp;
2076*7c478bd9Sstevel@tonic-gate 	    bp = itree_next_bubble(ep, bp)) {
2077*7c478bd9Sstevel@tonic-gate 		if (bp->t != B_TO)
2078*7c478bd9Sstevel@tonic-gate 			continue;
2079*7c478bd9Sstevel@tonic-gate 		for (ap = itree_next_arrow(bp, NULL); ap;
2080*7c478bd9Sstevel@tonic-gate 		    ap = itree_next_arrow(bp, ap)) {
2081*7c478bd9Sstevel@tonic-gate 			/* check count of marks against K in the bubble */
2082*7c478bd9Sstevel@tonic-gate 			if (ap->arrowp->tail->mark == mark &&
2083*7c478bd9Sstevel@tonic-gate 			    ++count >= bp->nork)
2084*7c478bd9Sstevel@tonic-gate 				return (1);
2085*7c478bd9Sstevel@tonic-gate 		}
2086*7c478bd9Sstevel@tonic-gate 	}
2087*7c478bd9Sstevel@tonic-gate 	return (0);
2088*7c478bd9Sstevel@tonic-gate }
2089*7c478bd9Sstevel@tonic-gate 
2090*7c478bd9Sstevel@tonic-gate static void
2091*7c478bd9Sstevel@tonic-gate mark_arrows(struct fme *fmep, struct event *ep, int mark)
2092*7c478bd9Sstevel@tonic-gate {
2093*7c478bd9Sstevel@tonic-gate 	struct bubble *bp;
2094*7c478bd9Sstevel@tonic-gate 	struct arrowlist *ap;
2095*7c478bd9Sstevel@tonic-gate 
2096*7c478bd9Sstevel@tonic-gate 	for (bp = itree_next_bubble(ep, NULL); bp;
2097*7c478bd9Sstevel@tonic-gate 	    bp = itree_next_bubble(ep, bp)) {
2098*7c478bd9Sstevel@tonic-gate 		if (bp->t != B_FROM)
2099*7c478bd9Sstevel@tonic-gate 			continue;
2100*7c478bd9Sstevel@tonic-gate 		if (bp->mark != mark) {
2101*7c478bd9Sstevel@tonic-gate 			stats_counter_bump(fmep->Marrowcount);
2102*7c478bd9Sstevel@tonic-gate 			bp->mark = mark;
2103*7c478bd9Sstevel@tonic-gate 			for (ap = itree_next_arrow(bp, NULL); ap;
2104*7c478bd9Sstevel@tonic-gate 			    ap = itree_next_arrow(bp, ap)) {
2105*7c478bd9Sstevel@tonic-gate 				struct constraintlist *ctp;
2106*7c478bd9Sstevel@tonic-gate 				struct evalue value;
2107*7c478bd9Sstevel@tonic-gate 				int do_not_follow = 0;
2108*7c478bd9Sstevel@tonic-gate 				/*
2109*7c478bd9Sstevel@tonic-gate 				 * see if false constraint prevents us
2110*7c478bd9Sstevel@tonic-gate 				 * from traversing this arrow, but don't
2111*7c478bd9Sstevel@tonic-gate 				 * bother if the event is an ereport we
2112*7c478bd9Sstevel@tonic-gate 				 * haven't seen
2113*7c478bd9Sstevel@tonic-gate 				 */
2114*7c478bd9Sstevel@tonic-gate 				if (ap->arrowp->head->myevent->t != N_EREPORT ||
2115*7c478bd9Sstevel@tonic-gate 				    ap->arrowp->head->myevent->count != 0) {
2116*7c478bd9Sstevel@tonic-gate 					platform_set_payloadnvp(
2117*7c478bd9Sstevel@tonic-gate 					    ap->arrowp->head->myevent->nvp);
2118*7c478bd9Sstevel@tonic-gate 					for (ctp = ap->arrowp->constraints;
2119*7c478bd9Sstevel@tonic-gate 					    ctp != NULL; ctp = ctp->next) {
2120*7c478bd9Sstevel@tonic-gate 						if (eval_expr(ctp->cnode,
2121*7c478bd9Sstevel@tonic-gate 						    NULL, NULL,
2122*7c478bd9Sstevel@tonic-gate 						    &fmep->globals,
2123*7c478bd9Sstevel@tonic-gate 						    fmep->cfgdata->cooked,
2124*7c478bd9Sstevel@tonic-gate 						    ap->arrowp, 0,
2125*7c478bd9Sstevel@tonic-gate 						    &value) == 0 ||
2126*7c478bd9Sstevel@tonic-gate 						    value.t == UNDEFINED ||
2127*7c478bd9Sstevel@tonic-gate 						    value.v == 0) {
2128*7c478bd9Sstevel@tonic-gate 							do_not_follow = 1;
2129*7c478bd9Sstevel@tonic-gate 							break;
2130*7c478bd9Sstevel@tonic-gate 						}
2131*7c478bd9Sstevel@tonic-gate 					}
2132*7c478bd9Sstevel@tonic-gate 					platform_set_payloadnvp(NULL);
2133*7c478bd9Sstevel@tonic-gate 				}
2134*7c478bd9Sstevel@tonic-gate 
2135*7c478bd9Sstevel@tonic-gate 				if (do_not_follow) {
2136*7c478bd9Sstevel@tonic-gate 					indent();
2137*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_VERB|O_NONL,
2138*7c478bd9Sstevel@tonic-gate 					    "  False arrow to ");
2139*7c478bd9Sstevel@tonic-gate 					itree_pevent_brief(
2140*7c478bd9Sstevel@tonic-gate 					    O_ALTFP|O_VERB|O_NONL,
2141*7c478bd9Sstevel@tonic-gate 					    ap->arrowp->head->myevent);
2142*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_VERB|O_NONL, " ");
2143*7c478bd9Sstevel@tonic-gate 					ptree(O_ALTFP|O_VERB|O_NONL,
2144*7c478bd9Sstevel@tonic-gate 					    ctp->cnode, 1, 0);
2145*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_VERB, NULL);
2146*7c478bd9Sstevel@tonic-gate 					continue;
2147*7c478bd9Sstevel@tonic-gate 				}
2148*7c478bd9Sstevel@tonic-gate 
2149*7c478bd9Sstevel@tonic-gate 				if (triggered(fmep, ap->arrowp->head->myevent,
2150*7c478bd9Sstevel@tonic-gate 				    mark))
2151*7c478bd9Sstevel@tonic-gate 					mark_arrows(fmep,
2152*7c478bd9Sstevel@tonic-gate 					    ap->arrowp->head->myevent, mark);
2153*7c478bd9Sstevel@tonic-gate 			}
2154*7c478bd9Sstevel@tonic-gate 		}
2155*7c478bd9Sstevel@tonic-gate 	}
2156*7c478bd9Sstevel@tonic-gate }
2157*7c478bd9Sstevel@tonic-gate 
2158*7c478bd9Sstevel@tonic-gate static enum fme_state
2159*7c478bd9Sstevel@tonic-gate effects_test(struct fme *fmep, struct event *fault_event)
2160*7c478bd9Sstevel@tonic-gate {
2161*7c478bd9Sstevel@tonic-gate 	struct event *error_event;
2162*7c478bd9Sstevel@tonic-gate 	enum fme_state return_value = FME_CREDIBLE;
2163*7c478bd9Sstevel@tonic-gate 
2164*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->Ecallcount);
2165*7c478bd9Sstevel@tonic-gate 	indent_push("  E");
2166*7c478bd9Sstevel@tonic-gate 	indent();
2167*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "->");
2168*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, fault_event);
2169*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2170*7c478bd9Sstevel@tonic-gate 
2171*7c478bd9Sstevel@tonic-gate 	mark_arrows(fmep, fault_event, 1);
2172*7c478bd9Sstevel@tonic-gate 	for (error_event = fmep->observations;
2173*7c478bd9Sstevel@tonic-gate 	    error_event; error_event = error_event->observations) {
2174*7c478bd9Sstevel@tonic-gate 		indent();
2175*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, " ");
2176*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, error_event);
2177*7c478bd9Sstevel@tonic-gate 		if (!triggered(fmep, error_event, 1)) {
2178*7c478bd9Sstevel@tonic-gate 			return_value = FME_DISPROVED;
2179*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB, " NOT triggered");
2180*7c478bd9Sstevel@tonic-gate 			break;
2181*7c478bd9Sstevel@tonic-gate 		} else {
2182*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB, " triggered");
2183*7c478bd9Sstevel@tonic-gate 		}
2184*7c478bd9Sstevel@tonic-gate 	}
2185*7c478bd9Sstevel@tonic-gate 	mark_arrows(fmep, fault_event, 0);
2186*7c478bd9Sstevel@tonic-gate 
2187*7c478bd9Sstevel@tonic-gate 	indent();
2188*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "<-%s ", fme_state2str(return_value));
2189*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, fault_event);
2190*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2191*7c478bd9Sstevel@tonic-gate 	indent_pop();
2192*7c478bd9Sstevel@tonic-gate 	return (return_value);
2193*7c478bd9Sstevel@tonic-gate }
2194*7c478bd9Sstevel@tonic-gate 
2195*7c478bd9Sstevel@tonic-gate static enum fme_state
2196*7c478bd9Sstevel@tonic-gate requirements_test(struct fme *fmep, struct event *ep,
2197*7c478bd9Sstevel@tonic-gate     unsigned long long at_latest_by, unsigned long long *pdelay,
2198*7c478bd9Sstevel@tonic-gate     struct arrow *arrowp)
2199*7c478bd9Sstevel@tonic-gate {
2200*7c478bd9Sstevel@tonic-gate 	int waiting_events;
2201*7c478bd9Sstevel@tonic-gate 	int credible_events;
2202*7c478bd9Sstevel@tonic-gate 	enum fme_state return_value = FME_CREDIBLE;
2203*7c478bd9Sstevel@tonic-gate 	unsigned long long overall_delay = TIMEVAL_EVENTUALLY;
2204*7c478bd9Sstevel@tonic-gate 	unsigned long long arrow_delay;
2205*7c478bd9Sstevel@tonic-gate 	unsigned long long my_delay;
2206*7c478bd9Sstevel@tonic-gate 	struct event *ep2;
2207*7c478bd9Sstevel@tonic-gate 	struct bubble *bp;
2208*7c478bd9Sstevel@tonic-gate 	struct arrowlist *ap;
2209*7c478bd9Sstevel@tonic-gate 
2210*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->Rcallcount);
2211*7c478bd9Sstevel@tonic-gate 	indent_push("  R");
2212*7c478bd9Sstevel@tonic-gate 	indent();
2213*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "->");
2214*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2215*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, ", at latest by: ");
2216*7c478bd9Sstevel@tonic-gate 	ptree_timeval(O_ALTFP|O_VERB|O_NONL, &at_latest_by);
2217*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2218*7c478bd9Sstevel@tonic-gate 
2219*7c478bd9Sstevel@tonic-gate 	if (ep->t == N_EREPORT) {
2220*7c478bd9Sstevel@tonic-gate 		if (ep->count == 0) {
2221*7c478bd9Sstevel@tonic-gate 			if (fmep->pull >= at_latest_by) {
2222*7c478bd9Sstevel@tonic-gate 				return_value = FME_DISPROVED;
2223*7c478bd9Sstevel@tonic-gate 			} else {
2224*7c478bd9Sstevel@tonic-gate 				*pdelay = at_latest_by;
2225*7c478bd9Sstevel@tonic-gate 				return_value = FME_WAIT;
2226*7c478bd9Sstevel@tonic-gate 			}
2227*7c478bd9Sstevel@tonic-gate 		} else if (arrowp != NULL) {
2228*7c478bd9Sstevel@tonic-gate 			/*
2229*7c478bd9Sstevel@tonic-gate 			 * evaluate constraints only for current observation
2230*7c478bd9Sstevel@tonic-gate 			 */
2231*7c478bd9Sstevel@tonic-gate 			struct constraintlist *ctp;
2232*7c478bd9Sstevel@tonic-gate 			struct evalue value;
2233*7c478bd9Sstevel@tonic-gate 
2234*7c478bd9Sstevel@tonic-gate 			platform_set_payloadnvp(ep->nvp);
2235*7c478bd9Sstevel@tonic-gate 			for (ctp = arrowp->constraints; ctp != NULL;
2236*7c478bd9Sstevel@tonic-gate 				ctp = ctp->next) {
2237*7c478bd9Sstevel@tonic-gate 				if (eval_expr(ctp->cnode, NULL, NULL,
2238*7c478bd9Sstevel@tonic-gate 				    &fmep->globals, fmep->cfgdata->cooked,
2239*7c478bd9Sstevel@tonic-gate 				    arrowp, 0, &value) == 0 ||
2240*7c478bd9Sstevel@tonic-gate 				    value.t == UNDEFINED || value.v == 0) {
2241*7c478bd9Sstevel@tonic-gate 					indent();
2242*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_VERB|O_NONL,
2243*7c478bd9Sstevel@tonic-gate 					    "  False constraint ");
2244*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_VERB|O_NONL, " ");
2245*7c478bd9Sstevel@tonic-gate 					ptree(O_ALTFP|O_VERB|O_NONL,
2246*7c478bd9Sstevel@tonic-gate 					    ctp->cnode, 1, 0);
2247*7c478bd9Sstevel@tonic-gate 					out(O_ALTFP|O_VERB, NULL);
2248*7c478bd9Sstevel@tonic-gate 					return_value = FME_DISPROVED;
2249*7c478bd9Sstevel@tonic-gate 					break;
2250*7c478bd9Sstevel@tonic-gate 				}
2251*7c478bd9Sstevel@tonic-gate 			}
2252*7c478bd9Sstevel@tonic-gate 			platform_set_payloadnvp(NULL);
2253*7c478bd9Sstevel@tonic-gate 		}
2254*7c478bd9Sstevel@tonic-gate 
2255*7c478bd9Sstevel@tonic-gate 		indent();
2256*7c478bd9Sstevel@tonic-gate 		switch (return_value) {
2257*7c478bd9Sstevel@tonic-gate 		case FME_CREDIBLE:
2258*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB|O_NONL, "<-CREDIBLE ");
2259*7c478bd9Sstevel@tonic-gate 			itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2260*7c478bd9Sstevel@tonic-gate 			break;
2261*7c478bd9Sstevel@tonic-gate 		case FME_DISPROVED:
2262*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB|O_NONL, "<-DISPROVED ");
2263*7c478bd9Sstevel@tonic-gate 			itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2264*7c478bd9Sstevel@tonic-gate 			break;
2265*7c478bd9Sstevel@tonic-gate 		case FME_WAIT:
2266*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB|O_NONL, "<-WAIT ");
2267*7c478bd9Sstevel@tonic-gate 			itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2268*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB|O_NONL, " to ");
2269*7c478bd9Sstevel@tonic-gate 			ptree_timeval(O_ALTFP|O_VERB|O_NONL, &at_latest_by);
2270*7c478bd9Sstevel@tonic-gate 			break;
2271*7c478bd9Sstevel@tonic-gate 		default:
2272*7c478bd9Sstevel@tonic-gate 			out(O_DIE, "requirements_test: unexpected fme_state");
2273*7c478bd9Sstevel@tonic-gate 			break;
2274*7c478bd9Sstevel@tonic-gate 		}
2275*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
2276*7c478bd9Sstevel@tonic-gate 		indent_pop();
2277*7c478bd9Sstevel@tonic-gate 
2278*7c478bd9Sstevel@tonic-gate 		return (return_value);
2279*7c478bd9Sstevel@tonic-gate 	}
2280*7c478bd9Sstevel@tonic-gate 
2281*7c478bd9Sstevel@tonic-gate 	/* this event is not a report, descend the tree */
2282*7c478bd9Sstevel@tonic-gate 	for (bp = itree_next_bubble(ep, NULL); bp;
2283*7c478bd9Sstevel@tonic-gate 	    bp = itree_next_bubble(ep, bp)) {
2284*7c478bd9Sstevel@tonic-gate 		if (bp->t != B_FROM)
2285*7c478bd9Sstevel@tonic-gate 			continue;
2286*7c478bd9Sstevel@tonic-gate 		if (bp->mark == 0) {
2287*7c478bd9Sstevel@tonic-gate 			int n = bp->nork;
2288*7c478bd9Sstevel@tonic-gate 
2289*7c478bd9Sstevel@tonic-gate 			bp->mark = 1;
2290*7c478bd9Sstevel@tonic-gate 			credible_events = 0;
2291*7c478bd9Sstevel@tonic-gate 			waiting_events = 0;
2292*7c478bd9Sstevel@tonic-gate 			arrow_delay = TIMEVAL_EVENTUALLY;
2293*7c478bd9Sstevel@tonic-gate 			/*
2294*7c478bd9Sstevel@tonic-gate 			 * n is -1 for 'A' so adjust it.
2295*7c478bd9Sstevel@tonic-gate 			 * XXX just count up the arrows for now.
2296*7c478bd9Sstevel@tonic-gate 			 */
2297*7c478bd9Sstevel@tonic-gate 			if (n < 0) {
2298*7c478bd9Sstevel@tonic-gate 				n = 0;
2299*7c478bd9Sstevel@tonic-gate 				for (ap = itree_next_arrow(bp, NULL); ap;
2300*7c478bd9Sstevel@tonic-gate 				    ap = itree_next_arrow(bp, ap))
2301*7c478bd9Sstevel@tonic-gate 					n++;
2302*7c478bd9Sstevel@tonic-gate 				indent();
2303*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB, " Bubble Counted N=%d", n);
2304*7c478bd9Sstevel@tonic-gate 			} else {
2305*7c478bd9Sstevel@tonic-gate 				indent();
2306*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB, " Bubble N=%d", n);
2307*7c478bd9Sstevel@tonic-gate 			}
2308*7c478bd9Sstevel@tonic-gate 
2309*7c478bd9Sstevel@tonic-gate 			for (ap = itree_next_arrow(bp, NULL); ap;
2310*7c478bd9Sstevel@tonic-gate 			    ap = itree_next_arrow(bp, ap)) {
2311*7c478bd9Sstevel@tonic-gate 				ep2 = ap->arrowp->head->myevent;
2312*7c478bd9Sstevel@tonic-gate 				if (n <= credible_events)
2313*7c478bd9Sstevel@tonic-gate 					break;
2314*7c478bd9Sstevel@tonic-gate 
2315*7c478bd9Sstevel@tonic-gate 				if (triggered(fmep, ep2, 1))
2316*7c478bd9Sstevel@tonic-gate 					/* XXX adding max timevals! */
2317*7c478bd9Sstevel@tonic-gate 					switch (requirements_test(fmep, ep2,
2318*7c478bd9Sstevel@tonic-gate 					    at_latest_by + ap->arrowp->maxdelay,
2319*7c478bd9Sstevel@tonic-gate 					    &my_delay, ap->arrowp)) {
2320*7c478bd9Sstevel@tonic-gate 					case FME_CREDIBLE:
2321*7c478bd9Sstevel@tonic-gate 						credible_events++;
2322*7c478bd9Sstevel@tonic-gate 						break;
2323*7c478bd9Sstevel@tonic-gate 					case FME_DISPROVED:
2324*7c478bd9Sstevel@tonic-gate 						break;
2325*7c478bd9Sstevel@tonic-gate 					case FME_WAIT:
2326*7c478bd9Sstevel@tonic-gate 						if (my_delay < arrow_delay)
2327*7c478bd9Sstevel@tonic-gate 							arrow_delay = my_delay;
2328*7c478bd9Sstevel@tonic-gate 						waiting_events++;
2329*7c478bd9Sstevel@tonic-gate 						break;
2330*7c478bd9Sstevel@tonic-gate 					default:
2331*7c478bd9Sstevel@tonic-gate 						out(O_DIE,
2332*7c478bd9Sstevel@tonic-gate 						"Bug in requirements_test.");
2333*7c478bd9Sstevel@tonic-gate 					}
2334*7c478bd9Sstevel@tonic-gate 				else
2335*7c478bd9Sstevel@tonic-gate 					credible_events++;
2336*7c478bd9Sstevel@tonic-gate 			}
2337*7c478bd9Sstevel@tonic-gate 			indent();
2338*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB, " Credible: %d Waiting %d",
2339*7c478bd9Sstevel@tonic-gate 			    credible_events, waiting_events);
2340*7c478bd9Sstevel@tonic-gate 			if (credible_events + waiting_events < n) {
2341*7c478bd9Sstevel@tonic-gate 				/* Can never meet requirements */
2342*7c478bd9Sstevel@tonic-gate 				indent();
2343*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB|O_NONL, "<-DISPROVED ");
2344*7c478bd9Sstevel@tonic-gate 				itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2345*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB, NULL);
2346*7c478bd9Sstevel@tonic-gate 				indent_pop();
2347*7c478bd9Sstevel@tonic-gate 				return (FME_DISPROVED);
2348*7c478bd9Sstevel@tonic-gate 			}
2349*7c478bd9Sstevel@tonic-gate 			if (credible_events < n) { /* will have to wait */
2350*7c478bd9Sstevel@tonic-gate 				/* wait time is shortest known */
2351*7c478bd9Sstevel@tonic-gate 				if (arrow_delay < overall_delay)
2352*7c478bd9Sstevel@tonic-gate 					overall_delay = arrow_delay;
2353*7c478bd9Sstevel@tonic-gate 				return_value = FME_WAIT;
2354*7c478bd9Sstevel@tonic-gate 			}
2355*7c478bd9Sstevel@tonic-gate 		} else {
2356*7c478bd9Sstevel@tonic-gate 			indent();
2357*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB|O_NONL, " Mark was set: ");
2358*7c478bd9Sstevel@tonic-gate 			itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2359*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB|O_NONL, " to");
2360*7c478bd9Sstevel@tonic-gate 			for (ap = itree_next_arrow(bp, NULL); ap;
2361*7c478bd9Sstevel@tonic-gate 			    ap = itree_next_arrow(bp, ap)) {
2362*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB|O_NONL, " ");
2363*7c478bd9Sstevel@tonic-gate 				itree_pevent_brief(O_ALTFP|O_VERB|O_NONL,
2364*7c478bd9Sstevel@tonic-gate 				    ap->arrowp->head->myevent);
2365*7c478bd9Sstevel@tonic-gate 			}
2366*7c478bd9Sstevel@tonic-gate 			out(O_ALTFP|O_VERB, NULL);
2367*7c478bd9Sstevel@tonic-gate 		}
2368*7c478bd9Sstevel@tonic-gate 	}
2369*7c478bd9Sstevel@tonic-gate 
2370*7c478bd9Sstevel@tonic-gate 	/*
2371*7c478bd9Sstevel@tonic-gate 	 * evaluate constraints for ctlist, which is the list of
2372*7c478bd9Sstevel@tonic-gate 	 * constraints for the arrow pointing into this node of the tree
2373*7c478bd9Sstevel@tonic-gate 	 */
2374*7c478bd9Sstevel@tonic-gate 	if (return_value == FME_CREDIBLE && arrowp != NULL) {
2375*7c478bd9Sstevel@tonic-gate 		struct constraintlist *ctp;
2376*7c478bd9Sstevel@tonic-gate 		struct evalue value;
2377*7c478bd9Sstevel@tonic-gate 
2378*7c478bd9Sstevel@tonic-gate 		platform_set_payloadnvp(ep->nvp);
2379*7c478bd9Sstevel@tonic-gate 		for (ctp = arrowp->constraints; ctp != NULL;
2380*7c478bd9Sstevel@tonic-gate 			ctp = ctp->next) {
2381*7c478bd9Sstevel@tonic-gate 			if (eval_expr(ctp->cnode, NULL,	NULL, &fmep->globals,
2382*7c478bd9Sstevel@tonic-gate 			    fmep->cfgdata->cooked, arrowp, 0, &value) == 0 ||
2383*7c478bd9Sstevel@tonic-gate 			    value.t == UNDEFINED || value.v == 0) {
2384*7c478bd9Sstevel@tonic-gate 				indent();
2385*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB|O_NONL,
2386*7c478bd9Sstevel@tonic-gate 				    "  False constraint ");
2387*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB|O_NONL, " ");
2388*7c478bd9Sstevel@tonic-gate 				ptree(O_ALTFP|O_VERB|O_NONL,
2389*7c478bd9Sstevel@tonic-gate 				    ctp->cnode, 1, 0);
2390*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB, NULL);
2391*7c478bd9Sstevel@tonic-gate 				return_value = FME_DISPROVED;
2392*7c478bd9Sstevel@tonic-gate 				break;
2393*7c478bd9Sstevel@tonic-gate 			}
2394*7c478bd9Sstevel@tonic-gate 		}
2395*7c478bd9Sstevel@tonic-gate 		platform_set_payloadnvp(NULL);
2396*7c478bd9Sstevel@tonic-gate 	}
2397*7c478bd9Sstevel@tonic-gate 
2398*7c478bd9Sstevel@tonic-gate 	if (return_value == FME_WAIT)
2399*7c478bd9Sstevel@tonic-gate 		*pdelay = overall_delay;
2400*7c478bd9Sstevel@tonic-gate 	indent();
2401*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "<-%s ", fme_state2str(return_value));
2402*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2403*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2404*7c478bd9Sstevel@tonic-gate 	indent_pop();
2405*7c478bd9Sstevel@tonic-gate 	return (return_value);
2406*7c478bd9Sstevel@tonic-gate }
2407*7c478bd9Sstevel@tonic-gate 
2408*7c478bd9Sstevel@tonic-gate static enum fme_state
2409*7c478bd9Sstevel@tonic-gate causes_test(struct fme *fmep, struct event *ep,
2410*7c478bd9Sstevel@tonic-gate     unsigned long long at_latest_by, unsigned long long *pdelay)
2411*7c478bd9Sstevel@tonic-gate {
2412*7c478bd9Sstevel@tonic-gate 	unsigned long long overall_delay = TIMEVAL_EVENTUALLY;
2413*7c478bd9Sstevel@tonic-gate 	unsigned long long my_delay;
2414*7c478bd9Sstevel@tonic-gate 	int credible_results = 0;
2415*7c478bd9Sstevel@tonic-gate 	int waiting_results = 0;
2416*7c478bd9Sstevel@tonic-gate 	enum fme_state fstate;
2417*7c478bd9Sstevel@tonic-gate 	struct event *tail_event;
2418*7c478bd9Sstevel@tonic-gate 	struct bubble *bp;
2419*7c478bd9Sstevel@tonic-gate 	struct arrowlist *ap;
2420*7c478bd9Sstevel@tonic-gate 	int k = 1;
2421*7c478bd9Sstevel@tonic-gate 
2422*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->Ccallcount);
2423*7c478bd9Sstevel@tonic-gate 	indent_push("  C");
2424*7c478bd9Sstevel@tonic-gate 	indent();
2425*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "->");
2426*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2427*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2428*7c478bd9Sstevel@tonic-gate 
2429*7c478bd9Sstevel@tonic-gate 	for (bp = itree_next_bubble(ep, NULL); bp;
2430*7c478bd9Sstevel@tonic-gate 	    bp = itree_next_bubble(ep, bp)) {
2431*7c478bd9Sstevel@tonic-gate 		if (bp->t != B_TO)
2432*7c478bd9Sstevel@tonic-gate 			continue;
2433*7c478bd9Sstevel@tonic-gate 		k = bp->nork;	/* remember the K value */
2434*7c478bd9Sstevel@tonic-gate 		for (ap = itree_next_arrow(bp, NULL); ap;
2435*7c478bd9Sstevel@tonic-gate 		    ap = itree_next_arrow(bp, ap)) {
2436*7c478bd9Sstevel@tonic-gate 			struct constraintlist *ctp;
2437*7c478bd9Sstevel@tonic-gate 			struct evalue value;
2438*7c478bd9Sstevel@tonic-gate 			int do_not_follow = 0;
2439*7c478bd9Sstevel@tonic-gate 			/*
2440*7c478bd9Sstevel@tonic-gate 			 * see if false constraint prevents us
2441*7c478bd9Sstevel@tonic-gate 			 * from traversing this arrow
2442*7c478bd9Sstevel@tonic-gate 			 */
2443*7c478bd9Sstevel@tonic-gate 			platform_set_payloadnvp(ep->nvp);
2444*7c478bd9Sstevel@tonic-gate 			for (ctp = ap->arrowp->constraints;
2445*7c478bd9Sstevel@tonic-gate 			    ctp != NULL; ctp = ctp->next) {
2446*7c478bd9Sstevel@tonic-gate 				if (eval_expr(ctp->cnode, NULL, NULL,
2447*7c478bd9Sstevel@tonic-gate 				    &fmep->globals,
2448*7c478bd9Sstevel@tonic-gate 				    fmep->cfgdata->cooked,
2449*7c478bd9Sstevel@tonic-gate 				    ap->arrowp, 0,
2450*7c478bd9Sstevel@tonic-gate 				    &value) == 0 ||
2451*7c478bd9Sstevel@tonic-gate 				    value.t == UNDEFINED ||
2452*7c478bd9Sstevel@tonic-gate 				    value.v == 0) {
2453*7c478bd9Sstevel@tonic-gate 					do_not_follow = 1;
2454*7c478bd9Sstevel@tonic-gate 					break;
2455*7c478bd9Sstevel@tonic-gate 				}
2456*7c478bd9Sstevel@tonic-gate 			}
2457*7c478bd9Sstevel@tonic-gate 			platform_set_payloadnvp(NULL);
2458*7c478bd9Sstevel@tonic-gate 			if (do_not_follow) {
2459*7c478bd9Sstevel@tonic-gate 				indent();
2460*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB|O_NONL,
2461*7c478bd9Sstevel@tonic-gate 				    "  False arrow from ");
2462*7c478bd9Sstevel@tonic-gate 				itree_pevent_brief(O_ALTFP|O_VERB|O_NONL,
2463*7c478bd9Sstevel@tonic-gate 				    ap->arrowp->tail->myevent);
2464*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB|O_NONL, " ");
2465*7c478bd9Sstevel@tonic-gate 				ptree(O_ALTFP|O_VERB|O_NONL, ctp->cnode, 1, 0);
2466*7c478bd9Sstevel@tonic-gate 				out(O_ALTFP|O_VERB, NULL);
2467*7c478bd9Sstevel@tonic-gate 				continue;
2468*7c478bd9Sstevel@tonic-gate 			}
2469*7c478bd9Sstevel@tonic-gate 
2470*7c478bd9Sstevel@tonic-gate 			if (ap->arrowp->causes_tested++ > 0) {
2471*7c478bd9Sstevel@tonic-gate 				/*
2472*7c478bd9Sstevel@tonic-gate 				 * get to this point if this is not the
2473*7c478bd9Sstevel@tonic-gate 				 * first time we're going through this
2474*7c478bd9Sstevel@tonic-gate 				 * arrow in the causes test.  consider this
2475*7c478bd9Sstevel@tonic-gate 				 * branch to be credible and let the
2476*7c478bd9Sstevel@tonic-gate 				 * credible/noncredible outcome depend on
2477*7c478bd9Sstevel@tonic-gate 				 * the other branches in this cycle.
2478*7c478bd9Sstevel@tonic-gate 				 */
2479*7c478bd9Sstevel@tonic-gate 				fstate = FME_CREDIBLE;
2480*7c478bd9Sstevel@tonic-gate 			} else {
2481*7c478bd9Sstevel@tonic-gate 				/*
2482*7c478bd9Sstevel@tonic-gate 				 * get to this point if this is the first
2483*7c478bd9Sstevel@tonic-gate 				 * time we're going through this arrow.
2484*7c478bd9Sstevel@tonic-gate 				 */
2485*7c478bd9Sstevel@tonic-gate 				tail_event = ap->arrowp->tail->myevent;
2486*7c478bd9Sstevel@tonic-gate 				fstate = hypothesise(fmep, tail_event,
2487*7c478bd9Sstevel@tonic-gate 						    at_latest_by,
2488*7c478bd9Sstevel@tonic-gate 						    &my_delay, ap->arrowp);
2489*7c478bd9Sstevel@tonic-gate 			}
2490*7c478bd9Sstevel@tonic-gate 
2491*7c478bd9Sstevel@tonic-gate 			switch (fstate) {
2492*7c478bd9Sstevel@tonic-gate 			case FME_WAIT:
2493*7c478bd9Sstevel@tonic-gate 				if (my_delay < overall_delay)
2494*7c478bd9Sstevel@tonic-gate 					overall_delay = my_delay;
2495*7c478bd9Sstevel@tonic-gate 				waiting_results++;
2496*7c478bd9Sstevel@tonic-gate 				break;
2497*7c478bd9Sstevel@tonic-gate 			case FME_CREDIBLE:
2498*7c478bd9Sstevel@tonic-gate 				credible_results++;
2499*7c478bd9Sstevel@tonic-gate 				break;
2500*7c478bd9Sstevel@tonic-gate 			case FME_DISPROVED:
2501*7c478bd9Sstevel@tonic-gate 				break;
2502*7c478bd9Sstevel@tonic-gate 			default:
2503*7c478bd9Sstevel@tonic-gate 				out(O_DIE, "Bug in causes_test");
2504*7c478bd9Sstevel@tonic-gate 			}
2505*7c478bd9Sstevel@tonic-gate 
2506*7c478bd9Sstevel@tonic-gate 			ap->arrowp->causes_tested--;
2507*7c478bd9Sstevel@tonic-gate 			ASSERT(ap->arrowp->causes_tested >= 0);
2508*7c478bd9Sstevel@tonic-gate 		}
2509*7c478bd9Sstevel@tonic-gate 	}
2510*7c478bd9Sstevel@tonic-gate 	/* compare against K */
2511*7c478bd9Sstevel@tonic-gate 	if (credible_results + waiting_results < k) {
2512*7c478bd9Sstevel@tonic-gate 		indent();
2513*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "<-DISPROVED ");
2514*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2515*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
2516*7c478bd9Sstevel@tonic-gate 		indent_pop();
2517*7c478bd9Sstevel@tonic-gate 		return (FME_DISPROVED);
2518*7c478bd9Sstevel@tonic-gate 	}
2519*7c478bd9Sstevel@tonic-gate 	if (waiting_results != 0) {
2520*7c478bd9Sstevel@tonic-gate 		*pdelay = overall_delay;
2521*7c478bd9Sstevel@tonic-gate 		indent();
2522*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "<-WAIT ");
2523*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2524*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, " to ");
2525*7c478bd9Sstevel@tonic-gate 		ptree_timeval(O_ALTFP|O_VERB|O_NONL, &at_latest_by);
2526*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
2527*7c478bd9Sstevel@tonic-gate 		indent_pop();
2528*7c478bd9Sstevel@tonic-gate 		return (FME_WAIT);
2529*7c478bd9Sstevel@tonic-gate 	}
2530*7c478bd9Sstevel@tonic-gate 	indent();
2531*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "<-CREDIBLE ");
2532*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2533*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2534*7c478bd9Sstevel@tonic-gate 	indent_pop();
2535*7c478bd9Sstevel@tonic-gate 	return (FME_CREDIBLE);
2536*7c478bd9Sstevel@tonic-gate }
2537*7c478bd9Sstevel@tonic-gate 
2538*7c478bd9Sstevel@tonic-gate static enum fme_state
2539*7c478bd9Sstevel@tonic-gate hypothesise(struct fme *fmep, struct event *ep,
2540*7c478bd9Sstevel@tonic-gate 	unsigned long long at_latest_by, unsigned long long *pdelay,
2541*7c478bd9Sstevel@tonic-gate 	struct arrow *arrowp)
2542*7c478bd9Sstevel@tonic-gate {
2543*7c478bd9Sstevel@tonic-gate 	enum fme_state rtr, otr;
2544*7c478bd9Sstevel@tonic-gate 	unsigned long long my_delay;
2545*7c478bd9Sstevel@tonic-gate 	unsigned long long overall_delay = TIMEVAL_EVENTUALLY;
2546*7c478bd9Sstevel@tonic-gate 
2547*7c478bd9Sstevel@tonic-gate 	stats_counter_bump(fmep->Hcallcount);
2548*7c478bd9Sstevel@tonic-gate 	indent_push("  H");
2549*7c478bd9Sstevel@tonic-gate 	indent();
2550*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "->");
2551*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2552*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, ", at latest by: ");
2553*7c478bd9Sstevel@tonic-gate 	ptree_timeval(O_ALTFP|O_VERB|O_NONL, &at_latest_by);
2554*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2555*7c478bd9Sstevel@tonic-gate 
2556*7c478bd9Sstevel@tonic-gate 	rtr = requirements_test(fmep, ep, at_latest_by, &my_delay, arrowp);
2557*7c478bd9Sstevel@tonic-gate 	mark_arrows(fmep, ep, 0); /* clean up after requirements test */
2558*7c478bd9Sstevel@tonic-gate 	if ((rtr == FME_WAIT) && (my_delay < overall_delay))
2559*7c478bd9Sstevel@tonic-gate 		overall_delay = my_delay;
2560*7c478bd9Sstevel@tonic-gate 	if (rtr != FME_DISPROVED) {
2561*7c478bd9Sstevel@tonic-gate 		if (is_problem(ep->t)) {
2562*7c478bd9Sstevel@tonic-gate 			otr = effects_test(fmep, ep);
2563*7c478bd9Sstevel@tonic-gate 			if (otr != FME_DISPROVED) {
2564*7c478bd9Sstevel@tonic-gate 				if (fmep->peek == 0 && ep->is_suspect++ == 0) {
2565*7c478bd9Sstevel@tonic-gate 					ep->suspects = fmep->suspects;
2566*7c478bd9Sstevel@tonic-gate 					fmep->suspects = ep;
2567*7c478bd9Sstevel@tonic-gate 					fmep->nsuspects++;
2568*7c478bd9Sstevel@tonic-gate 					if (!is_fault(ep->t))
2569*7c478bd9Sstevel@tonic-gate 						fmep->nonfault++;
2570*7c478bd9Sstevel@tonic-gate 				}
2571*7c478bd9Sstevel@tonic-gate 			}
2572*7c478bd9Sstevel@tonic-gate 		} else
2573*7c478bd9Sstevel@tonic-gate 			otr = causes_test(fmep, ep, at_latest_by, &my_delay);
2574*7c478bd9Sstevel@tonic-gate 		if ((otr == FME_WAIT) && (my_delay < overall_delay))
2575*7c478bd9Sstevel@tonic-gate 			overall_delay = my_delay;
2576*7c478bd9Sstevel@tonic-gate 		if ((otr != FME_DISPROVED) &&
2577*7c478bd9Sstevel@tonic-gate 		    ((rtr == FME_WAIT) || (otr == FME_WAIT)))
2578*7c478bd9Sstevel@tonic-gate 			*pdelay = overall_delay;
2579*7c478bd9Sstevel@tonic-gate 	}
2580*7c478bd9Sstevel@tonic-gate 	if (rtr == FME_DISPROVED) {
2581*7c478bd9Sstevel@tonic-gate 		indent();
2582*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "<-DISPROVED ");
2583*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2584*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, " (doesn't meet requirements)");
2585*7c478bd9Sstevel@tonic-gate 		indent_pop();
2586*7c478bd9Sstevel@tonic-gate 		return (FME_DISPROVED);
2587*7c478bd9Sstevel@tonic-gate 	}
2588*7c478bd9Sstevel@tonic-gate 	if ((otr == FME_DISPROVED) && is_problem(ep->t)) {
2589*7c478bd9Sstevel@tonic-gate 		indent();
2590*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "<-DISPROVED ");
2591*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2592*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, " (doesn't explain all reports)");
2593*7c478bd9Sstevel@tonic-gate 		indent_pop();
2594*7c478bd9Sstevel@tonic-gate 		return (FME_DISPROVED);
2595*7c478bd9Sstevel@tonic-gate 	}
2596*7c478bd9Sstevel@tonic-gate 	if (otr == FME_DISPROVED) {
2597*7c478bd9Sstevel@tonic-gate 		indent();
2598*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "<-DISPROVED ");
2599*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2600*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, " (causes are not credible)");
2601*7c478bd9Sstevel@tonic-gate 		indent_pop();
2602*7c478bd9Sstevel@tonic-gate 		return (FME_DISPROVED);
2603*7c478bd9Sstevel@tonic-gate 	}
2604*7c478bd9Sstevel@tonic-gate 	if ((rtr == FME_WAIT) || (otr == FME_WAIT)) {
2605*7c478bd9Sstevel@tonic-gate 		indent();
2606*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, "<-WAIT ");
2607*7c478bd9Sstevel@tonic-gate 		itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2608*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB|O_NONL, " to ");
2609*7c478bd9Sstevel@tonic-gate 		ptree_timeval(O_ALTFP|O_VERB|O_NONL, &overall_delay);
2610*7c478bd9Sstevel@tonic-gate 		out(O_ALTFP|O_VERB, NULL);
2611*7c478bd9Sstevel@tonic-gate 		indent_pop();
2612*7c478bd9Sstevel@tonic-gate 		return (FME_WAIT);
2613*7c478bd9Sstevel@tonic-gate 	}
2614*7c478bd9Sstevel@tonic-gate 	indent();
2615*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB|O_NONL, "<-CREDIBLE ");
2616*7c478bd9Sstevel@tonic-gate 	itree_pevent_brief(O_ALTFP|O_VERB|O_NONL, ep);
2617*7c478bd9Sstevel@tonic-gate 	out(O_ALTFP|O_VERB, NULL);
2618*7c478bd9Sstevel@tonic-gate 	indent_pop();
2619*7c478bd9Sstevel@tonic-gate 	return (FME_CREDIBLE);
2620*7c478bd9Sstevel@tonic-gate }
2621