1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * SMF software-response subsidiary
28  */
29 
30 #include <strings.h>
31 #include <fm/libtopo.h>
32 #include <libscf.h>
33 #include <sys/fm/protocol.h>
34 #include <fm/fmd_fmri.h>
35 
36 #include "../../common/sw.h"
37 #include "smf.h"
38 
39 static struct {
40 	fmd_stat_t swrp_smf_repairs;
41 	fmd_stat_t swrp_smf_clears;
42 	fmd_stat_t swrp_smf_closed;
43 	fmd_stat_t swrp_smf_wrongclass;
44 	fmd_stat_t swrp_smf_badlist;
45 	fmd_stat_t swrp_smf_badresource;
46 	fmd_stat_t swrp_smf_badclrevent;
47 	fmd_stat_t swrp_smf_noloop;
48 	fmd_stat_t swrp_smf_suppressed;
49 	fmd_stat_t swrp_smf_cachefull;
50 } swrp_smf_stats = {
51 	{ "swrp_smf_repairs", FMD_TYPE_UINT64,
52 	    "repair events received for propogation to SMF" },
53 	{ "swrp_smf_clears", FMD_TYPE_UINT64,
54 	    "notifications from SMF of exiting maint state" },
55 	{ "swrp_smf_closed", FMD_TYPE_UINT64,
56 	    "cases closed" },
57 	{ "swrp_smf_wrongclass", FMD_TYPE_UINT64,
58 	    "unexpected event class received" },
59 	{ "swrp_smf_badlist", FMD_TYPE_UINT64,
60 	    "list event with invalid structure" },
61 	{ "swrp_smf_badresource", FMD_TYPE_UINT64,
62 	    "list.repaired with smf fault but bad svc fmri" },
63 	{ "swrp_smf_badclrevent", FMD_TYPE_UINT64,
64 	    "maint clear event from SMF malformed" },
65 	{ "swrp_smf_noloop", FMD_TYPE_UINT64,
66 	    "avoidance of smf->fmd->smf repairs propogations" },
67 	{ "swrp_smf_suppressed", FMD_TYPE_UINT64,
68 	    "not propogated to smf because no longer in maint" },
69 	{ "swrp_smf_cachefull", FMD_TYPE_UINT64,
70 	    "uuid cache full" },
71 };
72 
73 #define	BUMPSTAT(stat)		swrp_smf_stats.stat.fmds_value.ui64++
74 
75 #define	CACHE_NENT_INC		16
76 #define	CACHE_NENT_MAX		128
77 
78 struct smf_uuid_cache_ent {
79 	char uuid[37];
80 	char fmristr[90];
81 	uint8_t mark;
82 };
83 
84 #define	CACHE_VERSION		1
85 
86 struct smf_uuid_cache {
87 	uint32_t version;			/* Version */
88 	uint32_t nentries;			/* Real size of array below */
89 	struct smf_uuid_cache_ent entry[1];	/* Cache entries */
90 };
91 
92 static struct smf_uuid_cache *uuid_cache;
93 
94 #define	UUID_CACHE_BUFNAME	"uuid_cache"
95 
96 static void
uuid_cache_grow(fmd_hdl_t * hdl)97 uuid_cache_grow(fmd_hdl_t *hdl)
98 {
99 	struct smf_uuid_cache *newcache;
100 	size_t newsz;
101 	uint32_t n;
102 
103 	n = (uuid_cache == NULL ? 0 : uuid_cache->nentries) + CACHE_NENT_INC;
104 	newsz = sizeof (struct smf_uuid_cache) + (n - 1) *
105 	    sizeof (struct smf_uuid_cache_ent);
106 
107 	newcache = fmd_hdl_zalloc(hdl, newsz, FMD_SLEEP);
108 	newcache->version = CACHE_VERSION;
109 	newcache->nentries = n;
110 
111 	if (uuid_cache != NULL) {
112 		uint32_t oldn = uuid_cache->nentries;
113 		size_t oldsz = sizeof (struct smf_uuid_cache) +
114 		    (oldn - 1) * sizeof (struct smf_uuid_cache_ent);
115 
116 		bcopy(&uuid_cache->entry[0], &newcache->entry[0], oldsz);
117 		fmd_hdl_free(hdl, uuid_cache, oldsz);
118 		fmd_buf_destroy(hdl, NULL, UUID_CACHE_BUFNAME);
119 	}
120 
121 	uuid_cache = newcache;
122 	fmd_buf_create(hdl, NULL, UUID_CACHE_BUFNAME, newsz);
123 }
124 
125 static void
uuid_cache_persist(fmd_hdl_t * hdl)126 uuid_cache_persist(fmd_hdl_t *hdl)
127 {
128 	size_t sz = sizeof (struct smf_uuid_cache) +
129 	    (uuid_cache->nentries - 1) * sizeof (struct smf_uuid_cache_ent);
130 
131 	fmd_buf_write(hdl, NULL, UUID_CACHE_BUFNAME, uuid_cache, sz);
132 }
133 
134 /*
135  * Garbage-collect the uuid cache.  Any cases that are already resolved
136  * we do not need an entry for.  If a case is not resolved but the
137  * service involved in that case is no longer in maintenance state
138  * then we've lost sync somehow, so repair the asru (which will
139  * also resolve the case).
140  */
141 static void
uuid_cache_gc(fmd_hdl_t * hdl)142 uuid_cache_gc(fmd_hdl_t *hdl)
143 {
144 	struct smf_uuid_cache_ent *entp;
145 	topo_hdl_t *thp = NULL;
146 	nvlist_t *svcfmri;
147 	char *svcname;
148 	int err, i;
149 
150 	for (i = 0; i < uuid_cache->nentries; i++) {
151 		entp = &uuid_cache->entry[i];
152 
153 		if (entp->uuid[0] == '\0')
154 			continue;
155 
156 		if (fmd_case_uuisresolved(hdl, entp->uuid)) {
157 			bzero(entp->uuid, sizeof (entp->uuid));
158 			bzero(entp->fmristr, sizeof (entp->fmristr));
159 			entp->mark = 0;
160 		} else {
161 			if (thp == NULL)
162 				thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
163 
164 			if (topo_fmri_str2nvl(thp, entp->fmristr, &svcfmri,
165 			    &err) != 0) {
166 				fmd_hdl_error(hdl, "str2nvl failed for %s\n",
167 				    entp->fmristr);
168 				continue;
169 			}
170 
171 			if (fmd_nvl_fmri_service_state(hdl, svcfmri) !=
172 			    FMD_SERVICE_STATE_UNUSABLE) {
173 				svcname = sw_smf_svcfmri2shortstr(hdl, svcfmri);
174 				(void) fmd_repair_asru(hdl, entp->fmristr);
175 				fmd_hdl_strfree(hdl, svcname);
176 			}
177 
178 			nvlist_free(svcfmri);
179 		}
180 	}
181 
182 	if (thp)
183 		fmd_hdl_topo_rele(hdl, thp);
184 
185 	uuid_cache_persist(hdl);
186 }
187 
188 static void
uuid_cache_restore(fmd_hdl_t * hdl)189 uuid_cache_restore(fmd_hdl_t *hdl)
190 {
191 	size_t sz = fmd_buf_size(hdl, NULL, UUID_CACHE_BUFNAME);
192 
193 	if (sz == 0)
194 		return;
195 
196 	uuid_cache = fmd_hdl_alloc(hdl, sz, FMD_SLEEP);
197 	fmd_buf_read(hdl, NULL, UUID_CACHE_BUFNAME, uuid_cache, sz);
198 
199 	/*
200 	 * Garbage collect now, not just for tidiness but also to help
201 	 * fmd and smf state stay in sync at module startup.
202 	 */
203 	uuid_cache_gc(hdl);
204 }
205 
206 /*
207  * Add the UUID of an SMF maintenance defect case to our cache and
208  * record the associated full svc FMRI string for the case.
209  */
210 static void
swrp_smf_cache_add(fmd_hdl_t * hdl,char * uuid,char * fmristr)211 swrp_smf_cache_add(fmd_hdl_t *hdl, char *uuid, char *fmristr)
212 {
213 	struct smf_uuid_cache_ent *entp = NULL;
214 	int gced = 0;
215 	int i;
216 
217 	if (uuid_cache == NULL)
218 		uuid_cache_grow(hdl);
219 
220 	/*
221 	 * If we somehow already have an entry for this uuid then
222 	 * return leaving it undisturbed.
223 	 */
224 	for (i = 0; i < uuid_cache->nentries; i++) {
225 		if (strcmp(uuid, uuid_cache->entry[i].uuid) == 0)
226 			return;
227 	}
228 
229 scan:
230 	for (i = 0; i < uuid_cache->nentries; i++) {
231 		if (uuid_cache->entry[i].uuid[0] == '\0') {
232 			entp = &uuid_cache->entry[i];
233 			break;
234 		}
235 	}
236 
237 	if (entp == NULL) {
238 		uint32_t oldn = uuid_cache->nentries;
239 
240 		/*
241 		 * Before growing the cache we try again after first
242 		 * garbage-collecting the existing cache for any cases
243 		 * that are confirmed as resolved.
244 		 */
245 		if (!gced) {
246 			uuid_cache_gc(hdl);
247 			gced = 1;
248 			goto scan;
249 		}
250 
251 		if (oldn < CACHE_NENT_MAX) {
252 			uuid_cache_grow(hdl);
253 			entp = &uuid_cache->entry[oldn];
254 		} else {
255 			BUMPSTAT(swrp_smf_cachefull);
256 			return;
257 		}
258 	}
259 
260 	(void) strncpy(entp->uuid, uuid, sizeof (entp->uuid));
261 	(void) strncpy(entp->fmristr, fmristr, sizeof (entp->fmristr));
262 	uuid_cache_persist(hdl);
263 }
264 
265 /*
266  * Mark cache entry/entries as resolved - if they match in either uuid
267  * (if not NULL) or fmristr (if not NULL) mark as resolved.  Return 1 iff
268  * an entry that matched on uuid was already marked, otherwise (entry
269  * matched on either, matched on uuid but not marked, not found).
270  */
271 static int
swrp_smf_cache_mark(fmd_hdl_t * hdl,char * uuid,char * fmristr)272 swrp_smf_cache_mark(fmd_hdl_t *hdl, char *uuid, char *fmristr)
273 {
274 	int dirty = 0;
275 	int rv = 0;
276 	int i;
277 
278 	if (uuid_cache == NULL)
279 		return (0);
280 
281 	for (i = 0; i < uuid_cache->nentries; i++) {
282 		struct smf_uuid_cache_ent *entp = &uuid_cache->entry[i];
283 
284 		if (entp->uuid[0] == '\0')
285 			continue;
286 
287 		if (uuid && strcmp(uuid, entp->uuid) == 0) {
288 			if (entp->mark)
289 				rv = 1;
290 			entp->mark = 1;
291 			dirty++;
292 		} else if (fmristr && strcmp(fmristr, entp->fmristr) == 0) {
293 			entp->mark = 1;
294 			dirty++;
295 		}
296 	}
297 
298 	if (dirty)
299 		uuid_cache_persist(hdl);
300 
301 	return (rv);
302 }
303 
304 /*
305  * We will receive list events for cases we are not interested in.  Test
306  * that this list has exactly one suspect and that it matches the maintenance
307  * defect.  Return the defect to the caller in the second argument,
308  * and the defect resource element in the third arg.
309  */
310 static int
suspect_is_maint_defect(fmd_hdl_t * hdl,nvlist_t * nvl,nvlist_t ** defectnvl,nvlist_t ** rsrcnvl)311 suspect_is_maint_defect(fmd_hdl_t *hdl, nvlist_t *nvl,
312     nvlist_t **defectnvl, nvlist_t **rsrcnvl)
313 {
314 	nvlist_t **faults;
315 	uint_t nfaults;
316 
317 	if (nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
318 	    &faults, &nfaults) != 0) {
319 		BUMPSTAT(swrp_smf_badlist);
320 		return (0);
321 	}
322 
323 	if (nfaults != 1 ||
324 	    !fmd_nvl_class_match(hdl, faults[0], SW_SMF_MAINT_DEFECT))
325 		return (0);
326 
327 	if (nvlist_lookup_nvlist(faults[0], FM_FAULT_RESOURCE, rsrcnvl) != 0) {
328 		BUMPSTAT(swrp_smf_badlist);
329 		return (0);
330 	}
331 
332 	*defectnvl = faults[0];
333 
334 	return (1);
335 }
336 
337 /*
338  * Received newly-diagnosed list.suspect events that are for the
339  * maintenane defect we diagnose.  Close the case (the resource was already
340  * isolated by SMF) after cachng the case UUID.
341  */
342 /*ARGSUSED*/
343 static void
swrp_smf_cacheuuid(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class,void * arg)344 swrp_smf_cacheuuid(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
345     const char *class, void *arg)
346 {
347 	nvlist_t *defect, *rsrc;
348 	char *fmristr, *uuid;
349 
350 	if (nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) != 0) {
351 		BUMPSTAT(swrp_smf_badlist);
352 		return;
353 	}
354 
355 	if (!suspect_is_maint_defect(hdl, nvl, &defect, &rsrc))
356 		return;
357 
358 	if ((fmristr = sw_smf_svcfmri2str(hdl, rsrc)) == NULL) {
359 		BUMPSTAT(swrp_smf_badlist);
360 		return;
361 	}
362 
363 	swrp_smf_cache_add(hdl, uuid, fmristr);
364 	fmd_hdl_strfree(hdl, fmristr);
365 
366 	if (!fmd_case_uuclosed(hdl, uuid)) {
367 		fmd_case_uuclose(hdl, uuid);
368 		BUMPSTAT(swrp_smf_closed);
369 	}
370 }
371 
372 /*ARGSUSED*/
373 static void
swrp_smf2fmd(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class,void * arg)374 swrp_smf2fmd(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
375     const char *class, void *arg)
376 {
377 	nvlist_t *attr, *fmri;
378 	char *fromstate;
379 	char *fmristr;
380 
381 	if (!fmd_nvl_class_match(hdl, nvl, TRANCLASS("*"))) {
382 		BUMPSTAT(swrp_smf_wrongclass);
383 		return;
384 	}
385 
386 	if (nvlist_lookup_nvlist(nvl, FM_IREPORT_ATTRIBUTES, &attr) != 0 ||
387 	    nvlist_lookup_string(attr, "from-state", &fromstate) != 0) {
388 		BUMPSTAT(swrp_smf_badclrevent);
389 		return;
390 	}
391 
392 	/*
393 	 * Filter those not describing a transition out of maintenance.
394 	 */
395 	if (strcmp(fromstate, "maintenance") != 0)
396 		return;
397 
398 	if (nvlist_lookup_nvlist(attr, "svc", &fmri) != 0) {
399 		BUMPSTAT(swrp_smf_badclrevent);
400 		return;
401 	}
402 
403 	if ((fmristr = sw_smf_svcfmri2str(hdl, fmri)) == NULL) {
404 		BUMPSTAT(swrp_smf_badclrevent);
405 		return;
406 	}
407 
408 	/*
409 	 * Mark any UUID for a case against this service as resolved
410 	 * in our cache.  When we fmd_repair_asru below fmd will emit
411 	 * a list.repaired as a result, and our handling of that event
412 	 * must not propogate the repair towards SMF (since the repair
413 	 * was initiated via SMF itself and not via fmadm).
414 	 */
415 	(void) swrp_smf_cache_mark(hdl, NULL, fmristr);
416 
417 	(void) fmd_repair_asru(hdl, fmristr);
418 	fmd_hdl_strfree(hdl, fmristr);
419 	BUMPSTAT(swrp_smf_clears);
420 }
421 
422 /*ARGSUSED*/
423 static void
swrp_fmd2smf(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class,void * arg)424 swrp_fmd2smf(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
425     const char *class, void *arg)
426 {
427 	char *fmristr, *shrtfmristr;
428 	nvlist_t *defect, *rsrc;
429 	char *uuid;
430 	int already;
431 
432 	if (strcmp(class, FM_LIST_REPAIRED_CLASS) != 0) {
433 		BUMPSTAT(swrp_smf_wrongclass);
434 		return;
435 	}
436 
437 	if (nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) != 0) {
438 		BUMPSTAT(swrp_smf_badlist);
439 		return;
440 	}
441 
442 	if (!suspect_is_maint_defect(hdl, nvl, &defect, &rsrc))
443 		return;
444 
445 	if ((fmristr = sw_smf_svcfmri2str(hdl, rsrc)) == NULL) {
446 		BUMPSTAT(swrp_smf_badresource);
447 		return;
448 	}
449 
450 	already = swrp_smf_cache_mark(hdl, uuid, fmristr);
451 	fmd_hdl_strfree(hdl, fmristr);
452 
453 	/*
454 	 * If the cache already had a marked entry for this UUID then
455 	 * this is a list.repaired arising from a SMF-initiated maintenance
456 	 * clear (propogated with fmd_repair_asru above which then results
457 	 * in a list.repaired) and so we should not propogate the repair
458 	 * back towards SMF.  But do still force the case to RESOLVED state in
459 	 * case fmd is unable to confirm the service no longer in maintenance
460 	 * state (it may have failed again) so that a new case can be opened.
461 	 */
462 	fmd_case_uuresolved(hdl, uuid);
463 	if (already) {
464 		BUMPSTAT(swrp_smf_noloop);
465 		return;
466 	}
467 
468 	/*
469 	 * Only propogate to SMF if we can see that service still
470 	 * in maintenance state.  We're not synchronized with SMF
471 	 * and this state could change at any time, but if we can
472 	 * see it's not in maintenance state then things are obviously
473 	 * moving (e.g., external svcadm active) so we don't poke
474 	 * at SMF otherwise we confuse things or duplicate operations.
475 	 */
476 
477 	if (fmd_nvl_fmri_service_state(hdl, rsrc) ==
478 	    FMD_SERVICE_STATE_UNUSABLE) {
479 		shrtfmristr = sw_smf_svcfmri2shortstr(hdl, rsrc);
480 
481 		if (shrtfmristr != NULL) {
482 			(void) smf_restore_instance(shrtfmristr);
483 			fmd_hdl_strfree(hdl, shrtfmristr);
484 			BUMPSTAT(swrp_smf_repairs);
485 		} else {
486 			BUMPSTAT(swrp_smf_badresource);
487 		}
488 	} else {
489 		BUMPSTAT(swrp_smf_suppressed);
490 	}
491 }
492 
493 const struct sw_disp swrp_smf_disp[] = {
494 	{ TRANCLASS("*"), swrp_smf2fmd, NULL },
495 	{ FM_LIST_SUSPECT_CLASS, swrp_smf_cacheuuid, NULL },
496 	{ FM_LIST_REPAIRED_CLASS, swrp_fmd2smf, NULL },
497 	{ NULL, NULL, NULL }
498 };
499 
500 /*ARGSUSED*/
501 int
swrp_smf_init(fmd_hdl_t * hdl,id_t id,const struct sw_disp ** dpp,int * nelemp)502 swrp_smf_init(fmd_hdl_t *hdl, id_t id, const struct sw_disp **dpp, int *nelemp)
503 {
504 	(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (swrp_smf_stats) /
505 	    sizeof (fmd_stat_t), (fmd_stat_t *)&swrp_smf_stats);
506 
507 	uuid_cache_restore(hdl);
508 
509 	/*
510 	 * We need to subscribe to all SMF transition class events because
511 	 * we need to look inside the payload to see which events indicate
512 	 * a transition out of maintenance state.
513 	 */
514 	fmd_hdl_subscribe(hdl, TRANCLASS("*"));
515 
516 	/*
517 	 * Subscribe to the defect class diagnosed for maintenance events.
518 	 * The module will then receive list.suspect events including
519 	 * these defects, and in our dispatch table above we list routing
520 	 * for list.suspect.
521 	 */
522 	fmd_hdl_subscribe(hdl, SW_SMF_MAINT_DEFECT);
523 
524 	*dpp = &swrp_smf_disp[0];
525 	*nelemp = sizeof (swrp_smf_disp) / sizeof (swrp_smf_disp[0]);
526 	return (SW_SUB_INIT_SUCCESS);
527 }
528 
529 /*ARGSUSED*/
530 void
swrp_smf_fini(fmd_hdl_t * hdl)531 swrp_smf_fini(fmd_hdl_t *hdl)
532 {
533 }
534 
535 const struct sw_subinfo smf_response_info = {
536 	"smf repair",			/* swsub_name */
537 	SW_CASE_NONE,			/* swsub_casetype */
538 	swrp_smf_init,			/* swsub_init */
539 	swrp_smf_fini,			/* swsub_fini */
540 	NULL,				/* swsub_timeout */
541 	NULL,				/* swsub_case_close */
542 	NULL,				/* swsub_case_vrfy */
543 };
544