199653d4eSeschrock /*
299653d4eSeschrock  * CDDL HEADER START
399653d4eSeschrock  *
499653d4eSeschrock  * The contents of this file are subject to the terms of the
599653d4eSeschrock  * Common Development and Distribution License (the "License").
699653d4eSeschrock  * You may not use this file except in compliance with the License.
799653d4eSeschrock  *
899653d4eSeschrock  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
999653d4eSeschrock  * or http://www.opensolaris.org/os/licensing.
1099653d4eSeschrock  * See the License for the specific language governing permissions
1199653d4eSeschrock  * and limitations under the License.
1299653d4eSeschrock  *
1399653d4eSeschrock  * When distributing Covered Code, include this CDDL HEADER in each
1499653d4eSeschrock  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1599653d4eSeschrock  * If applicable, add the following below this CDDL HEADER, with the
1699653d4eSeschrock  * fields enclosed by brackets "[]" replaced with your own identifying
1799653d4eSeschrock  * information: Portions Copyright [yyyy] [name of copyright owner]
1899653d4eSeschrock  *
1999653d4eSeschrock  * CDDL HEADER END
2099653d4eSeschrock  */
2199653d4eSeschrock /*
222a417b23SRobert Johnston  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
2399653d4eSeschrock  */
245711d393Sloli /*
255711d393Sloli  * Copyright 2019 Joyent, Inc.
265711d393Sloli  */
2799653d4eSeschrock 
2899653d4eSeschrock /*
2999653d4eSeschrock  * The ZFS retire agent is responsible for managing hot spares across all pools.
303d7072f8Seschrock  * When we see a device fault or a device removal, we try to open the associated
313d7072f8Seschrock  * pool and look for any hot spares.  We iterate over any available hot spares
323d7072f8Seschrock  * and attempt a 'zpool replace' for each one.
333d7072f8Seschrock  *
343d7072f8Seschrock  * For vdevs diagnosed as faulty, the agent is also responsible for proactively
353d7072f8Seschrock  * marking the vdev FAULTY (for I/O errors) or DEGRADED (for checksum errors).
3699653d4eSeschrock  */
3799653d4eSeschrock 
3899653d4eSeschrock #include <fm/fmd_api.h>
3999653d4eSeschrock #include <sys/fs/zfs.h>
4099653d4eSeschrock #include <sys/fm/protocol.h>
4199653d4eSeschrock #include <sys/fm/fs/zfs.h>
4299653d4eSeschrock #include <libzfs.h>
43069f55e2SEric Schrock #include <fm/libtopo.h>
443d7072f8Seschrock #include <string.h>
4599653d4eSeschrock 
46069f55e2SEric Schrock typedef struct zfs_retire_repaired {
47069f55e2SEric Schrock 	struct zfs_retire_repaired	*zrr_next;
48069f55e2SEric Schrock 	uint64_t			zrr_pool;
49069f55e2SEric Schrock 	uint64_t			zrr_vdev;
50069f55e2SEric Schrock } zfs_retire_repaired_t;
51069f55e2SEric Schrock 
52069f55e2SEric Schrock typedef struct zfs_retire_data {
53069f55e2SEric Schrock 	libzfs_handle_t			*zrd_hdl;
54069f55e2SEric Schrock 	zfs_retire_repaired_t		*zrd_repaired;
55069f55e2SEric Schrock } zfs_retire_data_t;
56069f55e2SEric Schrock 
57069f55e2SEric Schrock static void
zfs_retire_clear_data(fmd_hdl_t * hdl,zfs_retire_data_t * zdp)58069f55e2SEric Schrock zfs_retire_clear_data(fmd_hdl_t *hdl, zfs_retire_data_t *zdp)
59069f55e2SEric Schrock {
60069f55e2SEric Schrock 	zfs_retire_repaired_t *zrp;
61069f55e2SEric Schrock 
62069f55e2SEric Schrock 	while ((zrp = zdp->zrd_repaired) != NULL) {
63069f55e2SEric Schrock 		zdp->zrd_repaired = zrp->zrr_next;
64069f55e2SEric Schrock 		fmd_hdl_free(hdl, zrp, sizeof (zfs_retire_repaired_t));
65069f55e2SEric Schrock 	}
66069f55e2SEric Schrock }
67069f55e2SEric Schrock 
6899653d4eSeschrock /*
6999653d4eSeschrock  * Find a pool with a matching GUID.
7099653d4eSeschrock  */
7199653d4eSeschrock typedef struct find_cbdata {
7299653d4eSeschrock 	uint64_t	cb_guid;
73069f55e2SEric Schrock 	const char	*cb_fru;
7499653d4eSeschrock 	zpool_handle_t	*cb_zhp;
75069f55e2SEric Schrock 	nvlist_t	*cb_vdev;
7699653d4eSeschrock } find_cbdata_t;
7799653d4eSeschrock 
7899653d4eSeschrock static int
find_pool(zpool_handle_t * zhp,void * data)7999653d4eSeschrock find_pool(zpool_handle_t *zhp, void *data)
8099653d4eSeschrock {
8199653d4eSeschrock 	find_cbdata_t *cbp = data;
8299653d4eSeschrock 
83990b4856Slling 	if (cbp->cb_guid ==
84990b4856Slling 	    zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL)) {
8599653d4eSeschrock 		cbp->cb_zhp = zhp;
8699653d4eSeschrock 		return (1);
8799653d4eSeschrock 	}
8899653d4eSeschrock 
8999653d4eSeschrock 	zpool_close(zhp);
9099653d4eSeschrock 	return (0);
9199653d4eSeschrock }
9299653d4eSeschrock 
9399653d4eSeschrock /*
9499653d4eSeschrock  * Find a vdev within a tree with a matching GUID.
9599653d4eSeschrock  */
9699653d4eSeschrock static nvlist_t *
find_vdev(libzfs_handle_t * zhdl,nvlist_t * nv,const char * search_fru,uint64_t search_guid)97069f55e2SEric Schrock find_vdev(libzfs_handle_t *zhdl, nvlist_t *nv, const char *search_fru,
98069f55e2SEric Schrock     uint64_t search_guid)
9999653d4eSeschrock {
10099653d4eSeschrock 	uint64_t guid;
10199653d4eSeschrock 	nvlist_t **child;
10299653d4eSeschrock 	uint_t c, children;
10399653d4eSeschrock 	nvlist_t *ret;
104069f55e2SEric Schrock 	char *fru;
105069f55e2SEric Schrock 
106069f55e2SEric Schrock 	if (search_fru != NULL) {
107069f55e2SEric Schrock 		if (nvlist_lookup_string(nv, ZPOOL_CONFIG_FRU, &fru) == 0 &&
108069f55e2SEric Schrock 		    libzfs_fru_compare(zhdl, fru, search_fru))
109069f55e2SEric Schrock 			return (nv);
110069f55e2SEric Schrock 	} else {
111069f55e2SEric Schrock 		if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
112069f55e2SEric Schrock 		    guid == search_guid)
113069f55e2SEric Schrock 			return (nv);
114069f55e2SEric Schrock 	}
11599653d4eSeschrock 
11699653d4eSeschrock 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
11799653d4eSeschrock 	    &child, &children) != 0)
11899653d4eSeschrock 		return (NULL);
11999653d4eSeschrock 
12099653d4eSeschrock 	for (c = 0; c < children; c++) {
121069f55e2SEric Schrock 		if ((ret = find_vdev(zhdl, child[c], search_fru,
122069f55e2SEric Schrock 		    search_guid)) != NULL)
12399653d4eSeschrock 			return (ret);
12499653d4eSeschrock 	}
12599653d4eSeschrock 
126c5904d13Seschrock 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
127*e830fb12SKody A Kantor 	    &child, &children) == 0) {
128*e830fb12SKody A Kantor 		for (c = 0; c < children; c++) {
129*e830fb12SKody A Kantor 			if ((ret = find_vdev(zhdl, child[c], search_fru,
130*e830fb12SKody A Kantor 			    search_guid)) != NULL)
131*e830fb12SKody A Kantor 				return (ret);
132*e830fb12SKody A Kantor 		}
133*e830fb12SKody A Kantor 	}
134c5904d13Seschrock 
135*e830fb12SKody A Kantor 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
136*e830fb12SKody A Kantor 	    &child, &children) == 0) {
137*e830fb12SKody A Kantor 		for (c = 0; c < children; c++) {
138*e830fb12SKody A Kantor 			if ((ret = find_vdev(zhdl, child[c], search_fru,
139*e830fb12SKody A Kantor 			    search_guid)) != NULL)
140*e830fb12SKody A Kantor 				return (ret);
141*e830fb12SKody A Kantor 		}
142c5904d13Seschrock 	}
143c5904d13Seschrock 
14499653d4eSeschrock 	return (NULL);
14599653d4eSeschrock }
14699653d4eSeschrock 
1473d7072f8Seschrock /*
1483d7072f8Seschrock  * Given a (pool, vdev) GUID pair, find the matching pool and vdev.
1493d7072f8Seschrock  */
1503d7072f8Seschrock static zpool_handle_t *
find_by_guid(libzfs_handle_t * zhdl,uint64_t pool_guid,uint64_t vdev_guid,nvlist_t ** vdevp)1513d7072f8Seschrock find_by_guid(libzfs_handle_t *zhdl, uint64_t pool_guid, uint64_t vdev_guid,
1523d7072f8Seschrock     nvlist_t **vdevp)
1533d7072f8Seschrock {
1543d7072f8Seschrock 	find_cbdata_t cb;
1553d7072f8Seschrock 	zpool_handle_t *zhp;
1563d7072f8Seschrock 	nvlist_t *config, *nvroot;
1573d7072f8Seschrock 
1583d7072f8Seschrock 	/*
1593d7072f8Seschrock 	 * Find the corresponding pool and make sure the vdev still exists.
1603d7072f8Seschrock 	 */
1613d7072f8Seschrock 	cb.cb_guid = pool_guid;
1623d7072f8Seschrock 	if (zpool_iter(zhdl, find_pool, &cb) != 1)
1633d7072f8Seschrock 		return (NULL);
1643d7072f8Seschrock 
1653d7072f8Seschrock 	zhp = cb.cb_zhp;
1663d7072f8Seschrock 	config = zpool_get_config(zhp, NULL);
1673d7072f8Seschrock 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1683d7072f8Seschrock 	    &nvroot) != 0) {
1693d7072f8Seschrock 		zpool_close(zhp);
1703d7072f8Seschrock 		return (NULL);
1713d7072f8Seschrock 	}
1723d7072f8Seschrock 
173069f55e2SEric Schrock 	if (vdev_guid != 0) {
174069f55e2SEric Schrock 		if ((*vdevp = find_vdev(zhdl, nvroot, NULL,
175069f55e2SEric Schrock 		    vdev_guid)) == NULL) {
176069f55e2SEric Schrock 			zpool_close(zhp);
177069f55e2SEric Schrock 			return (NULL);
178069f55e2SEric Schrock 		}
1793d7072f8Seschrock 	}
1803d7072f8Seschrock 
1813d7072f8Seschrock 	return (zhp);
1823d7072f8Seschrock }
1833d7072f8Seschrock 
184069f55e2SEric Schrock static int
search_pool(zpool_handle_t * zhp,void * data)185069f55e2SEric Schrock search_pool(zpool_handle_t *zhp, void *data)
186069f55e2SEric Schrock {
187069f55e2SEric Schrock 	find_cbdata_t *cbp = data;
188069f55e2SEric Schrock 	nvlist_t *config;
189069f55e2SEric Schrock 	nvlist_t *nvroot;
190069f55e2SEric Schrock 
191069f55e2SEric Schrock 	config = zpool_get_config(zhp, NULL);
192069f55e2SEric Schrock 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
193069f55e2SEric Schrock 	    &nvroot) != 0) {
194069f55e2SEric Schrock 		zpool_close(zhp);
195069f55e2SEric Schrock 		return (0);
196069f55e2SEric Schrock 	}
197069f55e2SEric Schrock 
198069f55e2SEric Schrock 	if ((cbp->cb_vdev = find_vdev(zpool_get_handle(zhp), nvroot,
199069f55e2SEric Schrock 	    cbp->cb_fru, 0)) != NULL) {
200069f55e2SEric Schrock 		cbp->cb_zhp = zhp;
201069f55e2SEric Schrock 		return (1);
202069f55e2SEric Schrock 	}
203069f55e2SEric Schrock 
204069f55e2SEric Schrock 	zpool_close(zhp);
205069f55e2SEric Schrock 	return (0);
206069f55e2SEric Schrock }
207069f55e2SEric Schrock 
208069f55e2SEric Schrock /*
209069f55e2SEric Schrock  * Given a FRU FMRI, find the matching pool and vdev.
210069f55e2SEric Schrock  */
211069f55e2SEric Schrock static zpool_handle_t *
find_by_fru(libzfs_handle_t * zhdl,const char * fru,nvlist_t ** vdevp)212069f55e2SEric Schrock find_by_fru(libzfs_handle_t *zhdl, const char *fru, nvlist_t **vdevp)
213069f55e2SEric Schrock {
214069f55e2SEric Schrock 	find_cbdata_t cb;
215069f55e2SEric Schrock 
216069f55e2SEric Schrock 	cb.cb_fru = fru;
217069f55e2SEric Schrock 	cb.cb_zhp = NULL;
218069f55e2SEric Schrock 	if (zpool_iter(zhdl, search_pool, &cb) != 1)
219069f55e2SEric Schrock 		return (NULL);
220069f55e2SEric Schrock 
221069f55e2SEric Schrock 	*vdevp = cb.cb_vdev;
222069f55e2SEric Schrock 	return (cb.cb_zhp);
223069f55e2SEric Schrock }
224069f55e2SEric Schrock 
2253d7072f8Seschrock /*
2263d7072f8Seschrock  * Given a vdev, attempt to replace it with every known spare until one
2273d7072f8Seschrock  * succeeds.
2283d7072f8Seschrock  */
2293d7072f8Seschrock static void
replace_with_spare(fmd_hdl_t * hdl,zpool_handle_t * zhp,nvlist_t * vdev)230069f55e2SEric Schrock replace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
2313d7072f8Seschrock {
2323d7072f8Seschrock 	nvlist_t *config, *nvroot, *replacement;
2333d7072f8Seschrock 	nvlist_t **spares;
2343d7072f8Seschrock 	uint_t s, nspares;
2353d7072f8Seschrock 	char *dev_name;
2365711d393Sloli 	zprop_source_t source;
2375711d393Sloli 	int ashift;
238*e830fb12SKody A Kantor 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
239*e830fb12SKody A Kantor 	libzfs_handle_t *zhdl = zdp->zrd_hdl;
2403d7072f8Seschrock 
2413d7072f8Seschrock 	config = zpool_get_config(zhp, NULL);
2423d7072f8Seschrock 	if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2433d7072f8Seschrock 	    &nvroot) != 0)
2443d7072f8Seschrock 		return;
2453d7072f8Seschrock 
2463d7072f8Seschrock 	/*
2473d7072f8Seschrock 	 * Find out if there are any hot spares available in the pool.
2483d7072f8Seschrock 	 */
2493d7072f8Seschrock 	if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
2503d7072f8Seschrock 	    &spares, &nspares) != 0)
2513d7072f8Seschrock 		return;
2523d7072f8Seschrock 
2535711d393Sloli 	/*
2545711d393Sloli 	 * lookup "ashift" pool property, we may need it for the replacement
2555711d393Sloli 	 */
2565711d393Sloli 	ashift = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &source);
2575711d393Sloli 
258069f55e2SEric Schrock 	replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);
2593d7072f8Seschrock 
260069f55e2SEric Schrock 	(void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
261069f55e2SEric Schrock 	    VDEV_TYPE_ROOT);
2623d7072f8Seschrock 
263*e830fb12SKody A Kantor 	dev_name = zpool_vdev_name(zhdl, zhp, vdev, B_FALSE);
2643d7072f8Seschrock 
2653d7072f8Seschrock 	/*
2663d7072f8Seschrock 	 * Try to replace each spare, ending when we successfully
2673d7072f8Seschrock 	 * replace it.
2683d7072f8Seschrock 	 */
2693d7072f8Seschrock 	for (s = 0; s < nspares; s++) {
2703d7072f8Seschrock 		char *spare_name;
2713d7072f8Seschrock 
2723d7072f8Seschrock 		if (nvlist_lookup_string(spares[s], ZPOOL_CONFIG_PATH,
2733d7072f8Seschrock 		    &spare_name) != 0)
2743d7072f8Seschrock 			continue;
2753d7072f8Seschrock 
2765711d393Sloli 		/* if set, add the "ashift" pool property to the spare nvlist */
2775711d393Sloli 		if (source != ZPROP_SRC_DEFAULT)
2785711d393Sloli 			(void) nvlist_add_uint64(spares[s],
2795711d393Sloli 			    ZPOOL_CONFIG_ASHIFT, ashift);
2805711d393Sloli 
281069f55e2SEric Schrock 		(void) nvlist_add_nvlist_array(replacement,
282069f55e2SEric Schrock 		    ZPOOL_CONFIG_CHILDREN, &spares[s], 1);
2833d7072f8Seschrock 
2843d7072f8Seschrock 		if (zpool_vdev_attach(zhp, dev_name, spare_name,
2853d7072f8Seschrock 		    replacement, B_TRUE) == 0)
2863d7072f8Seschrock 			break;
2873d7072f8Seschrock 	}
2883d7072f8Seschrock 
2893d7072f8Seschrock 	free(dev_name);
2903d7072f8Seschrock 	nvlist_free(replacement);
2913d7072f8Seschrock }
2923d7072f8Seschrock 
293069f55e2SEric Schrock /*
294069f55e2SEric Schrock  * Repair this vdev if we had diagnosed a 'fault.fs.zfs.device' and
295069f55e2SEric Schrock  * ASRU is now usable.  ZFS has found the device to be present and
296069f55e2SEric Schrock  * functioning.
297069f55e2SEric Schrock  */
298069f55e2SEric Schrock /*ARGSUSED*/
299069f55e2SEric Schrock void
zfs_vdev_repair(fmd_hdl_t * hdl,nvlist_t * nvl)300069f55e2SEric Schrock zfs_vdev_repair(fmd_hdl_t *hdl, nvlist_t *nvl)
301069f55e2SEric Schrock {
302069f55e2SEric Schrock 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
303069f55e2SEric Schrock 	zfs_retire_repaired_t *zrp;
304069f55e2SEric Schrock 	uint64_t pool_guid, vdev_guid;
305069f55e2SEric Schrock 	nvlist_t *asru;
306069f55e2SEric Schrock 
307069f55e2SEric Schrock 	if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
308069f55e2SEric Schrock 	    &pool_guid) != 0 || nvlist_lookup_uint64(nvl,
309069f55e2SEric Schrock 	    FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
310069f55e2SEric Schrock 		return;
311069f55e2SEric Schrock 
312069f55e2SEric Schrock 	/*
313069f55e2SEric Schrock 	 * Before checking the state of the ASRU, go through and see if we've
314069f55e2SEric Schrock 	 * already made an attempt to repair this ASRU.  This list is cleared
315069f55e2SEric Schrock 	 * whenever we receive any kind of list event, and is designed to
316069f55e2SEric Schrock 	 * prevent us from generating a feedback loop when we attempt repairs
317069f55e2SEric Schrock 	 * against a faulted pool.  The problem is that checking the unusable
318069f55e2SEric Schrock 	 * state of the ASRU can involve opening the pool, which can post
319069f55e2SEric Schrock 	 * statechange events but otherwise leave the pool in the faulted
320069f55e2SEric Schrock 	 * state.  This list allows us to detect when a statechange event is
321069f55e2SEric Schrock 	 * due to our own request.
322069f55e2SEric Schrock 	 */
323069f55e2SEric Schrock 	for (zrp = zdp->zrd_repaired; zrp != NULL; zrp = zrp->zrr_next) {
324069f55e2SEric Schrock 		if (zrp->zrr_pool == pool_guid &&
325069f55e2SEric Schrock 		    zrp->zrr_vdev == vdev_guid)
326069f55e2SEric Schrock 			return;
327069f55e2SEric Schrock 	}
328069f55e2SEric Schrock 
329069f55e2SEric Schrock 	asru = fmd_nvl_alloc(hdl, FMD_SLEEP);
330069f55e2SEric Schrock 
331069f55e2SEric Schrock 	(void) nvlist_add_uint8(asru, FM_VERSION, ZFS_SCHEME_VERSION0);
332069f55e2SEric Schrock 	(void) nvlist_add_string(asru, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
333069f55e2SEric Schrock 	(void) nvlist_add_uint64(asru, FM_FMRI_ZFS_POOL, pool_guid);
334069f55e2SEric Schrock 	(void) nvlist_add_uint64(asru, FM_FMRI_ZFS_VDEV, vdev_guid);
335069f55e2SEric Schrock 
336069f55e2SEric Schrock 	/*
337069f55e2SEric Schrock 	 * We explicitly check for the unusable state here to make sure we
338069f55e2SEric Schrock 	 * aren't responding to a transient state change.  As part of opening a
339069f55e2SEric Schrock 	 * vdev, it's possible to see the 'statechange' event, only to be
340069f55e2SEric Schrock 	 * followed by a vdev failure later.  If we don't check the current
341069f55e2SEric Schrock 	 * state of the vdev (or pool) before marking it repaired, then we risk
342069f55e2SEric Schrock 	 * generating spurious repair events followed immediately by the same
343069f55e2SEric Schrock 	 * diagnosis.
344069f55e2SEric Schrock 	 *
345069f55e2SEric Schrock 	 * This assumes that the ZFS scheme code associated unusable (i.e.
346069f55e2SEric Schrock 	 * isolated) with its own definition of faulty state.  In the case of a
347069f55e2SEric Schrock 	 * DEGRADED leaf vdev (due to checksum errors), this is not the case.
348069f55e2SEric Schrock 	 * This works, however, because the transient state change is not
349069f55e2SEric Schrock 	 * posted in this case.  This could be made more explicit by not
350069f55e2SEric Schrock 	 * relying on the scheme's unusable callback and instead directly
351069f55e2SEric Schrock 	 * checking the vdev state, where we could correctly account for
352069f55e2SEric Schrock 	 * DEGRADED state.
353069f55e2SEric Schrock 	 */
354069f55e2SEric Schrock 	if (!fmd_nvl_fmri_unusable(hdl, asru) && fmd_nvl_fmri_has_fault(hdl,
355069f55e2SEric Schrock 	    asru, FMD_HAS_FAULT_ASRU, NULL)) {
356069f55e2SEric Schrock 		topo_hdl_t *thp;
357069f55e2SEric Schrock 		char *fmri = NULL;
358069f55e2SEric Schrock 		int err;
359069f55e2SEric Schrock 
360069f55e2SEric Schrock 		thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
361069f55e2SEric Schrock 		if (topo_fmri_nvl2str(thp, asru, &fmri, &err) == 0)
362eb04386eSEric Schrock 			(void) fmd_repair_asru(hdl, fmri);
363069f55e2SEric Schrock 		fmd_hdl_topo_rele(hdl, thp);
364069f55e2SEric Schrock 
365069f55e2SEric Schrock 		topo_hdl_strfree(thp, fmri);
366069f55e2SEric Schrock 	}
3672a417b23SRobert Johnston 	nvlist_free(asru);
368069f55e2SEric Schrock 	zrp = fmd_hdl_alloc(hdl, sizeof (zfs_retire_repaired_t), FMD_SLEEP);
369069f55e2SEric Schrock 	zrp->zrr_next = zdp->zrd_repaired;
370069f55e2SEric Schrock 	zrp->zrr_pool = pool_guid;
371069f55e2SEric Schrock 	zrp->zrr_vdev = vdev_guid;
372069f55e2SEric Schrock 	zdp->zrd_repaired = zrp;
373069f55e2SEric Schrock }
374069f55e2SEric Schrock 
37599653d4eSeschrock /*ARGSUSED*/
37699653d4eSeschrock static void
zfs_retire_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)37799653d4eSeschrock zfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
37899653d4eSeschrock     const char *class)
37999653d4eSeschrock {
38099653d4eSeschrock 	uint64_t pool_guid, vdev_guid;
38199653d4eSeschrock 	zpool_handle_t *zhp;
382069f55e2SEric Schrock 	nvlist_t *resource, *fault, *fru;
3833d7072f8Seschrock 	nvlist_t **faults;
3843d7072f8Seschrock 	uint_t f, nfaults;
385069f55e2SEric Schrock 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
386069f55e2SEric Schrock 	libzfs_handle_t *zhdl = zdp->zrd_hdl;
3873d7072f8Seschrock 	boolean_t fault_device, degrade_device;
3883d7072f8Seschrock 	boolean_t is_repair;
389069f55e2SEric Schrock 	char *scheme, *fmri;
3903d7072f8Seschrock 	nvlist_t *vdev;
39125c6ff4bSstephh 	char *uuid;
39225c6ff4bSstephh 	int repair_done = 0;
393cbf75e67SStephen Hanson 	boolean_t retire;
394069f55e2SEric Schrock 	boolean_t is_disk;
395069f55e2SEric Schrock 	vdev_aux_t aux;
396069f55e2SEric Schrock 	topo_hdl_t *thp;
397069f55e2SEric Schrock 	int err;
3983d7072f8Seschrock 
3993d7072f8Seschrock 	/*
4003d7072f8Seschrock 	 * If this is a resource notifying us of device removal, then simply
4013d7072f8Seschrock 	 * check for an available spare and continue.
4023d7072f8Seschrock 	 */
4033d7072f8Seschrock 	if (strcmp(class, "resource.fs.zfs.removed") == 0) {
4043d7072f8Seschrock 		if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
4053d7072f8Seschrock 		    &pool_guid) != 0 ||
4063d7072f8Seschrock 		    nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
4073d7072f8Seschrock 		    &vdev_guid) != 0)
4083d7072f8Seschrock 			return;
4093d7072f8Seschrock 
4103d7072f8Seschrock 		if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
4113d7072f8Seschrock 		    &vdev)) == NULL)
4123d7072f8Seschrock 			return;
4133d7072f8Seschrock 
4143d7072f8Seschrock 		if (fmd_prop_get_int32(hdl, "spare_on_remove"))
415069f55e2SEric Schrock 			replace_with_spare(hdl, zhp, vdev);
4163d7072f8Seschrock 		zpool_close(zhp);
4173d7072f8Seschrock 		return;
4183d7072f8Seschrock 	}
4193d7072f8Seschrock 
420cbf75e67SStephen Hanson 	if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0)
421cbf75e67SStephen Hanson 		return;
422cbf75e67SStephen Hanson 
423069f55e2SEric Schrock 	if (strcmp(class, "resource.fs.zfs.statechange") == 0 ||
424069f55e2SEric Schrock 	    strcmp(class,
425069f55e2SEric Schrock 	    "resource.sysevent.EC_zfs.ESC_ZFS_vdev_remove") == 0) {
426069f55e2SEric Schrock 		zfs_vdev_repair(hdl, nvl);
427069f55e2SEric Schrock 		return;
428069f55e2SEric Schrock 	}
429069f55e2SEric Schrock 
430069f55e2SEric Schrock 	zfs_retire_clear_data(hdl, zdp);
431069f55e2SEric Schrock 
43225c6ff4bSstephh 	if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0)
4333d7072f8Seschrock 		is_repair = B_TRUE;
4343d7072f8Seschrock 	else
4353d7072f8Seschrock 		is_repair = B_FALSE;
43699653d4eSeschrock 
43799653d4eSeschrock 	/*
4383d7072f8Seschrock 	 * We subscribe to zfs faults as well as all repair events.
43999653d4eSeschrock 	 */
44099653d4eSeschrock 	if (nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
44199653d4eSeschrock 	    &faults, &nfaults) != 0)
44299653d4eSeschrock 		return;
44399653d4eSeschrock 
44499653d4eSeschrock 	for (f = 0; f < nfaults; f++) {
4453d7072f8Seschrock 		fault = faults[f];
4463d7072f8Seschrock 
4473d7072f8Seschrock 		fault_device = B_FALSE;
4483d7072f8Seschrock 		degrade_device = B_FALSE;
449069f55e2SEric Schrock 		is_disk = B_FALSE;
45099653d4eSeschrock 
451cbf75e67SStephen Hanson 		if (nvlist_lookup_boolean_value(fault, FM_SUSPECT_RETIRE,
452cbf75e67SStephen Hanson 		    &retire) == 0 && retire == 0)
453cbf75e67SStephen Hanson 			continue;
454cbf75e67SStephen Hanson 
4550244979bSAlek Pinchuk 		if (fmd_nvl_class_match(hdl, fault,
4560244979bSAlek Pinchuk 		    "fault.io.disk.ssm-wearout") &&
4570244979bSAlek Pinchuk 		    fmd_prop_get_int32(hdl, "ssm_wearout_skip_retire") ==
4580244979bSAlek Pinchuk 		    FMD_B_TRUE) {
4590244979bSAlek Pinchuk 			fmd_hdl_debug(hdl, "zfs-retire: ignoring SSM fault");
4600244979bSAlek Pinchuk 			continue;
4610244979bSAlek Pinchuk 		}
4620244979bSAlek Pinchuk 
46399653d4eSeschrock 		/*
4643d7072f8Seschrock 		 * While we subscribe to fault.fs.zfs.*, we only take action
4653d7072f8Seschrock 		 * for faults targeting a specific vdev (open failure or SERD
466069f55e2SEric Schrock 		 * failure).  We also subscribe to fault.io.* events, so that
467069f55e2SEric Schrock 		 * faulty disks will be faulted in the ZFS configuration.
46899653d4eSeschrock 		 */
469069f55e2SEric Schrock 		if (fmd_nvl_class_match(hdl, fault, "fault.fs.zfs.vdev.io")) {
4703d7072f8Seschrock 			fault_device = B_TRUE;
471069f55e2SEric Schrock 		} else if (fmd_nvl_class_match(hdl, fault,
472069f55e2SEric Schrock 		    "fault.fs.zfs.vdev.checksum")) {
4733d7072f8Seschrock 			degrade_device = B_TRUE;
474069f55e2SEric Schrock 		} else if (fmd_nvl_class_match(hdl, fault,
475069f55e2SEric Schrock 		    "fault.fs.zfs.device")) {
4763d7072f8Seschrock 			fault_device = B_FALSE;
477069f55e2SEric Schrock 		} else if (fmd_nvl_class_match(hdl, fault, "fault.io.*")) {
478069f55e2SEric Schrock 			is_disk = B_TRUE;
479069f55e2SEric Schrock 			fault_device = B_TRUE;
480069f55e2SEric Schrock 		} else {
48199653d4eSeschrock 			continue;
482069f55e2SEric Schrock 		}
48399653d4eSeschrock 
484069f55e2SEric Schrock 		if (is_disk) {
485069f55e2SEric Schrock 			/*
486069f55e2SEric Schrock 			 * This is a disk fault.  Lookup the FRU, convert it to
487069f55e2SEric Schrock 			 * an FMRI string, and attempt to find a matching vdev.
488069f55e2SEric Schrock 			 */
489069f55e2SEric Schrock 			if (nvlist_lookup_nvlist(fault, FM_FAULT_FRU,
490069f55e2SEric Schrock 			    &fru) != 0 ||
491069f55e2SEric Schrock 			    nvlist_lookup_string(fru, FM_FMRI_SCHEME,
492069f55e2SEric Schrock 			    &scheme) != 0)
493069f55e2SEric Schrock 				continue;
494069f55e2SEric Schrock 
495069f55e2SEric Schrock 			if (strcmp(scheme, FM_FMRI_SCHEME_HC) != 0)
496069f55e2SEric Schrock 				continue;
497069f55e2SEric Schrock 
498069f55e2SEric Schrock 			thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
499069f55e2SEric Schrock 			if (topo_fmri_nvl2str(thp, fru, &fmri, &err) != 0) {
500069f55e2SEric Schrock 				fmd_hdl_topo_rele(hdl, thp);
501069f55e2SEric Schrock 				continue;
502069f55e2SEric Schrock 			}
503069f55e2SEric Schrock 
504069f55e2SEric Schrock 			zhp = find_by_fru(zhdl, fmri, &vdev);
505069f55e2SEric Schrock 			topo_hdl_strfree(thp, fmri);
506069f55e2SEric Schrock 			fmd_hdl_topo_rele(hdl, thp);
507069f55e2SEric Schrock 
508069f55e2SEric Schrock 			if (zhp == NULL)
509069f55e2SEric Schrock 				continue;
510069f55e2SEric Schrock 
511069f55e2SEric Schrock 			(void) nvlist_lookup_uint64(vdev,
512069f55e2SEric Schrock 			    ZPOOL_CONFIG_GUID, &vdev_guid);
513069f55e2SEric Schrock 			aux = VDEV_AUX_EXTERNAL;
514069f55e2SEric Schrock 		} else {
515069f55e2SEric Schrock 			/*
516069f55e2SEric Schrock 			 * This is a ZFS fault.  Lookup the resource, and
517069f55e2SEric Schrock 			 * attempt to find the matching vdev.
518069f55e2SEric Schrock 			 */
519069f55e2SEric Schrock 			if (nvlist_lookup_nvlist(fault, FM_FAULT_RESOURCE,
520069f55e2SEric Schrock 			    &resource) != 0 ||
521069f55e2SEric Schrock 			    nvlist_lookup_string(resource, FM_FMRI_SCHEME,
522069f55e2SEric Schrock 			    &scheme) != 0)
523069f55e2SEric Schrock 				continue;
524069f55e2SEric Schrock 
525069f55e2SEric Schrock 			if (strcmp(scheme, FM_FMRI_SCHEME_ZFS) != 0)
526069f55e2SEric Schrock 				continue;
527069f55e2SEric Schrock 
528069f55e2SEric Schrock 			if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_POOL,
529069f55e2SEric Schrock 			    &pool_guid) != 0)
530069f55e2SEric Schrock 				continue;
531069f55e2SEric Schrock 
532069f55e2SEric Schrock 			if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_VDEV,
533069f55e2SEric Schrock 			    &vdev_guid) != 0) {
534069f55e2SEric Schrock 				if (is_repair)
535069f55e2SEric Schrock 					vdev_guid = 0;
536069f55e2SEric Schrock 				else
537069f55e2SEric Schrock 					continue;
538069f55e2SEric Schrock 			}
539069f55e2SEric Schrock 
540069f55e2SEric Schrock 			if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
541069f55e2SEric Schrock 			    &vdev)) == NULL)
542069f55e2SEric Schrock 				continue;
543069f55e2SEric Schrock 
544069f55e2SEric Schrock 			aux = VDEV_AUX_ERR_EXCEEDED;
545069f55e2SEric Schrock 		}
54699653d4eSeschrock 
547069f55e2SEric Schrock 		if (vdev_guid == 0) {
548069f55e2SEric Schrock 			/*
549069f55e2SEric Schrock 			 * For pool-level repair events, clear the entire pool.
550069f55e2SEric Schrock 			 */
551468c413aSTim Haley 			(void) zpool_clear(zhp, NULL, NULL);
552069f55e2SEric Schrock 			zpool_close(zhp);
55399653d4eSeschrock 			continue;
554069f55e2SEric Schrock 		}
55599653d4eSeschrock 
5563d7072f8Seschrock 		/*
5573d7072f8Seschrock 		 * If this is a repair event, then mark the vdev as repaired and
5583d7072f8Seschrock 		 * continue.
5593d7072f8Seschrock 		 */
5603d7072f8Seschrock 		if (is_repair) {
56125c6ff4bSstephh 			repair_done = 1;
5623d7072f8Seschrock 			(void) zpool_vdev_clear(zhp, vdev_guid);
56399653d4eSeschrock 			zpool_close(zhp);
56499653d4eSeschrock 			continue;
56599653d4eSeschrock 		}
56699653d4eSeschrock 
56799653d4eSeschrock 		/*
5683d7072f8Seschrock 		 * Actively fault the device if needed.
56999653d4eSeschrock 		 */
5703d7072f8Seschrock 		if (fault_device)
571069f55e2SEric Schrock 			(void) zpool_vdev_fault(zhp, vdev_guid, aux);
5723d7072f8Seschrock 		if (degrade_device)
573069f55e2SEric Schrock 			(void) zpool_vdev_degrade(zhp, vdev_guid, aux);
57499653d4eSeschrock 
5753d7072f8Seschrock 		/*
5763d7072f8Seschrock 		 * Attempt to substitute a hot spare.
5773d7072f8Seschrock 		 */
578069f55e2SEric Schrock 		replace_with_spare(hdl, zhp, vdev);
57999653d4eSeschrock 		zpool_close(zhp);
58099653d4eSeschrock 	}
58125c6ff4bSstephh 
58225c6ff4bSstephh 	if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 && repair_done &&
58325c6ff4bSstephh 	    nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) == 0)
58425c6ff4bSstephh 		fmd_case_uuresolved(hdl, uuid);
58599653d4eSeschrock }
58699653d4eSeschrock 
58799653d4eSeschrock static const fmd_hdl_ops_t fmd_ops = {
58899653d4eSeschrock 	zfs_retire_recv,	/* fmdo_recv */
58999653d4eSeschrock 	NULL,			/* fmdo_timeout */
59099653d4eSeschrock 	NULL,			/* fmdo_close */
59199653d4eSeschrock 	NULL,			/* fmdo_stats */
59299653d4eSeschrock 	NULL,			/* fmdo_gc */
59399653d4eSeschrock };
59499653d4eSeschrock 
59599653d4eSeschrock static const fmd_prop_t fmd_props[] = {
5963d7072f8Seschrock 	{ "spare_on_remove", FMD_TYPE_BOOL, "true" },
5970244979bSAlek Pinchuk 	{ "ssm_wearout_skip_retire", FMD_TYPE_BOOL, "true"},
59899653d4eSeschrock 	{ NULL, 0, NULL }
59999653d4eSeschrock };
60099653d4eSeschrock 
60199653d4eSeschrock static const fmd_hdl_info_t fmd_info = {
60299653d4eSeschrock 	"ZFS Retire Agent", "1.0", &fmd_ops, fmd_props
60399653d4eSeschrock };
60499653d4eSeschrock 
60599653d4eSeschrock void
_fmd_init(fmd_hdl_t * hdl)60699653d4eSeschrock _fmd_init(fmd_hdl_t *hdl)
60799653d4eSeschrock {
608069f55e2SEric Schrock 	zfs_retire_data_t *zdp;
60999653d4eSeschrock 	libzfs_handle_t *zhdl;
61099653d4eSeschrock 
61199653d4eSeschrock 	if ((zhdl = libzfs_init()) == NULL)
61299653d4eSeschrock 		return;
61399653d4eSeschrock 
61499653d4eSeschrock 	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
61599653d4eSeschrock 		libzfs_fini(zhdl);
61699653d4eSeschrock 		return;
61799653d4eSeschrock 	}
61899653d4eSeschrock 
619069f55e2SEric Schrock 	zdp = fmd_hdl_zalloc(hdl, sizeof (zfs_retire_data_t), FMD_SLEEP);
620069f55e2SEric Schrock 	zdp->zrd_hdl = zhdl;
621069f55e2SEric Schrock 
622069f55e2SEric Schrock 	fmd_hdl_setspecific(hdl, zdp);
62399653d4eSeschrock }
62499653d4eSeschrock 
62599653d4eSeschrock void
_fmd_fini(fmd_hdl_t * hdl)62699653d4eSeschrock _fmd_fini(fmd_hdl_t *hdl)
62799653d4eSeschrock {
628069f55e2SEric Schrock 	zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
62999653d4eSeschrock 
630069f55e2SEric Schrock 	if (zdp != NULL) {
631069f55e2SEric Schrock 		zfs_retire_clear_data(hdl, zdp);
632069f55e2SEric Schrock 		libzfs_fini(zdp->zrd_hdl);
633069f55e2SEric Schrock 		fmd_hdl_free(hdl, zdp, sizeof (zfs_retire_data_t));
634069f55e2SEric Schrock 	}
63599653d4eSeschrock }
636