1ea8dc4b6Seschrock /*
2ea8dc4b6Seschrock * CDDL HEADER START
3ea8dc4b6Seschrock *
4ea8dc4b6Seschrock * The contents of this file are subject to the terms of the
5ea8dc4b6Seschrock * Common Development and Distribution License (the "License").
6ea8dc4b6Seschrock * You may not use this file except in compliance with the License.
7ea8dc4b6Seschrock *
8ea8dc4b6Seschrock * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9ea8dc4b6Seschrock * or http://www.opensolaris.org/os/licensing.
10ea8dc4b6Seschrock * See the License for the specific language governing permissions
11ea8dc4b6Seschrock * and limitations under the License.
12ea8dc4b6Seschrock *
13ea8dc4b6Seschrock * When distributing Covered Code, include this CDDL HEADER in each
14ea8dc4b6Seschrock * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15ea8dc4b6Seschrock * If applicable, add the following below this CDDL HEADER, with the
16ea8dc4b6Seschrock * fields enclosed by brackets "[]" replaced with your own identifying
17ea8dc4b6Seschrock * information: Portions Copyright [yyyy] [name of copyright owner]
18ea8dc4b6Seschrock *
19ea8dc4b6Seschrock * CDDL HEADER END
20ea8dc4b6Seschrock */
2169a6f717SMarcel Telka
22ea8dc4b6Seschrock /*
2369a6f717SMarcel Telka * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
2411027bc7STim Haley * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
25*e830fb12SKody A Kantor * Copyright 2019 Joyent, Inc.
26ea8dc4b6Seschrock */
27ea8dc4b6Seschrock
28ea8dc4b6Seschrock #include <assert.h>
29ea8dc4b6Seschrock #include <stddef.h>
30ea8dc4b6Seschrock #include <strings.h>
31ea8dc4b6Seschrock #include <libuutil.h>
323d7072f8Seschrock #include <libzfs.h>
33ea8dc4b6Seschrock #include <fm/fmd_api.h>
34069f55e2SEric Schrock #include <fm/libtopo.h>
3511027bc7STim Haley #include <sys/types.h>
3611027bc7STim Haley #include <sys/time.h>
37ea8dc4b6Seschrock #include <sys/fs/zfs.h>
38ea8dc4b6Seschrock #include <sys/fm/protocol.h>
39ea8dc4b6Seschrock #include <sys/fm/fs/zfs.h>
40ea8dc4b6Seschrock
413d7072f8Seschrock /*
42*e830fb12SKody A Kantor * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io,probe}'.
43*e830fb12SKody A Kantor * This #define reserves enough space for two 64-bit hex values plus the length
44*e830fb12SKody A Kantor * of the longest string.
453d7072f8Seschrock */
463d7072f8Seschrock #define MAX_SERDLEN (16 * 2 + sizeof ("zfs___checksum"))
473d7072f8Seschrock
4851ece835Seschrock /*
4951ece835Seschrock * On-disk case structure. This must maintain backwards compatibility with
5051ece835Seschrock * previous versions of the DE. By default, any members appended to the end
5151ece835Seschrock * will be filled with zeros if they don't exist in a previous version.
5251ece835Seschrock */
53ea8dc4b6Seschrock typedef struct zfs_case_data {
54ea8dc4b6Seschrock uint64_t zc_version;
55ea8dc4b6Seschrock uint64_t zc_ena;
56ea8dc4b6Seschrock uint64_t zc_pool_guid;
57ea8dc4b6Seschrock uint64_t zc_vdev_guid;
5851ece835Seschrock int zc_has_timer; /* defunct */
59ea8dc4b6Seschrock int zc_pool_state;
603d7072f8Seschrock char zc_serd_checksum[MAX_SERDLEN];
613d7072f8Seschrock char zc_serd_io[MAX_SERDLEN];
6251ece835Seschrock int zc_has_remove_timer;
63*e830fb12SKody A Kantor char zc_serd_probe[MAX_SERDLEN];
64ea8dc4b6Seschrock } zfs_case_data_t;
65ea8dc4b6Seschrock
6611027bc7STim Haley /*
6711027bc7STim Haley * Time-of-day
6811027bc7STim Haley */
6911027bc7STim Haley typedef struct er_timeval {
7011027bc7STim Haley uint64_t ertv_sec;
7111027bc7STim Haley uint64_t ertv_nsec;
7211027bc7STim Haley } er_timeval_t;
7311027bc7STim Haley
7451ece835Seschrock /*
7551ece835Seschrock * In-core case structure.
7651ece835Seschrock */
77ea8dc4b6Seschrock typedef struct zfs_case {
783d7072f8Seschrock boolean_t zc_present;
793d7072f8Seschrock uint32_t zc_version;
80ea8dc4b6Seschrock zfs_case_data_t zc_data;
81ea8dc4b6Seschrock fmd_case_t *zc_case;
82ea8dc4b6Seschrock uu_list_node_t zc_node;
8351ece835Seschrock id_t zc_remove_timer;
84069f55e2SEric Schrock char *zc_fru;
8511027bc7STim Haley er_timeval_t zc_when;
86ea8dc4b6Seschrock } zfs_case_t;
87ea8dc4b6Seschrock
883d7072f8Seschrock #define CASE_DATA "data"
89069f55e2SEric Schrock #define CASE_FRU "fru"
903d7072f8Seschrock #define CASE_DATA_VERSION_INITIAL 1
913d7072f8Seschrock #define CASE_DATA_VERSION_SERD 2
92ea8dc4b6Seschrock
93*e830fb12SKody A Kantor /* The length of the maximum uint64 rendered as a decimal string. */
94*e830fb12SKody A Kantor #define MAX_ULL_STR 21
95*e830fb12SKody A Kantor
9611027bc7STim Haley typedef struct zfs_de_stats {
9711027bc7STim Haley fmd_stat_t old_drops;
9811027bc7STim Haley fmd_stat_t dev_drops;
9911027bc7STim Haley fmd_stat_t vdev_drops;
10011027bc7STim Haley fmd_stat_t import_drops;
10111027bc7STim Haley fmd_stat_t resource_drops;
102*e830fb12SKody A Kantor fmd_stat_t pool_drops;
10311027bc7STim Haley } zfs_de_stats_t;
10411027bc7STim Haley
10511027bc7STim Haley zfs_de_stats_t zfs_stats = {
10611027bc7STim Haley { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
10711027bc7STim Haley { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
10811027bc7STim Haley { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
10911027bc7STim Haley { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
110*e830fb12SKody A Kantor { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" },
111*e830fb12SKody A Kantor { "pool_drops", FMD_TYPE_UINT64, "ereports dropped (pool iter failed)"},
11211027bc7STim Haley };
11311027bc7STim Haley
11451ece835Seschrock static hrtime_t zfs_remove_timeout;
115ea8dc4b6Seschrock
116ea8dc4b6Seschrock uu_list_pool_t *zfs_case_pool;
117ea8dc4b6Seschrock uu_list_t *zfs_cases;
118ea8dc4b6Seschrock
11951ece835Seschrock #define ZFS_MAKE_RSRC(type) \
12051ece835Seschrock FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
12151ece835Seschrock #define ZFS_MAKE_EREPORT(type) \
12251ece835Seschrock FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
12351ece835Seschrock
1243d7072f8Seschrock /*
1253d7072f8Seschrock * Write out the persistent representation of an active case.
1263d7072f8Seschrock */
127ea8dc4b6Seschrock static void
zfs_case_serialize(fmd_hdl_t * hdl,zfs_case_t * zcp)128ea8dc4b6Seschrock zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp)
129ea8dc4b6Seschrock {
1303d7072f8Seschrock /*
1313d7072f8Seschrock * Always update cases to the latest version, even if they were the
1323d7072f8Seschrock * previous version when unserialized.
1333d7072f8Seschrock */
1343d7072f8Seschrock zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
135ea8dc4b6Seschrock fmd_buf_write(hdl, zcp->zc_case, CASE_DATA, &zcp->zc_data,
136ea8dc4b6Seschrock sizeof (zcp->zc_data));
137069f55e2SEric Schrock
138069f55e2SEric Schrock if (zcp->zc_fru != NULL)
139069f55e2SEric Schrock fmd_buf_write(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
140069f55e2SEric Schrock strlen(zcp->zc_fru));
141ea8dc4b6Seschrock }
142ea8dc4b6Seschrock
1433d7072f8Seschrock /*
1443d7072f8Seschrock * Read back the persistent representation of an active case.
1453d7072f8Seschrock */
146ea8dc4b6Seschrock static zfs_case_t *
zfs_case_unserialize(fmd_hdl_t * hdl,fmd_case_t * cp)147ea8dc4b6Seschrock zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
148ea8dc4b6Seschrock {
149ea8dc4b6Seschrock zfs_case_t *zcp;
150069f55e2SEric Schrock size_t frulen;
151ea8dc4b6Seschrock
152ea8dc4b6Seschrock zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
153ea8dc4b6Seschrock zcp->zc_case = cp;
154ea8dc4b6Seschrock
155ea8dc4b6Seschrock fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
156ea8dc4b6Seschrock sizeof (zcp->zc_data));
157ea8dc4b6Seschrock
1583d7072f8Seschrock if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
159ea8dc4b6Seschrock fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
160ea8dc4b6Seschrock return (NULL);
161ea8dc4b6Seschrock }
162ea8dc4b6Seschrock
163069f55e2SEric Schrock if ((frulen = fmd_buf_size(hdl, zcp->zc_case, CASE_FRU)) > 0) {
164069f55e2SEric Schrock zcp->zc_fru = fmd_hdl_alloc(hdl, frulen + 1, FMD_SLEEP);
165069f55e2SEric Schrock fmd_buf_read(hdl, zcp->zc_case, CASE_FRU, zcp->zc_fru,
166069f55e2SEric Schrock frulen);
167069f55e2SEric Schrock zcp->zc_fru[frulen] = '\0';
168069f55e2SEric Schrock }
169069f55e2SEric Schrock
1703d7072f8Seschrock /*
1713d7072f8Seschrock * fmd_buf_read() will have already zeroed out the remainder of the
1723d7072f8Seschrock * buffer, so we don't have to do anything special if the version
1733d7072f8Seschrock * doesn't include the SERD engine name.
1743d7072f8Seschrock */
1753d7072f8Seschrock
17651ece835Seschrock if (zcp->zc_data.zc_has_remove_timer)
17751ece835Seschrock zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
17851ece835Seschrock NULL, zfs_remove_timeout);
179ea8dc4b6Seschrock
180ea8dc4b6Seschrock (void) uu_list_insert_before(zfs_cases, NULL, zcp);
181ea8dc4b6Seschrock
182ea8dc4b6Seschrock fmd_case_setspecific(hdl, cp, zcp);
183ea8dc4b6Seschrock
184ea8dc4b6Seschrock return (zcp);
185ea8dc4b6Seschrock }
186ea8dc4b6Seschrock
1873d7072f8Seschrock /*
1883d7072f8Seschrock * Iterate over any active cases. If any cases are associated with a pool or
1893d7072f8Seschrock * vdev which is no longer present on the system, close the associated case.
1903d7072f8Seschrock */
1913d7072f8Seschrock static void
zfs_mark_vdev(uint64_t pool_guid,nvlist_t * vd,er_timeval_t * loaded)19211027bc7STim Haley zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
1933d7072f8Seschrock {
1943d7072f8Seschrock uint64_t vdev_guid;
1953d7072f8Seschrock uint_t c, children;
1963d7072f8Seschrock nvlist_t **child;
1973d7072f8Seschrock zfs_case_t *zcp;
1983d7072f8Seschrock int ret;
1993d7072f8Seschrock
2003d7072f8Seschrock ret = nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
2013d7072f8Seschrock assert(ret == 0);
2023d7072f8Seschrock
2033d7072f8Seschrock /*
2043d7072f8Seschrock * Mark any cases associated with this (pool, vdev) pair.
2053d7072f8Seschrock */
2063d7072f8Seschrock for (zcp = uu_list_first(zfs_cases); zcp != NULL;
2073d7072f8Seschrock zcp = uu_list_next(zfs_cases, zcp)) {
2083d7072f8Seschrock if (zcp->zc_data.zc_pool_guid == pool_guid &&
20911027bc7STim Haley zcp->zc_data.zc_vdev_guid == vdev_guid) {
2103d7072f8Seschrock zcp->zc_present = B_TRUE;
21111027bc7STim Haley zcp->zc_when = *loaded;
21211027bc7STim Haley }
2133d7072f8Seschrock }
2143d7072f8Seschrock
2153d7072f8Seschrock /*
2163d7072f8Seschrock * Iterate over all children.
2173d7072f8Seschrock */
2183d7072f8Seschrock if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
2190430f8daSeschrock &children) == 0) {
2200430f8daSeschrock for (c = 0; c < children; c++)
22111027bc7STim Haley zfs_mark_vdev(pool_guid, child[c], loaded);
2220430f8daSeschrock }
2230430f8daSeschrock
2240430f8daSeschrock if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
2250430f8daSeschrock &children) == 0) {
2260430f8daSeschrock for (c = 0; c < children; c++)
22711027bc7STim Haley zfs_mark_vdev(pool_guid, child[c], loaded);
2280430f8daSeschrock }
2290430f8daSeschrock
2300430f8daSeschrock if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
2310430f8daSeschrock &children) == 0) {
2323d7072f8Seschrock for (c = 0; c < children; c++)
23311027bc7STim Haley zfs_mark_vdev(pool_guid, child[c], loaded);
2343d7072f8Seschrock }
2353d7072f8Seschrock }
2363d7072f8Seschrock
237ea8dc4b6Seschrock /*ARGSUSED*/
2383d7072f8Seschrock static int
zfs_mark_pool(zpool_handle_t * zhp,void * unused)2393d7072f8Seschrock zfs_mark_pool(zpool_handle_t *zhp, void *unused)
2403d7072f8Seschrock {
2413d7072f8Seschrock zfs_case_t *zcp;
242990b4856Slling uint64_t pool_guid;
24311027bc7STim Haley uint64_t *tod;
24411027bc7STim Haley er_timeval_t loaded = { 0 };
2453d7072f8Seschrock nvlist_t *config, *vd;
24611027bc7STim Haley uint_t nelem = 0;
2473d7072f8Seschrock int ret;
2483d7072f8Seschrock
249990b4856Slling pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
2503d7072f8Seschrock /*
2513d7072f8Seschrock * Mark any cases associated with just this pool.
2523d7072f8Seschrock */
2533d7072f8Seschrock for (zcp = uu_list_first(zfs_cases); zcp != NULL;
2543d7072f8Seschrock zcp = uu_list_next(zfs_cases, zcp)) {
2553d7072f8Seschrock if (zcp->zc_data.zc_pool_guid == pool_guid &&
2563d7072f8Seschrock zcp->zc_data.zc_vdev_guid == 0)
2573d7072f8Seschrock zcp->zc_present = B_TRUE;
2583d7072f8Seschrock }
2593d7072f8Seschrock
260c5904d13Seschrock if ((config = zpool_get_config(zhp, NULL)) == NULL) {
261c5904d13Seschrock zpool_close(zhp);
2623d7072f8Seschrock return (-1);
263c5904d13Seschrock }
2643d7072f8Seschrock
26511027bc7STim Haley (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
26611027bc7STim Haley &tod, &nelem);
26711027bc7STim Haley if (nelem == 2) {
26811027bc7STim Haley loaded.ertv_sec = tod[0];
26911027bc7STim Haley loaded.ertv_nsec = tod[1];
27011027bc7STim Haley for (zcp = uu_list_first(zfs_cases); zcp != NULL;
27111027bc7STim Haley zcp = uu_list_next(zfs_cases, zcp)) {
27211027bc7STim Haley if (zcp->zc_data.zc_pool_guid == pool_guid &&
27311027bc7STim Haley zcp->zc_data.zc_vdev_guid == 0) {
27411027bc7STim Haley zcp->zc_when = loaded;
27511027bc7STim Haley }
27611027bc7STim Haley }
27711027bc7STim Haley }
27811027bc7STim Haley
2793d7072f8Seschrock ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
2803d7072f8Seschrock assert(ret == 0);
2813d7072f8Seschrock
28211027bc7STim Haley zfs_mark_vdev(pool_guid, vd, &loaded);
2833d7072f8Seschrock
284c5904d13Seschrock zpool_close(zhp);
285c5904d13Seschrock
2863d7072f8Seschrock return (0);
2873d7072f8Seschrock }
2883d7072f8Seschrock
289*e830fb12SKody A Kantor /*
290*e830fb12SKody A Kantor * Find a pool with a matching GUID.
291*e830fb12SKody A Kantor */
292*e830fb12SKody A Kantor typedef struct find_cbdata {
293*e830fb12SKody A Kantor uint64_t cb_guid;
294*e830fb12SKody A Kantor zpool_handle_t *cb_zhp;
295*e830fb12SKody A Kantor } find_cbdata_t;
296*e830fb12SKody A Kantor
297*e830fb12SKody A Kantor static int
find_pool(zpool_handle_t * zhp,void * data)298*e830fb12SKody A Kantor find_pool(zpool_handle_t *zhp, void *data)
299*e830fb12SKody A Kantor {
300*e830fb12SKody A Kantor find_cbdata_t *cbp = data;
301*e830fb12SKody A Kantor
302*e830fb12SKody A Kantor if (cbp->cb_guid ==
303*e830fb12SKody A Kantor zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL)) {
304*e830fb12SKody A Kantor cbp->cb_zhp = zhp;
305*e830fb12SKody A Kantor return (0);
306*e830fb12SKody A Kantor }
307*e830fb12SKody A Kantor
308*e830fb12SKody A Kantor zpool_close(zhp);
309*e830fb12SKody A Kantor return (0);
310*e830fb12SKody A Kantor }
311*e830fb12SKody A Kantor
31211027bc7STim Haley struct load_time_arg {
31311027bc7STim Haley uint64_t lt_guid;
31411027bc7STim Haley er_timeval_t *lt_time;
31511027bc7STim Haley boolean_t lt_found;
31611027bc7STim Haley };
31711027bc7STim Haley
31811027bc7STim Haley static int
zpool_find_load_time(zpool_handle_t * zhp,void * arg)31911027bc7STim Haley zpool_find_load_time(zpool_handle_t *zhp, void *arg)
32011027bc7STim Haley {
32111027bc7STim Haley struct load_time_arg *lta = arg;
32211027bc7STim Haley uint64_t pool_guid;
32311027bc7STim Haley uint64_t *tod;
32411027bc7STim Haley nvlist_t *config;
32511027bc7STim Haley uint_t nelem;
32611027bc7STim Haley
32769a6f717SMarcel Telka if (lta->lt_found) {
32869a6f717SMarcel Telka zpool_close(zhp);
32911027bc7STim Haley return (0);
33069a6f717SMarcel Telka }
33111027bc7STim Haley
33211027bc7STim Haley pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
33369a6f717SMarcel Telka if (pool_guid != lta->lt_guid) {
33469a6f717SMarcel Telka zpool_close(zhp);
33511027bc7STim Haley return (0);
33669a6f717SMarcel Telka }
33711027bc7STim Haley
33811027bc7STim Haley if ((config = zpool_get_config(zhp, NULL)) == NULL) {
33911027bc7STim Haley zpool_close(zhp);
34011027bc7STim Haley return (-1);
34111027bc7STim Haley }
34211027bc7STim Haley
34311027bc7STim Haley if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
34411027bc7STim Haley &tod, &nelem) == 0 && nelem == 2) {
34511027bc7STim Haley lta->lt_found = B_TRUE;
34611027bc7STim Haley lta->lt_time->ertv_sec = tod[0];
34711027bc7STim Haley lta->lt_time->ertv_nsec = tod[1];
34811027bc7STim Haley }
34911027bc7STim Haley
35069a6f717SMarcel Telka zpool_close(zhp);
35169a6f717SMarcel Telka
35211027bc7STim Haley return (0);
35311027bc7STim Haley }
35411027bc7STim Haley
355ea8dc4b6Seschrock static void
zfs_purge_cases(fmd_hdl_t * hdl)3563d7072f8Seschrock zfs_purge_cases(fmd_hdl_t *hdl)
3573d7072f8Seschrock {
3583d7072f8Seschrock zfs_case_t *zcp;
3593d7072f8Seschrock uu_list_walk_t *walk;
3603d7072f8Seschrock libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
3613d7072f8Seschrock
3623d7072f8Seschrock /*
3633d7072f8Seschrock * There is no way to open a pool by GUID, or lookup a vdev by GUID. No
3643d7072f8Seschrock * matter what we do, we're going to have to stomach a O(vdevs * cases)
3653d7072f8Seschrock * algorithm. In reality, both quantities are likely so small that
3663d7072f8Seschrock * neither will matter. Given that iterating over pools is more
3673d7072f8Seschrock * expensive than iterating over the in-memory case list, we opt for a
3683d7072f8Seschrock * 'present' flag in each case that starts off cleared. We then iterate
3693d7072f8Seschrock * over all pools, marking those that are still present, and removing
3703d7072f8Seschrock * those that aren't found.
3713d7072f8Seschrock *
3723d7072f8Seschrock * Note that we could also construct an FMRI and rely on
3733d7072f8Seschrock * fmd_nvl_fmri_present(), but this would end up doing the same search.
3743d7072f8Seschrock */
3753d7072f8Seschrock
3763d7072f8Seschrock /*
3773d7072f8Seschrock * Mark the cases an not present.
3783d7072f8Seschrock */
3793d7072f8Seschrock for (zcp = uu_list_first(zfs_cases); zcp != NULL;
3803d7072f8Seschrock zcp = uu_list_next(zfs_cases, zcp))
3813d7072f8Seschrock zcp->zc_present = B_FALSE;
3823d7072f8Seschrock
3833d7072f8Seschrock /*
3843d7072f8Seschrock * Iterate over all pools and mark the pools and vdevs found. If this
3853d7072f8Seschrock * fails (most probably because we're out of memory), then don't close
3863d7072f8Seschrock * any of the cases and we cannot be sure they are accurate.
3873d7072f8Seschrock */
3883d7072f8Seschrock if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
3893d7072f8Seschrock return;
3903d7072f8Seschrock
3913d7072f8Seschrock /*
3923d7072f8Seschrock * Remove those cases which were not found.
3933d7072f8Seschrock */
3943d7072f8Seschrock walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
3953d7072f8Seschrock while ((zcp = uu_list_walk_next(walk)) != NULL) {
3963d7072f8Seschrock if (!zcp->zc_present)
3973d7072f8Seschrock fmd_case_close(hdl, zcp->zc_case);
3983d7072f8Seschrock }
3993d7072f8Seschrock uu_list_walk_end(walk);
4003d7072f8Seschrock }
4013d7072f8Seschrock
4023d7072f8Seschrock /*
403*e830fb12SKody A Kantor * Construct the name of a serd engine given the pool/vdev GUID and type (io,
404*e830fb12SKody A Kantor * checksum, or probe).
4053d7072f8Seschrock */
4063d7072f8Seschrock static void
zfs_serd_name(char * buf,uint64_t pool_guid,uint64_t vdev_guid,const char * type)4073d7072f8Seschrock zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
4083d7072f8Seschrock const char *type)
4093d7072f8Seschrock {
4103d7072f8Seschrock (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s", pool_guid,
4113d7072f8Seschrock vdev_guid, type);
4123d7072f8Seschrock }
4133d7072f8Seschrock
4143d7072f8Seschrock /*
4153d7072f8Seschrock * Solve a given ZFS case. This first checks to make sure the diagnosis is
4163d7072f8Seschrock * still valid, as well as cleaning up any pending timer associated with the
4173d7072f8Seschrock * case.
4183d7072f8Seschrock */
4193d7072f8Seschrock static void
zfs_case_solve(fmd_hdl_t * hdl,zfs_case_t * zcp,const char * faultname,boolean_t checkunusable)4203d7072f8Seschrock zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
4213d7072f8Seschrock boolean_t checkunusable)
4223d7072f8Seschrock {
423069f55e2SEric Schrock libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
4243d7072f8Seschrock nvlist_t *detector, *fault;
4253d7072f8Seschrock boolean_t serialize;
426069f55e2SEric Schrock nvlist_t *fmri, *fru;
427069f55e2SEric Schrock topo_hdl_t *thp;
428069f55e2SEric Schrock int err;
4293d7072f8Seschrock
4303d7072f8Seschrock /*
4313d7072f8Seschrock * Construct the detector from the case data. The detector is in the
4323d7072f8Seschrock * ZFS scheme, and is either the pool or the vdev, depending on whether
4333d7072f8Seschrock * this is a vdev or pool fault.
4343d7072f8Seschrock */
435069f55e2SEric Schrock detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
436069f55e2SEric Schrock
437069f55e2SEric Schrock (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
438069f55e2SEric Schrock (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
439069f55e2SEric Schrock (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
440069f55e2SEric Schrock zcp->zc_data.zc_pool_guid);
441069f55e2SEric Schrock if (zcp->zc_data.zc_vdev_guid != 0) {
442069f55e2SEric Schrock (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
443069f55e2SEric Schrock zcp->zc_data.zc_vdev_guid);
4443d7072f8Seschrock }
4453d7072f8Seschrock
4463d7072f8Seschrock /*
4473d7072f8Seschrock * We also want to make sure that the detector (pool or vdev) properly
4483d7072f8Seschrock * reflects the diagnosed state, when the fault corresponds to internal
4493d7072f8Seschrock * ZFS state (i.e. not checksum or I/O error-induced). Otherwise, a
4503d7072f8Seschrock * device which was unavailable early in boot (because the driver/file
4513d7072f8Seschrock * wasn't available) and is now healthy will be mis-diagnosed.
4523d7072f8Seschrock */
4533d7072f8Seschrock if (!fmd_nvl_fmri_present(hdl, detector) ||
4543d7072f8Seschrock (checkunusable && !fmd_nvl_fmri_unusable(hdl, detector))) {
4553d7072f8Seschrock fmd_case_close(hdl, zcp->zc_case);
4563d7072f8Seschrock nvlist_free(detector);
4573d7072f8Seschrock return;
4583d7072f8Seschrock }
4593d7072f8Seschrock
460069f55e2SEric Schrock
461069f55e2SEric Schrock fru = NULL;
462069f55e2SEric Schrock if (zcp->zc_fru != NULL &&
463069f55e2SEric Schrock (thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION)) != NULL) {
464069f55e2SEric Schrock /*
465069f55e2SEric Schrock * If the vdev had an associated FRU, then get the FRU nvlist
466069f55e2SEric Schrock * from the topo handle and use that in the suspect list. We
467069f55e2SEric Schrock * explicitly lookup the FRU because the fmri reported from the
468069f55e2SEric Schrock * kernel may not have up to date details about the disk itself
469069f55e2SEric Schrock * (serial, part, etc).
470069f55e2SEric Schrock */
471069f55e2SEric Schrock if (topo_fmri_str2nvl(thp, zcp->zc_fru, &fmri, &err) == 0) {
472069f55e2SEric Schrock /*
473069f55e2SEric Schrock * If the disk is part of the system chassis, but the
474069f55e2SEric Schrock * FRU indicates a different chassis ID than our
475069f55e2SEric Schrock * current system, then ignore the error. This
476069f55e2SEric Schrock * indicates that the device was part of another
477069f55e2SEric Schrock * cluster head, and for obvious reasons cannot be
478069f55e2SEric Schrock * imported on this system.
479069f55e2SEric Schrock */
480069f55e2SEric Schrock if (libzfs_fru_notself(zhdl, zcp->zc_fru)) {
481069f55e2SEric Schrock fmd_case_close(hdl, zcp->zc_case);
482069f55e2SEric Schrock nvlist_free(fmri);
483069f55e2SEric Schrock fmd_hdl_topo_rele(hdl, thp);
484069f55e2SEric Schrock nvlist_free(detector);
485069f55e2SEric Schrock return;
486069f55e2SEric Schrock }
487069f55e2SEric Schrock
488069f55e2SEric Schrock /*
489069f55e2SEric Schrock * If the device is no longer present on the system, or
490069f55e2SEric Schrock * topo_fmri_fru() fails for other reasons, then fall
491069f55e2SEric Schrock * back to the fmri specified in the vdev.
492069f55e2SEric Schrock */
493069f55e2SEric Schrock if (topo_fmri_fru(thp, fmri, &fru, &err) != 0)
494069f55e2SEric Schrock fru = fmd_nvl_dup(hdl, fmri, FMD_SLEEP);
495069f55e2SEric Schrock nvlist_free(fmri);
496069f55e2SEric Schrock }
497069f55e2SEric Schrock
498069f55e2SEric Schrock fmd_hdl_topo_rele(hdl, thp);
499069f55e2SEric Schrock }
500069f55e2SEric Schrock
501069f55e2SEric Schrock fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
502069f55e2SEric Schrock fru, detector);
5033d7072f8Seschrock fmd_case_add_suspect(hdl, zcp->zc_case, fault);
504069f55e2SEric Schrock
505069f55e2SEric Schrock nvlist_free(fru);
506069f55e2SEric Schrock
5073d7072f8Seschrock fmd_case_solve(hdl, zcp->zc_case);
5083d7072f8Seschrock
5093d7072f8Seschrock serialize = B_FALSE;
51051ece835Seschrock if (zcp->zc_data.zc_has_remove_timer) {
51151ece835Seschrock fmd_timer_remove(hdl, zcp->zc_remove_timer);
51251ece835Seschrock zcp->zc_data.zc_has_remove_timer = 0;
5133d7072f8Seschrock serialize = B_TRUE;
5143d7072f8Seschrock }
5153d7072f8Seschrock if (serialize)
5163d7072f8Seschrock zfs_case_serialize(hdl, zcp);
5173d7072f8Seschrock
5183d7072f8Seschrock nvlist_free(detector);
5193d7072f8Seschrock }
5203d7072f8Seschrock
52111027bc7STim Haley /*
52211027bc7STim Haley * This #define and function access a private interface of the FMA
52311027bc7STim Haley * framework. Ereports include a time-of-day upper bound.
52411027bc7STim Haley * We want to look at that so we can compare it to when pools get
52511027bc7STim Haley * loaded.
52611027bc7STim Haley */
52711027bc7STim Haley #define FMD_EVN_TOD "__tod"
52811027bc7STim Haley
52911027bc7STim Haley static boolean_t
timeval_earlier(er_timeval_t * a,er_timeval_t * b)53011027bc7STim Haley timeval_earlier(er_timeval_t *a, er_timeval_t *b)
53111027bc7STim Haley {
53211027bc7STim Haley return (a->ertv_sec < b->ertv_sec ||
53311027bc7STim Haley (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
53411027bc7STim Haley }
53511027bc7STim Haley
53611027bc7STim Haley /*ARGSUSED*/
53711027bc7STim Haley static void
zfs_ereport_when(fmd_hdl_t * hdl,nvlist_t * nvl,er_timeval_t * when)53811027bc7STim Haley zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
53911027bc7STim Haley {
54011027bc7STim Haley uint64_t *tod;
54111027bc7STim Haley uint_t nelem;
54211027bc7STim Haley
54311027bc7STim Haley if (nvlist_lookup_uint64_array(nvl, FMD_EVN_TOD, &tod, &nelem) == 0 &&
54411027bc7STim Haley nelem == 2) {
54511027bc7STim Haley when->ertv_sec = tod[0];
54611027bc7STim Haley when->ertv_nsec = tod[1];
54711027bc7STim Haley } else {
54811027bc7STim Haley when->ertv_sec = when->ertv_nsec = UINT64_MAX;
54911027bc7STim Haley }
55011027bc7STim Haley }
55111027bc7STim Haley
5523d7072f8Seschrock /*
5533d7072f8Seschrock * Main fmd entry point.
5543d7072f8Seschrock */
5553d7072f8Seschrock /*ARGSUSED*/
5563d7072f8Seschrock static void
zfs_fm_recv(fmd_hdl_t * hdl,fmd_event_t * ep,nvlist_t * nvl,const char * class)5573d7072f8Seschrock zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
558ea8dc4b6Seschrock {
559c5904d13Seschrock zfs_case_t *zcp, *dcp;
560*e830fb12SKody A Kantor libzfs_handle_t *zhdl;
561*e830fb12SKody A Kantor zpool_handle_t *zhp;
562*e830fb12SKody A Kantor
563ea8dc4b6Seschrock int32_t pool_state;
564ea8dc4b6Seschrock uint64_t ena, pool_guid, vdev_guid;
56511027bc7STim Haley er_timeval_t pool_load;
56611027bc7STim Haley er_timeval_t er_when;
567ea8dc4b6Seschrock nvlist_t *detector;
56811027bc7STim Haley boolean_t pool_found = B_FALSE;
569ea8dc4b6Seschrock boolean_t isresource;
570*e830fb12SKody A Kantor boolean_t is_inactive_spare, islog, iscache;
571*e830fb12SKody A Kantor nvlist_t *vd_nvl = NULL;
572*e830fb12SKody A Kantor char *fru, *type, *vdg;
573*e830fb12SKody A Kantor find_cbdata_t cb;
574069f55e2SEric Schrock
575069f55e2SEric Schrock /*
576069f55e2SEric Schrock * We subscribe to notifications for vdev or pool removal. In these
577069f55e2SEric Schrock * cases, there may be cases that no longer apply. Purge any cases
578069f55e2SEric Schrock * that no longer apply.
579069f55e2SEric Schrock */
580069f55e2SEric Schrock if (fmd_nvl_class_match(hdl, nvl, "resource.sysevent.EC_zfs.*")) {
581069f55e2SEric Schrock zfs_purge_cases(hdl);
58211027bc7STim Haley zfs_stats.resource_drops.fmds_value.ui64++;
583069f55e2SEric Schrock return;
584069f55e2SEric Schrock }
585ea8dc4b6Seschrock
586ea8dc4b6Seschrock isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
587ea8dc4b6Seschrock
588ea8dc4b6Seschrock if (isresource) {
589ea8dc4b6Seschrock /*
5903d7072f8Seschrock * For resources, we don't have a normal payload.
591ea8dc4b6Seschrock */
592ea8dc4b6Seschrock if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
593ea8dc4b6Seschrock &vdev_guid) != 0)
594ea8dc4b6Seschrock pool_state = SPA_LOAD_OPEN;
595ea8dc4b6Seschrock else
596ea8dc4b6Seschrock pool_state = SPA_LOAD_NONE;
597ea8dc4b6Seschrock detector = NULL;
598ea8dc4b6Seschrock } else {
599ea8dc4b6Seschrock (void) nvlist_lookup_nvlist(nvl,
600ea8dc4b6Seschrock FM_EREPORT_DETECTOR, &detector);
601ea8dc4b6Seschrock (void) nvlist_lookup_int32(nvl,
602ea8dc4b6Seschrock FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
603ea8dc4b6Seschrock }
604ea8dc4b6Seschrock
605ea8dc4b6Seschrock /*
606ea8dc4b6Seschrock * We also ignore all ereports generated during an import of a pool,
607ea8dc4b6Seschrock * since the only possible fault (.pool) would result in import failure,
608ea8dc4b6Seschrock * and hence no persistent fault. Some day we may want to do something
609ea8dc4b6Seschrock * with these ereports, so we continue generating them internally.
610ea8dc4b6Seschrock */
61111027bc7STim Haley if (pool_state == SPA_LOAD_IMPORT) {
61211027bc7STim Haley zfs_stats.import_drops.fmds_value.ui64++;
613ea8dc4b6Seschrock return;
61411027bc7STim Haley }
615ea8dc4b6Seschrock
616c5904d13Seschrock /*
617c5904d13Seschrock * Device I/O errors are ignored during pool open.
618c5904d13Seschrock */
619c5904d13Seschrock if (pool_state == SPA_LOAD_OPEN &&
6200430f8daSeschrock (fmd_nvl_class_match(hdl, nvl,
6210430f8daSeschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
6220430f8daSeschrock fmd_nvl_class_match(hdl, nvl,
6230430f8daSeschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
6240430f8daSeschrock fmd_nvl_class_match(hdl, nvl,
62511027bc7STim Haley ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
62611027bc7STim Haley zfs_stats.dev_drops.fmds_value.ui64++;
627c5904d13Seschrock return;
62811027bc7STim Haley }
629c5904d13Seschrock
6300430f8daSeschrock /*
6310430f8daSeschrock * We ignore ereports for anything except disks and files.
6320430f8daSeschrock */
6330430f8daSeschrock if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
6340430f8daSeschrock &type) == 0) {
6350430f8daSeschrock if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
63611027bc7STim Haley strcmp(type, VDEV_TYPE_FILE) != 0) {
63711027bc7STim Haley zfs_stats.vdev_drops.fmds_value.ui64++;
6380430f8daSeschrock return;
63911027bc7STim Haley }
6400430f8daSeschrock }
6410430f8daSeschrock
642ea8dc4b6Seschrock /*
643b693757aSEric Schrock * Determine if this ereport corresponds to an open case. Previous
644b693757aSEric Schrock * incarnations of this DE used the ENA to chain events together as
645b693757aSEric Schrock * part of the same case. The problem with this is that we rely on
646b693757aSEric Schrock * global uniqueness of cases based on (pool_guid, vdev_guid) pair when
647b693757aSEric Schrock * generating SERD engines. Instead, we have a case for each vdev or
648b693757aSEric Schrock * pool, regardless of the ENA.
649ea8dc4b6Seschrock */
6503d7072f8Seschrock (void) nvlist_lookup_uint64(nvl,
6513d7072f8Seschrock FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
6523d7072f8Seschrock if (nvlist_lookup_uint64(nvl,
6533d7072f8Seschrock FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
6543d7072f8Seschrock vdev_guid = 0;
6553d7072f8Seschrock if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
656e3c62504Seschrock ena = 0;
657ea8dc4b6Seschrock
65811027bc7STim Haley zfs_ereport_when(hdl, nvl, &er_when);
65911027bc7STim Haley
660ea8dc4b6Seschrock for (zcp = uu_list_first(zfs_cases); zcp != NULL;
661ea8dc4b6Seschrock zcp = uu_list_next(zfs_cases, zcp)) {
66211027bc7STim Haley if (zcp->zc_data.zc_pool_guid == pool_guid) {
66311027bc7STim Haley pool_found = B_TRUE;
66411027bc7STim Haley pool_load = zcp->zc_when;
66511027bc7STim Haley }
666*e830fb12SKody A Kantor if (zcp->zc_data.zc_vdev_guid == vdev_guid &&
667*e830fb12SKody A Kantor zcp->zc_data.zc_pool_guid == pool_guid)
668ea8dc4b6Seschrock break;
669ea8dc4b6Seschrock }
670ea8dc4b6Seschrock
67111027bc7STim Haley if (pool_found) {
67211027bc7STim Haley fmd_hdl_debug(hdl, "pool %llx, "
67311027bc7STim Haley "ereport time %lld.%lld, pool load time = %lld.%lld\n",
67411027bc7STim Haley pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
67511027bc7STim Haley pool_load.ertv_sec, pool_load.ertv_nsec);
67611027bc7STim Haley }
67711027bc7STim Haley
67811027bc7STim Haley /*
67911027bc7STim Haley * Avoid falsely accusing a pool of being faulty. Do so by
68011027bc7STim Haley * not replaying ereports that were generated prior to the
68111027bc7STim Haley * current import. If the failure that generated them was
68211027bc7STim Haley * transient because the device was actually removed but we
68311027bc7STim Haley * didn't receive the normal asynchronous notification, we
68411027bc7STim Haley * don't want to mark it as faulted and potentially panic. If
68511027bc7STim Haley * there is still a problem we'd expect not to be able to
68611027bc7STim Haley * import the pool, or that new ereports will be generated
68711027bc7STim Haley * once the pool is used.
68811027bc7STim Haley */
68911027bc7STim Haley if (pool_found && timeval_earlier(&er_when, &pool_load)) {
69011027bc7STim Haley zfs_stats.old_drops.fmds_value.ui64++;
69111027bc7STim Haley return;
69211027bc7STim Haley }
69311027bc7STim Haley
69411027bc7STim Haley if (!pool_found) {
69511027bc7STim Haley /*
69611027bc7STim Haley * Haven't yet seen this pool, but same situation
69711027bc7STim Haley * may apply.
69811027bc7STim Haley */
69911027bc7STim Haley libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
70011027bc7STim Haley struct load_time_arg la;
70111027bc7STim Haley
70211027bc7STim Haley la.lt_guid = pool_guid;
70311027bc7STim Haley la.lt_time = &pool_load;
70411027bc7STim Haley la.lt_found = B_FALSE;
70511027bc7STim Haley
70611027bc7STim Haley if (zhdl != NULL &&
70711027bc7STim Haley zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
70811027bc7STim Haley la.lt_found == B_TRUE) {
70911027bc7STim Haley pool_found = B_TRUE;
71011027bc7STim Haley fmd_hdl_debug(hdl, "pool %llx, "
71111027bc7STim Haley "ereport time %lld.%lld, "
71211027bc7STim Haley "pool load time = %lld.%lld\n",
71311027bc7STim Haley pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
71411027bc7STim Haley pool_load.ertv_sec, pool_load.ertv_nsec);
71511027bc7STim Haley if (timeval_earlier(&er_when, &pool_load)) {
71611027bc7STim Haley zfs_stats.old_drops.fmds_value.ui64++;
71711027bc7STim Haley return;
71811027bc7STim Haley }
71911027bc7STim Haley }
72011027bc7STim Haley }
72111027bc7STim Haley
722ea8dc4b6Seschrock if (zcp == NULL) {
723ea8dc4b6Seschrock fmd_case_t *cs;
7243d7072f8Seschrock zfs_case_data_t data = { 0 };
725ea8dc4b6Seschrock
726ea8dc4b6Seschrock /*
727ea8dc4b6Seschrock * If this is one of our 'fake' resource ereports, and there is
728ea8dc4b6Seschrock * no case open, simply discard it.
729ea8dc4b6Seschrock */
73011027bc7STim Haley if (isresource) {
73111027bc7STim Haley zfs_stats.resource_drops.fmds_value.ui64++;
732ea8dc4b6Seschrock return;
73311027bc7STim Haley }
734ea8dc4b6Seschrock
735ea8dc4b6Seschrock /*
736ea8dc4b6Seschrock * Open a new case.
737ea8dc4b6Seschrock */
738ea8dc4b6Seschrock cs = fmd_case_open(hdl, NULL);
739ea8dc4b6Seschrock
740ea8dc4b6Seschrock /*
741ea8dc4b6Seschrock * Initialize the case buffer. To commonize code, we actually
742ea8dc4b6Seschrock * create the buffer with existing data, and then call
743ea8dc4b6Seschrock * zfs_case_unserialize() to instantiate the in-core structure.
744ea8dc4b6Seschrock */
745ea8dc4b6Seschrock fmd_buf_create(hdl, cs, CASE_DATA,
746ea8dc4b6Seschrock sizeof (zfs_case_data_t));
747ea8dc4b6Seschrock
7483d7072f8Seschrock data.zc_version = CASE_DATA_VERSION_SERD;
749ea8dc4b6Seschrock data.zc_ena = ena;
750ea8dc4b6Seschrock data.zc_pool_guid = pool_guid;
751ea8dc4b6Seschrock data.zc_vdev_guid = vdev_guid;
752ea8dc4b6Seschrock data.zc_pool_state = (int)pool_state;
753ea8dc4b6Seschrock
754ea8dc4b6Seschrock fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
755ea8dc4b6Seschrock
756ea8dc4b6Seschrock zcp = zfs_case_unserialize(hdl, cs);
757ea8dc4b6Seschrock assert(zcp != NULL);
75811027bc7STim Haley if (pool_found)
75911027bc7STim Haley zcp->zc_when = pool_load;
760ea8dc4b6Seschrock }
761ea8dc4b6Seschrock
76211027bc7STim Haley
763069f55e2SEric Schrock /*
764069f55e2SEric Schrock * If this is an ereport for a case with an associated vdev FRU, make
765069f55e2SEric Schrock * sure it is accurate and up to date.
766069f55e2SEric Schrock */
767069f55e2SEric Schrock if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU,
768069f55e2SEric Schrock &fru) == 0) {
769069f55e2SEric Schrock topo_hdl_t *thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
770069f55e2SEric Schrock if (zcp->zc_fru == NULL ||
771069f55e2SEric Schrock !topo_fmri_strcmp(thp, zcp->zc_fru, fru)) {
772069f55e2SEric Schrock if (zcp->zc_fru != NULL) {
773069f55e2SEric Schrock fmd_hdl_strfree(hdl, zcp->zc_fru);
774069f55e2SEric Schrock fmd_buf_destroy(hdl, zcp->zc_case, CASE_FRU);
775069f55e2SEric Schrock }
776069f55e2SEric Schrock zcp->zc_fru = fmd_hdl_strdup(hdl, fru, FMD_SLEEP);
777069f55e2SEric Schrock zfs_case_serialize(hdl, zcp);
778069f55e2SEric Schrock }
779069f55e2SEric Schrock fmd_hdl_topo_rele(hdl, thp);
780069f55e2SEric Schrock }
781069f55e2SEric Schrock
782ea8dc4b6Seschrock if (isresource) {
783c5904d13Seschrock if (fmd_nvl_class_match(hdl, nvl,
78451ece835Seschrock ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
7853d7072f8Seschrock /*
7863d7072f8Seschrock * The 'resource.fs.zfs.autoreplace' event indicates
7873d7072f8Seschrock * that the pool was loaded with the 'autoreplace'
7883d7072f8Seschrock * property set. In this case, any pending device
7893d7072f8Seschrock * failures should be ignored, as the asynchronous
7903d7072f8Seschrock * autoreplace handling will take care of them.
7913d7072f8Seschrock */
7923d7072f8Seschrock fmd_case_close(hdl, zcp->zc_case);
793c5904d13Seschrock } else if (fmd_nvl_class_match(hdl, nvl,
79451ece835Seschrock ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
7953d7072f8Seschrock /*
7963d7072f8Seschrock * The 'resource.fs.zfs.removed' event indicates that
7973d7072f8Seschrock * device removal was detected, and the device was
7983d7072f8Seschrock * closed asynchronously. If this is the case, we
7993d7072f8Seschrock * assume that any recent I/O errors were due to the
8003d7072f8Seschrock * device removal, not any fault of the device itself.
8013d7072f8Seschrock * We reset the SERD engine, and cancel any pending
8023d7072f8Seschrock * timers.
8033d7072f8Seschrock */
80451ece835Seschrock if (zcp->zc_data.zc_has_remove_timer) {
80551ece835Seschrock fmd_timer_remove(hdl, zcp->zc_remove_timer);
80651ece835Seschrock zcp->zc_data.zc_has_remove_timer = 0;
8073d7072f8Seschrock zfs_case_serialize(hdl, zcp);
8083d7072f8Seschrock }
8093d7072f8Seschrock if (zcp->zc_data.zc_serd_io[0] != '\0')
8103d7072f8Seschrock fmd_serd_reset(hdl,
8113d7072f8Seschrock zcp->zc_data.zc_serd_io);
8123d7072f8Seschrock if (zcp->zc_data.zc_serd_checksum[0] != '\0')
8133d7072f8Seschrock fmd_serd_reset(hdl,
8143d7072f8Seschrock zcp->zc_data.zc_serd_checksum);
815*e830fb12SKody A Kantor if (zcp->zc_data.zc_serd_probe[0] != '\0')
816*e830fb12SKody A Kantor fmd_serd_reset(hdl, zcp->zc_data.zc_serd_probe);
8173d7072f8Seschrock }
81811027bc7STim Haley zfs_stats.resource_drops.fmds_value.ui64++;
819ea8dc4b6Seschrock return;
820ea8dc4b6Seschrock }
821ea8dc4b6Seschrock
822ea8dc4b6Seschrock /*
823ea8dc4b6Seschrock * Associate the ereport with this case.
824ea8dc4b6Seschrock */
825ea8dc4b6Seschrock fmd_case_add_ereport(hdl, zcp->zc_case, ep);
826ea8dc4b6Seschrock
827ea8dc4b6Seschrock /*
828ea8dc4b6Seschrock * Don't do anything else if this case is already solved.
829ea8dc4b6Seschrock */
830ea8dc4b6Seschrock if (fmd_case_solved(hdl, zcp->zc_case))
831ea8dc4b6Seschrock return;
832ea8dc4b6Seschrock
833*e830fb12SKody A Kantor zhdl = fmd_hdl_getspecific(hdl);
834*e830fb12SKody A Kantor
835*e830fb12SKody A Kantor /*
836*e830fb12SKody A Kantor * Find the corresponding pool.
837*e830fb12SKody A Kantor */
838*e830fb12SKody A Kantor cb.cb_guid = pool_guid;
839*e830fb12SKody A Kantor cb.cb_zhp = NULL;
840*e830fb12SKody A Kantor if (zhdl != NULL && zpool_iter(zhdl, find_pool, &cb) != 0) {
841*e830fb12SKody A Kantor zfs_stats.pool_drops.fmds_value.ui64++;
842*e830fb12SKody A Kantor return;
843*e830fb12SKody A Kantor }
844*e830fb12SKody A Kantor
845*e830fb12SKody A Kantor zhp = cb.cb_zhp; /* NULL if pool was not found. */
846*e830fb12SKody A Kantor if (zhp != NULL) {
847*e830fb12SKody A Kantor /*
848*e830fb12SKody A Kantor * The libzfs API takes a string representation of a base-10
849*e830fb12SKody A Kantor * guid here instead of a number, likely because the primary
850*e830fb12SKody A Kantor * libzfs consumers are the CLI tools.
851*e830fb12SKody A Kantor */
852*e830fb12SKody A Kantor vdg = fmd_hdl_zalloc(hdl, MAX_ULL_STR, FMD_SLEEP);
853*e830fb12SKody A Kantor (void) snprintf(vdg, MAX_ULL_STR, "%" PRIx64, vdev_guid);
854*e830fb12SKody A Kantor
855*e830fb12SKody A Kantor /*
856*e830fb12SKody A Kantor * According to libzfs the 'spare' bit is set when the spare is
857*e830fb12SKody A Kantor * unused, and unset when in use.
858*e830fb12SKody A Kantor *
859*e830fb12SKody A Kantor * We don't really care about the returned nvlist. We're only
860*e830fb12SKody A Kantor * interested in the boolean flags.
861*e830fb12SKody A Kantor */
862*e830fb12SKody A Kantor if ((vd_nvl = zpool_find_vdev(zhp, vdg,
863*e830fb12SKody A Kantor &is_inactive_spare, &islog, &iscache)) != NULL) {
864*e830fb12SKody A Kantor nvlist_free(vd_nvl);
865*e830fb12SKody A Kantor }
866*e830fb12SKody A Kantor fmd_hdl_free(hdl, vdg, MAX_ULL_STR);
867*e830fb12SKody A Kantor }
868*e830fb12SKody A Kantor
869ea8dc4b6Seschrock /*
870ea8dc4b6Seschrock * Determine if we should solve the case and generate a fault. We solve
871ea8dc4b6Seschrock * a case if:
872ea8dc4b6Seschrock *
873*e830fb12SKody A Kantor * a. A pool failed to open (ereport.fs.zfs.pool)
874*e830fb12SKody A Kantor * b. A device failed to open (ereport.fs.zfs.pool) while a pool
875ea8dc4b6Seschrock * was up and running.
876ea8dc4b6Seschrock *
877ea8dc4b6Seschrock * We may see a series of ereports associated with a pool open, all
878ea8dc4b6Seschrock * chained together by the same ENA. If the pool open succeeds, then
879ea8dc4b6Seschrock * we'll see no further ereports. To detect when a pool open has
880ea8dc4b6Seschrock * succeeded, we associate a timer with the event. When it expires, we
881ea8dc4b6Seschrock * close the case.
882ea8dc4b6Seschrock */
88351ece835Seschrock if (fmd_nvl_class_match(hdl, nvl,
88451ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
885ea8dc4b6Seschrock /*
886c5904d13Seschrock * Pool level fault. Before solving the case, go through and
887c5904d13Seschrock * close any open device cases that may be pending.
888ea8dc4b6Seschrock */
889c5904d13Seschrock for (dcp = uu_list_first(zfs_cases); dcp != NULL;
890c5904d13Seschrock dcp = uu_list_next(zfs_cases, dcp)) {
891c5904d13Seschrock if (dcp->zc_data.zc_pool_guid ==
892c5904d13Seschrock zcp->zc_data.zc_pool_guid &&
893c5904d13Seschrock dcp->zc_data.zc_vdev_guid != 0)
894c5904d13Seschrock fmd_case_close(hdl, dcp->zc_case);
895c5904d13Seschrock }
896c5904d13Seschrock
8973d7072f8Seschrock zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE);
898b87f3af3Sperrin } else if (fmd_nvl_class_match(hdl, nvl,
899b87f3af3Sperrin ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
900b87f3af3Sperrin /*
901b87f3af3Sperrin * Pool level fault for reading the intent logs.
902b87f3af3Sperrin */
903b87f3af3Sperrin zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE);
904c5904d13Seschrock } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
905ea8dc4b6Seschrock /*
906b693757aSEric Schrock * Device fault.
907ea8dc4b6Seschrock */
9083d7072f8Seschrock zfs_case_solve(hdl, zcp, "fault.fs.zfs.device", B_TRUE);
90951ece835Seschrock } else if (fmd_nvl_class_match(hdl, nvl,
91051ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
91151ece835Seschrock fmd_nvl_class_match(hdl, nvl,
91251ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
91351ece835Seschrock fmd_nvl_class_match(hdl, nvl,
91451ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
91551ece835Seschrock fmd_nvl_class_match(hdl, nvl,
91651ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
91732b87932Sek char *failmode = NULL;
9180430f8daSeschrock boolean_t checkremove = B_FALSE;
91932b87932Sek
9203d7072f8Seschrock /*
921*e830fb12SKody A Kantor * If this is a checksum, I/O, or probe error, then toss it into
922*e830fb12SKody A Kantor * the appropriate SERD engine and check to see if it has fired.
9233d7072f8Seschrock * Ideally, we want to do something more sophisticated,
9243d7072f8Seschrock * (persistent errors for a single data block, etc). For now,
9253d7072f8Seschrock * a single SERD engine is sufficient.
9263d7072f8Seschrock */
92751ece835Seschrock if (fmd_nvl_class_match(hdl, nvl,
92851ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
9293d7072f8Seschrock if (zcp->zc_data.zc_serd_io[0] == '\0') {
9303d7072f8Seschrock zfs_serd_name(zcp->zc_data.zc_serd_io,
9313d7072f8Seschrock pool_guid, vdev_guid, "io");
9323d7072f8Seschrock fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
9333d7072f8Seschrock fmd_prop_get_int32(hdl, "io_N"),
9343d7072f8Seschrock fmd_prop_get_int64(hdl, "io_T"));
9353d7072f8Seschrock zfs_case_serialize(hdl, zcp);
9363d7072f8Seschrock }
93751ece835Seschrock if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
93851ece835Seschrock checkremove = B_TRUE;
9393d7072f8Seschrock } else if (fmd_nvl_class_match(hdl, nvl,
94051ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
9413d7072f8Seschrock if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
9423d7072f8Seschrock zfs_serd_name(zcp->zc_data.zc_serd_checksum,
9433d7072f8Seschrock pool_guid, vdev_guid, "checksum");
9443d7072f8Seschrock fmd_serd_create(hdl,
9453d7072f8Seschrock zcp->zc_data.zc_serd_checksum,
9463d7072f8Seschrock fmd_prop_get_int32(hdl, "checksum_N"),
9473d7072f8Seschrock fmd_prop_get_int64(hdl, "checksum_T"));
9483d7072f8Seschrock zfs_case_serialize(hdl, zcp);
9493d7072f8Seschrock }
95051ece835Seschrock if (fmd_serd_record(hdl,
95151ece835Seschrock zcp->zc_data.zc_serd_checksum, ep)) {
95251ece835Seschrock zfs_case_solve(hdl, zcp,
95351ece835Seschrock "fault.fs.zfs.vdev.checksum", B_FALSE);
95451ece835Seschrock }
95532b87932Sek } else if (fmd_nvl_class_match(hdl, nvl,
95651ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
95751ece835Seschrock (nvlist_lookup_string(nvl,
95832b87932Sek FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
95932b87932Sek failmode != NULL) {
96032b87932Sek if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
96132b87932Sek strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
96232b87932Sek zfs_case_solve(hdl, zcp,
96332b87932Sek "fault.fs.zfs.io_failure_continue",
96432b87932Sek B_FALSE);
96532b87932Sek } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
96632b87932Sek strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
96732b87932Sek zfs_case_solve(hdl, zcp,
96832b87932Sek "fault.fs.zfs.io_failure_wait", B_FALSE);
96932b87932Sek }
97051ece835Seschrock } else if (fmd_nvl_class_match(hdl, nvl,
97151ece835Seschrock ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
972*e830fb12SKody A Kantor if (zcp->zc_data.zc_serd_probe[0] == '\0') {
973*e830fb12SKody A Kantor zfs_serd_name(zcp->zc_data.zc_serd_probe,
974*e830fb12SKody A Kantor pool_guid, vdev_guid, "probe");
975*e830fb12SKody A Kantor fmd_serd_create(hdl, zcp->zc_data.zc_serd_probe,
976*e830fb12SKody A Kantor fmd_prop_get_int32(hdl, "probe_N"),
977*e830fb12SKody A Kantor fmd_prop_get_int64(hdl, "probe_T"));
978*e830fb12SKody A Kantor zfs_case_serialize(hdl, zcp);
979*e830fb12SKody A Kantor }
980*e830fb12SKody A Kantor
981*e830fb12SKody A Kantor /*
982*e830fb12SKody A Kantor * We only want to wait for SERD triggers for spare
983*e830fb12SKody A Kantor * vdevs. Normal pool vdevs should be diagnosed
984*e830fb12SKody A Kantor * immediately if a probe failure is received.
985*e830fb12SKody A Kantor */
986*e830fb12SKody A Kantor if (!is_inactive_spare || fmd_serd_record(hdl,
987*e830fb12SKody A Kantor zcp->zc_data.zc_serd_probe, ep)) {
988*e830fb12SKody A Kantor checkremove = B_TRUE;
989*e830fb12SKody A Kantor }
990ea8dc4b6Seschrock }
991ea8dc4b6Seschrock
992ea8dc4b6Seschrock /*
9933d7072f8Seschrock * Because I/O errors may be due to device removal, we postpone
9943d7072f8Seschrock * any diagnosis until we're sure that we aren't about to
9953d7072f8Seschrock * receive a 'resource.fs.zfs.removed' event.
996ea8dc4b6Seschrock */
99751ece835Seschrock if (checkremove) {
99851ece835Seschrock if (zcp->zc_data.zc_has_remove_timer)
99951ece835Seschrock fmd_timer_remove(hdl, zcp->zc_remove_timer);
100051ece835Seschrock zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
100151ece835Seschrock zfs_remove_timeout);
100251ece835Seschrock if (!zcp->zc_data.zc_has_remove_timer) {
100351ece835Seschrock zcp->zc_data.zc_has_remove_timer = 1;
10043d7072f8Seschrock zfs_case_serialize(hdl, zcp);
10053d7072f8Seschrock }
1006ea8dc4b6Seschrock }
1007ea8dc4b6Seschrock }
1008ea8dc4b6Seschrock }
1009ea8dc4b6Seschrock
1010ea8dc4b6Seschrock /*
101151ece835Seschrock * The timeout is fired when we diagnosed an I/O error, and it was not due to
101251ece835Seschrock * device removal (which would cause the timeout to be cancelled).
1013ea8dc4b6Seschrock */
1014ea8dc4b6Seschrock /* ARGSUSED */
1015ea8dc4b6Seschrock static void
zfs_fm_timeout(fmd_hdl_t * hdl,id_t id,void * data)10163d7072f8Seschrock zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
1017ea8dc4b6Seschrock {
1018ea8dc4b6Seschrock zfs_case_t *zcp = data;
1019ea8dc4b6Seschrock
102051ece835Seschrock if (id == zcp->zc_remove_timer)
102151ece835Seschrock zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE);
1022ea8dc4b6Seschrock }
1023ea8dc4b6Seschrock
1024ea8dc4b6Seschrock static void
zfs_fm_close(fmd_hdl_t * hdl,fmd_case_t * cs)10253d7072f8Seschrock zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
1026ea8dc4b6Seschrock {
1027ea8dc4b6Seschrock zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
1028ea8dc4b6Seschrock
10293d7072f8Seschrock if (zcp->zc_data.zc_serd_checksum[0] != '\0')
10303d7072f8Seschrock fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
10313d7072f8Seschrock if (zcp->zc_data.zc_serd_io[0] != '\0')
10323d7072f8Seschrock fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
1033*e830fb12SKody A Kantor if (zcp->zc_data.zc_serd_probe[0] != '\0')
1034*e830fb12SKody A Kantor fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_probe);
103551ece835Seschrock if (zcp->zc_data.zc_has_remove_timer)
103651ece835Seschrock fmd_timer_remove(hdl, zcp->zc_remove_timer);
1037ea8dc4b6Seschrock uu_list_remove(zfs_cases, zcp);
1038ea8dc4b6Seschrock fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1039ea8dc4b6Seschrock }
1040ea8dc4b6Seschrock
10413d7072f8Seschrock /*
10423d7072f8Seschrock * We use the fmd gc entry point to look for old cases that no longer apply.
10433d7072f8Seschrock * This allows us to keep our set of case data small in a long running system.
10443d7072f8Seschrock */
10453d7072f8Seschrock static void
zfs_fm_gc(fmd_hdl_t * hdl)10463d7072f8Seschrock zfs_fm_gc(fmd_hdl_t *hdl)
10473d7072f8Seschrock {
10483d7072f8Seschrock zfs_purge_cases(hdl);
10493d7072f8Seschrock }
10503d7072f8Seschrock
1051ea8dc4b6Seschrock static const fmd_hdl_ops_t fmd_ops = {
10523d7072f8Seschrock zfs_fm_recv, /* fmdo_recv */
10533d7072f8Seschrock zfs_fm_timeout, /* fmdo_timeout */
10543d7072f8Seschrock zfs_fm_close, /* fmdo_close */
1055ea8dc4b6Seschrock NULL, /* fmdo_stats */
10563d7072f8Seschrock zfs_fm_gc, /* fmdo_gc */
1057ea8dc4b6Seschrock };
1058ea8dc4b6Seschrock
1059ea8dc4b6Seschrock static const fmd_prop_t fmd_props[] = {
10603d7072f8Seschrock { "checksum_N", FMD_TYPE_UINT32, "10" },
10613d7072f8Seschrock { "checksum_T", FMD_TYPE_TIME, "10min" },
10623d7072f8Seschrock { "io_N", FMD_TYPE_UINT32, "10" },
10633d7072f8Seschrock { "io_T", FMD_TYPE_TIME, "10min" },
1064*e830fb12SKody A Kantor { "probe_N", FMD_TYPE_UINT32, "5" },
1065*e830fb12SKody A Kantor { "probe_T", FMD_TYPE_TIME, "24hour" },
10661d713200SEric Schrock { "remove_timeout", FMD_TYPE_TIME, "15sec" },
1067ea8dc4b6Seschrock { NULL, 0, NULL }
1068ea8dc4b6Seschrock };
1069ea8dc4b6Seschrock
1070ea8dc4b6Seschrock static const fmd_hdl_info_t fmd_info = {
1071ea8dc4b6Seschrock "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
1072ea8dc4b6Seschrock };
1073ea8dc4b6Seschrock
1074ea8dc4b6Seschrock void
_fmd_init(fmd_hdl_t * hdl)1075ea8dc4b6Seschrock _fmd_init(fmd_hdl_t *hdl)
1076ea8dc4b6Seschrock {
1077ea8dc4b6Seschrock fmd_case_t *cp;
10783d7072f8Seschrock libzfs_handle_t *zhdl;
10793d7072f8Seschrock
10803d7072f8Seschrock if ((zhdl = libzfs_init()) == NULL)
10813d7072f8Seschrock return;
1082ea8dc4b6Seschrock
1083ea8dc4b6Seschrock if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
1084ea8dc4b6Seschrock sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
10853d7072f8Seschrock NULL, 0)) == NULL) {
10863d7072f8Seschrock libzfs_fini(zhdl);
1087ea8dc4b6Seschrock return;
10883d7072f8Seschrock }
1089ea8dc4b6Seschrock
1090ea8dc4b6Seschrock if ((zfs_cases = uu_list_create(zfs_case_pool, NULL, 0)) == NULL) {
1091ea8dc4b6Seschrock uu_list_pool_destroy(zfs_case_pool);
10923d7072f8Seschrock libzfs_fini(zhdl);
1093ea8dc4b6Seschrock return;
1094ea8dc4b6Seschrock }
1095ea8dc4b6Seschrock
1096ea8dc4b6Seschrock if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
1097ea8dc4b6Seschrock uu_list_destroy(zfs_cases);
1098ea8dc4b6Seschrock uu_list_pool_destroy(zfs_case_pool);
10993d7072f8Seschrock libzfs_fini(zhdl);
1100ea8dc4b6Seschrock return;
1101ea8dc4b6Seschrock }
1102ea8dc4b6Seschrock
11033d7072f8Seschrock fmd_hdl_setspecific(hdl, zhdl);
11043d7072f8Seschrock
110511027bc7STim Haley (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
110611027bc7STim Haley sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
110711027bc7STim Haley
1108ea8dc4b6Seschrock /*
1109ea8dc4b6Seschrock * Iterate over all active cases and unserialize the associated buffers,
1110ea8dc4b6Seschrock * adding them to our list of open cases.
1111ea8dc4b6Seschrock */
1112ea8dc4b6Seschrock for (cp = fmd_case_next(hdl, NULL);
1113ea8dc4b6Seschrock cp != NULL; cp = fmd_case_next(hdl, cp))
1114ea8dc4b6Seschrock (void) zfs_case_unserialize(hdl, cp);
1115ea8dc4b6Seschrock
11163d7072f8Seschrock /*
11173d7072f8Seschrock * Clear out any old cases that are no longer valid.
11183d7072f8Seschrock */
11193d7072f8Seschrock zfs_purge_cases(hdl);
11203d7072f8Seschrock
112151ece835Seschrock zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout");
1122ea8dc4b6Seschrock }
1123ea8dc4b6Seschrock
1124ea8dc4b6Seschrock void
_fmd_fini(fmd_hdl_t * hdl)1125ea8dc4b6Seschrock _fmd_fini(fmd_hdl_t *hdl)
1126ea8dc4b6Seschrock {
1127ea8dc4b6Seschrock zfs_case_t *zcp;
1128ea8dc4b6Seschrock uu_list_walk_t *walk;
11293d7072f8Seschrock libzfs_handle_t *zhdl;
1130ea8dc4b6Seschrock
1131ea8dc4b6Seschrock /*
1132ea8dc4b6Seschrock * Remove all active cases.
1133ea8dc4b6Seschrock */
1134ea8dc4b6Seschrock walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
1135ea8dc4b6Seschrock while ((zcp = uu_list_walk_next(walk)) != NULL) {
1136ea8dc4b6Seschrock uu_list_remove(zfs_cases, zcp);
1137ea8dc4b6Seschrock fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
1138ea8dc4b6Seschrock }
1139ea8dc4b6Seschrock uu_list_walk_end(walk);
1140ea8dc4b6Seschrock
1141ea8dc4b6Seschrock uu_list_destroy(zfs_cases);
1142ea8dc4b6Seschrock uu_list_pool_destroy(zfs_case_pool);
11433d7072f8Seschrock
11443d7072f8Seschrock zhdl = fmd_hdl_getspecific(hdl);
11453d7072f8Seschrock libzfs_fini(zhdl);
1146ea8dc4b6Seschrock }
1147