xref: /illumos-gate/usr/src/uts/common/fs/zfs/dmu_traverse.c (revision eb633035c80613ec93d62f90482837adaaf21a0a)
1fa9e4066Sahrens /*
2fa9e4066Sahrens  * CDDL HEADER START
3fa9e4066Sahrens  *
4fa9e4066Sahrens  * The contents of this file are subject to the terms of the
5ea8dc4b6Seschrock  * Common Development and Distribution License (the "License").
6ea8dc4b6Seschrock  * You may not use this file except in compliance with the License.
7fa9e4066Sahrens  *
8fa9e4066Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e4066Sahrens  * or http://www.opensolaris.org/os/licensing.
10fa9e4066Sahrens  * See the License for the specific language governing permissions
11fa9e4066Sahrens  * and limitations under the License.
12fa9e4066Sahrens  *
13fa9e4066Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14fa9e4066Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e4066Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16fa9e4066Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e4066Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e4066Sahrens  *
19fa9e4066Sahrens  * CDDL HEADER END
20fa9e4066Sahrens  */
21fa9e4066Sahrens /*
223f9d6ad7SLin Ling  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
2386714001SSerapheim Dimitropoulos  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
24fa9e4066Sahrens  */
25fa9e4066Sahrens 
26fa9e4066Sahrens #include <sys/zfs_context.h>
27fa9e4066Sahrens #include <sys/dmu_objset.h>
28fa9e4066Sahrens #include <sys/dmu_traverse.h>
29fa9e4066Sahrens #include <sys/dsl_dataset.h>
30fa9e4066Sahrens #include <sys/dsl_dir.h>
31fa9e4066Sahrens #include <sys/dsl_pool.h>
32fa9e4066Sahrens #include <sys/dnode.h>
33fa9e4066Sahrens #include <sys/spa.h>
3486714001SSerapheim Dimitropoulos #include <sys/spa_impl.h>
35fa9e4066Sahrens #include <sys/zio.h>
36fa9e4066Sahrens #include <sys/dmu_impl.h>
370a586ceaSMark Shellenbaum #include <sys/sa.h>
380a586ceaSMark Shellenbaum #include <sys/sa_impl.h>
3988b7b0f2SMatthew Ahrens #include <sys/callb.h>
4043466aaeSMax Grossman #include <sys/zfeature.h>
4188b7b0f2SMatthew Ahrens 
4234d7ce05SGeorge Wilson int32_t zfs_pd_bytes_max = 50 * 1024 * 1024;	/* 50MB */
43f76886deSPaul Dagnelie boolean_t send_holes_without_birth_time = B_TRUE;
4444f92b7dSChris Kirby 
456e0cbcaaSMatthew Ahrens typedef struct prefetch_data {
4688b7b0f2SMatthew Ahrens 	kmutex_t pd_mtx;
4788b7b0f2SMatthew Ahrens 	kcondvar_t pd_cv;
4834d7ce05SGeorge Wilson 	int32_t pd_bytes_fetched;
4988b7b0f2SMatthew Ahrens 	int pd_flags;
5088b7b0f2SMatthew Ahrens 	boolean_t pd_cancel;
5188b7b0f2SMatthew Ahrens 	boolean_t pd_exited;
529c3fd121SMatthew Ahrens 	zbookmark_phys_t pd_resume;
536e0cbcaaSMatthew Ahrens } prefetch_data_t;
5488b7b0f2SMatthew Ahrens 
556e0cbcaaSMatthew Ahrens typedef struct traverse_data {
5688b7b0f2SMatthew Ahrens 	spa_t *td_spa;
5788b7b0f2SMatthew Ahrens 	uint64_t td_objset;
5888b7b0f2SMatthew Ahrens 	blkptr_t *td_rootbp;
5988b7b0f2SMatthew Ahrens 	uint64_t td_min_txg;
607802d7bfSMatthew Ahrens 	zbookmark_phys_t *td_resume;
6188b7b0f2SMatthew Ahrens 	int td_flags;
626e0cbcaaSMatthew Ahrens 	prefetch_data_t *td_pfd;
637fd05ac4SMatthew Ahrens 	boolean_t td_paused;
64f7950bf1SMatthew Ahrens 	uint64_t td_hole_birth_enabled_txg;
6588b7b0f2SMatthew Ahrens 	blkptr_cb_t *td_func;
6688b7b0f2SMatthew Ahrens 	void *td_arg;
67286ef713SPaul Dagnelie 	boolean_t td_realloc_possible;
686e0cbcaaSMatthew Ahrens } traverse_data_t;
69fa9e4066Sahrens 
706e0cbcaaSMatthew Ahrens static int traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
711b912ec7SGeorge Wilson     uint64_t objset, uint64_t object);
72b4709335SMatthew Ahrens static void prefetch_dnode_metadata(traverse_data_t *td, const dnode_phys_t *,
731b912ec7SGeorge Wilson     uint64_t objset, uint64_t object);
7414843421SMatthew Ahrens 
75b24ab676SJeff Bonwick static int
765dabedeeSbonwick traverse_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg)
77ea8dc4b6Seschrock {
786e0cbcaaSMatthew Ahrens 	traverse_data_t *td = arg;
797802d7bfSMatthew Ahrens 	zbookmark_phys_t zb;
80ea8dc4b6Seschrock 
8143466aaeSMax Grossman 	if (BP_IS_HOLE(bp))
82b24ab676SJeff Bonwick 		return (0);
835dabedeeSbonwick 
8486714001SSerapheim Dimitropoulos 	if (claim_txg == 0 && bp->blk_birth >= spa_min_claim_txg(td->td_spa))
8586714001SSerapheim Dimitropoulos 		return (-1);
8688b7b0f2SMatthew Ahrens 
87b24ab676SJeff Bonwick 	SET_BOOKMARK(&zb, td->td_objset, ZB_ZIL_OBJECT, ZB_ZIL_LEVEL,
88b24ab676SJeff Bonwick 	    bp->blk_cksum.zc_word[ZIL_ZC_SEQ]);
89b24ab676SJeff Bonwick 
901b912ec7SGeorge Wilson 	(void) td->td_func(td->td_spa, zilog, bp, &zb, NULL, td->td_arg);
91b24ab676SJeff Bonwick 
92b24ab676SJeff Bonwick 	return (0);
93ea8dc4b6Seschrock }
94ea8dc4b6Seschrock 
95b24ab676SJeff Bonwick static int
965dabedeeSbonwick traverse_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg)
97ea8dc4b6Seschrock {
986e0cbcaaSMatthew Ahrens 	traverse_data_t *td = arg;
99ea8dc4b6Seschrock 
100ea8dc4b6Seschrock 	if (lrc->lrc_txtype == TX_WRITE) {
101ea8dc4b6Seschrock 		lr_write_t *lr = (lr_write_t *)lrc;
102ea8dc4b6Seschrock 		blkptr_t *bp = &lr->lr_blkptr;
1037802d7bfSMatthew Ahrens 		zbookmark_phys_t zb;
104ea8dc4b6Seschrock 
10543466aaeSMax Grossman 		if (BP_IS_HOLE(bp))
106b24ab676SJeff Bonwick 			return (0);
1075dabedeeSbonwick 
10888b7b0f2SMatthew Ahrens 		if (claim_txg == 0 || bp->blk_birth < claim_txg)
109b24ab676SJeff Bonwick 			return (0);
110b24ab676SJeff Bonwick 
1116e0cbcaaSMatthew Ahrens 		SET_BOOKMARK(&zb, td->td_objset, lr->lr_foid,
1126e0cbcaaSMatthew Ahrens 		    ZB_ZIL_LEVEL, lr->lr_offset / BP_GET_LSIZE(bp));
11388b7b0f2SMatthew Ahrens 
1141b912ec7SGeorge Wilson 		(void) td->td_func(td->td_spa, zilog, bp, &zb, NULL,
115b24ab676SJeff Bonwick 		    td->td_arg);
116ea8dc4b6Seschrock 	}
117b24ab676SJeff Bonwick 	return (0);
118ea8dc4b6Seschrock }
119ea8dc4b6Seschrock 
120ea8dc4b6Seschrock static void
1216e0cbcaaSMatthew Ahrens traverse_zil(traverse_data_t *td, zil_header_t *zh)
122ea8dc4b6Seschrock {
1235dabedeeSbonwick 	uint64_t claim_txg = zh->zh_claim_txg;
124ea8dc4b6Seschrock 
1255dabedeeSbonwick 	/*
1265dabedeeSbonwick 	 * We only want to visit blocks that have been claimed but not yet
12786714001SSerapheim Dimitropoulos 	 * replayed; plus blocks that are already stable in read-only mode.
1285dabedeeSbonwick 	 */
1298ad4d6ddSJeff Bonwick 	if (claim_txg == 0 && spa_writeable(td->td_spa))
1305dabedeeSbonwick 		return;
1315dabedeeSbonwick 
13286714001SSerapheim Dimitropoulos 	zilog_t *zilog = zil_alloc(spa_get_dsl(td->td_spa)->dp_meta_objset, zh);
13388b7b0f2SMatthew Ahrens 	(void) zil_parse(zilog, traverse_zil_block, traverse_zil_record, td,
134*eb633035STom Caputi 	    claim_txg, !(td->td_flags & TRAVERSE_NO_DECRYPT));
135ea8dc4b6Seschrock 	zil_free(zilog);
136ea8dc4b6Seschrock }
137ea8dc4b6Seschrock 
138ad135b5dSChristopher Siden typedef enum resume_skip {
139ad135b5dSChristopher Siden 	RESUME_SKIP_ALL,
140ad135b5dSChristopher Siden 	RESUME_SKIP_NONE,
141ad135b5dSChristopher Siden 	RESUME_SKIP_CHILDREN
142ad135b5dSChristopher Siden } resume_skip_t;
143ad135b5dSChristopher Siden 
144ad135b5dSChristopher Siden /*
145ad135b5dSChristopher Siden  * Returns RESUME_SKIP_ALL if td indicates that we are resuming a traversal and
146ad135b5dSChristopher Siden  * the block indicated by zb does not need to be visited at all. Returns
147ad135b5dSChristopher Siden  * RESUME_SKIP_CHILDREN if we are resuming a post traversal and we reach the
148ad135b5dSChristopher Siden  * resume point. This indicates that this block should be visited but not its
149ad135b5dSChristopher Siden  * children (since they must have been visited in a previous traversal).
150ad135b5dSChristopher Siden  * Otherwise returns RESUME_SKIP_NONE.
151ad135b5dSChristopher Siden  */
152ad135b5dSChristopher Siden static resume_skip_t
153ad135b5dSChristopher Siden resume_skip_check(traverse_data_t *td, const dnode_phys_t *dnp,
1547802d7bfSMatthew Ahrens     const zbookmark_phys_t *zb)
155ad135b5dSChristopher Siden {
156ad135b5dSChristopher Siden 	if (td->td_resume != NULL && !ZB_IS_ZERO(td->td_resume)) {
157ad135b5dSChristopher Siden 		/*
158ad135b5dSChristopher Siden 		 * If we already visited this bp & everything below,
159ad135b5dSChristopher Siden 		 * don't bother doing it again.
160ad135b5dSChristopher Siden 		 */
161a2cdcdd2SPaul Dagnelie 		if (zbookmark_subtree_completed(dnp, zb, td->td_resume))
162ad135b5dSChristopher Siden 			return (RESUME_SKIP_ALL);
163ad135b5dSChristopher Siden 
164ad135b5dSChristopher Siden 		/*
165ad135b5dSChristopher Siden 		 * If we found the block we're trying to resume from, zero
166ad135b5dSChristopher Siden 		 * the bookmark out to indicate that we have resumed.
167ad135b5dSChristopher Siden 		 */
168ad135b5dSChristopher Siden 		if (bcmp(zb, td->td_resume, sizeof (*zb)) == 0) {
169ad135b5dSChristopher Siden 			bzero(td->td_resume, sizeof (*zb));
170ad135b5dSChristopher Siden 			if (td->td_flags & TRAVERSE_POST)
171ad135b5dSChristopher Siden 				return (RESUME_SKIP_CHILDREN);
172ad135b5dSChristopher Siden 		}
173ad135b5dSChristopher Siden 	}
174ad135b5dSChristopher Siden 	return (RESUME_SKIP_NONE);
175ad135b5dSChristopher Siden }
176ad135b5dSChristopher Siden 
177b4709335SMatthew Ahrens static void
178b4709335SMatthew Ahrens traverse_prefetch_metadata(traverse_data_t *td,
1797802d7bfSMatthew Ahrens     const blkptr_t *bp, const zbookmark_phys_t *zb)
180b4709335SMatthew Ahrens {
1817adb730bSGeorge Wilson 	arc_flags_t flags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
182*eb633035STom Caputi 	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE;
183b4709335SMatthew Ahrens 
184b4709335SMatthew Ahrens 	if (!(td->td_flags & TRAVERSE_PREFETCH_METADATA))
185b4709335SMatthew Ahrens 		return;
186b4709335SMatthew Ahrens 	/*
187b4709335SMatthew Ahrens 	 * If we are in the process of resuming, don't prefetch, because
188b4709335SMatthew Ahrens 	 * some children will not be needed (and in fact may have already
189b4709335SMatthew Ahrens 	 * been freed).
190b4709335SMatthew Ahrens 	 */
191b4709335SMatthew Ahrens 	if (td->td_resume != NULL && !ZB_IS_ZERO(td->td_resume))
192b4709335SMatthew Ahrens 		return;
193b4709335SMatthew Ahrens 	if (BP_IS_HOLE(bp) || bp->blk_birth <= td->td_min_txg)
194b4709335SMatthew Ahrens 		return;
195b4709335SMatthew Ahrens 	if (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE)
196b4709335SMatthew Ahrens 		return;
197b4709335SMatthew Ahrens 
198*eb633035STom Caputi 	if ((td->td_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))
199*eb633035STom Caputi 		zio_flags |= ZIO_FLAG_RAW;
200*eb633035STom Caputi 
2011b912ec7SGeorge Wilson 	(void) arc_read(NULL, td->td_spa, bp, NULL, NULL,
202*eb633035STom Caputi 	    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
203b4709335SMatthew Ahrens }
204b4709335SMatthew Ahrens 
20506315b79SMatthew Ahrens static boolean_t
20606315b79SMatthew Ahrens prefetch_needed(prefetch_data_t *pfd, const blkptr_t *bp)
20706315b79SMatthew Ahrens {
20806315b79SMatthew Ahrens 	ASSERT(pfd->pd_flags & TRAVERSE_PREFETCH_DATA);
20906315b79SMatthew Ahrens 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp) ||
21006315b79SMatthew Ahrens 	    BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG)
21106315b79SMatthew Ahrens 		return (B_FALSE);
21206315b79SMatthew Ahrens 	return (B_TRUE);
21306315b79SMatthew Ahrens }
21406315b79SMatthew Ahrens 
215fa9e4066Sahrens static int
2166e0cbcaaSMatthew Ahrens traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
2177802d7bfSMatthew Ahrens     const blkptr_t *bp, const zbookmark_phys_t *zb)
218fa9e4066Sahrens {
2197802d7bfSMatthew Ahrens 	zbookmark_phys_t czb;
2207fd05ac4SMatthew Ahrens 	int err = 0;
22188b7b0f2SMatthew Ahrens 	arc_buf_t *buf = NULL;
2226e0cbcaaSMatthew Ahrens 	prefetch_data_t *pd = td->td_pfd;
223cd088ea4SVictor Latushkin 	boolean_t hard = td->td_flags & TRAVERSE_HARD;
224ad135b5dSChristopher Siden 
225ad135b5dSChristopher Siden 	switch (resume_skip_check(td, dnp, zb)) {
226ad135b5dSChristopher Siden 	case RESUME_SKIP_ALL:
227ad135b5dSChristopher Siden 		return (0);
228ad135b5dSChristopher Siden 	case RESUME_SKIP_CHILDREN:
229ad135b5dSChristopher Siden 		goto post;
230ad135b5dSChristopher Siden 	case RESUME_SKIP_NONE:
231ad135b5dSChristopher Siden 		break;
232ad135b5dSChristopher Siden 	default:
233ad135b5dSChristopher Siden 		ASSERT(0);
234ad135b5dSChristopher Siden 	}
235fa9e4066Sahrens 
23643466aaeSMax Grossman 	if (bp->blk_birth == 0) {
237f7950bf1SMatthew Ahrens 		/*
238286ef713SPaul Dagnelie 		 * Since this block has a birth time of 0 it must be one of
239286ef713SPaul Dagnelie 		 * two things: a hole created before the
240286ef713SPaul Dagnelie 		 * SPA_FEATURE_HOLE_BIRTH feature was enabled, or a hole
241286ef713SPaul Dagnelie 		 * which has always been a hole in an object.
242286ef713SPaul Dagnelie 		 *
243286ef713SPaul Dagnelie 		 * If a file is written sparsely, then the unwritten parts of
244286ef713SPaul Dagnelie 		 * the file were "always holes" -- that is, they have been
245286ef713SPaul Dagnelie 		 * holes since this object was allocated.  However, we (and
246286ef713SPaul Dagnelie 		 * our callers) can not necessarily tell when an object was
247286ef713SPaul Dagnelie 		 * allocated.  Therefore, if it's possible that this object
248286ef713SPaul Dagnelie 		 * was freed and then its object number reused, we need to
249286ef713SPaul Dagnelie 		 * visit all the holes with birth==0.
250286ef713SPaul Dagnelie 		 *
251286ef713SPaul Dagnelie 		 * If it isn't possible that the object number was reused,
252286ef713SPaul Dagnelie 		 * then if SPA_FEATURE_HOLE_BIRTH was enabled before we wrote
253286ef713SPaul Dagnelie 		 * all the blocks we will visit as part of this traversal,
254286ef713SPaul Dagnelie 		 * then this hole must have always existed, so we can skip
255286ef713SPaul Dagnelie 		 * it.  We visit blocks born after (exclusive) td_min_txg.
256286ef713SPaul Dagnelie 		 *
257286ef713SPaul Dagnelie 		 * Note that the meta-dnode cannot be reallocated.
258f7950bf1SMatthew Ahrens 		 */
259f76886deSPaul Dagnelie 		if (!send_holes_without_birth_time &&
260f76886deSPaul Dagnelie 		    (!td->td_realloc_possible ||
261286ef713SPaul Dagnelie 		    zb->zb_object == DMU_META_DNODE_OBJECT) &&
262286ef713SPaul Dagnelie 		    td->td_hole_birth_enabled_txg <= td->td_min_txg)
263f7950bf1SMatthew Ahrens 			return (0);
26443466aaeSMax Grossman 	} else if (bp->blk_birth <= td->td_min_txg) {
26543466aaeSMax Grossman 		return (0);
26643466aaeSMax Grossman 	}
26743466aaeSMax Grossman 
26806315b79SMatthew Ahrens 	if (pd != NULL && !pd->pd_exited && prefetch_needed(pd, bp)) {
26934d7ce05SGeorge Wilson 		uint64_t size = BP_GET_LSIZE(bp);
27088b7b0f2SMatthew Ahrens 		mutex_enter(&pd->pd_mtx);
27134d7ce05SGeorge Wilson 		ASSERT(pd->pd_bytes_fetched >= 0);
27234d7ce05SGeorge Wilson 		while (pd->pd_bytes_fetched < size && !pd->pd_exited)
27388b7b0f2SMatthew Ahrens 			cv_wait(&pd->pd_cv, &pd->pd_mtx);
27434d7ce05SGeorge Wilson 		pd->pd_bytes_fetched -= size;
27588b7b0f2SMatthew Ahrens 		cv_broadcast(&pd->pd_cv);
27688b7b0f2SMatthew Ahrens 		mutex_exit(&pd->pd_mtx);
277fa9e4066Sahrens 	}
278fa9e4066Sahrens 
27906315b79SMatthew Ahrens 	if (BP_IS_HOLE(bp)) {
28006315b79SMatthew Ahrens 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
28106315b79SMatthew Ahrens 		if (err != 0)
28206315b79SMatthew Ahrens 			goto post;
28306315b79SMatthew Ahrens 		return (0);
28406315b79SMatthew Ahrens 	}
28506315b79SMatthew Ahrens 
28688b7b0f2SMatthew Ahrens 	if (td->td_flags & TRAVERSE_PRE) {
2871b912ec7SGeorge Wilson 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp,
2883f9d6ad7SLin Ling 		    td->td_arg);
28999d5e173STim Haley 		if (err == TRAVERSE_VISIT_NO_CHILDREN)
29099d5e173STim Haley 			return (0);
291ad135b5dSChristopher Siden 		if (err != 0)
292ad135b5dSChristopher Siden 			goto post;
293fa9e4066Sahrens 	}
294fa9e4066Sahrens 
29588b7b0f2SMatthew Ahrens 	if (BP_GET_LEVEL(bp) > 0) {
2967adb730bSGeorge Wilson 		arc_flags_t flags = ARC_FLAG_WAIT;
29788b7b0f2SMatthew Ahrens 		int i;
29888b7b0f2SMatthew Ahrens 		blkptr_t *cbp;
29988b7b0f2SMatthew Ahrens 		int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
30088b7b0f2SMatthew Ahrens 
301*eb633035STom Caputi 		ASSERT(!BP_IS_PROTECTED(bp));
302*eb633035STom Caputi 
3031b912ec7SGeorge Wilson 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
30488b7b0f2SMatthew Ahrens 		    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
3053b2aab18SMatthew Ahrens 		if (err != 0)
3067fd05ac4SMatthew Ahrens 			goto post;
307b4709335SMatthew Ahrens 		cbp = buf->b_data;
308b4709335SMatthew Ahrens 
309b4709335SMatthew Ahrens 		for (i = 0; i < epb; i++) {
310b4709335SMatthew Ahrens 			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
311b4709335SMatthew Ahrens 			    zb->zb_level - 1,
312b4709335SMatthew Ahrens 			    zb->zb_blkid * epb + i);
3131b912ec7SGeorge Wilson 			traverse_prefetch_metadata(td, &cbp[i], &czb);
314b4709335SMatthew Ahrens 		}
31588b7b0f2SMatthew Ahrens 
31688b7b0f2SMatthew Ahrens 		/* recursively visitbp() blocks below this */
317b4709335SMatthew Ahrens 		for (i = 0; i < epb; i++) {
31888b7b0f2SMatthew Ahrens 			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
31988b7b0f2SMatthew Ahrens 			    zb->zb_level - 1,
32088b7b0f2SMatthew Ahrens 			    zb->zb_blkid * epb + i);
3211b912ec7SGeorge Wilson 			err = traverse_visitbp(td, dnp, &cbp[i], &czb);
3227fd05ac4SMatthew Ahrens 			if (err != 0)
3237fd05ac4SMatthew Ahrens 				break;
32488b7b0f2SMatthew Ahrens 		}
32588b7b0f2SMatthew Ahrens 	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
3267adb730bSGeorge Wilson 		arc_flags_t flags = ARC_FLAG_WAIT;
327*eb633035STom Caputi 		uint32_t zio_flags = ZIO_FLAG_CANFAIL;
32814843421SMatthew Ahrens 		int i;
32988b7b0f2SMatthew Ahrens 		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
33088b7b0f2SMatthew Ahrens 
331*eb633035STom Caputi 		/*
332*eb633035STom Caputi 		 * dnode blocks might have their bonus buffers encrypted, so
333*eb633035STom Caputi 		 * we must be careful to honor TRAVERSE_NO_DECRYPT
334*eb633035STom Caputi 		 */
335*eb633035STom Caputi 		if ((td->td_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))
336*eb633035STom Caputi 			zio_flags |= ZIO_FLAG_RAW;
3371b912ec7SGeorge Wilson 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
338*eb633035STom Caputi 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
3393b2aab18SMatthew Ahrens 		if (err != 0)
3407fd05ac4SMatthew Ahrens 			goto post;
3419c3fd121SMatthew Ahrens 		dnode_phys_t *child_dnp = buf->b_data;
342b4709335SMatthew Ahrens 
34354811da5SToomas Soome 		for (i = 0; i < epb; i += child_dnp[i].dn_extra_slots + 1) {
3449c3fd121SMatthew Ahrens 			prefetch_dnode_metadata(td, &child_dnp[i],
3459c3fd121SMatthew Ahrens 			    zb->zb_objset, zb->zb_blkid * epb + i);
346b4709335SMatthew Ahrens 		}
34788b7b0f2SMatthew Ahrens 
34888b7b0f2SMatthew Ahrens 		/* recursively visitbp() blocks below this */
34954811da5SToomas Soome 		for (i = 0; i < epb; i += child_dnp[i].dn_extra_slots + 1) {
3509c3fd121SMatthew Ahrens 			err = traverse_dnode(td, &child_dnp[i],
3519c3fd121SMatthew Ahrens 			    zb->zb_objset, zb->zb_blkid * epb + i);
3527fd05ac4SMatthew Ahrens 			if (err != 0)
3537fd05ac4SMatthew Ahrens 				break;
354fa9e4066Sahrens 		}
35588b7b0f2SMatthew Ahrens 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
356*eb633035STom Caputi 		uint32_t zio_flags = ZIO_FLAG_CANFAIL;
3577adb730bSGeorge Wilson 		arc_flags_t flags = ARC_FLAG_WAIT;
35888b7b0f2SMatthew Ahrens 
359*eb633035STom Caputi 		if ((td->td_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))
360*eb633035STom Caputi 			zio_flags |= ZIO_FLAG_RAW;
361*eb633035STom Caputi 
3621b912ec7SGeorge Wilson 		err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
363*eb633035STom Caputi 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
3643b2aab18SMatthew Ahrens 		if (err != 0)
3657fd05ac4SMatthew Ahrens 			goto post;
36688b7b0f2SMatthew Ahrens 
3679c3fd121SMatthew Ahrens 		objset_phys_t *osp = buf->b_data;
3689c3fd121SMatthew Ahrens 		prefetch_dnode_metadata(td, &osp->os_meta_dnode, zb->zb_objset,
369b4709335SMatthew Ahrens 		    DMU_META_DNODE_OBJECT);
370286ef713SPaul Dagnelie 		/*
371286ef713SPaul Dagnelie 		 * See the block comment above for the goal of this variable.
372286ef713SPaul Dagnelie 		 * If the maxblkid of the meta-dnode is 0, then we know that
373286ef713SPaul Dagnelie 		 * we've never had more than DNODES_PER_BLOCK objects in the
374286ef713SPaul Dagnelie 		 * dataset, which means we can't have reused any object ids.
375286ef713SPaul Dagnelie 		 */
376286ef713SPaul Dagnelie 		if (osp->os_meta_dnode.dn_maxblkid == 0)
377286ef713SPaul Dagnelie 			td->td_realloc_possible = B_FALSE;
378286ef713SPaul Dagnelie 
379b4709335SMatthew Ahrens 		if (arc_buf_size(buf) >= sizeof (objset_phys_t)) {
380b4709335SMatthew Ahrens 			prefetch_dnode_metadata(td, &osp->os_groupused_dnode,
38148f1b90eSMatthew Ahrens 			    zb->zb_objset, DMU_GROUPUSED_OBJECT);
38248f1b90eSMatthew Ahrens 			prefetch_dnode_metadata(td, &osp->os_userused_dnode,
3831b912ec7SGeorge Wilson 			    zb->zb_objset, DMU_USERUSED_OBJECT);
384b4709335SMatthew Ahrens 		}
385b4709335SMatthew Ahrens 
3869c3fd121SMatthew Ahrens 		err = traverse_dnode(td, &osp->os_meta_dnode, zb->zb_objset,
387b24ab676SJeff Bonwick 		    DMU_META_DNODE_OBJECT);
38814843421SMatthew Ahrens 		if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
3899c3fd121SMatthew Ahrens 			err = traverse_dnode(td, &osp->os_groupused_dnode,
3909c3fd121SMatthew Ahrens 			    zb->zb_objset, DMU_GROUPUSED_OBJECT);
39114843421SMatthew Ahrens 		}
39214843421SMatthew Ahrens 		if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) {
3939c3fd121SMatthew Ahrens 			err = traverse_dnode(td, &osp->os_userused_dnode,
3949c3fd121SMatthew Ahrens 			    zb->zb_objset, DMU_USERUSED_OBJECT);
39588b7b0f2SMatthew Ahrens 		}
39688b7b0f2SMatthew Ahrens 	}
397fa9e4066Sahrens 
39888b7b0f2SMatthew Ahrens 	if (buf)
399dcbf3bd6SGeorge Wilson 		arc_buf_destroy(buf, &buf);
400fa9e4066Sahrens 
401ad135b5dSChristopher Siden post:
4027fd05ac4SMatthew Ahrens 	if (err == 0 && (td->td_flags & TRAVERSE_POST))
4031b912ec7SGeorge Wilson 		err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
4047fd05ac4SMatthew Ahrens 
4057fd05ac4SMatthew Ahrens 	if (hard && (err == EIO || err == ECKSUM)) {
4067fd05ac4SMatthew Ahrens 		/*
4077fd05ac4SMatthew Ahrens 		 * Ignore this disk error as requested by the HARD flag,
4087fd05ac4SMatthew Ahrens 		 * and continue traversal.
4097fd05ac4SMatthew Ahrens 		 */
4107fd05ac4SMatthew Ahrens 		err = 0;
411ad135b5dSChristopher Siden 	}
412ad135b5dSChristopher Siden 
4137fd05ac4SMatthew Ahrens 	/*
4147fd05ac4SMatthew Ahrens 	 * If we are stopping here, set td_resume.
4157fd05ac4SMatthew Ahrens 	 */
4167fd05ac4SMatthew Ahrens 	if (td->td_resume != NULL && err != 0 && !td->td_paused) {
4177fd05ac4SMatthew Ahrens 		td->td_resume->zb_objset = zb->zb_objset;
4187fd05ac4SMatthew Ahrens 		td->td_resume->zb_object = zb->zb_object;
4197fd05ac4SMatthew Ahrens 		td->td_resume->zb_level = 0;
4207fd05ac4SMatthew Ahrens 		/*
4217fd05ac4SMatthew Ahrens 		 * If we have stopped on an indirect block (e.g. due to
4227fd05ac4SMatthew Ahrens 		 * i/o error), we have not visited anything below it.
4237fd05ac4SMatthew Ahrens 		 * Set the bookmark to the first level-0 block that we need
4247fd05ac4SMatthew Ahrens 		 * to visit.  This way, the resuming code does not need to
4257fd05ac4SMatthew Ahrens 		 * deal with resuming from indirect blocks.
4269c3fd121SMatthew Ahrens 		 *
4279c3fd121SMatthew Ahrens 		 * Note, if zb_level <= 0, dnp may be NULL, so we don't want
4289c3fd121SMatthew Ahrens 		 * to dereference it.
4297fd05ac4SMatthew Ahrens 		 */
4309c3fd121SMatthew Ahrens 		td->td_resume->zb_blkid = zb->zb_blkid;
4319c3fd121SMatthew Ahrens 		if (zb->zb_level > 0) {
4329c3fd121SMatthew Ahrens 			td->td_resume->zb_blkid <<= zb->zb_level *
4339c3fd121SMatthew Ahrens 			    (dnp->dn_indblkshift - SPA_BLKPTRSHIFT);
4349c3fd121SMatthew Ahrens 		}
4357fd05ac4SMatthew Ahrens 		td->td_paused = B_TRUE;
4363f9d6ad7SLin Ling 	}
437fa9e4066Sahrens 
4387fd05ac4SMatthew Ahrens 	return (err);
439fa9e4066Sahrens }
440fa9e4066Sahrens 
441b4709335SMatthew Ahrens static void
442b4709335SMatthew Ahrens prefetch_dnode_metadata(traverse_data_t *td, const dnode_phys_t *dnp,
4431b912ec7SGeorge Wilson     uint64_t objset, uint64_t object)
444b4709335SMatthew Ahrens {
445b4709335SMatthew Ahrens 	int j;
4467802d7bfSMatthew Ahrens 	zbookmark_phys_t czb;
447b4709335SMatthew Ahrens 
448b4709335SMatthew Ahrens 	for (j = 0; j < dnp->dn_nblkptr; j++) {
449b4709335SMatthew Ahrens 		SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
4501b912ec7SGeorge Wilson 		traverse_prefetch_metadata(td, &dnp->dn_blkptr[j], &czb);
451b4709335SMatthew Ahrens 	}
452b4709335SMatthew Ahrens 
453b4709335SMatthew Ahrens 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
454b4709335SMatthew Ahrens 		SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID);
45554811da5SToomas Soome 		traverse_prefetch_metadata(td, DN_SPILL_BLKPTR(dnp), &czb);
456b4709335SMatthew Ahrens 	}
457b4709335SMatthew Ahrens }
458b4709335SMatthew Ahrens 
45914843421SMatthew Ahrens static int
4606e0cbcaaSMatthew Ahrens traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp,
4611b912ec7SGeorge Wilson     uint64_t objset, uint64_t object)
46214843421SMatthew Ahrens {
4637fd05ac4SMatthew Ahrens 	int j, err = 0;
4647802d7bfSMatthew Ahrens 	zbookmark_phys_t czb;
46514843421SMatthew Ahrens 
4669c3fd121SMatthew Ahrens 	if (object != DMU_META_DNODE_OBJECT && td->td_resume != NULL &&
4679c3fd121SMatthew Ahrens 	    object < td->td_resume->zb_object)
4689c3fd121SMatthew Ahrens 		return (0);
4699c3fd121SMatthew Ahrens 
470a2cdcdd2SPaul Dagnelie 	if (td->td_flags & TRAVERSE_PRE) {
471a2cdcdd2SPaul Dagnelie 		SET_BOOKMARK(&czb, objset, object, ZB_DNODE_LEVEL,
472a2cdcdd2SPaul Dagnelie 		    ZB_DNODE_BLKID);
473a2cdcdd2SPaul Dagnelie 		err = td->td_func(td->td_spa, NULL, NULL, &czb, dnp,
474a2cdcdd2SPaul Dagnelie 		    td->td_arg);
475a2cdcdd2SPaul Dagnelie 		if (err == TRAVERSE_VISIT_NO_CHILDREN)
476a2cdcdd2SPaul Dagnelie 			return (0);
477a2cdcdd2SPaul Dagnelie 		if (err != 0)
478a2cdcdd2SPaul Dagnelie 			return (err);
479a2cdcdd2SPaul Dagnelie 	}
480a2cdcdd2SPaul Dagnelie 
48114843421SMatthew Ahrens 	for (j = 0; j < dnp->dn_nblkptr; j++) {
48214843421SMatthew Ahrens 		SET_BOOKMARK(&czb, objset, object, dnp->dn_nlevels - 1, j);
4831b912ec7SGeorge Wilson 		err = traverse_visitbp(td, dnp, &dnp->dn_blkptr[j], &czb);
4847fd05ac4SMatthew Ahrens 		if (err != 0)
4857fd05ac4SMatthew Ahrens 			break;
4863f9d6ad7SLin Ling 	}
4873f9d6ad7SLin Ling 
488a2cdcdd2SPaul Dagnelie 	if (err == 0 && (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) {
489b4709335SMatthew Ahrens 		SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID);
49054811da5SToomas Soome 		err = traverse_visitbp(td, dnp, DN_SPILL_BLKPTR(dnp), &czb);
49114843421SMatthew Ahrens 	}
492a2cdcdd2SPaul Dagnelie 
493a2cdcdd2SPaul Dagnelie 	if (err == 0 && (td->td_flags & TRAVERSE_POST)) {
494a2cdcdd2SPaul Dagnelie 		SET_BOOKMARK(&czb, objset, object, ZB_DNODE_LEVEL,
495a2cdcdd2SPaul Dagnelie 		    ZB_DNODE_BLKID);
496a2cdcdd2SPaul Dagnelie 		err = td->td_func(td->td_spa, NULL, NULL, &czb, dnp,
497a2cdcdd2SPaul Dagnelie 		    td->td_arg);
498a2cdcdd2SPaul Dagnelie 		if (err == TRAVERSE_VISIT_NO_CHILDREN)
499a2cdcdd2SPaul Dagnelie 			return (0);
500a2cdcdd2SPaul Dagnelie 		if (err != 0)
501a2cdcdd2SPaul Dagnelie 			return (err);
502a2cdcdd2SPaul Dagnelie 	}
5037fd05ac4SMatthew Ahrens 	return (err);
50414843421SMatthew Ahrens }
50514843421SMatthew Ahrens 
50688b7b0f2SMatthew Ahrens /* ARGSUSED */
50788b7b0f2SMatthew Ahrens static int
508b24ab676SJeff Bonwick traverse_prefetcher(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
5097802d7bfSMatthew Ahrens     const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
510e7cbe64fSgw {
5116e0cbcaaSMatthew Ahrens 	prefetch_data_t *pfd = arg;
512*eb633035STom Caputi 	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE;
513a3874b8bSToomas Soome 	arc_flags_t aflags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH |
514a3874b8bSToomas Soome 	    ARC_FLAG_PRESCIENT_PREFETCH;
515e7cbe64fSgw 
51634d7ce05SGeorge Wilson 	ASSERT(pfd->pd_bytes_fetched >= 0);
517a2cdcdd2SPaul Dagnelie 	if (bp == NULL)
518a2cdcdd2SPaul Dagnelie 		return (0);
51988b7b0f2SMatthew Ahrens 	if (pfd->pd_cancel)
520be6fd75aSMatthew Ahrens 		return (SET_ERROR(EINTR));
521e7cbe64fSgw 
52206315b79SMatthew Ahrens 	if (!prefetch_needed(pfd, bp))
523fa9e4066Sahrens 		return (0);
524fa9e4066Sahrens 
52588b7b0f2SMatthew Ahrens 	mutex_enter(&pfd->pd_mtx);
52634d7ce05SGeorge Wilson 	while (!pfd->pd_cancel && pfd->pd_bytes_fetched >= zfs_pd_bytes_max)
52788b7b0f2SMatthew Ahrens 		cv_wait(&pfd->pd_cv, &pfd->pd_mtx);
52834d7ce05SGeorge Wilson 	pfd->pd_bytes_fetched += BP_GET_LSIZE(bp);
52988b7b0f2SMatthew Ahrens 	cv_broadcast(&pfd->pd_cv);
53088b7b0f2SMatthew Ahrens 	mutex_exit(&pfd->pd_mtx);
531fa9e4066Sahrens 
532*eb633035STom Caputi 	if ((pfd->pd_flags & TRAVERSE_NO_DECRYPT) && BP_IS_PROTECTED(bp))
533*eb633035STom Caputi 		zio_flags |= ZIO_FLAG_RAW;
534*eb633035STom Caputi 
5351b912ec7SGeorge Wilson 	(void) arc_read(NULL, spa, bp, NULL, NULL, ZIO_PRIORITY_ASYNC_READ,
536*eb633035STom Caputi 	    zio_flags, &aflags, zb);
537fa9e4066Sahrens 
53888b7b0f2SMatthew Ahrens 	return (0);
539fa9e4066Sahrens }
540fa9e4066Sahrens 
541fa9e4066Sahrens static void
54288b7b0f2SMatthew Ahrens traverse_prefetch_thread(void *arg)
543fa9e4066Sahrens {
5446e0cbcaaSMatthew Ahrens 	traverse_data_t *td_main = arg;
5456e0cbcaaSMatthew Ahrens 	traverse_data_t td = *td_main;
5467802d7bfSMatthew Ahrens 	zbookmark_phys_t czb;
547fa9e4066Sahrens 
54888b7b0f2SMatthew Ahrens 	td.td_func = traverse_prefetcher;
54988b7b0f2SMatthew Ahrens 	td.td_arg = td_main->td_pfd;
55088b7b0f2SMatthew Ahrens 	td.td_pfd = NULL;
5519c3fd121SMatthew Ahrens 	td.td_resume = &td_main->td_pfd->pd_resume;
552fa9e4066Sahrens 
553b24ab676SJeff Bonwick 	SET_BOOKMARK(&czb, td.td_objset,
554b24ab676SJeff Bonwick 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
5551b912ec7SGeorge Wilson 	(void) traverse_visitbp(&td, NULL, td.td_rootbp, &czb);
556fa9e4066Sahrens 
55788b7b0f2SMatthew Ahrens 	mutex_enter(&td_main->td_pfd->pd_mtx);
55888b7b0f2SMatthew Ahrens 	td_main->td_pfd->pd_exited = B_TRUE;
55988b7b0f2SMatthew Ahrens 	cv_broadcast(&td_main->td_pfd->pd_cv);
56088b7b0f2SMatthew Ahrens 	mutex_exit(&td_main->td_pfd->pd_mtx);
561fa9e4066Sahrens }
562fa9e4066Sahrens 
56388b7b0f2SMatthew Ahrens /*
56488b7b0f2SMatthew Ahrens  * NB: dataset must not be changing on-disk (eg, is a snapshot or we are
56588b7b0f2SMatthew Ahrens  * in syncing context).
56688b7b0f2SMatthew Ahrens  */
56788b7b0f2SMatthew Ahrens static int
568ad135b5dSChristopher Siden traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
5697802d7bfSMatthew Ahrens     uint64_t txg_start, zbookmark_phys_t *resume, int flags,
570ad135b5dSChristopher Siden     blkptr_cb_t func, void *arg)
571fa9e4066Sahrens {
5726e0cbcaaSMatthew Ahrens 	traverse_data_t td;
5736e0cbcaaSMatthew Ahrens 	prefetch_data_t pd = { 0 };
5747802d7bfSMatthew Ahrens 	zbookmark_phys_t czb;
57588b7b0f2SMatthew Ahrens 	int err;
576fa9e4066Sahrens 
577ad135b5dSChristopher Siden 	ASSERT(ds == NULL || objset == ds->ds_object);
578ad135b5dSChristopher Siden 	ASSERT(!(flags & TRAVERSE_PRE) || !(flags & TRAVERSE_POST));
579ad135b5dSChristopher Siden 
58088b7b0f2SMatthew Ahrens 	td.td_spa = spa;
581ad135b5dSChristopher Siden 	td.td_objset = objset;
58288b7b0f2SMatthew Ahrens 	td.td_rootbp = rootbp;
58388b7b0f2SMatthew Ahrens 	td.td_min_txg = txg_start;
584ad135b5dSChristopher Siden 	td.td_resume = resume;
58588b7b0f2SMatthew Ahrens 	td.td_func = func;
58688b7b0f2SMatthew Ahrens 	td.td_arg = arg;
58788b7b0f2SMatthew Ahrens 	td.td_pfd = &pd;
58888b7b0f2SMatthew Ahrens 	td.td_flags = flags;
5897fd05ac4SMatthew Ahrens 	td.td_paused = B_FALSE;
590286ef713SPaul Dagnelie 	td.td_realloc_possible = (txg_start == 0 ? B_FALSE : B_TRUE);
59188b7b0f2SMatthew Ahrens 
592f7950bf1SMatthew Ahrens 	if (spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) {
593f7950bf1SMatthew Ahrens 		VERIFY(spa_feature_enabled_txg(spa,
594f7950bf1SMatthew Ahrens 		    SPA_FEATURE_HOLE_BIRTH, &td.td_hole_birth_enabled_txg));
595f7950bf1SMatthew Ahrens 	} else {
596286ef713SPaul Dagnelie 		td.td_hole_birth_enabled_txg = UINT64_MAX;
597f7950bf1SMatthew Ahrens 	}
598f7950bf1SMatthew Ahrens 
59988b7b0f2SMatthew Ahrens 	pd.pd_flags = flags;
6009c3fd121SMatthew Ahrens 	if (resume != NULL)
6019c3fd121SMatthew Ahrens 		pd.pd_resume = *resume;
60288b7b0f2SMatthew Ahrens 	mutex_init(&pd.pd_mtx, NULL, MUTEX_DEFAULT, NULL);
60388b7b0f2SMatthew Ahrens 	cv_init(&pd.pd_cv, NULL, CV_DEFAULT, NULL);
60488b7b0f2SMatthew Ahrens 
605*eb633035STom Caputi 	SET_BOOKMARK(&czb, td.td_objset,
606*eb633035STom Caputi 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
607*eb633035STom Caputi 
6086e0cbcaaSMatthew Ahrens 	/* See comment on ZIL traversal in dsl_scan_visitds. */
609bc9014e6SJustin Gibbs 	if (ds != NULL && !ds->ds_is_snapshot && !BP_IS_HOLE(rootbp)) {
610*eb633035STom Caputi 		enum zio_flag zio_flags = ZIO_FLAG_CANFAIL;
6117adb730bSGeorge Wilson 		arc_flags_t flags = ARC_FLAG_WAIT;
6123b2aab18SMatthew Ahrens 		objset_phys_t *osp;
6133b2aab18SMatthew Ahrens 		arc_buf_t *buf;
6146e0cbcaaSMatthew Ahrens 
615*eb633035STom Caputi 		if ((td.td_flags & TRAVERSE_NO_DECRYPT) &&
616*eb633035STom Caputi 		    BP_IS_PROTECTED(rootbp))
617*eb633035STom Caputi 			zio_flags |= ZIO_FLAG_RAW;
618*eb633035STom Caputi 
619*eb633035STom Caputi 		err = arc_read(NULL, td.td_spa, rootbp, arc_getbuf_func,
620*eb633035STom Caputi 		    &buf, ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, &czb);
6213b2aab18SMatthew Ahrens 		if (err != 0)
6226e0cbcaaSMatthew Ahrens 			return (err);
6236e0cbcaaSMatthew Ahrens 
6243b2aab18SMatthew Ahrens 		osp = buf->b_data;
6253b2aab18SMatthew Ahrens 		traverse_zil(&td, &osp->os_zil_header);
626dcbf3bd6SGeorge Wilson 		arc_buf_destroy(buf, &buf);
6276e0cbcaaSMatthew Ahrens 	}
6286e0cbcaaSMatthew Ahrens 
629b4709335SMatthew Ahrens 	if (!(flags & TRAVERSE_PREFETCH_DATA) ||
630fc8ae2ecSToomas Soome 	    taskq_dispatch(system_taskq, traverse_prefetch_thread,
631fc8ae2ecSToomas Soome 	    &td, TQ_NOQUEUE) == TASKQID_INVALID)
63288b7b0f2SMatthew Ahrens 		pd.pd_exited = B_TRUE;
63388b7b0f2SMatthew Ahrens 
6341b912ec7SGeorge Wilson 	err = traverse_visitbp(&td, NULL, rootbp, &czb);
63588b7b0f2SMatthew Ahrens 
63688b7b0f2SMatthew Ahrens 	mutex_enter(&pd.pd_mtx);
63788b7b0f2SMatthew Ahrens 	pd.pd_cancel = B_TRUE;
63888b7b0f2SMatthew Ahrens 	cv_broadcast(&pd.pd_cv);
63988b7b0f2SMatthew Ahrens 	while (!pd.pd_exited)
64088b7b0f2SMatthew Ahrens 		cv_wait(&pd.pd_cv, &pd.pd_mtx);
64188b7b0f2SMatthew Ahrens 	mutex_exit(&pd.pd_mtx);
64288b7b0f2SMatthew Ahrens 
64388b7b0f2SMatthew Ahrens 	mutex_destroy(&pd.pd_mtx);
64488b7b0f2SMatthew Ahrens 	cv_destroy(&pd.pd_cv);
645fa9e4066Sahrens 
64688b7b0f2SMatthew Ahrens 	return (err);
647fa9e4066Sahrens }
648fa9e4066Sahrens 
64988b7b0f2SMatthew Ahrens /*
65088b7b0f2SMatthew Ahrens  * NB: dataset must not be changing on-disk (eg, is a snapshot or we are
65188b7b0f2SMatthew Ahrens  * in syncing context).
65288b7b0f2SMatthew Ahrens  */
65388b7b0f2SMatthew Ahrens int
6549c3fd121SMatthew Ahrens traverse_dataset_resume(dsl_dataset_t *ds, uint64_t txg_start,
6559c3fd121SMatthew Ahrens     zbookmark_phys_t *resume,
6569c3fd121SMatthew Ahrens     int flags, blkptr_cb_t func, void *arg)
657fa9e4066Sahrens {
658ad135b5dSChristopher Siden 	return (traverse_impl(ds->ds_dir->dd_pool->dp_spa, ds, ds->ds_object,
6599c3fd121SMatthew Ahrens 	    &dsl_dataset_phys(ds)->ds_bp, txg_start, resume, flags, func, arg));
6609c3fd121SMatthew Ahrens }
6619c3fd121SMatthew Ahrens 
6629c3fd121SMatthew Ahrens int
6639c3fd121SMatthew Ahrens traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start,
6649c3fd121SMatthew Ahrens     int flags, blkptr_cb_t func, void *arg)
6659c3fd121SMatthew Ahrens {
6669c3fd121SMatthew Ahrens 	return (traverse_dataset_resume(ds, txg_start, NULL, flags, func, arg));
667ad135b5dSChristopher Siden }
668ad135b5dSChristopher Siden 
669ad135b5dSChristopher Siden int
670ad135b5dSChristopher Siden traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr,
6717802d7bfSMatthew Ahrens     uint64_t txg_start, zbookmark_phys_t *resume, int flags,
672ad135b5dSChristopher Siden     blkptr_cb_t func, void *arg)
673ad135b5dSChristopher Siden {
674ad135b5dSChristopher Siden 	return (traverse_impl(spa, NULL, ZB_DESTROYED_OBJSET,
675ad135b5dSChristopher Siden 	    blkptr, txg_start, resume, flags, func, arg));
676fa9e4066Sahrens }
677fa9e4066Sahrens 
67888b7b0f2SMatthew Ahrens /*
67988b7b0f2SMatthew Ahrens  * NB: pool must not be changing on-disk (eg, from zdb or sync context).
68088b7b0f2SMatthew Ahrens  */
68188b7b0f2SMatthew Ahrens int
682bbfd46c4SJeff Bonwick traverse_pool(spa_t *spa, uint64_t txg_start, int flags,
683bbfd46c4SJeff Bonwick     blkptr_cb_t func, void *arg)
684fa9e4066Sahrens {
6857fd05ac4SMatthew Ahrens 	int err;
68688b7b0f2SMatthew Ahrens 	dsl_pool_t *dp = spa_get_dsl(spa);
68788b7b0f2SMatthew Ahrens 	objset_t *mos = dp->dp_meta_objset;
688cd088ea4SVictor Latushkin 	boolean_t hard = (flags & TRAVERSE_HARD);
68988b7b0f2SMatthew Ahrens 
69088b7b0f2SMatthew Ahrens 	/* visit the MOS */
691ad135b5dSChristopher Siden 	err = traverse_impl(spa, NULL, 0, spa_get_rootblkptr(spa),
692ad135b5dSChristopher Siden 	    txg_start, NULL, flags, func, arg);
6933b2aab18SMatthew Ahrens 	if (err != 0)
69488b7b0f2SMatthew Ahrens 		return (err);
69588b7b0f2SMatthew Ahrens 
69688b7b0f2SMatthew Ahrens 	/* visit each dataset */
6979c3fd121SMatthew Ahrens 	for (uint64_t obj = 1; err == 0;
6989c3fd121SMatthew Ahrens 	    err = dmu_object_next(mos, &obj, B_FALSE, txg_start)) {
69988b7b0f2SMatthew Ahrens 		dmu_object_info_t doi;
70088b7b0f2SMatthew Ahrens 
70188b7b0f2SMatthew Ahrens 		err = dmu_object_info(mos, obj, &doi);
7023b2aab18SMatthew Ahrens 		if (err != 0) {
7037fd05ac4SMatthew Ahrens 			if (hard)
7047fd05ac4SMatthew Ahrens 				continue;
7057fd05ac4SMatthew Ahrens 			break;
706cd088ea4SVictor Latushkin 		}
70788b7b0f2SMatthew Ahrens 
7082acef22dSMatthew Ahrens 		if (doi.doi_bonus_type == DMU_OT_DSL_DATASET) {
70988b7b0f2SMatthew Ahrens 			dsl_dataset_t *ds;
710468c413aSTim Haley 			uint64_t txg = txg_start;
711468c413aSTim Haley 
7123b2aab18SMatthew Ahrens 			dsl_pool_config_enter(dp, FTAG);
71388b7b0f2SMatthew Ahrens 			err = dsl_dataset_hold_obj(dp, obj, FTAG, &ds);
7143b2aab18SMatthew Ahrens 			dsl_pool_config_exit(dp, FTAG);
7153b2aab18SMatthew Ahrens 			if (err != 0) {
7167fd05ac4SMatthew Ahrens 				if (hard)
7177fd05ac4SMatthew Ahrens 					continue;
7187fd05ac4SMatthew Ahrens 				break;
719cd088ea4SVictor Latushkin 			}
720c1379625SJustin T. Gibbs 			if (dsl_dataset_phys(ds)->ds_prev_snap_txg > txg)
721c1379625SJustin T. Gibbs 				txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
722bbfd46c4SJeff Bonwick 			err = traverse_dataset(ds, txg, flags, func, arg);
72388b7b0f2SMatthew Ahrens 			dsl_dataset_rele(ds, FTAG);
7247fd05ac4SMatthew Ahrens 			if (err != 0)
7257fd05ac4SMatthew Ahrens 				break;
72688b7b0f2SMatthew Ahrens 		}
727fa9e4066Sahrens 	}
72888b7b0f2SMatthew Ahrens 	if (err == ESRCH)
72988b7b0f2SMatthew Ahrens 		err = 0;
7307fd05ac4SMatthew Ahrens 	return (err);
731fa9e4066Sahrens }
732