xref: /illumos-gate/usr/src/uts/common/fs/zfs/dsl_scan.c (revision 6e0cbcaa0c6f2bc34634a4cc17b099f9ecef03d1)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/dsl_scan.h>
26 #include <sys/dsl_pool.h>
27 #include <sys/dsl_dataset.h>
28 #include <sys/dsl_prop.h>
29 #include <sys/dsl_dir.h>
30 #include <sys/dsl_synctask.h>
31 #include <sys/dnode.h>
32 #include <sys/dmu_tx.h>
33 #include <sys/dmu_objset.h>
34 #include <sys/arc.h>
35 #include <sys/zap.h>
36 #include <sys/zio.h>
37 #include <sys/zfs_context.h>
38 #include <sys/fs/zfs.h>
39 #include <sys/zfs_znode.h>
40 #include <sys/spa_impl.h>
41 #include <sys/vdev_impl.h>
42 #include <sys/zil_impl.h>
43 #include <sys/zio_checksum.h>
44 #include <sys/ddt.h>
45 #include <sys/sa.h>
46 #include <sys/sa_impl.h>
47 #ifdef _KERNEL
48 #include <sys/zfs_vfsops.h>
49 #endif
50 
51 typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, const zbookmark_t *);
52 
53 static scan_cb_t dsl_scan_defrag_cb;
54 static scan_cb_t dsl_scan_scrub_cb;
55 static scan_cb_t dsl_scan_remove_cb;
56 static dsl_syncfunc_t dsl_scan_cancel_sync;
57 static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *tx);
58 
59 int zfs_top_maxinflight = 32;		/* maximum I/Os per top-level */
60 int zfs_resilver_delay = 2;		/* number of ticks to delay resilver */
61 int zfs_scrub_delay = 4;		/* number of ticks to delay scrub */
62 int zfs_scan_idle = 50;			/* idle window in clock ticks */
63 
64 int zfs_scan_min_time_ms = 1000; /* min millisecs to scrub per txg */
65 int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */
66 int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver per txg */
67 boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */
68 boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable srub prefetching */
69 enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE;
70 int dsl_scan_delay_completion = B_FALSE; /* set to delay scan completion */
71 
72 #define	DSL_SCAN_IS_SCRUB_RESILVER(scn) \
73 	((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
74 	(scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
75 
76 extern int zfs_txg_timeout;
77 
78 /* the order has to match pool_scan_type */
79 static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = {
80 	NULL,
81 	dsl_scan_scrub_cb,	/* POOL_SCAN_SCRUB */
82 	dsl_scan_scrub_cb,	/* POOL_SCAN_RESILVER */
83 };
84 
85 int
86 dsl_scan_init(dsl_pool_t *dp, uint64_t txg)
87 {
88 	int err;
89 	dsl_scan_t *scn;
90 	spa_t *spa = dp->dp_spa;
91 	uint64_t f;
92 
93 	scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP);
94 	scn->scn_dp = dp;
95 
96 	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
97 	    "scrub_func", sizeof (uint64_t), 1, &f);
98 	if (err == 0) {
99 		/*
100 		 * There was an old-style scrub in progress.  Restart a
101 		 * new-style scrub from the beginning.
102 		 */
103 		scn->scn_restart_txg = txg;
104 		zfs_dbgmsg("old-style scrub was in progress; "
105 		    "restarting new-style scrub in txg %llu",
106 		    scn->scn_restart_txg);
107 
108 		/*
109 		 * Load the queue obj from the old location so that it
110 		 * can be freed by dsl_scan_done().
111 		 */
112 		(void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
113 		    "scrub_queue", sizeof (uint64_t), 1,
114 		    &scn->scn_phys.scn_queue_obj);
115 	} else {
116 		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
117 		    DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS,
118 		    &scn->scn_phys);
119 		if (err == ENOENT)
120 			return (0);
121 		else if (err)
122 			return (err);
123 
124 		if (scn->scn_phys.scn_state == DSS_SCANNING &&
125 		    spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) {
126 			/*
127 			 * A new-type scrub was in progress on an old
128 			 * pool, and the pool was accessed by old
129 			 * software.  Restart from the beginning, since
130 			 * the old software may have changed the pool in
131 			 * the meantime.
132 			 */
133 			scn->scn_restart_txg = txg;
134 			zfs_dbgmsg("new-style scrub was modified "
135 			    "by old software; restarting in txg %llu",
136 			    scn->scn_restart_txg);
137 		}
138 	}
139 
140 	spa_scan_stat_init(spa);
141 	return (0);
142 }
143 
144 void
145 dsl_scan_fini(dsl_pool_t *dp)
146 {
147 	if (dp->dp_scan) {
148 		kmem_free(dp->dp_scan, sizeof (dsl_scan_t));
149 		dp->dp_scan = NULL;
150 	}
151 }
152 
153 /* ARGSUSED */
154 static int
155 dsl_scan_setup_check(void *arg1, void *arg2, dmu_tx_t *tx)
156 {
157 	dsl_scan_t *scn = arg1;
158 
159 	if (scn->scn_phys.scn_state == DSS_SCANNING)
160 		return (EBUSY);
161 
162 	return (0);
163 }
164 
165 /* ARGSUSED */
166 static void
167 dsl_scan_setup_sync(void *arg1, void *arg2, dmu_tx_t *tx)
168 {
169 	dsl_scan_t *scn = arg1;
170 	pool_scan_func_t *funcp = arg2;
171 	dmu_object_type_t ot = 0;
172 	dsl_pool_t *dp = scn->scn_dp;
173 	spa_t *spa = dp->dp_spa;
174 
175 	ASSERT(scn->scn_phys.scn_state != DSS_SCANNING);
176 	ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS);
177 	bzero(&scn->scn_phys, sizeof (scn->scn_phys));
178 	scn->scn_phys.scn_func = *funcp;
179 	scn->scn_phys.scn_state = DSS_SCANNING;
180 	scn->scn_phys.scn_min_txg = 0;
181 	scn->scn_phys.scn_max_txg = tx->tx_txg;
182 	scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */
183 	scn->scn_phys.scn_start_time = gethrestime_sec();
184 	scn->scn_phys.scn_errors = 0;
185 	scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc;
186 	scn->scn_restart_txg = 0;
187 	spa_scan_stat_init(spa);
188 
189 	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
190 		scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max;
191 
192 		/* rewrite all disk labels */
193 		vdev_config_dirty(spa->spa_root_vdev);
194 
195 		if (vdev_resilver_needed(spa->spa_root_vdev,
196 		    &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) {
197 			spa_event_notify(spa, NULL, ESC_ZFS_RESILVER_START);
198 		} else {
199 			spa_event_notify(spa, NULL, ESC_ZFS_SCRUB_START);
200 		}
201 
202 		spa->spa_scrub_started = B_TRUE;
203 		/*
204 		 * If this is an incremental scrub, limit the DDT scrub phase
205 		 * to just the auto-ditto class (for correctness); the rest
206 		 * of the scrub should go faster using top-down pruning.
207 		 */
208 		if (scn->scn_phys.scn_min_txg > TXG_INITIAL)
209 			scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO;
210 
211 	}
212 
213 	/* back to the generic stuff */
214 
215 	if (dp->dp_blkstats == NULL) {
216 		dp->dp_blkstats =
217 		    kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP);
218 	}
219 	bzero(dp->dp_blkstats, sizeof (zfs_all_blkstats_t));
220 
221 	if (spa_version(spa) < SPA_VERSION_DSL_SCRUB)
222 		ot = DMU_OT_ZAP_OTHER;
223 
224 	scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset,
225 	    ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx);
226 
227 	dsl_scan_sync_state(scn, tx);
228 
229 	spa_history_log_internal(LOG_POOL_SCAN, spa, tx,
230 	    "func=%u mintxg=%llu maxtxg=%llu",
231 	    *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg);
232 }
233 
234 /* ARGSUSED */
235 static void
236 dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
237 {
238 	static const char *old_names[] = {
239 		"scrub_bookmark",
240 		"scrub_ddt_bookmark",
241 		"scrub_ddt_class_max",
242 		"scrub_queue",
243 		"scrub_min_txg",
244 		"scrub_max_txg",
245 		"scrub_func",
246 		"scrub_errors",
247 		NULL
248 	};
249 
250 	dsl_pool_t *dp = scn->scn_dp;
251 	spa_t *spa = dp->dp_spa;
252 	int i;
253 
254 	/* Remove any remnants of an old-style scrub. */
255 	for (i = 0; old_names[i]; i++) {
256 		(void) zap_remove(dp->dp_meta_objset,
257 		    DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx);
258 	}
259 
260 	if (scn->scn_phys.scn_queue_obj != 0) {
261 		VERIFY(0 == dmu_object_free(dp->dp_meta_objset,
262 		    scn->scn_phys.scn_queue_obj, tx));
263 		scn->scn_phys.scn_queue_obj = 0;
264 	}
265 
266 	/*
267 	 * If we were "restarted" from a stopped state, don't bother
268 	 * with anything else.
269 	 */
270 	if (scn->scn_phys.scn_state != DSS_SCANNING)
271 		return;
272 
273 	if (complete)
274 		scn->scn_phys.scn_state = DSS_FINISHED;
275 	else
276 		scn->scn_phys.scn_state = DSS_CANCELED;
277 
278 	spa_history_log_internal(LOG_POOL_SCAN_DONE, spa, tx,
279 	    "complete=%u", complete);
280 
281 	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
282 		mutex_enter(&spa->spa_scrub_lock);
283 		while (spa->spa_scrub_inflight > 0) {
284 			cv_wait(&spa->spa_scrub_io_cv,
285 			    &spa->spa_scrub_lock);
286 		}
287 		mutex_exit(&spa->spa_scrub_lock);
288 		spa->spa_scrub_started = B_FALSE;
289 		spa->spa_scrub_active = B_FALSE;
290 
291 		/*
292 		 * If the scrub/resilver completed, update all DTLs to
293 		 * reflect this.  Whether it succeeded or not, vacate
294 		 * all temporary scrub DTLs.
295 		 */
296 		vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg,
297 		    complete ? scn->scn_phys.scn_max_txg : 0, B_TRUE);
298 		if (complete) {
299 			spa_event_notify(spa, NULL, scn->scn_phys.scn_min_txg ?
300 			    ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH);
301 		}
302 		spa_errlog_rotate(spa);
303 
304 		/*
305 		 * We may have finished replacing a device.
306 		 * Let the async thread assess this and handle the detach.
307 		 */
308 		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
309 	}
310 
311 	scn->scn_phys.scn_end_time = gethrestime_sec();
312 }
313 
314 /* ARGSUSED */
315 static int
316 dsl_scan_cancel_check(void *arg1, void *arg2, dmu_tx_t *tx)
317 {
318 	dsl_scan_t *scn = arg1;
319 
320 	if (scn->scn_phys.scn_state != DSS_SCANNING)
321 		return (ENOENT);
322 	return (0);
323 }
324 
325 /* ARGSUSED */
326 static void
327 dsl_scan_cancel_sync(void *arg1, void *arg2, dmu_tx_t *tx)
328 {
329 	dsl_scan_t *scn = arg1;
330 
331 	dsl_scan_done(scn, B_FALSE, tx);
332 	dsl_scan_sync_state(scn, tx);
333 }
334 
335 int
336 dsl_scan_cancel(dsl_pool_t *dp)
337 {
338 	boolean_t complete = B_FALSE;
339 	int err;
340 
341 	err = dsl_sync_task_do(dp, dsl_scan_cancel_check,
342 	    dsl_scan_cancel_sync, dp->dp_scan, &complete, 3);
343 	return (err);
344 }
345 
346 static void dsl_scan_visitbp(blkptr_t *bp,
347     const zbookmark_t *zb, dnode_phys_t *dnp, arc_buf_t *pbuf,
348     dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype,
349     dmu_tx_t *tx);
350 static void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds,
351     dmu_objset_type_t ostype,
352     dnode_phys_t *dnp, arc_buf_t *buf, uint64_t object, dmu_tx_t *tx);
353 
354 void
355 dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp)
356 {
357 	zio_free(dp->dp_spa, txg, bp);
358 }
359 
360 void
361 dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp)
362 {
363 	ASSERT(dsl_pool_sync_context(dp));
364 	zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, pio->io_flags));
365 }
366 
367 int
368 dsl_read(zio_t *pio, spa_t *spa, const blkptr_t *bpp, arc_buf_t *pbuf,
369     arc_done_func_t *done, void *private, int priority, int zio_flags,
370     uint32_t *arc_flags, const zbookmark_t *zb)
371 {
372 	return (arc_read(pio, spa, bpp, pbuf, done, private,
373 	    priority, zio_flags, arc_flags, zb));
374 }
375 
376 int
377 dsl_read_nolock(zio_t *pio, spa_t *spa, const blkptr_t *bpp,
378     arc_done_func_t *done, void *private, int priority, int zio_flags,
379     uint32_t *arc_flags, const zbookmark_t *zb)
380 {
381 	return (arc_read_nolock(pio, spa, bpp, done, private,
382 	    priority, zio_flags, arc_flags, zb));
383 }
384 
385 static boolean_t
386 bookmark_is_zero(const zbookmark_t *zb)
387 {
388 	return (zb->zb_objset == 0 && zb->zb_object == 0 &&
389 	    zb->zb_level == 0 && zb->zb_blkid == 0);
390 }
391 
392 /* dnp is the dnode for zb1->zb_object */
393 static boolean_t
394 bookmark_is_before(const dnode_phys_t *dnp, const zbookmark_t *zb1,
395     const zbookmark_t *zb2)
396 {
397 	uint64_t zb1nextL0, zb2thisobj;
398 
399 	ASSERT(zb1->zb_objset == zb2->zb_objset);
400 	ASSERT(zb2->zb_level == 0);
401 
402 	/*
403 	 * A bookmark in the deadlist is considered to be after
404 	 * everything else.
405 	 */
406 	if (zb2->zb_object == DMU_DEADLIST_OBJECT)
407 		return (B_TRUE);
408 
409 	/* The objset_phys_t isn't before anything. */
410 	if (dnp == NULL)
411 		return (B_FALSE);
412 
413 	zb1nextL0 = (zb1->zb_blkid + 1) <<
414 	    ((zb1->zb_level) * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT));
415 
416 	zb2thisobj = zb2->zb_object ? zb2->zb_object :
417 	    zb2->zb_blkid << (DNODE_BLOCK_SHIFT - DNODE_SHIFT);
418 
419 	if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
420 		uint64_t nextobj = zb1nextL0 *
421 		    (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT) >> DNODE_SHIFT;
422 		return (nextobj <= zb2thisobj);
423 	}
424 
425 	if (zb1->zb_object < zb2thisobj)
426 		return (B_TRUE);
427 	if (zb1->zb_object > zb2thisobj)
428 		return (B_FALSE);
429 	if (zb2->zb_object == DMU_META_DNODE_OBJECT)
430 		return (B_FALSE);
431 	return (zb1nextL0 <= zb2->zb_blkid);
432 }
433 
434 static uint64_t
435 dsl_scan_ds_maxtxg(dsl_dataset_t *ds)
436 {
437 	uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg;
438 	if (dsl_dataset_is_snapshot(ds))
439 		return (MIN(smt, ds->ds_phys->ds_creation_txg));
440 	return (smt);
441 }
442 
443 static void
444 dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx)
445 {
446 	VERIFY(0 == zap_update(scn->scn_dp->dp_meta_objset,
447 	    DMU_POOL_DIRECTORY_OBJECT,
448 	    DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS,
449 	    &scn->scn_phys, tx));
450 }
451 
452 static boolean_t
453 dsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_t *zb)
454 {
455 	uint64_t elapsed_nanosecs;
456 	int mintime;
457 
458 	/* we never skip user/group accounting objects */
459 	if (zb && (int64_t)zb->zb_object < 0)
460 		return (B_FALSE);
461 
462 	if (scn->scn_pausing)
463 		return (B_TRUE); /* we're already pausing */
464 
465 	if (!bookmark_is_zero(&scn->scn_phys.scn_bookmark))
466 		return (B_FALSE); /* we're resuming */
467 
468 	/* We only know how to resume from level-0 blocks. */
469 	if (zb && zb->zb_level != 0)
470 		return (B_FALSE);
471 
472 	mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ?
473 	    zfs_resilver_min_time_ms : zfs_scan_min_time_ms;
474 	elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
475 	if (elapsed_nanosecs / NANOSEC > zfs_txg_timeout ||
476 	    (elapsed_nanosecs / MICROSEC > mintime &&
477 	    txg_sync_waiting(scn->scn_dp)) ||
478 	    spa_shutting_down(scn->scn_dp->dp_spa)) {
479 		if (zb) {
480 			dprintf("pausing at bookmark %llx/%llx/%llx/%llx\n",
481 			    (longlong_t)zb->zb_objset,
482 			    (longlong_t)zb->zb_object,
483 			    (longlong_t)zb->zb_level,
484 			    (longlong_t)zb->zb_blkid);
485 			scn->scn_phys.scn_bookmark = *zb;
486 		}
487 		dprintf("pausing at DDT bookmark %llx/%llx/%llx/%llx\n",
488 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class,
489 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type,
490 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum,
491 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor);
492 		scn->scn_pausing = B_TRUE;
493 		return (B_TRUE);
494 	}
495 	return (B_FALSE);
496 }
497 
498 typedef struct zil_scan_arg {
499 	dsl_pool_t	*zsa_dp;
500 	zil_header_t	*zsa_zh;
501 } zil_scan_arg_t;
502 
503 /* ARGSUSED */
504 static int
505 dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg)
506 {
507 	zil_scan_arg_t *zsa = arg;
508 	dsl_pool_t *dp = zsa->zsa_dp;
509 	dsl_scan_t *scn = dp->dp_scan;
510 	zil_header_t *zh = zsa->zsa_zh;
511 	zbookmark_t zb;
512 
513 	if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
514 		return (0);
515 
516 	/*
517 	 * One block ("stubby") can be allocated a long time ago; we
518 	 * want to visit that one because it has been allocated
519 	 * (on-disk) even if it hasn't been claimed (even though for
520 	 * scrub there's nothing to do to it).
521 	 */
522 	if (claim_txg == 0 && bp->blk_birth >= spa_first_txg(dp->dp_spa))
523 		return (0);
524 
525 	SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET],
526 	    ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]);
527 
528 	VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
529 	return (0);
530 }
531 
532 /* ARGSUSED */
533 static int
534 dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg)
535 {
536 	if (lrc->lrc_txtype == TX_WRITE) {
537 		zil_scan_arg_t *zsa = arg;
538 		dsl_pool_t *dp = zsa->zsa_dp;
539 		dsl_scan_t *scn = dp->dp_scan;
540 		zil_header_t *zh = zsa->zsa_zh;
541 		lr_write_t *lr = (lr_write_t *)lrc;
542 		blkptr_t *bp = &lr->lr_blkptr;
543 		zbookmark_t zb;
544 
545 		if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
546 			return (0);
547 
548 		/*
549 		 * birth can be < claim_txg if this record's txg is
550 		 * already txg sync'ed (but this log block contains
551 		 * other records that are not synced)
552 		 */
553 		if (claim_txg == 0 || bp->blk_birth < claim_txg)
554 			return (0);
555 
556 		SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET],
557 		    lr->lr_foid, ZB_ZIL_LEVEL,
558 		    lr->lr_offset / BP_GET_LSIZE(bp));
559 
560 		VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
561 	}
562 	return (0);
563 }
564 
565 static void
566 dsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh)
567 {
568 	uint64_t claim_txg = zh->zh_claim_txg;
569 	zil_scan_arg_t zsa = { dp, zh };
570 	zilog_t *zilog;
571 
572 	/*
573 	 * We only want to visit blocks that have been claimed but not yet
574 	 * replayed (or, in read-only mode, blocks that *would* be claimed).
575 	 */
576 	if (claim_txg == 0 && spa_writeable(dp->dp_spa))
577 		return;
578 
579 	zilog = zil_alloc(dp->dp_meta_objset, zh);
580 
581 	(void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa,
582 	    claim_txg);
583 
584 	zil_free(zilog);
585 }
586 
587 /* ARGSUSED */
588 static void
589 dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp,
590     uint64_t objset, uint64_t object, uint64_t blkid)
591 {
592 	zbookmark_t czb;
593 	uint32_t flags = ARC_NOWAIT | ARC_PREFETCH;
594 
595 	if (zfs_no_scrub_prefetch)
596 		return;
597 
598 	if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_min_txg ||
599 	    (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE))
600 		return;
601 
602 	SET_BOOKMARK(&czb, objset, object, BP_GET_LEVEL(bp), blkid);
603 
604 	/*
605 	 * XXX need to make sure all of these arc_read() prefetches are
606 	 * done before setting xlateall (similar to dsl_read())
607 	 */
608 	(void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, bp,
609 	    buf, NULL, NULL, ZIO_PRIORITY_ASYNC_READ,
610 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD, &flags, &czb);
611 }
612 
613 static boolean_t
614 dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
615     const zbookmark_t *zb)
616 {
617 	/*
618 	 * We never skip over user/group accounting objects (obj<0)
619 	 */
620 	if (!bookmark_is_zero(&scn->scn_phys.scn_bookmark) &&
621 	    (int64_t)zb->zb_object >= 0) {
622 		/*
623 		 * If we already visited this bp & everything below (in
624 		 * a prior txg sync), don't bother doing it again.
625 		 */
626 		if (bookmark_is_before(dnp, zb, &scn->scn_phys.scn_bookmark))
627 			return (B_TRUE);
628 
629 		/*
630 		 * If we found the block we're trying to resume from, or
631 		 * we went past it to a different object, zero it out to
632 		 * indicate that it's OK to start checking for pausing
633 		 * again.
634 		 */
635 		if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 ||
636 		    zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) {
637 			dprintf("resuming at %llx/%llx/%llx/%llx\n",
638 			    (longlong_t)zb->zb_objset,
639 			    (longlong_t)zb->zb_object,
640 			    (longlong_t)zb->zb_level,
641 			    (longlong_t)zb->zb_blkid);
642 			bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb));
643 		}
644 	}
645 	return (B_FALSE);
646 }
647 
648 /*
649  * Return nonzero on i/o error.
650  * Return new buf to write out in *bufp.
651  */
652 static int
653 dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
654     dnode_phys_t *dnp, const blkptr_t *bp,
655     const zbookmark_t *zb, dmu_tx_t *tx, arc_buf_t **bufp)
656 {
657 	dsl_pool_t *dp = scn->scn_dp;
658 	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD;
659 	int err;
660 
661 	if (BP_GET_LEVEL(bp) > 0) {
662 		uint32_t flags = ARC_WAIT;
663 		int i;
664 		blkptr_t *cbp;
665 		int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
666 
667 		err = arc_read_nolock(NULL, dp->dp_spa, bp,
668 		    arc_getbuf_func, bufp,
669 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
670 		if (err) {
671 			scn->scn_phys.scn_errors++;
672 			return (err);
673 		}
674 		for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) {
675 			dsl_scan_prefetch(scn, *bufp, cbp, zb->zb_objset,
676 			    zb->zb_object, zb->zb_blkid * epb + i);
677 		}
678 		for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) {
679 			zbookmark_t czb;
680 
681 			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
682 			    zb->zb_level - 1,
683 			    zb->zb_blkid * epb + i);
684 			dsl_scan_visitbp(cbp, &czb, dnp,
685 			    *bufp, ds, scn, ostype, tx);
686 		}
687 	} else if (BP_GET_TYPE(bp) == DMU_OT_USERGROUP_USED) {
688 		uint32_t flags = ARC_WAIT;
689 
690 		err = arc_read_nolock(NULL, dp->dp_spa, bp,
691 		    arc_getbuf_func, bufp,
692 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
693 		if (err) {
694 			scn->scn_phys.scn_errors++;
695 			return (err);
696 		}
697 	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
698 		uint32_t flags = ARC_WAIT;
699 		dnode_phys_t *cdnp;
700 		int i, j;
701 		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
702 
703 		err = arc_read_nolock(NULL, dp->dp_spa, bp,
704 		    arc_getbuf_func, bufp,
705 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
706 		if (err) {
707 			scn->scn_phys.scn_errors++;
708 			return (err);
709 		}
710 		for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) {
711 			for (j = 0; j < cdnp->dn_nblkptr; j++) {
712 				blkptr_t *cbp = &cdnp->dn_blkptr[j];
713 				dsl_scan_prefetch(scn, *bufp, cbp,
714 				    zb->zb_objset, zb->zb_blkid * epb + i, j);
715 			}
716 		}
717 		for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) {
718 			dsl_scan_visitdnode(scn, ds, ostype,
719 			    cdnp, *bufp, zb->zb_blkid * epb + i, tx);
720 		}
721 
722 	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
723 		uint32_t flags = ARC_WAIT;
724 		objset_phys_t *osp;
725 
726 		err = arc_read_nolock(NULL, dp->dp_spa, bp,
727 		    arc_getbuf_func, bufp,
728 		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
729 		if (err) {
730 			scn->scn_phys.scn_errors++;
731 			return (err);
732 		}
733 
734 		osp = (*bufp)->b_data;
735 
736 		dsl_scan_visitdnode(scn, ds, osp->os_type,
737 		    &osp->os_meta_dnode, *bufp, DMU_META_DNODE_OBJECT, tx);
738 
739 		if (OBJSET_BUF_HAS_USERUSED(*bufp)) {
740 			/*
741 			 * We also always visit user/group accounting
742 			 * objects, and never skip them, even if we are
743 			 * pausing.  This is necessary so that the space
744 			 * deltas from this txg get integrated.
745 			 */
746 			dsl_scan_visitdnode(scn, ds, osp->os_type,
747 			    &osp->os_groupused_dnode, *bufp,
748 			    DMU_GROUPUSED_OBJECT, tx);
749 			dsl_scan_visitdnode(scn, ds, osp->os_type,
750 			    &osp->os_userused_dnode, *bufp,
751 			    DMU_USERUSED_OBJECT, tx);
752 		}
753 	}
754 
755 	return (0);
756 }
757 
758 static void
759 dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
760     dmu_objset_type_t ostype, dnode_phys_t *dnp, arc_buf_t *buf,
761     uint64_t object, dmu_tx_t *tx)
762 {
763 	int j;
764 
765 	for (j = 0; j < dnp->dn_nblkptr; j++) {
766 		zbookmark_t czb;
767 
768 		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
769 		    dnp->dn_nlevels - 1, j);
770 		dsl_scan_visitbp(&dnp->dn_blkptr[j],
771 		    &czb, dnp, buf, ds, scn, ostype, tx);
772 	}
773 
774 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
775 		zbookmark_t czb;
776 		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
777 		    0, DMU_SPILL_BLKID);
778 		dsl_scan_visitbp(&dnp->dn_spill,
779 		    &czb, dnp, buf, ds, scn, ostype, tx);
780 	}
781 }
782 
783 /*
784  * The arguments are in this order because mdb can only print the
785  * first 5; we want them to be useful.
786  */
787 static void
788 dsl_scan_visitbp(blkptr_t *bp, const zbookmark_t *zb,
789     dnode_phys_t *dnp, arc_buf_t *pbuf,
790     dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype,
791     dmu_tx_t *tx)
792 {
793 	dsl_pool_t *dp = scn->scn_dp;
794 	arc_buf_t *buf = NULL;
795 	blkptr_t bp_toread = *bp;
796 
797 	/* ASSERT(pbuf == NULL || arc_released(pbuf)); */
798 
799 	if (dsl_scan_check_pause(scn, zb))
800 		return;
801 
802 	if (dsl_scan_check_resume(scn, dnp, zb))
803 		return;
804 
805 	if (bp->blk_birth == 0)
806 		return;
807 
808 	scn->scn_visited_this_txg++;
809 
810 	dprintf_bp(bp,
811 	    "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx buf=%p bp=%p",
812 	    ds, ds ? ds->ds_object : 0,
813 	    zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid,
814 	    pbuf, bp);
815 
816 	if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
817 		return;
818 
819 	if (BP_GET_TYPE(bp) != DMU_OT_USERGROUP_USED) {
820 		/*
821 		 * For non-user-accounting blocks, we need to read the
822 		 * new bp (from a deleted snapshot, found in
823 		 * check_existing_xlation).  If we used the old bp,
824 		 * pointers inside this block from before we resumed
825 		 * would be untranslated.
826 		 *
827 		 * For user-accounting blocks, we need to read the old
828 		 * bp, because we will apply the entire space delta to
829 		 * it (original untranslated -> translations from
830 		 * deleted snap -> now).
831 		 */
832 		bp_toread = *bp;
833 	}
834 
835 	if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx,
836 	    &buf) != 0)
837 		return;
838 
839 	/*
840 	 * If dsl_scan_ddt() has aready visited this block, it will have
841 	 * already done any translations or scrubbing, so don't call the
842 	 * callback again.
843 	 */
844 	if (ddt_class_contains(dp->dp_spa,
845 	    scn->scn_phys.scn_ddt_class_max, bp)) {
846 		ASSERT(buf == NULL);
847 		return;
848 	}
849 
850 	/*
851 	 * If this block is from the future (after cur_max_txg), then we
852 	 * are doing this on behalf of a deleted snapshot, and we will
853 	 * revisit the future block on the next pass of this dataset.
854 	 * Don't scan it now unless we need to because something
855 	 * under it was modified.
856 	 */
857 	if (bp->blk_birth <= scn->scn_phys.scn_cur_max_txg) {
858 		scan_funcs[scn->scn_phys.scn_func](dp, bp, zb);
859 	}
860 	if (buf)
861 		(void) arc_buf_remove_ref(buf, &buf);
862 }
863 
864 static void
865 dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp,
866     dmu_tx_t *tx)
867 {
868 	zbookmark_t zb;
869 
870 	SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
871 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
872 	dsl_scan_visitbp(bp, &zb, NULL, NULL,
873 	    ds, scn, DMU_OST_NONE, tx);
874 
875 	dprintf_ds(ds, "finished scan%s", "");
876 }
877 
878 void
879 dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
880 {
881 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
882 	dsl_scan_t *scn = dp->dp_scan;
883 	uint64_t mintxg;
884 
885 	if (scn->scn_phys.scn_state != DSS_SCANNING)
886 		return;
887 
888 	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
889 		if (dsl_dataset_is_snapshot(ds)) {
890 			/* Note, scn_cur_{min,max}_txg stays the same. */
891 			scn->scn_phys.scn_bookmark.zb_objset =
892 			    ds->ds_phys->ds_next_snap_obj;
893 			zfs_dbgmsg("destroying ds %llu; currently traversing; "
894 			    "reset zb_objset to %llu",
895 			    (u_longlong_t)ds->ds_object,
896 			    (u_longlong_t)ds->ds_phys->ds_next_snap_obj);
897 			scn->scn_phys.scn_flags |= DSF_VISIT_DS_AGAIN;
898 		} else {
899 			SET_BOOKMARK(&scn->scn_phys.scn_bookmark,
900 			    ZB_DESTROYED_OBJSET, 0, 0, 0);
901 			zfs_dbgmsg("destroying ds %llu; currently traversing; "
902 			    "reset bookmark to -1,0,0,0",
903 			    (u_longlong_t)ds->ds_object);
904 		}
905 	} else if (zap_lookup_int_key(dp->dp_meta_objset,
906 	    scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) {
907 		ASSERT3U(ds->ds_phys->ds_num_children, <=, 1);
908 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
909 		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
910 		if (dsl_dataset_is_snapshot(ds)) {
911 			/*
912 			 * We keep the same mintxg; it could be >
913 			 * ds_creation_txg if the previous snapshot was
914 			 * deleted too.
915 			 */
916 			VERIFY(zap_add_int_key(dp->dp_meta_objset,
917 			    scn->scn_phys.scn_queue_obj,
918 			    ds->ds_phys->ds_next_snap_obj, mintxg, tx) == 0);
919 			zfs_dbgmsg("destroying ds %llu; in queue; "
920 			    "replacing with %llu",
921 			    (u_longlong_t)ds->ds_object,
922 			    (u_longlong_t)ds->ds_phys->ds_next_snap_obj);
923 		} else {
924 			zfs_dbgmsg("destroying ds %llu; in queue; removing",
925 			    (u_longlong_t)ds->ds_object);
926 		}
927 	} else {
928 		zfs_dbgmsg("destroying ds %llu; ignoring",
929 		    (u_longlong_t)ds->ds_object);
930 	}
931 
932 	/*
933 	 * dsl_scan_sync() should be called after this, and should sync
934 	 * out our changed state, but just to be safe, do it here.
935 	 */
936 	dsl_scan_sync_state(scn, tx);
937 }
938 
939 void
940 dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx)
941 {
942 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
943 	dsl_scan_t *scn = dp->dp_scan;
944 	uint64_t mintxg;
945 
946 	if (scn->scn_phys.scn_state != DSS_SCANNING)
947 		return;
948 
949 	ASSERT(ds->ds_phys->ds_prev_snap_obj != 0);
950 
951 	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
952 		scn->scn_phys.scn_bookmark.zb_objset =
953 		    ds->ds_phys->ds_prev_snap_obj;
954 		zfs_dbgmsg("snapshotting ds %llu; currently traversing; "
955 		    "reset zb_objset to %llu",
956 		    (u_longlong_t)ds->ds_object,
957 		    (u_longlong_t)ds->ds_phys->ds_prev_snap_obj);
958 	} else if (zap_lookup_int_key(dp->dp_meta_objset,
959 	    scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) {
960 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
961 		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
962 		VERIFY(zap_add_int_key(dp->dp_meta_objset,
963 		    scn->scn_phys.scn_queue_obj,
964 		    ds->ds_phys->ds_prev_snap_obj, mintxg, tx) == 0);
965 		zfs_dbgmsg("snapshotting ds %llu; in queue; "
966 		    "replacing with %llu",
967 		    (u_longlong_t)ds->ds_object,
968 		    (u_longlong_t)ds->ds_phys->ds_prev_snap_obj);
969 	}
970 	dsl_scan_sync_state(scn, tx);
971 }
972 
973 void
974 dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx)
975 {
976 	dsl_pool_t *dp = ds1->ds_dir->dd_pool;
977 	dsl_scan_t *scn = dp->dp_scan;
978 	uint64_t mintxg;
979 
980 	if (scn->scn_phys.scn_state != DSS_SCANNING)
981 		return;
982 
983 	if (scn->scn_phys.scn_bookmark.zb_objset == ds1->ds_object) {
984 		scn->scn_phys.scn_bookmark.zb_objset = ds2->ds_object;
985 		zfs_dbgmsg("clone_swap ds %llu; currently traversing; "
986 		    "reset zb_objset to %llu",
987 		    (u_longlong_t)ds1->ds_object,
988 		    (u_longlong_t)ds2->ds_object);
989 	} else if (scn->scn_phys.scn_bookmark.zb_objset == ds2->ds_object) {
990 		scn->scn_phys.scn_bookmark.zb_objset = ds1->ds_object;
991 		zfs_dbgmsg("clone_swap ds %llu; currently traversing; "
992 		    "reset zb_objset to %llu",
993 		    (u_longlong_t)ds2->ds_object,
994 		    (u_longlong_t)ds1->ds_object);
995 	}
996 
997 	if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
998 	    ds1->ds_object, &mintxg) == 0) {
999 		int err;
1000 
1001 		ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg);
1002 		ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg);
1003 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1004 		    scn->scn_phys.scn_queue_obj, ds1->ds_object, tx));
1005 		err = zap_add_int_key(dp->dp_meta_objset,
1006 		    scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx);
1007 		VERIFY(err == 0 || err == EEXIST);
1008 		if (err == EEXIST) {
1009 			/* Both were there to begin with */
1010 			VERIFY(0 == zap_add_int_key(dp->dp_meta_objset,
1011 			    scn->scn_phys.scn_queue_obj,
1012 			    ds1->ds_object, mintxg, tx));
1013 		}
1014 		zfs_dbgmsg("clone_swap ds %llu; in queue; "
1015 		    "replacing with %llu",
1016 		    (u_longlong_t)ds1->ds_object,
1017 		    (u_longlong_t)ds2->ds_object);
1018 	} else if (zap_lookup_int_key(dp->dp_meta_objset,
1019 	    scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg) == 0) {
1020 		ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg);
1021 		ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg);
1022 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1023 		    scn->scn_phys.scn_queue_obj, ds2->ds_object, tx));
1024 		VERIFY(0 == zap_add_int_key(dp->dp_meta_objset,
1025 		    scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx));
1026 		zfs_dbgmsg("clone_swap ds %llu; in queue; "
1027 		    "replacing with %llu",
1028 		    (u_longlong_t)ds2->ds_object,
1029 		    (u_longlong_t)ds1->ds_object);
1030 	}
1031 
1032 	dsl_scan_sync_state(scn, tx);
1033 }
1034 
1035 struct enqueue_clones_arg {
1036 	dmu_tx_t *tx;
1037 	uint64_t originobj;
1038 };
1039 
1040 /* ARGSUSED */
1041 static int
1042 enqueue_clones_cb(spa_t *spa, uint64_t dsobj, const char *dsname, void *arg)
1043 {
1044 	struct enqueue_clones_arg *eca = arg;
1045 	dsl_dataset_t *ds;
1046 	int err;
1047 	dsl_pool_t *dp = spa->spa_dsl_pool;
1048 	dsl_scan_t *scn = dp->dp_scan;
1049 
1050 	err = dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds);
1051 	if (err)
1052 		return (err);
1053 
1054 	if (ds->ds_dir->dd_phys->dd_origin_obj == eca->originobj) {
1055 		while (ds->ds_phys->ds_prev_snap_obj != eca->originobj) {
1056 			dsl_dataset_t *prev;
1057 			err = dsl_dataset_hold_obj(dp,
1058 			    ds->ds_phys->ds_prev_snap_obj, FTAG, &prev);
1059 
1060 			dsl_dataset_rele(ds, FTAG);
1061 			if (err)
1062 				return (err);
1063 			ds = prev;
1064 		}
1065 		VERIFY(zap_add_int_key(dp->dp_meta_objset,
1066 		    scn->scn_phys.scn_queue_obj, ds->ds_object,
1067 		    ds->ds_phys->ds_prev_snap_txg, eca->tx) == 0);
1068 	}
1069 	dsl_dataset_rele(ds, FTAG);
1070 	return (0);
1071 }
1072 
1073 static void
1074 dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
1075 {
1076 	dsl_pool_t *dp = scn->scn_dp;
1077 	dsl_dataset_t *ds;
1078 	objset_t *os;
1079 
1080 	VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
1081 
1082 	if (dmu_objset_from_ds(ds, &os))
1083 		goto out;
1084 
1085 	/*
1086 	 * Only the ZIL in the head (non-snapshot) is valid.  Even though
1087 	 * snapshots can have ZIL block pointers (which may be the same
1088 	 * BP as in the head), they must be ignored.  So we traverse the
1089 	 * ZIL here, rather than in scan_recurse(), because the regular
1090 	 * snapshot block-sharing rules don't apply to it.
1091 	 */
1092 	if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds))
1093 		dsl_scan_zil(dp, &os->os_zil_header);
1094 
1095 	/*
1096 	 * Iterate over the bps in this ds.
1097 	 */
1098 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
1099 	dsl_scan_visit_rootbp(scn, ds, &ds->ds_phys->ds_bp, tx);
1100 
1101 	char *dsname = kmem_alloc(ZFS_MAXNAMELEN, KM_SLEEP);
1102 	dsl_dataset_name(ds, dsname);
1103 	zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; "
1104 	    "pausing=%u",
1105 	    (longlong_t)dsobj, dsname,
1106 	    (longlong_t)scn->scn_phys.scn_cur_min_txg,
1107 	    (longlong_t)scn->scn_phys.scn_cur_max_txg,
1108 	    (int)scn->scn_pausing);
1109 	kmem_free(dsname, ZFS_MAXNAMELEN);
1110 
1111 	if (scn->scn_pausing)
1112 		goto out;
1113 
1114 	/*
1115 	 * We've finished this pass over this dataset.
1116 	 */
1117 
1118 	/*
1119 	 * If we did not completely visit this dataset, do another pass.
1120 	 */
1121 	if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) {
1122 		zfs_dbgmsg("incomplete pass; visiting again");
1123 		scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN;
1124 		VERIFY(zap_add_int_key(dp->dp_meta_objset,
1125 		    scn->scn_phys.scn_queue_obj, ds->ds_object,
1126 		    scn->scn_phys.scn_cur_max_txg, tx) == 0);
1127 		goto out;
1128 	}
1129 
1130 	/*
1131 	 * Add descendent datasets to work queue.
1132 	 */
1133 	if (ds->ds_phys->ds_next_snap_obj != 0) {
1134 		VERIFY(zap_add_int_key(dp->dp_meta_objset,
1135 		    scn->scn_phys.scn_queue_obj, ds->ds_phys->ds_next_snap_obj,
1136 		    ds->ds_phys->ds_creation_txg, tx) == 0);
1137 	}
1138 	if (ds->ds_phys->ds_num_children > 1) {
1139 		boolean_t usenext = B_FALSE;
1140 		if (ds->ds_phys->ds_next_clones_obj != 0) {
1141 			uint64_t count;
1142 			/*
1143 			 * A bug in a previous version of the code could
1144 			 * cause upgrade_clones_cb() to not set
1145 			 * ds_next_snap_obj when it should, leading to a
1146 			 * missing entry.  Therefore we can only use the
1147 			 * next_clones_obj when its count is correct.
1148 			 */
1149 			int err = zap_count(dp->dp_meta_objset,
1150 			    ds->ds_phys->ds_next_clones_obj, &count);
1151 			if (err == 0 &&
1152 			    count == ds->ds_phys->ds_num_children - 1)
1153 				usenext = B_TRUE;
1154 		}
1155 
1156 		if (usenext) {
1157 			VERIFY(zap_join_key(dp->dp_meta_objset,
1158 			    ds->ds_phys->ds_next_clones_obj,
1159 			    scn->scn_phys.scn_queue_obj,
1160 			    ds->ds_phys->ds_creation_txg, tx) == 0);
1161 		} else {
1162 			struct enqueue_clones_arg eca;
1163 			eca.tx = tx;
1164 			eca.originobj = ds->ds_object;
1165 
1166 			(void) dmu_objset_find_spa(ds->ds_dir->dd_pool->dp_spa,
1167 			    NULL, enqueue_clones_cb, &eca, DS_FIND_CHILDREN);
1168 		}
1169 	}
1170 
1171 out:
1172 	dsl_dataset_rele(ds, FTAG);
1173 }
1174 
1175 /* ARGSUSED */
1176 static int
1177 enqueue_cb(spa_t *spa, uint64_t dsobj, const char *dsname, void *arg)
1178 {
1179 	dmu_tx_t *tx = arg;
1180 	dsl_dataset_t *ds;
1181 	int err;
1182 	dsl_pool_t *dp = spa->spa_dsl_pool;
1183 	dsl_scan_t *scn = dp->dp_scan;
1184 
1185 	err = dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds);
1186 	if (err)
1187 		return (err);
1188 
1189 	while (ds->ds_phys->ds_prev_snap_obj != 0) {
1190 		dsl_dataset_t *prev;
1191 		err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj,
1192 		    FTAG, &prev);
1193 		if (err) {
1194 			dsl_dataset_rele(ds, FTAG);
1195 			return (err);
1196 		}
1197 
1198 		/*
1199 		 * If this is a clone, we don't need to worry about it for now.
1200 		 */
1201 		if (prev->ds_phys->ds_next_snap_obj != ds->ds_object) {
1202 			dsl_dataset_rele(ds, FTAG);
1203 			dsl_dataset_rele(prev, FTAG);
1204 			return (0);
1205 		}
1206 		dsl_dataset_rele(ds, FTAG);
1207 		ds = prev;
1208 	}
1209 
1210 	VERIFY(zap_add_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
1211 	    ds->ds_object, ds->ds_phys->ds_prev_snap_txg, tx) == 0);
1212 	dsl_dataset_rele(ds, FTAG);
1213 	return (0);
1214 }
1215 
1216 /*
1217  * Scrub/dedup interaction.
1218  *
1219  * If there are N references to a deduped block, we don't want to scrub it
1220  * N times -- ideally, we should scrub it exactly once.
1221  *
1222  * We leverage the fact that the dde's replication class (enum ddt_class)
1223  * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest
1224  * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order.
1225  *
1226  * To prevent excess scrubbing, the scrub begins by walking the DDT
1227  * to find all blocks with refcnt > 1, and scrubs each of these once.
1228  * Since there are two replication classes which contain blocks with
1229  * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first.
1230  * Finally the top-down scrub begins, only visiting blocks with refcnt == 1.
1231  *
1232  * There would be nothing more to say if a block's refcnt couldn't change
1233  * during a scrub, but of course it can so we must account for changes
1234  * in a block's replication class.
1235  *
1236  * Here's an example of what can occur:
1237  *
1238  * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1
1239  * when visited during the top-down scrub phase, it will be scrubbed twice.
1240  * This negates our scrub optimization, but is otherwise harmless.
1241  *
1242  * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1
1243  * on each visit during the top-down scrub phase, it will never be scrubbed.
1244  * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's
1245  * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to
1246  * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1
1247  * while a scrub is in progress, it scrubs the block right then.
1248  */
1249 static void
1250 dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx)
1251 {
1252 	ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark;
1253 	ddt_entry_t dde = { 0 };
1254 	int error;
1255 	uint64_t n = 0;
1256 
1257 	while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) {
1258 		ddt_t *ddt;
1259 
1260 		if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max)
1261 			break;
1262 		dprintf("visiting ddb=%llu/%llu/%llu/%llx\n",
1263 		    (longlong_t)ddb->ddb_class,
1264 		    (longlong_t)ddb->ddb_type,
1265 		    (longlong_t)ddb->ddb_checksum,
1266 		    (longlong_t)ddb->ddb_cursor);
1267 
1268 		/* There should be no pending changes to the dedup table */
1269 		ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum];
1270 		ASSERT(avl_first(&ddt->ddt_tree) == NULL);
1271 
1272 		dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx);
1273 		n++;
1274 
1275 		if (dsl_scan_check_pause(scn, NULL))
1276 			break;
1277 	}
1278 
1279 	zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; pausing=%u",
1280 	    (longlong_t)n, (int)scn->scn_phys.scn_ddt_class_max,
1281 	    (int)scn->scn_pausing);
1282 
1283 	ASSERT(error == 0 || error == ENOENT);
1284 	ASSERT(error != ENOENT ||
1285 	    ddb->ddb_class > scn->scn_phys.scn_ddt_class_max);
1286 }
1287 
1288 /* ARGSUSED */
1289 void
1290 dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum,
1291     ddt_entry_t *dde, dmu_tx_t *tx)
1292 {
1293 	const ddt_key_t *ddk = &dde->dde_key;
1294 	ddt_phys_t *ddp = dde->dde_phys;
1295 	blkptr_t bp;
1296 	zbookmark_t zb = { 0 };
1297 
1298 	if (scn->scn_phys.scn_state != DSS_SCANNING)
1299 		return;
1300 
1301 	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
1302 		if (ddp->ddp_phys_birth == 0 ||
1303 		    ddp->ddp_phys_birth > scn->scn_phys.scn_cur_max_txg)
1304 			continue;
1305 		ddt_bp_create(checksum, ddk, ddp, &bp);
1306 
1307 		scn->scn_visited_this_txg++;
1308 		scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb);
1309 	}
1310 }
1311 
1312 static void
1313 dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx)
1314 {
1315 	dsl_pool_t *dp = scn->scn_dp;
1316 	zap_cursor_t zc;
1317 	zap_attribute_t za;
1318 
1319 	if (scn->scn_phys.scn_ddt_bookmark.ddb_class <=
1320 	    scn->scn_phys.scn_ddt_class_max) {
1321 		scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg;
1322 		scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg;
1323 		dsl_scan_ddt(scn, tx);
1324 		if (scn->scn_pausing)
1325 			return;
1326 	}
1327 
1328 	if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) {
1329 		/* First do the MOS & ORIGIN */
1330 
1331 		scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg;
1332 		scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg;
1333 		dsl_scan_visit_rootbp(scn, NULL,
1334 		    &dp->dp_meta_rootbp, tx);
1335 		spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp);
1336 		if (scn->scn_pausing)
1337 			return;
1338 
1339 		if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) {
1340 			VERIFY(0 == dmu_objset_find_spa(dp->dp_spa,
1341 			    NULL, enqueue_cb, tx, DS_FIND_CHILDREN));
1342 		} else {
1343 			dsl_scan_visitds(scn,
1344 			    dp->dp_origin_snap->ds_object, tx);
1345 		}
1346 		ASSERT(!scn->scn_pausing);
1347 	} else if (scn->scn_phys.scn_bookmark.zb_objset !=
1348 	    ZB_DESTROYED_OBJSET) {
1349 		/*
1350 		 * If we were paused, continue from here.  Note if the
1351 		 * ds we were paused on was deleted, the zb_objset may
1352 		 * be -1, so we will skip this and find a new objset
1353 		 * below.
1354 		 */
1355 		dsl_scan_visitds(scn, scn->scn_phys.scn_bookmark.zb_objset, tx);
1356 		if (scn->scn_pausing)
1357 			return;
1358 	}
1359 
1360 	/*
1361 	 * In case we were paused right at the end of the ds, zero the
1362 	 * bookmark so we don't think that we're still trying to resume.
1363 	 */
1364 	bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_t));
1365 
1366 	/* keep pulling things out of the zap-object-as-queue */
1367 	while (zap_cursor_init(&zc, dp->dp_meta_objset,
1368 	    scn->scn_phys.scn_queue_obj),
1369 	    zap_cursor_retrieve(&zc, &za) == 0) {
1370 		dsl_dataset_t *ds;
1371 		uint64_t dsobj;
1372 
1373 		dsobj = strtonum(za.za_name, NULL);
1374 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1375 		    scn->scn_phys.scn_queue_obj, dsobj, tx));
1376 
1377 		/* Set up min/max txg */
1378 		VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
1379 		if (za.za_first_integer != 0) {
1380 			scn->scn_phys.scn_cur_min_txg =
1381 			    MAX(scn->scn_phys.scn_min_txg,
1382 			    za.za_first_integer);
1383 		} else {
1384 			scn->scn_phys.scn_cur_min_txg =
1385 			    MAX(scn->scn_phys.scn_min_txg,
1386 			    ds->ds_phys->ds_prev_snap_txg);
1387 		}
1388 		scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds);
1389 		dsl_dataset_rele(ds, FTAG);
1390 
1391 		dsl_scan_visitds(scn, dsobj, tx);
1392 		zap_cursor_fini(&zc);
1393 		if (scn->scn_pausing)
1394 			return;
1395 	}
1396 	zap_cursor_fini(&zc);
1397 }
1398 
1399 static int
1400 dsl_scan_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
1401 {
1402 	dsl_scan_t *scn = arg;
1403 	uint64_t elapsed_nanosecs;
1404 
1405 	elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
1406 
1407 	if (elapsed_nanosecs / NANOSEC > zfs_txg_timeout ||
1408 	    (elapsed_nanosecs / MICROSEC > zfs_free_min_time_ms &&
1409 	    txg_sync_waiting(scn->scn_dp)) ||
1410 	    spa_shutting_down(scn->scn_dp->dp_spa))
1411 		return (ERESTART);
1412 
1413 	zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa,
1414 	    dmu_tx_get_txg(tx), bp, 0));
1415 	dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD,
1416 	    -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp),
1417 	    -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx);
1418 	scn->scn_visited_this_txg++;
1419 	return (0);
1420 }
1421 
1422 boolean_t
1423 dsl_scan_active(dsl_scan_t *scn)
1424 {
1425 	spa_t *spa = scn->scn_dp->dp_spa;
1426 	uint64_t used = 0, comp, uncomp;
1427 
1428 	if (spa->spa_load_state != SPA_LOAD_NONE)
1429 		return (B_FALSE);
1430 	if (spa_shutting_down(spa))
1431 		return (B_FALSE);
1432 
1433 	if (scn->scn_phys.scn_state == DSS_SCANNING)
1434 		return (B_TRUE);
1435 
1436 	if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
1437 		(void) bpobj_space(&scn->scn_dp->dp_free_bpobj,
1438 		    &used, &comp, &uncomp);
1439 	}
1440 	return (used != 0);
1441 }
1442 
1443 void
1444 dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
1445 {
1446 	dsl_scan_t *scn = dp->dp_scan;
1447 	spa_t *spa = dp->dp_spa;
1448 	int err;
1449 
1450 	/*
1451 	 * Check for scn_restart_txg before checking spa_load_state, so
1452 	 * that we can restart an old-style scan while the pool is being
1453 	 * imported (see dsl_scan_init).
1454 	 */
1455 	if (scn->scn_restart_txg != 0 &&
1456 	    scn->scn_restart_txg <= tx->tx_txg) {
1457 		pool_scan_func_t func = POOL_SCAN_SCRUB;
1458 		dsl_scan_done(scn, B_FALSE, tx);
1459 		if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL))
1460 			func = POOL_SCAN_RESILVER;
1461 		zfs_dbgmsg("restarting scan func=%u txg=%llu",
1462 		    func, tx->tx_txg);
1463 		dsl_scan_setup_sync(scn, &func, tx);
1464 	}
1465 
1466 	if (!dsl_scan_active(scn) ||
1467 	    spa_sync_pass(dp->dp_spa) > 1)
1468 		return;
1469 
1470 	scn->scn_visited_this_txg = 0;
1471 	scn->scn_pausing = B_FALSE;
1472 	scn->scn_sync_start_time = gethrtime();
1473 	spa->spa_scrub_active = B_TRUE;
1474 
1475 	/*
1476 	 * First process the free list.  If we pause the free, don't do
1477 	 * any scanning.  This ensures that there is no free list when
1478 	 * we are scanning, so the scan code doesn't have to worry about
1479 	 * traversing it.
1480 	 */
1481 	if (spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
1482 		scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
1483 		    NULL, ZIO_FLAG_MUSTSUCCEED);
1484 		err = bpobj_iterate(&dp->dp_free_bpobj,
1485 		    dsl_scan_free_cb, scn, tx);
1486 		VERIFY3U(0, ==, zio_wait(scn->scn_zio_root));
1487 		if (scn->scn_visited_this_txg) {
1488 			zfs_dbgmsg("freed %llu blocks in %llums from "
1489 			    "free_bpobj txg %llu",
1490 			    (longlong_t)scn->scn_visited_this_txg,
1491 			    (longlong_t)
1492 			    (gethrtime() - scn->scn_sync_start_time) / MICROSEC,
1493 			    (longlong_t)tx->tx_txg);
1494 			scn->scn_visited_this_txg = 0;
1495 			/*
1496 			 * Re-sync the ddt so that we can further modify
1497 			 * it when doing bprewrite.
1498 			 */
1499 			ddt_sync(spa, tx->tx_txg);
1500 		}
1501 		if (err == ERESTART)
1502 			return;
1503 	}
1504 
1505 	if (scn->scn_phys.scn_state != DSS_SCANNING)
1506 		return;
1507 
1508 	if (scn->scn_phys.scn_ddt_bookmark.ddb_class <=
1509 	    scn->scn_phys.scn_ddt_class_max) {
1510 		zfs_dbgmsg("doing scan sync txg %llu; "
1511 		    "ddt bm=%llu/%llu/%llu/%llx",
1512 		    (longlong_t)tx->tx_txg,
1513 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class,
1514 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type,
1515 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum,
1516 		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor);
1517 		ASSERT(scn->scn_phys.scn_bookmark.zb_objset == 0);
1518 		ASSERT(scn->scn_phys.scn_bookmark.zb_object == 0);
1519 		ASSERT(scn->scn_phys.scn_bookmark.zb_level == 0);
1520 		ASSERT(scn->scn_phys.scn_bookmark.zb_blkid == 0);
1521 	} else {
1522 		zfs_dbgmsg("doing scan sync txg %llu; bm=%llu/%llu/%llu/%llu",
1523 		    (longlong_t)tx->tx_txg,
1524 		    (longlong_t)scn->scn_phys.scn_bookmark.zb_objset,
1525 		    (longlong_t)scn->scn_phys.scn_bookmark.zb_object,
1526 		    (longlong_t)scn->scn_phys.scn_bookmark.zb_level,
1527 		    (longlong_t)scn->scn_phys.scn_bookmark.zb_blkid);
1528 	}
1529 
1530 	scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
1531 	    NULL, ZIO_FLAG_CANFAIL);
1532 	dsl_scan_visit(scn, tx);
1533 	(void) zio_wait(scn->scn_zio_root);
1534 	scn->scn_zio_root = NULL;
1535 
1536 	zfs_dbgmsg("visited %llu blocks in %llums",
1537 	    (longlong_t)scn->scn_visited_this_txg,
1538 	    (longlong_t)(gethrtime() - scn->scn_sync_start_time) / MICROSEC);
1539 
1540 	if (!scn->scn_pausing) {
1541 		/* finished with scan. */
1542 		zfs_dbgmsg("finished scan txg %llu", (longlong_t)tx->tx_txg);
1543 		dsl_scan_done(scn, B_TRUE, tx);
1544 	}
1545 
1546 	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
1547 		mutex_enter(&spa->spa_scrub_lock);
1548 		while (spa->spa_scrub_inflight > 0) {
1549 			cv_wait(&spa->spa_scrub_io_cv,
1550 			    &spa->spa_scrub_lock);
1551 		}
1552 		mutex_exit(&spa->spa_scrub_lock);
1553 	}
1554 
1555 	dsl_scan_sync_state(scn, tx);
1556 }
1557 
1558 /*
1559  * This will start a new scan, or restart an existing one.
1560  */
1561 void
1562 dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg)
1563 {
1564 	if (txg == 0) {
1565 		dmu_tx_t *tx;
1566 		tx = dmu_tx_create_dd(dp->dp_mos_dir);
1567 		VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT));
1568 
1569 		txg = dmu_tx_get_txg(tx);
1570 		dp->dp_scan->scn_restart_txg = txg;
1571 		dmu_tx_commit(tx);
1572 	} else {
1573 		dp->dp_scan->scn_restart_txg = txg;
1574 	}
1575 	zfs_dbgmsg("restarting resilver txg=%llu", txg);
1576 }
1577 
1578 boolean_t
1579 dsl_scan_resilvering(dsl_pool_t *dp)
1580 {
1581 	return (dp->dp_scan->scn_phys.scn_state == DSS_SCANNING &&
1582 	    dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER);
1583 }
1584 
1585 /*
1586  * scrub consumers
1587  */
1588 
1589 static void
1590 count_block(zfs_all_blkstats_t *zab, const blkptr_t *bp)
1591 {
1592 	int i;
1593 
1594 	/*
1595 	 * If we resume after a reboot, zab will be NULL; don't record
1596 	 * incomplete stats in that case.
1597 	 */
1598 	if (zab == NULL)
1599 		return;
1600 
1601 	for (i = 0; i < 4; i++) {
1602 		int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS;
1603 		int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL;
1604 		zfs_blkstat_t *zb = &zab->zab_type[l][t];
1605 		int equal;
1606 
1607 		zb->zb_count++;
1608 		zb->zb_asize += BP_GET_ASIZE(bp);
1609 		zb->zb_lsize += BP_GET_LSIZE(bp);
1610 		zb->zb_psize += BP_GET_PSIZE(bp);
1611 		zb->zb_gangs += BP_COUNT_GANG(bp);
1612 
1613 		switch (BP_GET_NDVAS(bp)) {
1614 		case 2:
1615 			if (DVA_GET_VDEV(&bp->blk_dva[0]) ==
1616 			    DVA_GET_VDEV(&bp->blk_dva[1]))
1617 				zb->zb_ditto_2_of_2_samevdev++;
1618 			break;
1619 		case 3:
1620 			equal = (DVA_GET_VDEV(&bp->blk_dva[0]) ==
1621 			    DVA_GET_VDEV(&bp->blk_dva[1])) +
1622 			    (DVA_GET_VDEV(&bp->blk_dva[0]) ==
1623 			    DVA_GET_VDEV(&bp->blk_dva[2])) +
1624 			    (DVA_GET_VDEV(&bp->blk_dva[1]) ==
1625 			    DVA_GET_VDEV(&bp->blk_dva[2]));
1626 			if (equal == 1)
1627 				zb->zb_ditto_2_of_3_samevdev++;
1628 			else if (equal == 3)
1629 				zb->zb_ditto_3_of_3_samevdev++;
1630 			break;
1631 		}
1632 	}
1633 }
1634 
1635 static void
1636 dsl_scan_scrub_done(zio_t *zio)
1637 {
1638 	spa_t *spa = zio->io_spa;
1639 
1640 	zio_data_buf_free(zio->io_data, zio->io_size);
1641 
1642 	mutex_enter(&spa->spa_scrub_lock);
1643 	spa->spa_scrub_inflight--;
1644 	cv_broadcast(&spa->spa_scrub_io_cv);
1645 
1646 	if (zio->io_error && (zio->io_error != ECKSUM ||
1647 	    !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) {
1648 		spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors++;
1649 	}
1650 	mutex_exit(&spa->spa_scrub_lock);
1651 }
1652 
1653 static int
1654 dsl_scan_scrub_cb(dsl_pool_t *dp,
1655     const blkptr_t *bp, const zbookmark_t *zb)
1656 {
1657 	dsl_scan_t *scn = dp->dp_scan;
1658 	size_t size = BP_GET_PSIZE(bp);
1659 	spa_t *spa = dp->dp_spa;
1660 	uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp);
1661 	boolean_t needs_io;
1662 	int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL;
1663 	int zio_priority;
1664 	int scan_delay = 0;
1665 
1666 	if (phys_birth <= scn->scn_phys.scn_min_txg ||
1667 	    phys_birth >= scn->scn_phys.scn_max_txg)
1668 		return (0);
1669 
1670 	count_block(dp->dp_blkstats, bp);
1671 
1672 	ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn));
1673 	if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) {
1674 		zio_flags |= ZIO_FLAG_SCRUB;
1675 		zio_priority = ZIO_PRIORITY_SCRUB;
1676 		needs_io = B_TRUE;
1677 		scan_delay = zfs_scrub_delay;
1678 	} else if (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) {
1679 		zio_flags |= ZIO_FLAG_RESILVER;
1680 		zio_priority = ZIO_PRIORITY_RESILVER;
1681 		needs_io = B_FALSE;
1682 		scan_delay = zfs_resilver_delay;
1683 	}
1684 
1685 	/* If it's an intent log block, failure is expected. */
1686 	if (zb->zb_level == ZB_ZIL_LEVEL)
1687 		zio_flags |= ZIO_FLAG_SPECULATIVE;
1688 
1689 	for (int d = 0; d < BP_GET_NDVAS(bp); d++) {
1690 		vdev_t *vd = vdev_lookup_top(spa,
1691 		    DVA_GET_VDEV(&bp->blk_dva[d]));
1692 
1693 		/*
1694 		 * Keep track of how much data we've examined so that
1695 		 * zpool(1M) status can make useful progress reports.
1696 		 */
1697 		scn->scn_phys.scn_examined += DVA_GET_ASIZE(&bp->blk_dva[d]);
1698 		spa->spa_scan_pass_exam += DVA_GET_ASIZE(&bp->blk_dva[d]);
1699 
1700 		/* if it's a resilver, this may not be in the target range */
1701 		if (!needs_io) {
1702 			if (DVA_GET_GANG(&bp->blk_dva[d])) {
1703 				/*
1704 				 * Gang members may be spread across multiple
1705 				 * vdevs, so the best estimate we have is the
1706 				 * scrub range, which has already been checked.
1707 				 * XXX -- it would be better to change our
1708 				 * allocation policy to ensure that all
1709 				 * gang members reside on the same vdev.
1710 				 */
1711 				needs_io = B_TRUE;
1712 			} else {
1713 				needs_io = vdev_dtl_contains(vd, DTL_PARTIAL,
1714 				    phys_birth, 1);
1715 			}
1716 		}
1717 	}
1718 
1719 	if (needs_io && !zfs_no_scrub_io) {
1720 		vdev_t *rvd = spa->spa_root_vdev;
1721 		uint64_t maxinflight = rvd->vdev_children * zfs_top_maxinflight;
1722 		void *data = zio_data_buf_alloc(size);
1723 
1724 		mutex_enter(&spa->spa_scrub_lock);
1725 		while (spa->spa_scrub_inflight >= maxinflight)
1726 			cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
1727 		spa->spa_scrub_inflight++;
1728 		mutex_exit(&spa->spa_scrub_lock);
1729 
1730 		/*
1731 		 * If we're seeing recent (zfs_scan_idle) "important" I/Os
1732 		 * then throttle our workload to limit the impact of a scan.
1733 		 */
1734 		if (ddi_get_lbolt64() - spa->spa_last_io <= zfs_scan_idle)
1735 			delay(scan_delay);
1736 
1737 		zio_nowait(zio_read(NULL, spa, bp, data, size,
1738 		    dsl_scan_scrub_done, NULL, zio_priority,
1739 		    zio_flags, zb));
1740 	}
1741 
1742 	/* do not relocate this block */
1743 	return (0);
1744 }
1745 
1746 int
1747 dsl_scan(dsl_pool_t *dp, pool_scan_func_t func)
1748 {
1749 	spa_t *spa = dp->dp_spa;
1750 
1751 	/*
1752 	 * Purge all vdev caches and probe all devices.  We do this here
1753 	 * rather than in sync context because this requires a writer lock
1754 	 * on the spa_config lock, which we can't do from sync context.  The
1755 	 * spa_scrub_reopen flag indicates that vdev_open() should not
1756 	 * attempt to start another scrub.
1757 	 */
1758 	spa_vdev_state_enter(spa, SCL_NONE);
1759 	spa->spa_scrub_reopen = B_TRUE;
1760 	vdev_reopen(spa->spa_root_vdev);
1761 	spa->spa_scrub_reopen = B_FALSE;
1762 	(void) spa_vdev_state_exit(spa, NULL, 0);
1763 
1764 	return (dsl_sync_task_do(dp, dsl_scan_setup_check,
1765 	    dsl_scan_setup_sync, dp->dp_scan, &func, 0));
1766 }
1767