1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2016 Gary Mills
24 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
25 * Copyright 2017 Joyent, Inc.
26 * Copyright (c) 2017 Datto Inc.
27 */
28
29#include <sys/dsl_scan.h>
30#include <sys/dsl_pool.h>
31#include <sys/dsl_dataset.h>
32#include <sys/dsl_prop.h>
33#include <sys/dsl_dir.h>
34#include <sys/dsl_synctask.h>
35#include <sys/dnode.h>
36#include <sys/dmu_tx.h>
37#include <sys/dmu_objset.h>
38#include <sys/arc.h>
39#include <sys/zap.h>
40#include <sys/zio.h>
41#include <sys/zfs_context.h>
42#include <sys/fs/zfs.h>
43#include <sys/zfs_znode.h>
44#include <sys/spa_impl.h>
45#include <sys/vdev_impl.h>
46#include <sys/zil_impl.h>
47#include <sys/zio_checksum.h>
48#include <sys/ddt.h>
49#include <sys/sa.h>
50#include <sys/sa_impl.h>
51#include <sys/zfeature.h>
52#include <sys/abd.h>
53#ifdef _KERNEL
54#include <sys/zfs_vfsops.h>
55#endif
56
57typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *,
58    const zbookmark_phys_t *);
59
60static scan_cb_t dsl_scan_scrub_cb;
61static void dsl_scan_cancel_sync(void *, dmu_tx_t *);
62static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *);
63static boolean_t dsl_scan_restarting(dsl_scan_t *, dmu_tx_t *);
64
65int zfs_top_maxinflight = 32;		/* maximum I/Os per top-level */
66int zfs_resilver_delay = 2;		/* number of ticks to delay resilver */
67int zfs_scrub_delay = 4;		/* number of ticks to delay scrub */
68int zfs_scan_idle = 50;			/* idle window in clock ticks */
69
70int zfs_scan_min_time_ms = 1000; /* min millisecs to scrub per txg */
71int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */
72int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver per txg */
73boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */
74boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */
75enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE;
76int dsl_scan_delay_completion = B_FALSE; /* set to delay scan completion */
77/* max number of blocks to free in a single TXG */
78uint64_t zfs_free_max_blocks = UINT64_MAX;
79
80#define	DSL_SCAN_IS_SCRUB_RESILVER(scn) \
81	((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
82	(scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
83
84extern int zfs_txg_timeout;
85
86/*
87 * Enable/disable the processing of the free_bpobj object.
88 */
89boolean_t zfs_free_bpobj_enabled = B_TRUE;
90
91/* the order has to match pool_scan_type */
92static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = {
93	NULL,
94	dsl_scan_scrub_cb,	/* POOL_SCAN_SCRUB */
95	dsl_scan_scrub_cb,	/* POOL_SCAN_RESILVER */
96};
97
98int
99dsl_scan_init(dsl_pool_t *dp, uint64_t txg)
100{
101	int err;
102	dsl_scan_t *scn;
103	spa_t *spa = dp->dp_spa;
104	uint64_t f;
105
106	scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP);
107	scn->scn_dp = dp;
108
109	/*
110	 * It's possible that we're resuming a scan after a reboot so
111	 * make sure that the scan_async_destroying flag is initialized
112	 * appropriately.
113	 */
114	ASSERT(!scn->scn_async_destroying);
115	scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa,
116	    SPA_FEATURE_ASYNC_DESTROY);
117
118	err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
119	    "scrub_func", sizeof (uint64_t), 1, &f);
120	if (err == 0) {
121		/*
122		 * There was an old-style scrub in progress.  Restart a
123		 * new-style scrub from the beginning.
124		 */
125		scn->scn_restart_txg = txg;
126		zfs_dbgmsg("old-style scrub was in progress; "
127		    "restarting new-style scrub in txg %llu",
128		    scn->scn_restart_txg);
129
130		/*
131		 * Load the queue obj from the old location so that it
132		 * can be freed by dsl_scan_done().
133		 */
134		(void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
135		    "scrub_queue", sizeof (uint64_t), 1,
136		    &scn->scn_phys.scn_queue_obj);
137	} else {
138		err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
139		    DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS,
140		    &scn->scn_phys);
141		if (err == ENOENT)
142			return (0);
143		else if (err)
144			return (err);
145
146		if (scn->scn_phys.scn_state == DSS_SCANNING &&
147		    spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) {
148			/*
149			 * A new-type scrub was in progress on an old
150			 * pool, and the pool was accessed by old
151			 * software.  Restart from the beginning, since
152			 * the old software may have changed the pool in
153			 * the meantime.
154			 */
155			scn->scn_restart_txg = txg;
156			zfs_dbgmsg("new-style scrub was modified "
157			    "by old software; restarting in txg %llu",
158			    scn->scn_restart_txg);
159		}
160	}
161
162	spa_scan_stat_init(spa);
163	return (0);
164}
165
166void
167dsl_scan_fini(dsl_pool_t *dp)
168{
169	if (dp->dp_scan) {
170		kmem_free(dp->dp_scan, sizeof (dsl_scan_t));
171		dp->dp_scan = NULL;
172	}
173}
174
175/* ARGSUSED */
176static int
177dsl_scan_setup_check(void *arg, dmu_tx_t *tx)
178{
179	dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
180
181	if (scn->scn_phys.scn_state == DSS_SCANNING)
182		return (SET_ERROR(EBUSY));
183
184	return (0);
185}
186
187static void
188dsl_scan_setup_sync(void *arg, dmu_tx_t *tx)
189{
190	dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
191	pool_scan_func_t *funcp = arg;
192	dmu_object_type_t ot = 0;
193	dsl_pool_t *dp = scn->scn_dp;
194	spa_t *spa = dp->dp_spa;
195
196	ASSERT(scn->scn_phys.scn_state != DSS_SCANNING);
197	ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS);
198	bzero(&scn->scn_phys, sizeof (scn->scn_phys));
199	scn->scn_phys.scn_func = *funcp;
200	scn->scn_phys.scn_state = DSS_SCANNING;
201	scn->scn_phys.scn_min_txg = 0;
202	scn->scn_phys.scn_max_txg = tx->tx_txg;
203	scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */
204	scn->scn_phys.scn_start_time = gethrestime_sec();
205	scn->scn_phys.scn_errors = 0;
206	scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc;
207	scn->scn_restart_txg = 0;
208	scn->scn_done_txg = 0;
209	spa_scan_stat_init(spa);
210
211	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
212		scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max;
213
214		/* rewrite all disk labels */
215		vdev_config_dirty(spa->spa_root_vdev);
216
217		if (vdev_resilver_needed(spa->spa_root_vdev,
218		    &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) {
219			spa_event_notify(spa, NULL, NULL,
220			    ESC_ZFS_RESILVER_START);
221		} else {
222			spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_START);
223		}
224
225		spa->spa_scrub_started = B_TRUE;
226		/*
227		 * If this is an incremental scrub, limit the DDT scrub phase
228		 * to just the auto-ditto class (for correctness); the rest
229		 * of the scrub should go faster using top-down pruning.
230		 */
231		if (scn->scn_phys.scn_min_txg > TXG_INITIAL)
232			scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO;
233
234	}
235
236	/* back to the generic stuff */
237
238	if (dp->dp_blkstats == NULL) {
239		dp->dp_blkstats =
240		    kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP);
241	}
242	bzero(dp->dp_blkstats, sizeof (zfs_all_blkstats_t));
243
244	if (spa_version(spa) < SPA_VERSION_DSL_SCRUB)
245		ot = DMU_OT_ZAP_OTHER;
246
247	scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset,
248	    ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx);
249
250	dsl_scan_sync_state(scn, tx);
251
252	spa_history_log_internal(spa, "scan setup", tx,
253	    "func=%u mintxg=%llu maxtxg=%llu",
254	    *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg);
255}
256
257/* ARGSUSED */
258static void
259dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
260{
261	static const char *old_names[] = {
262		"scrub_bookmark",
263		"scrub_ddt_bookmark",
264		"scrub_ddt_class_max",
265		"scrub_queue",
266		"scrub_min_txg",
267		"scrub_max_txg",
268		"scrub_func",
269		"scrub_errors",
270		NULL
271	};
272
273	dsl_pool_t *dp = scn->scn_dp;
274	spa_t *spa = dp->dp_spa;
275	int i;
276
277	/* Remove any remnants of an old-style scrub. */
278	for (i = 0; old_names[i]; i++) {
279		(void) zap_remove(dp->dp_meta_objset,
280		    DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx);
281	}
282
283	if (scn->scn_phys.scn_queue_obj != 0) {
284		VERIFY(0 == dmu_object_free(dp->dp_meta_objset,
285		    scn->scn_phys.scn_queue_obj, tx));
286		scn->scn_phys.scn_queue_obj = 0;
287	}
288
289	scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED;
290
291	/*
292	 * If we were "restarted" from a stopped state, don't bother
293	 * with anything else.
294	 */
295	if (scn->scn_phys.scn_state != DSS_SCANNING)
296		return;
297
298	if (complete)
299		scn->scn_phys.scn_state = DSS_FINISHED;
300	else
301		scn->scn_phys.scn_state = DSS_CANCELED;
302
303	if (dsl_scan_restarting(scn, tx))
304		spa_history_log_internal(spa, "scan aborted, restarting", tx,
305		    "errors=%llu", spa_get_errlog_size(spa));
306	else if (!complete)
307		spa_history_log_internal(spa, "scan cancelled", tx,
308		    "errors=%llu", spa_get_errlog_size(spa));
309	else
310		spa_history_log_internal(spa, "scan done", tx,
311		    "errors=%llu", spa_get_errlog_size(spa));
312
313	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
314		mutex_enter(&spa->spa_scrub_lock);
315		while (spa->spa_scrub_inflight > 0) {
316			cv_wait(&spa->spa_scrub_io_cv,
317			    &spa->spa_scrub_lock);
318		}
319		mutex_exit(&spa->spa_scrub_lock);
320		spa->spa_scrub_started = B_FALSE;
321		spa->spa_scrub_active = B_FALSE;
322
323		/*
324		 * If the scrub/resilver completed, update all DTLs to
325		 * reflect this.  Whether it succeeded or not, vacate
326		 * all temporary scrub DTLs.
327		 */
328		vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg,
329		    complete ? scn->scn_phys.scn_max_txg : 0, B_TRUE);
330		if (complete) {
331			spa_event_notify(spa, NULL, NULL,
332			    scn->scn_phys.scn_min_txg ?
333			    ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH);
334		}
335		spa_errlog_rotate(spa);
336
337		/*
338		 * We may have finished replacing a device.
339		 * Let the async thread assess this and handle the detach.
340		 */
341		spa_async_request(spa, SPA_ASYNC_RESILVER_DONE);
342	}
343
344	scn->scn_phys.scn_end_time = gethrestime_sec();
345}
346
347/* ARGSUSED */
348static int
349dsl_scan_cancel_check(void *arg, dmu_tx_t *tx)
350{
351	dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
352
353	if (scn->scn_phys.scn_state != DSS_SCANNING)
354		return (SET_ERROR(ENOENT));
355	return (0);
356}
357
358/* ARGSUSED */
359static void
360dsl_scan_cancel_sync(void *arg, dmu_tx_t *tx)
361{
362	dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
363
364	dsl_scan_done(scn, B_FALSE, tx);
365	dsl_scan_sync_state(scn, tx);
366}
367
368int
369dsl_scan_cancel(dsl_pool_t *dp)
370{
371	return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check,
372	    dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED));
373}
374
375boolean_t
376dsl_scan_is_paused_scrub(const dsl_scan_t *scn)
377{
378	if (dsl_scan_scrubbing(scn->scn_dp) &&
379	    scn->scn_phys.scn_flags & DSF_SCRUB_PAUSED)
380		return (B_TRUE);
381
382	return (B_FALSE);
383}
384
385static int
386dsl_scrub_pause_resume_check(void *arg, dmu_tx_t *tx)
387{
388	pool_scrub_cmd_t *cmd = arg;
389	dsl_pool_t *dp = dmu_tx_pool(tx);
390	dsl_scan_t *scn = dp->dp_scan;
391
392	if (*cmd == POOL_SCRUB_PAUSE) {
393		/* can't pause a scrub when there is no in-progress scrub */
394		if (!dsl_scan_scrubbing(dp))
395			return (SET_ERROR(ENOENT));
396
397		/* can't pause a paused scrub */
398		if (dsl_scan_is_paused_scrub(scn))
399			return (SET_ERROR(EBUSY));
400	} else if (*cmd != POOL_SCRUB_NORMAL) {
401		return (SET_ERROR(ENOTSUP));
402	}
403
404	return (0);
405}
406
407static void
408dsl_scrub_pause_resume_sync(void *arg, dmu_tx_t *tx)
409{
410	pool_scrub_cmd_t *cmd = arg;
411	dsl_pool_t *dp = dmu_tx_pool(tx);
412	spa_t *spa = dp->dp_spa;
413	dsl_scan_t *scn = dp->dp_scan;
414
415	if (*cmd == POOL_SCRUB_PAUSE) {
416		/* can't pause a scrub when there is no in-progress scrub */
417		spa->spa_scan_pass_scrub_pause = gethrestime_sec();
418		scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED;
419		dsl_scan_sync_state(scn, tx);
420	} else {
421		ASSERT3U(*cmd, ==, POOL_SCRUB_NORMAL);
422		if (dsl_scan_is_paused_scrub(scn)) {
423			/*
424			 * We need to keep track of how much time we spend
425			 * paused per pass so that we can adjust the scrub rate
426			 * shown in the output of 'zpool status'
427			 */
428			spa->spa_scan_pass_scrub_spent_paused +=
429			    gethrestime_sec() - spa->spa_scan_pass_scrub_pause;
430			spa->spa_scan_pass_scrub_pause = 0;
431			scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED;
432			dsl_scan_sync_state(scn, tx);
433		}
434	}
435}
436
437/*
438 * Set scrub pause/resume state if it makes sense to do so
439 */
440int
441dsl_scrub_set_pause_resume(const dsl_pool_t *dp, pool_scrub_cmd_t cmd)
442{
443	return (dsl_sync_task(spa_name(dp->dp_spa),
444	    dsl_scrub_pause_resume_check, dsl_scrub_pause_resume_sync, &cmd, 3,
445	    ZFS_SPACE_CHECK_RESERVED));
446}
447
448boolean_t
449dsl_scan_scrubbing(const dsl_pool_t *dp)
450{
451	dsl_scan_t *scn = dp->dp_scan;
452
453	if (scn->scn_phys.scn_state == DSS_SCANNING &&
454	    scn->scn_phys.scn_func == POOL_SCAN_SCRUB)
455		return (B_TRUE);
456
457	return (B_FALSE);
458}
459
460static void dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
461    dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
462    dmu_objset_type_t ostype, dmu_tx_t *tx);
463static void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds,
464    dmu_objset_type_t ostype,
465    dnode_phys_t *dnp, uint64_t object, dmu_tx_t *tx);
466
467void
468dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp)
469{
470	zio_free(dp->dp_spa, txg, bp);
471}
472
473void
474dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp)
475{
476	ASSERT(dsl_pool_sync_context(dp));
477	zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, pio->io_flags));
478}
479
480static uint64_t
481dsl_scan_ds_maxtxg(dsl_dataset_t *ds)
482{
483	uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg;
484	if (ds->ds_is_snapshot)
485		return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg));
486	return (smt);
487}
488
489static void
490dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx)
491{
492	VERIFY0(zap_update(scn->scn_dp->dp_meta_objset,
493	    DMU_POOL_DIRECTORY_OBJECT,
494	    DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS,
495	    &scn->scn_phys, tx));
496}
497
498extern int zfs_vdev_async_write_active_min_dirty_percent;
499
500static boolean_t
501dsl_scan_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb)
502{
503	/* we never skip user/group accounting objects */
504	if (zb && (int64_t)zb->zb_object < 0)
505		return (B_FALSE);
506
507	if (scn->scn_suspending)
508		return (B_TRUE); /* we're already suspending */
509
510	if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark))
511		return (B_FALSE); /* we're resuming */
512
513	/* We only know how to resume from level-0 blocks. */
514	if (zb && zb->zb_level != 0)
515		return (B_FALSE);
516
517	/*
518	 * We suspend if:
519	 *  - we have scanned for the maximum time: an entire txg
520	 *    timeout (default 5 sec)
521	 *  or
522	 *  - we have scanned for at least the minimum time (default 1 sec
523	 *    for scrub, 3 sec for resilver), and either we have sufficient
524	 *    dirty data that we are starting to write more quickly
525	 *    (default 30%), or someone is explicitly waiting for this txg
526	 *    to complete.
527	 *  or
528	 *  - the spa is shutting down because this pool is being exported
529	 *    or the machine is rebooting.
530	 */
531	int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ?
532	    zfs_resilver_min_time_ms : zfs_scan_min_time_ms;
533	uint64_t elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
534	int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max;
535	if (elapsed_nanosecs / NANOSEC >= zfs_txg_timeout ||
536	    (NSEC2MSEC(elapsed_nanosecs) > mintime &&
537	    (txg_sync_waiting(scn->scn_dp) ||
538	    dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent)) ||
539	    spa_shutting_down(scn->scn_dp->dp_spa)) {
540		if (zb) {
541			dprintf("suspending at bookmark %llx/%llx/%llx/%llx\n",
542			    (longlong_t)zb->zb_objset,
543			    (longlong_t)zb->zb_object,
544			    (longlong_t)zb->zb_level,
545			    (longlong_t)zb->zb_blkid);
546			scn->scn_phys.scn_bookmark = *zb;
547		}
548		dprintf("suspending at DDT bookmark %llx/%llx/%llx/%llx\n",
549		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class,
550		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type,
551		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum,
552		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor);
553		scn->scn_suspending = B_TRUE;
554		return (B_TRUE);
555	}
556	return (B_FALSE);
557}
558
559typedef struct zil_scan_arg {
560	dsl_pool_t	*zsa_dp;
561	zil_header_t	*zsa_zh;
562} zil_scan_arg_t;
563
564/* ARGSUSED */
565static int
566dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg)
567{
568	zil_scan_arg_t *zsa = arg;
569	dsl_pool_t *dp = zsa->zsa_dp;
570	dsl_scan_t *scn = dp->dp_scan;
571	zil_header_t *zh = zsa->zsa_zh;
572	zbookmark_phys_t zb;
573
574	if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
575		return (0);
576
577	/*
578	 * One block ("stubby") can be allocated a long time ago; we
579	 * want to visit that one because it has been allocated
580	 * (on-disk) even if it hasn't been claimed (even though for
581	 * scrub there's nothing to do to it).
582	 */
583	if (claim_txg == 0 && bp->blk_birth >= spa_first_txg(dp->dp_spa))
584		return (0);
585
586	SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET],
587	    ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]);
588
589	VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
590	return (0);
591}
592
593/* ARGSUSED */
594static int
595dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg)
596{
597	if (lrc->lrc_txtype == TX_WRITE) {
598		zil_scan_arg_t *zsa = arg;
599		dsl_pool_t *dp = zsa->zsa_dp;
600		dsl_scan_t *scn = dp->dp_scan;
601		zil_header_t *zh = zsa->zsa_zh;
602		lr_write_t *lr = (lr_write_t *)lrc;
603		blkptr_t *bp = &lr->lr_blkptr;
604		zbookmark_phys_t zb;
605
606		if (BP_IS_HOLE(bp) ||
607		    bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
608			return (0);
609
610		/*
611		 * birth can be < claim_txg if this record's txg is
612		 * already txg sync'ed (but this log block contains
613		 * other records that are not synced)
614		 */
615		if (claim_txg == 0 || bp->blk_birth < claim_txg)
616			return (0);
617
618		SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET],
619		    lr->lr_foid, ZB_ZIL_LEVEL,
620		    lr->lr_offset / BP_GET_LSIZE(bp));
621
622		VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
623	}
624	return (0);
625}
626
627static void
628dsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh)
629{
630	uint64_t claim_txg = zh->zh_claim_txg;
631	zil_scan_arg_t zsa = { dp, zh };
632	zilog_t *zilog;
633
634	/*
635	 * We only want to visit blocks that have been claimed but not yet
636	 * replayed (or, in read-only mode, blocks that *would* be claimed).
637	 */
638	if (claim_txg == 0 && spa_writeable(dp->dp_spa))
639		return;
640
641	zilog = zil_alloc(dp->dp_meta_objset, zh);
642
643	(void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa,
644	    claim_txg);
645
646	zil_free(zilog);
647}
648
649/* ARGSUSED */
650static void
651dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp,
652    uint64_t objset, uint64_t object, uint64_t blkid)
653{
654	zbookmark_phys_t czb;
655	arc_flags_t flags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH;
656
657	if (zfs_no_scrub_prefetch)
658		return;
659
660	if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_min_txg ||
661	    (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE))
662		return;
663
664	SET_BOOKMARK(&czb, objset, object, BP_GET_LEVEL(bp), blkid);
665
666	(void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, bp,
667	    NULL, NULL, ZIO_PRIORITY_ASYNC_READ,
668	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD, &flags, &czb);
669}
670
671static boolean_t
672dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
673    const zbookmark_phys_t *zb)
674{
675	/*
676	 * We never skip over user/group accounting objects (obj<0)
677	 */
678	if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) &&
679	    (int64_t)zb->zb_object >= 0) {
680		/*
681		 * If we already visited this bp & everything below (in
682		 * a prior txg sync), don't bother doing it again.
683		 */
684		if (zbookmark_subtree_completed(dnp, zb,
685		    &scn->scn_phys.scn_bookmark))
686			return (B_TRUE);
687
688		/*
689		 * If we found the block we're trying to resume from, or
690		 * we went past it to a different object, zero it out to
691		 * indicate that it's OK to start checking for suspending
692		 * again.
693		 */
694		if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 ||
695		    zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) {
696			dprintf("resuming at %llx/%llx/%llx/%llx\n",
697			    (longlong_t)zb->zb_objset,
698			    (longlong_t)zb->zb_object,
699			    (longlong_t)zb->zb_level,
700			    (longlong_t)zb->zb_blkid);
701			bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb));
702		}
703	}
704	return (B_FALSE);
705}
706
707/*
708 * Return nonzero on i/o error.
709 * Return new buf to write out in *bufp.
710 */
711static int
712dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
713    dnode_phys_t *dnp, const blkptr_t *bp,
714    const zbookmark_phys_t *zb, dmu_tx_t *tx)
715{
716	dsl_pool_t *dp = scn->scn_dp;
717	int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD;
718	int err;
719
720	if (BP_GET_LEVEL(bp) > 0) {
721		arc_flags_t flags = ARC_FLAG_WAIT;
722		int i;
723		blkptr_t *cbp;
724		int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
725		arc_buf_t *buf;
726
727		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf,
728		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
729		if (err) {
730			scn->scn_phys.scn_errors++;
731			return (err);
732		}
733		for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) {
734			dsl_scan_prefetch(scn, buf, cbp, zb->zb_objset,
735			    zb->zb_object, zb->zb_blkid * epb + i);
736		}
737		for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) {
738			zbookmark_phys_t czb;
739
740			SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
741			    zb->zb_level - 1,
742			    zb->zb_blkid * epb + i);
743			dsl_scan_visitbp(cbp, &czb, dnp,
744			    ds, scn, ostype, tx);
745		}
746		arc_buf_destroy(buf, &buf);
747	} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
748		arc_flags_t flags = ARC_FLAG_WAIT;
749		dnode_phys_t *cdnp;
750		int i, j;
751		int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT;
752		arc_buf_t *buf;
753
754		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf,
755		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
756		if (err) {
757			scn->scn_phys.scn_errors++;
758			return (err);
759		}
760		for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) {
761			for (j = 0; j < cdnp->dn_nblkptr; j++) {
762				blkptr_t *cbp = &cdnp->dn_blkptr[j];
763				dsl_scan_prefetch(scn, buf, cbp,
764				    zb->zb_objset, zb->zb_blkid * epb + i, j);
765			}
766		}
767		for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) {
768			dsl_scan_visitdnode(scn, ds, ostype,
769			    cdnp, zb->zb_blkid * epb + i, tx);
770		}
771
772		arc_buf_destroy(buf, &buf);
773	} else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
774		arc_flags_t flags = ARC_FLAG_WAIT;
775		objset_phys_t *osp;
776		arc_buf_t *buf;
777
778		err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf,
779		    ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb);
780		if (err) {
781			scn->scn_phys.scn_errors++;
782			return (err);
783		}
784
785		osp = buf->b_data;
786
787		dsl_scan_visitdnode(scn, ds, osp->os_type,
788		    &osp->os_meta_dnode, DMU_META_DNODE_OBJECT, tx);
789
790		if (OBJSET_BUF_HAS_USERUSED(buf)) {
791			/*
792			 * We also always visit user/group accounting
793			 * objects, and never skip them, even if we are
794			 * suspending.  This is necessary so that the space
795			 * deltas from this txg get integrated.
796			 */
797			dsl_scan_visitdnode(scn, ds, osp->os_type,
798			    &osp->os_groupused_dnode,
799			    DMU_GROUPUSED_OBJECT, tx);
800			dsl_scan_visitdnode(scn, ds, osp->os_type,
801			    &osp->os_userused_dnode,
802			    DMU_USERUSED_OBJECT, tx);
803		}
804		arc_buf_destroy(buf, &buf);
805	}
806
807	return (0);
808}
809
810static void
811dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
812    dmu_objset_type_t ostype, dnode_phys_t *dnp,
813    uint64_t object, dmu_tx_t *tx)
814{
815	int j;
816
817	for (j = 0; j < dnp->dn_nblkptr; j++) {
818		zbookmark_phys_t czb;
819
820		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
821		    dnp->dn_nlevels - 1, j);
822		dsl_scan_visitbp(&dnp->dn_blkptr[j],
823		    &czb, dnp, ds, scn, ostype, tx);
824	}
825
826	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
827		zbookmark_phys_t czb;
828		SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object,
829		    0, DMU_SPILL_BLKID);
830		dsl_scan_visitbp(&dnp->dn_spill,
831		    &czb, dnp, ds, scn, ostype, tx);
832	}
833}
834
835/*
836 * The arguments are in this order because mdb can only print the
837 * first 5; we want them to be useful.
838 */
839static void
840dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb,
841    dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
842    dmu_objset_type_t ostype, dmu_tx_t *tx)
843{
844	dsl_pool_t *dp = scn->scn_dp;
845	arc_buf_t *buf = NULL;
846	blkptr_t bp_toread = *bp;
847
848	/* ASSERT(pbuf == NULL || arc_released(pbuf)); */
849
850	if (dsl_scan_check_suspend(scn, zb))
851		return;
852
853	if (dsl_scan_check_resume(scn, dnp, zb))
854		return;
855
856	if (BP_IS_HOLE(bp))
857		return;
858
859	scn->scn_visited_this_txg++;
860
861	dprintf_bp(bp,
862	    "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx bp=%p",
863	    ds, ds ? ds->ds_object : 0,
864	    zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid,
865	    bp);
866
867	if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg)
868		return;
869
870	if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx) != 0)
871		return;
872
873	/*
874	 * If dsl_scan_ddt() has already visited this block, it will have
875	 * already done any translations or scrubbing, so don't call the
876	 * callback again.
877	 */
878	if (ddt_class_contains(dp->dp_spa,
879	    scn->scn_phys.scn_ddt_class_max, bp)) {
880		ASSERT(buf == NULL);
881		return;
882	}
883
884	/*
885	 * If this block is from the future (after cur_max_txg), then we
886	 * are doing this on behalf of a deleted snapshot, and we will
887	 * revisit the future block on the next pass of this dataset.
888	 * Don't scan it now unless we need to because something
889	 * under it was modified.
890	 */
891	if (BP_PHYSICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_max_txg) {
892		scan_funcs[scn->scn_phys.scn_func](dp, bp, zb);
893	}
894}
895
896static void
897dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp,
898    dmu_tx_t *tx)
899{
900	zbookmark_phys_t zb;
901
902	SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
903	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
904	dsl_scan_visitbp(bp, &zb, NULL,
905	    ds, scn, DMU_OST_NONE, tx);
906
907	dprintf_ds(ds, "finished scan%s", "");
908}
909
910void
911dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
912{
913	dsl_pool_t *dp = ds->ds_dir->dd_pool;
914	dsl_scan_t *scn = dp->dp_scan;
915	uint64_t mintxg;
916
917	if (scn->scn_phys.scn_state != DSS_SCANNING)
918		return;
919
920	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
921		if (ds->ds_is_snapshot) {
922			/*
923			 * Note:
924			 *  - scn_cur_{min,max}_txg stays the same.
925			 *  - Setting the flag is not really necessary if
926			 *    scn_cur_max_txg == scn_max_txg, because there
927			 *    is nothing after this snapshot that we care
928			 *    about.  However, we set it anyway and then
929			 *    ignore it when we retraverse it in
930			 *    dsl_scan_visitds().
931			 */
932			scn->scn_phys.scn_bookmark.zb_objset =
933			    dsl_dataset_phys(ds)->ds_next_snap_obj;
934			zfs_dbgmsg("destroying ds %llu; currently traversing; "
935			    "reset zb_objset to %llu",
936			    (u_longlong_t)ds->ds_object,
937			    (u_longlong_t)dsl_dataset_phys(ds)->
938			    ds_next_snap_obj);
939			scn->scn_phys.scn_flags |= DSF_VISIT_DS_AGAIN;
940		} else {
941			SET_BOOKMARK(&scn->scn_phys.scn_bookmark,
942			    ZB_DESTROYED_OBJSET, 0, 0, 0);
943			zfs_dbgmsg("destroying ds %llu; currently traversing; "
944			    "reset bookmark to -1,0,0,0",
945			    (u_longlong_t)ds->ds_object);
946		}
947	} else if (zap_lookup_int_key(dp->dp_meta_objset,
948	    scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) {
949		ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
950		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
951		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
952		if (ds->ds_is_snapshot) {
953			/*
954			 * We keep the same mintxg; it could be >
955			 * ds_creation_txg if the previous snapshot was
956			 * deleted too.
957			 */
958			VERIFY(zap_add_int_key(dp->dp_meta_objset,
959			    scn->scn_phys.scn_queue_obj,
960			    dsl_dataset_phys(ds)->ds_next_snap_obj,
961			    mintxg, tx) == 0);
962			zfs_dbgmsg("destroying ds %llu; in queue; "
963			    "replacing with %llu",
964			    (u_longlong_t)ds->ds_object,
965			    (u_longlong_t)dsl_dataset_phys(ds)->
966			    ds_next_snap_obj);
967		} else {
968			zfs_dbgmsg("destroying ds %llu; in queue; removing",
969			    (u_longlong_t)ds->ds_object);
970		}
971	}
972
973	/*
974	 * dsl_scan_sync() should be called after this, and should sync
975	 * out our changed state, but just to be safe, do it here.
976	 */
977	dsl_scan_sync_state(scn, tx);
978}
979
980void
981dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx)
982{
983	dsl_pool_t *dp = ds->ds_dir->dd_pool;
984	dsl_scan_t *scn = dp->dp_scan;
985	uint64_t mintxg;
986
987	if (scn->scn_phys.scn_state != DSS_SCANNING)
988		return;
989
990	ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
991
992	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
993		scn->scn_phys.scn_bookmark.zb_objset =
994		    dsl_dataset_phys(ds)->ds_prev_snap_obj;
995		zfs_dbgmsg("snapshotting ds %llu; currently traversing; "
996		    "reset zb_objset to %llu",
997		    (u_longlong_t)ds->ds_object,
998		    (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj);
999	} else if (zap_lookup_int_key(dp->dp_meta_objset,
1000	    scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) {
1001		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1002		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
1003		VERIFY(zap_add_int_key(dp->dp_meta_objset,
1004		    scn->scn_phys.scn_queue_obj,
1005		    dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg, tx) == 0);
1006		zfs_dbgmsg("snapshotting ds %llu; in queue; "
1007		    "replacing with %llu",
1008		    (u_longlong_t)ds->ds_object,
1009		    (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj);
1010	}
1011	dsl_scan_sync_state(scn, tx);
1012}
1013
1014void
1015dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx)
1016{
1017	dsl_pool_t *dp = ds1->ds_dir->dd_pool;
1018	dsl_scan_t *scn = dp->dp_scan;
1019	uint64_t mintxg;
1020
1021	if (scn->scn_phys.scn_state != DSS_SCANNING)
1022		return;
1023
1024	if (scn->scn_phys.scn_bookmark.zb_objset == ds1->ds_object) {
1025		scn->scn_phys.scn_bookmark.zb_objset = ds2->ds_object;
1026		zfs_dbgmsg("clone_swap ds %llu; currently traversing; "
1027		    "reset zb_objset to %llu",
1028		    (u_longlong_t)ds1->ds_object,
1029		    (u_longlong_t)ds2->ds_object);
1030	} else if (scn->scn_phys.scn_bookmark.zb_objset == ds2->ds_object) {
1031		scn->scn_phys.scn_bookmark.zb_objset = ds1->ds_object;
1032		zfs_dbgmsg("clone_swap ds %llu; currently traversing; "
1033		    "reset zb_objset to %llu",
1034		    (u_longlong_t)ds2->ds_object,
1035		    (u_longlong_t)ds1->ds_object);
1036	}
1037
1038	if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
1039	    ds1->ds_object, &mintxg) == 0) {
1040		int err;
1041
1042		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg);
1043		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg);
1044		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1045		    scn->scn_phys.scn_queue_obj, ds1->ds_object, tx));
1046		err = zap_add_int_key(dp->dp_meta_objset,
1047		    scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx);
1048		VERIFY(err == 0 || err == EEXIST);
1049		if (err == EEXIST) {
1050			/* Both were there to begin with */
1051			VERIFY(0 == zap_add_int_key(dp->dp_meta_objset,
1052			    scn->scn_phys.scn_queue_obj,
1053			    ds1->ds_object, mintxg, tx));
1054		}
1055		zfs_dbgmsg("clone_swap ds %llu; in queue; "
1056		    "replacing with %llu",
1057		    (u_longlong_t)ds1->ds_object,
1058		    (u_longlong_t)ds2->ds_object);
1059	} else if (zap_lookup_int_key(dp->dp_meta_objset,
1060	    scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg) == 0) {
1061		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg);
1062		ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg);
1063		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1064		    scn->scn_phys.scn_queue_obj, ds2->ds_object, tx));
1065		VERIFY(0 == zap_add_int_key(dp->dp_meta_objset,
1066		    scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx));
1067		zfs_dbgmsg("clone_swap ds %llu; in queue; "
1068		    "replacing with %llu",
1069		    (u_longlong_t)ds2->ds_object,
1070		    (u_longlong_t)ds1->ds_object);
1071	}
1072
1073	dsl_scan_sync_state(scn, tx);
1074}
1075
1076struct enqueue_clones_arg {
1077	dmu_tx_t *tx;
1078	uint64_t originobj;
1079};
1080
1081/* ARGSUSED */
1082static int
1083enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
1084{
1085	struct enqueue_clones_arg *eca = arg;
1086	dsl_dataset_t *ds;
1087	int err;
1088	dsl_scan_t *scn = dp->dp_scan;
1089
1090	if (dsl_dir_phys(hds->ds_dir)->dd_origin_obj != eca->originobj)
1091		return (0);
1092
1093	err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds);
1094	if (err)
1095		return (err);
1096
1097	while (dsl_dataset_phys(ds)->ds_prev_snap_obj != eca->originobj) {
1098		dsl_dataset_t *prev;
1099		err = dsl_dataset_hold_obj(dp,
1100		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
1101
1102		dsl_dataset_rele(ds, FTAG);
1103		if (err)
1104			return (err);
1105		ds = prev;
1106	}
1107	VERIFY(zap_add_int_key(dp->dp_meta_objset,
1108	    scn->scn_phys.scn_queue_obj, ds->ds_object,
1109	    dsl_dataset_phys(ds)->ds_prev_snap_txg, eca->tx) == 0);
1110	dsl_dataset_rele(ds, FTAG);
1111	return (0);
1112}
1113
1114static void
1115dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
1116{
1117	dsl_pool_t *dp = scn->scn_dp;
1118	dsl_dataset_t *ds;
1119	objset_t *os;
1120
1121	VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
1122
1123	if (scn->scn_phys.scn_cur_min_txg >=
1124	    scn->scn_phys.scn_max_txg) {
1125		/*
1126		 * This can happen if this snapshot was created after the
1127		 * scan started, and we already completed a previous snapshot
1128		 * that was created after the scan started.  This snapshot
1129		 * only references blocks with:
1130		 *
1131		 *	birth < our ds_creation_txg
1132		 *	cur_min_txg is no less than ds_creation_txg.
1133		 *	We have already visited these blocks.
1134		 * or
1135		 *	birth > scn_max_txg
1136		 *	The scan requested not to visit these blocks.
1137		 *
1138		 * Subsequent snapshots (and clones) can reference our
1139		 * blocks, or blocks with even higher birth times.
1140		 * Therefore we do not need to visit them either,
1141		 * so we do not add them to the work queue.
1142		 *
1143		 * Note that checking for cur_min_txg >= cur_max_txg
1144		 * is not sufficient, because in that case we may need to
1145		 * visit subsequent snapshots.  This happens when min_txg > 0,
1146		 * which raises cur_min_txg.  In this case we will visit
1147		 * this dataset but skip all of its blocks, because the
1148		 * rootbp's birth time is < cur_min_txg.  Then we will
1149		 * add the next snapshots/clones to the work queue.
1150		 */
1151		char *dsname = kmem_alloc(MAXNAMELEN, KM_SLEEP);
1152		dsl_dataset_name(ds, dsname);
1153		zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because "
1154		    "cur_min_txg (%llu) >= max_txg (%llu)",
1155		    dsobj, dsname,
1156		    scn->scn_phys.scn_cur_min_txg,
1157		    scn->scn_phys.scn_max_txg);
1158		kmem_free(dsname, MAXNAMELEN);
1159
1160		goto out;
1161	}
1162
1163	if (dmu_objset_from_ds(ds, &os))
1164		goto out;
1165
1166	/*
1167	 * Only the ZIL in the head (non-snapshot) is valid.  Even though
1168	 * snapshots can have ZIL block pointers (which may be the same
1169	 * BP as in the head), they must be ignored.  So we traverse the
1170	 * ZIL here, rather than in scan_recurse(), because the regular
1171	 * snapshot block-sharing rules don't apply to it.
1172	 */
1173	if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !ds->ds_is_snapshot)
1174		dsl_scan_zil(dp, &os->os_zil_header);
1175
1176	/*
1177	 * Iterate over the bps in this ds.
1178	 */
1179	dmu_buf_will_dirty(ds->ds_dbuf, tx);
1180	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
1181	dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx);
1182	rrw_exit(&ds->ds_bp_rwlock, FTAG);
1183
1184	char *dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
1185	dsl_dataset_name(ds, dsname);
1186	zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; "
1187	    "suspending=%u",
1188	    (longlong_t)dsobj, dsname,
1189	    (longlong_t)scn->scn_phys.scn_cur_min_txg,
1190	    (longlong_t)scn->scn_phys.scn_cur_max_txg,
1191	    (int)scn->scn_suspending);
1192	kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN);
1193
1194	if (scn->scn_suspending)
1195		goto out;
1196
1197	/*
1198	 * We've finished this pass over this dataset.
1199	 */
1200
1201	/*
1202	 * If we did not completely visit this dataset, do another pass.
1203	 */
1204	if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) {
1205		zfs_dbgmsg("incomplete pass; visiting again");
1206		scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN;
1207		VERIFY(zap_add_int_key(dp->dp_meta_objset,
1208		    scn->scn_phys.scn_queue_obj, ds->ds_object,
1209		    scn->scn_phys.scn_cur_max_txg, tx) == 0);
1210		goto out;
1211	}
1212
1213	/*
1214	 * Add descendent datasets to work queue.
1215	 */
1216	if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) {
1217		VERIFY(zap_add_int_key(dp->dp_meta_objset,
1218		    scn->scn_phys.scn_queue_obj,
1219		    dsl_dataset_phys(ds)->ds_next_snap_obj,
1220		    dsl_dataset_phys(ds)->ds_creation_txg, tx) == 0);
1221	}
1222	if (dsl_dataset_phys(ds)->ds_num_children > 1) {
1223		boolean_t usenext = B_FALSE;
1224		if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
1225			uint64_t count;
1226			/*
1227			 * A bug in a previous version of the code could
1228			 * cause upgrade_clones_cb() to not set
1229			 * ds_next_snap_obj when it should, leading to a
1230			 * missing entry.  Therefore we can only use the
1231			 * next_clones_obj when its count is correct.
1232			 */
1233			int err = zap_count(dp->dp_meta_objset,
1234			    dsl_dataset_phys(ds)->ds_next_clones_obj, &count);
1235			if (err == 0 &&
1236			    count == dsl_dataset_phys(ds)->ds_num_children - 1)
1237				usenext = B_TRUE;
1238		}
1239
1240		if (usenext) {
1241			VERIFY0(zap_join_key(dp->dp_meta_objset,
1242			    dsl_dataset_phys(ds)->ds_next_clones_obj,
1243			    scn->scn_phys.scn_queue_obj,
1244			    dsl_dataset_phys(ds)->ds_creation_txg, tx));
1245		} else {
1246			struct enqueue_clones_arg eca;
1247			eca.tx = tx;
1248			eca.originobj = ds->ds_object;
1249
1250			VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
1251			    enqueue_clones_cb, &eca, DS_FIND_CHILDREN));
1252		}
1253	}
1254
1255out:
1256	dsl_dataset_rele(ds, FTAG);
1257}
1258
1259/* ARGSUSED */
1260static int
1261enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
1262{
1263	dmu_tx_t *tx = arg;
1264	dsl_dataset_t *ds;
1265	int err;
1266	dsl_scan_t *scn = dp->dp_scan;
1267
1268	err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds);
1269	if (err)
1270		return (err);
1271
1272	while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
1273		dsl_dataset_t *prev;
1274		err = dsl_dataset_hold_obj(dp,
1275		    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
1276		if (err) {
1277			dsl_dataset_rele(ds, FTAG);
1278			return (err);
1279		}
1280
1281		/*
1282		 * If this is a clone, we don't need to worry about it for now.
1283		 */
1284		if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object) {
1285			dsl_dataset_rele(ds, FTAG);
1286			dsl_dataset_rele(prev, FTAG);
1287			return (0);
1288		}
1289		dsl_dataset_rele(ds, FTAG);
1290		ds = prev;
1291	}
1292
1293	VERIFY(zap_add_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
1294	    ds->ds_object, dsl_dataset_phys(ds)->ds_prev_snap_txg, tx) == 0);
1295	dsl_dataset_rele(ds, FTAG);
1296	return (0);
1297}
1298
1299/*
1300 * Scrub/dedup interaction.
1301 *
1302 * If there are N references to a deduped block, we don't want to scrub it
1303 * N times -- ideally, we should scrub it exactly once.
1304 *
1305 * We leverage the fact that the dde's replication class (enum ddt_class)
1306 * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest
1307 * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order.
1308 *
1309 * To prevent excess scrubbing, the scrub begins by walking the DDT
1310 * to find all blocks with refcnt > 1, and scrubs each of these once.
1311 * Since there are two replication classes which contain blocks with
1312 * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first.
1313 * Finally the top-down scrub begins, only visiting blocks with refcnt == 1.
1314 *
1315 * There would be nothing more to say if a block's refcnt couldn't change
1316 * during a scrub, but of course it can so we must account for changes
1317 * in a block's replication class.
1318 *
1319 * Here's an example of what can occur:
1320 *
1321 * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1
1322 * when visited during the top-down scrub phase, it will be scrubbed twice.
1323 * This negates our scrub optimization, but is otherwise harmless.
1324 *
1325 * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1
1326 * on each visit during the top-down scrub phase, it will never be scrubbed.
1327 * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's
1328 * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to
1329 * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1
1330 * while a scrub is in progress, it scrubs the block right then.
1331 */
1332static void
1333dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx)
1334{
1335	ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark;
1336	ddt_entry_t dde = { 0 };
1337	int error;
1338	uint64_t n = 0;
1339
1340	while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) {
1341		ddt_t *ddt;
1342
1343		if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max)
1344			break;
1345		dprintf("visiting ddb=%llu/%llu/%llu/%llx\n",
1346		    (longlong_t)ddb->ddb_class,
1347		    (longlong_t)ddb->ddb_type,
1348		    (longlong_t)ddb->ddb_checksum,
1349		    (longlong_t)ddb->ddb_cursor);
1350
1351		/* There should be no pending changes to the dedup table */
1352		ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum];
1353		ASSERT(avl_first(&ddt->ddt_tree) == NULL);
1354
1355		dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx);
1356		n++;
1357
1358		if (dsl_scan_check_suspend(scn, NULL))
1359			break;
1360	}
1361
1362	zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; "
1363	    "suspending=%u", (longlong_t)n,
1364	    (int)scn->scn_phys.scn_ddt_class_max, (int)scn->scn_suspending);
1365
1366	ASSERT(error == 0 || error == ENOENT);
1367	ASSERT(error != ENOENT ||
1368	    ddb->ddb_class > scn->scn_phys.scn_ddt_class_max);
1369}
1370
1371/* ARGSUSED */
1372void
1373dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum,
1374    ddt_entry_t *dde, dmu_tx_t *tx)
1375{
1376	const ddt_key_t *ddk = &dde->dde_key;
1377	ddt_phys_t *ddp = dde->dde_phys;
1378	blkptr_t bp;
1379	zbookmark_phys_t zb = { 0 };
1380
1381	if (scn->scn_phys.scn_state != DSS_SCANNING)
1382		return;
1383
1384	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
1385		if (ddp->ddp_phys_birth == 0 ||
1386		    ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg)
1387			continue;
1388		ddt_bp_create(checksum, ddk, ddp, &bp);
1389
1390		scn->scn_visited_this_txg++;
1391		scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb);
1392	}
1393}
1394
1395static void
1396dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx)
1397{
1398	dsl_pool_t *dp = scn->scn_dp;
1399	zap_cursor_t zc;
1400	zap_attribute_t za;
1401
1402	if (scn->scn_phys.scn_ddt_bookmark.ddb_class <=
1403	    scn->scn_phys.scn_ddt_class_max) {
1404		scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg;
1405		scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg;
1406		dsl_scan_ddt(scn, tx);
1407		if (scn->scn_suspending)
1408			return;
1409	}
1410
1411	if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) {
1412		/* First do the MOS & ORIGIN */
1413
1414		scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg;
1415		scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg;
1416		dsl_scan_visit_rootbp(scn, NULL,
1417		    &dp->dp_meta_rootbp, tx);
1418		spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp);
1419		if (scn->scn_suspending)
1420			return;
1421
1422		if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) {
1423			VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj,
1424			    enqueue_cb, tx, DS_FIND_CHILDREN));
1425		} else {
1426			dsl_scan_visitds(scn,
1427			    dp->dp_origin_snap->ds_object, tx);
1428		}
1429		ASSERT(!scn->scn_suspending);
1430	} else if (scn->scn_phys.scn_bookmark.zb_objset !=
1431	    ZB_DESTROYED_OBJSET) {
1432		/*
1433		 * If we were suspended, continue from here.  Note if the
1434		 * ds we were suspended on was deleted, the zb_objset may
1435		 * be -1, so we will skip this and find a new objset
1436		 * below.
1437		 */
1438		dsl_scan_visitds(scn, scn->scn_phys.scn_bookmark.zb_objset, tx);
1439		if (scn->scn_suspending)
1440			return;
1441	}
1442
1443	/*
1444	 * In case we were suspended right at the end of the ds, zero the
1445	 * bookmark so we don't think that we're still trying to resume.
1446	 */
1447	bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t));
1448
1449	/* keep pulling things out of the zap-object-as-queue */
1450	while (zap_cursor_init(&zc, dp->dp_meta_objset,
1451	    scn->scn_phys.scn_queue_obj),
1452	    zap_cursor_retrieve(&zc, &za) == 0) {
1453		dsl_dataset_t *ds;
1454		uint64_t dsobj;
1455
1456		dsobj = zfs_strtonum(za.za_name, NULL);
1457		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
1458		    scn->scn_phys.scn_queue_obj, dsobj, tx));
1459
1460		/* Set up min/max txg */
1461		VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
1462		if (za.za_first_integer != 0) {
1463			scn->scn_phys.scn_cur_min_txg =
1464			    MAX(scn->scn_phys.scn_min_txg,
1465			    za.za_first_integer);
1466		} else {
1467			scn->scn_phys.scn_cur_min_txg =
1468			    MAX(scn->scn_phys.scn_min_txg,
1469			    dsl_dataset_phys(ds)->ds_prev_snap_txg);
1470		}
1471		scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds);
1472		dsl_dataset_rele(ds, FTAG);
1473
1474		dsl_scan_visitds(scn, dsobj, tx);
1475		zap_cursor_fini(&zc);
1476		if (scn->scn_suspending)
1477			return;
1478	}
1479	zap_cursor_fini(&zc);
1480}
1481
1482static boolean_t
1483dsl_scan_free_should_suspend(dsl_scan_t *scn)
1484{
1485	uint64_t elapsed_nanosecs;
1486
1487	if (zfs_recover)
1488		return (B_FALSE);
1489
1490	if (scn->scn_visited_this_txg >= zfs_free_max_blocks)
1491		return (B_TRUE);
1492
1493	elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
1494	return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout ||
1495	    (NSEC2MSEC(elapsed_nanosecs) > zfs_free_min_time_ms &&
1496	    txg_sync_waiting(scn->scn_dp)) ||
1497	    spa_shutting_down(scn->scn_dp->dp_spa));
1498}
1499
1500static int
1501dsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
1502{
1503	dsl_scan_t *scn = arg;
1504
1505	if (!scn->scn_is_bptree ||
1506	    (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)) {
1507		if (dsl_scan_free_should_suspend(scn))
1508			return (SET_ERROR(ERESTART));
1509	}
1510
1511	zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa,
1512	    dmu_tx_get_txg(tx), bp, 0));
1513	dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD,
1514	    -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp),
1515	    -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx);
1516	scn->scn_visited_this_txg++;
1517	return (0);
1518}
1519
1520boolean_t
1521dsl_scan_active(dsl_scan_t *scn)
1522{
1523	spa_t *spa = scn->scn_dp->dp_spa;
1524	uint64_t used = 0, comp, uncomp;
1525
1526	if (spa->spa_load_state != SPA_LOAD_NONE)
1527		return (B_FALSE);
1528	if (spa_shutting_down(spa))
1529		return (B_FALSE);
1530	if ((scn->scn_phys.scn_state == DSS_SCANNING &&
1531	    !dsl_scan_is_paused_scrub(scn)) ||
1532	    (scn->scn_async_destroying && !scn->scn_async_stalled))
1533		return (B_TRUE);
1534
1535	if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
1536		(void) bpobj_space(&scn->scn_dp->dp_free_bpobj,
1537		    &used, &comp, &uncomp);
1538	}
1539	return (used != 0);
1540}
1541
1542/* Called whenever a txg syncs. */
1543void
1544dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx)
1545{
1546	dsl_scan_t *scn = dp->dp_scan;
1547	spa_t *spa = dp->dp_spa;
1548	int err = 0;
1549
1550	/*
1551	 * Check for scn_restart_txg before checking spa_load_state, so
1552	 * that we can restart an old-style scan while the pool is being
1553	 * imported (see dsl_scan_init).
1554	 */
1555	if (dsl_scan_restarting(scn, tx)) {
1556		pool_scan_func_t func = POOL_SCAN_SCRUB;
1557		dsl_scan_done(scn, B_FALSE, tx);
1558		if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL))
1559			func = POOL_SCAN_RESILVER;
1560		zfs_dbgmsg("restarting scan func=%u txg=%llu",
1561		    func, tx->tx_txg);
1562		dsl_scan_setup_sync(&func, tx);
1563	}
1564
1565	/*
1566	 * Only process scans in sync pass 1.
1567	 */
1568	if (spa_sync_pass(dp->dp_spa) > 1)
1569		return;
1570
1571	/*
1572	 * If the spa is shutting down, then stop scanning. This will
1573	 * ensure that the scan does not dirty any new data during the
1574	 * shutdown phase.
1575	 */
1576	if (spa_shutting_down(spa))
1577		return;
1578
1579	/*
1580	 * If the scan is inactive due to a stalled async destroy, try again.
1581	 */
1582	if (!scn->scn_async_stalled && !dsl_scan_active(scn))
1583		return;
1584
1585	scn->scn_visited_this_txg = 0;
1586	scn->scn_suspending = B_FALSE;
1587	scn->scn_sync_start_time = gethrtime();
1588	spa->spa_scrub_active = B_TRUE;
1589
1590	/*
1591	 * First process the async destroys.  If we suspend, don't do
1592	 * any scrubbing or resilvering.  This ensures that there are no
1593	 * async destroys while we are scanning, so the scan code doesn't
1594	 * have to worry about traversing it.  It is also faster to free the
1595	 * blocks than to scrub them.
1596	 */
1597	if (zfs_free_bpobj_enabled &&
1598	    spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
1599		scn->scn_is_bptree = B_FALSE;
1600		scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
1601		    NULL, ZIO_FLAG_MUSTSUCCEED);
1602		err = bpobj_iterate(&dp->dp_free_bpobj,
1603		    dsl_scan_free_block_cb, scn, tx);
1604		VERIFY3U(0, ==, zio_wait(scn->scn_zio_root));
1605
1606		if (err != 0 && err != ERESTART)
1607			zfs_panic_recover("error %u from bpobj_iterate()", err);
1608	}
1609
1610	if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) {
1611		ASSERT(scn->scn_async_destroying);
1612		scn->scn_is_bptree = B_TRUE;
1613		scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
1614		    NULL, ZIO_FLAG_MUSTSUCCEED);
1615		err = bptree_iterate(dp->dp_meta_objset,
1616		    dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx);
1617		VERIFY0(zio_wait(scn->scn_zio_root));
1618
1619		if (err == EIO || err == ECKSUM) {
1620			err = 0;
1621		} else if (err != 0 && err != ERESTART) {
1622			zfs_panic_recover("error %u from "
1623			    "traverse_dataset_destroyed()", err);
1624		}
1625
1626		if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) {
1627			/* finished; deactivate async destroy feature */
1628			spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx);
1629			ASSERT(!spa_feature_is_active(spa,
1630			    SPA_FEATURE_ASYNC_DESTROY));
1631			VERIFY0(zap_remove(dp->dp_meta_objset,
1632			    DMU_POOL_DIRECTORY_OBJECT,
1633			    DMU_POOL_BPTREE_OBJ, tx));
1634			VERIFY0(bptree_free(dp->dp_meta_objset,
1635			    dp->dp_bptree_obj, tx));
1636			dp->dp_bptree_obj = 0;
1637			scn->scn_async_destroying = B_FALSE;
1638			scn->scn_async_stalled = B_FALSE;
1639		} else {
1640			/*
1641			 * If we didn't make progress, mark the async
1642			 * destroy as stalled, so that we will not initiate
1643			 * a spa_sync() on its behalf.  Note that we only
1644			 * check this if we are not finished, because if the
1645			 * bptree had no blocks for us to visit, we can
1646			 * finish without "making progress".
1647			 */
1648			scn->scn_async_stalled =
1649			    (scn->scn_visited_this_txg == 0);
1650		}
1651	}
1652	if (scn->scn_visited_this_txg) {
1653		zfs_dbgmsg("freed %llu blocks in %llums from "
1654		    "free_bpobj/bptree txg %llu; err=%u",
1655		    (longlong_t)scn->scn_visited_this_txg,
1656		    (longlong_t)
1657		    NSEC2MSEC(gethrtime() - scn->scn_sync_start_time),
1658		    (longlong_t)tx->tx_txg, err);
1659		scn->scn_visited_this_txg = 0;
1660
1661		/*
1662		 * Write out changes to the DDT that may be required as a
1663		 * result of the blocks freed.  This ensures that the DDT
1664		 * is clean when a scrub/resilver runs.
1665		 */
1666		ddt_sync(spa, tx->tx_txg);
1667	}
1668	if (err != 0)
1669		return;
1670	if (dp->dp_free_dir != NULL && !scn->scn_async_destroying &&
1671	    zfs_free_leak_on_eio &&
1672	    (dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes != 0 ||
1673	    dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes != 0 ||
1674	    dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes != 0)) {
1675		/*
1676		 * We have finished background destroying, but there is still
1677		 * some space left in the dp_free_dir. Transfer this leaked
1678		 * space to the dp_leak_dir.
1679		 */
1680		if (dp->dp_leak_dir == NULL) {
1681			rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG);
1682			(void) dsl_dir_create_sync(dp, dp->dp_root_dir,
1683			    LEAK_DIR_NAME, tx);
1684			VERIFY0(dsl_pool_open_special_dir(dp,
1685			    LEAK_DIR_NAME, &dp->dp_leak_dir));
1686			rrw_exit(&dp->dp_config_rwlock, FTAG);
1687		}
1688		dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD,
1689		    dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes,
1690		    dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes,
1691		    dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx);
1692		dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD,
1693		    -dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes,
1694		    -dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes,
1695		    -dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx);
1696	}
1697	if (dp->dp_free_dir != NULL && !scn->scn_async_destroying) {
1698		/* finished; verify that space accounting went to zero */
1699		ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes);
1700		ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes);
1701		ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes);
1702	}
1703
1704	if (scn->scn_phys.scn_state != DSS_SCANNING)
1705		return;
1706
1707	if (scn->scn_done_txg == tx->tx_txg) {
1708		ASSERT(!scn->scn_suspending);
1709		/* finished with scan. */
1710		zfs_dbgmsg("txg %llu scan complete", tx->tx_txg);
1711		dsl_scan_done(scn, B_TRUE, tx);
1712		ASSERT3U(spa->spa_scrub_inflight, ==, 0);
1713		dsl_scan_sync_state(scn, tx);
1714		return;
1715	}
1716
1717	if (dsl_scan_is_paused_scrub(scn))
1718		return;
1719
1720	if (scn->scn_phys.scn_ddt_bookmark.ddb_class <=
1721	    scn->scn_phys.scn_ddt_class_max) {
1722		zfs_dbgmsg("doing scan sync txg %llu; "
1723		    "ddt bm=%llu/%llu/%llu/%llx",
1724		    (longlong_t)tx->tx_txg,
1725		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class,
1726		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type,
1727		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum,
1728		    (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor);
1729		ASSERT(scn->scn_phys.scn_bookmark.zb_objset == 0);
1730		ASSERT(scn->scn_phys.scn_bookmark.zb_object == 0);
1731		ASSERT(scn->scn_phys.scn_bookmark.zb_level == 0);
1732		ASSERT(scn->scn_phys.scn_bookmark.zb_blkid == 0);
1733	} else {
1734		zfs_dbgmsg("doing scan sync txg %llu; bm=%llu/%llu/%llu/%llu",
1735		    (longlong_t)tx->tx_txg,
1736		    (longlong_t)scn->scn_phys.scn_bookmark.zb_objset,
1737		    (longlong_t)scn->scn_phys.scn_bookmark.zb_object,
1738		    (longlong_t)scn->scn_phys.scn_bookmark.zb_level,
1739		    (longlong_t)scn->scn_phys.scn_bookmark.zb_blkid);
1740	}
1741
1742	scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
1743	    NULL, ZIO_FLAG_CANFAIL);
1744	dsl_pool_config_enter(dp, FTAG);
1745	dsl_scan_visit(scn, tx);
1746	dsl_pool_config_exit(dp, FTAG);
1747	(void) zio_wait(scn->scn_zio_root);
1748	scn->scn_zio_root = NULL;
1749
1750	zfs_dbgmsg("visited %llu blocks in %llums",
1751	    (longlong_t)scn->scn_visited_this_txg,
1752	    (longlong_t)NSEC2MSEC(gethrtime() - scn->scn_sync_start_time));
1753
1754	if (!scn->scn_suspending) {
1755		scn->scn_done_txg = tx->tx_txg + 1;
1756		zfs_dbgmsg("txg %llu traversal complete, waiting till txg %llu",
1757		    tx->tx_txg, scn->scn_done_txg);
1758	}
1759
1760	if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
1761		mutex_enter(&spa->spa_scrub_lock);
1762		while (spa->spa_scrub_inflight > 0) {
1763			cv_wait(&spa->spa_scrub_io_cv,
1764			    &spa->spa_scrub_lock);
1765		}
1766		mutex_exit(&spa->spa_scrub_lock);
1767	}
1768
1769	dsl_scan_sync_state(scn, tx);
1770}
1771
1772/*
1773 * This will start a new scan, or restart an existing one.
1774 */
1775void
1776dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg)
1777{
1778	if (txg == 0) {
1779		dmu_tx_t *tx;
1780		tx = dmu_tx_create_dd(dp->dp_mos_dir);
1781		VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT));
1782
1783		txg = dmu_tx_get_txg(tx);
1784		dp->dp_scan->scn_restart_txg = txg;
1785		dmu_tx_commit(tx);
1786	} else {
1787		dp->dp_scan->scn_restart_txg = txg;
1788	}
1789	zfs_dbgmsg("restarting resilver txg=%llu", txg);
1790}
1791
1792boolean_t
1793dsl_scan_resilvering(dsl_pool_t *dp)
1794{
1795	return (dp->dp_scan->scn_phys.scn_state == DSS_SCANNING &&
1796	    dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER);
1797}
1798
1799/*
1800 * scrub consumers
1801 */
1802
1803static void
1804count_block(zfs_all_blkstats_t *zab, const blkptr_t *bp)
1805{
1806	int i;
1807
1808	/*
1809	 * If we resume after a reboot, zab will be NULL; don't record
1810	 * incomplete stats in that case.
1811	 */
1812	if (zab == NULL)
1813		return;
1814
1815	for (i = 0; i < 4; i++) {
1816		int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS;
1817		int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL;
1818		if (t & DMU_OT_NEWTYPE)
1819			t = DMU_OT_OTHER;
1820		zfs_blkstat_t *zb = &zab->zab_type[l][t];
1821		int equal;
1822
1823		zb->zb_count++;
1824		zb->zb_asize += BP_GET_ASIZE(bp);
1825		zb->zb_lsize += BP_GET_LSIZE(bp);
1826		zb->zb_psize += BP_GET_PSIZE(bp);
1827		zb->zb_gangs += BP_COUNT_GANG(bp);
1828
1829		switch (BP_GET_NDVAS(bp)) {
1830		case 2:
1831			if (DVA_GET_VDEV(&bp->blk_dva[0]) ==
1832			    DVA_GET_VDEV(&bp->blk_dva[1]))
1833				zb->zb_ditto_2_of_2_samevdev++;
1834			break;
1835		case 3:
1836			equal = (DVA_GET_VDEV(&bp->blk_dva[0]) ==
1837			    DVA_GET_VDEV(&bp->blk_dva[1])) +
1838			    (DVA_GET_VDEV(&bp->blk_dva[0]) ==
1839			    DVA_GET_VDEV(&bp->blk_dva[2])) +
1840			    (DVA_GET_VDEV(&bp->blk_dva[1]) ==
1841			    DVA_GET_VDEV(&bp->blk_dva[2]));
1842			if (equal == 1)
1843				zb->zb_ditto_2_of_3_samevdev++;
1844			else if (equal == 3)
1845				zb->zb_ditto_3_of_3_samevdev++;
1846			break;
1847		}
1848	}
1849}
1850
1851static void
1852dsl_scan_scrub_done(zio_t *zio)
1853{
1854	spa_t *spa = zio->io_spa;
1855
1856	abd_free(zio->io_abd);
1857
1858	mutex_enter(&spa->spa_scrub_lock);
1859	spa->spa_scrub_inflight--;
1860	cv_broadcast(&spa->spa_scrub_io_cv);
1861
1862	if (zio->io_error && (zio->io_error != ECKSUM ||
1863	    !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) {
1864		spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors++;
1865	}
1866	mutex_exit(&spa->spa_scrub_lock);
1867}
1868
1869static int
1870dsl_scan_scrub_cb(dsl_pool_t *dp,
1871    const blkptr_t *bp, const zbookmark_phys_t *zb)
1872{
1873	dsl_scan_t *scn = dp->dp_scan;
1874	size_t size = BP_GET_PSIZE(bp);
1875	spa_t *spa = dp->dp_spa;
1876	uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp);
1877	boolean_t needs_io;
1878	int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL;
1879	int scan_delay = 0;
1880
1881	if (phys_birth <= scn->scn_phys.scn_min_txg ||
1882	    phys_birth >= scn->scn_phys.scn_max_txg)
1883		return (0);
1884
1885	count_block(dp->dp_blkstats, bp);
1886
1887	if (BP_IS_EMBEDDED(bp))
1888		return (0);
1889
1890	ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn));
1891	if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) {
1892		zio_flags |= ZIO_FLAG_SCRUB;
1893		needs_io = B_TRUE;
1894		scan_delay = zfs_scrub_delay;
1895	} else {
1896		ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER);
1897		zio_flags |= ZIO_FLAG_RESILVER;
1898		needs_io = B_FALSE;
1899		scan_delay = zfs_resilver_delay;
1900	}
1901
1902	/* If it's an intent log block, failure is expected. */
1903	if (zb->zb_level == ZB_ZIL_LEVEL)
1904		zio_flags |= ZIO_FLAG_SPECULATIVE;
1905
1906	for (int d = 0; d < BP_GET_NDVAS(bp); d++) {
1907		vdev_t *vd = vdev_lookup_top(spa,
1908		    DVA_GET_VDEV(&bp->blk_dva[d]));
1909
1910		/*
1911		 * Keep track of how much data we've examined so that
1912		 * zpool(1M) status can make useful progress reports.
1913		 */
1914		scn->scn_phys.scn_examined += DVA_GET_ASIZE(&bp->blk_dva[d]);
1915		spa->spa_scan_pass_exam += DVA_GET_ASIZE(&bp->blk_dva[d]);
1916
1917		/* if it's a resilver, this may not be in the target range */
1918		if (!needs_io) {
1919			if (DVA_GET_GANG(&bp->blk_dva[d])) {
1920				/*
1921				 * Gang members may be spread across multiple
1922				 * vdevs, so the best estimate we have is the
1923				 * scrub range, which has already been checked.
1924				 * XXX -- it would be better to change our
1925				 * allocation policy to ensure that all
1926				 * gang members reside on the same vdev.
1927				 */
1928				needs_io = B_TRUE;
1929			} else {
1930				needs_io = vdev_dtl_contains(vd, DTL_PARTIAL,
1931				    phys_birth, 1);
1932			}
1933		}
1934	}
1935
1936	if (needs_io && !zfs_no_scrub_io) {
1937		vdev_t *rvd = spa->spa_root_vdev;
1938		uint64_t maxinflight = rvd->vdev_children * zfs_top_maxinflight;
1939
1940		mutex_enter(&spa->spa_scrub_lock);
1941		while (spa->spa_scrub_inflight >= maxinflight)
1942			cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock);
1943		spa->spa_scrub_inflight++;
1944		mutex_exit(&spa->spa_scrub_lock);
1945
1946		/*
1947		 * If we're seeing recent (zfs_scan_idle) "important" I/Os
1948		 * then throttle our workload to limit the impact of a scan.
1949		 */
1950		if (ddi_get_lbolt64() - spa->spa_last_io <= zfs_scan_idle)
1951			delay(scan_delay);
1952
1953		zio_nowait(zio_read(NULL, spa, bp,
1954		    abd_alloc_for_io(size, B_FALSE), size, dsl_scan_scrub_done,
1955		    NULL, ZIO_PRIORITY_SCRUB, zio_flags, zb));
1956	}
1957
1958	/* do not relocate this block */
1959	return (0);
1960}
1961
1962/*
1963 * Called by the ZFS_IOC_POOL_SCAN ioctl to start a scrub or resilver.
1964 * Can also be called to resume a paused scrub.
1965 */
1966int
1967dsl_scan(dsl_pool_t *dp, pool_scan_func_t func)
1968{
1969	spa_t *spa = dp->dp_spa;
1970	dsl_scan_t *scn = dp->dp_scan;
1971
1972	/*
1973	 * Purge all vdev caches and probe all devices.  We do this here
1974	 * rather than in sync context because this requires a writer lock
1975	 * on the spa_config lock, which we can't do from sync context.  The
1976	 * spa_scrub_reopen flag indicates that vdev_open() should not
1977	 * attempt to start another scrub.
1978	 */
1979	spa_vdev_state_enter(spa, SCL_NONE);
1980	spa->spa_scrub_reopen = B_TRUE;
1981	vdev_reopen(spa->spa_root_vdev);
1982	spa->spa_scrub_reopen = B_FALSE;
1983	(void) spa_vdev_state_exit(spa, NULL, 0);
1984
1985	if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) {
1986		/* got scrub start cmd, resume paused scrub */
1987		int err = dsl_scrub_set_pause_resume(scn->scn_dp,
1988		    POOL_SCRUB_NORMAL);
1989		if (err == 0)
1990			return (ECANCELED);
1991
1992		return (SET_ERROR(err));
1993	}
1994
1995	return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check,
1996	    dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_NONE));
1997}
1998
1999static boolean_t
2000dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx)
2001{
2002	return (scn->scn_restart_txg != 0 &&
2003	    scn->scn_restart_txg <= tx->tx_txg);
2004}
2005