13f9d6ad7SLin Ling /* 23f9d6ad7SLin Ling * CDDL HEADER START 33f9d6ad7SLin Ling * 43f9d6ad7SLin Ling * The contents of this file are subject to the terms of the 53f9d6ad7SLin Ling * Common Development and Distribution License (the "License"). 63f9d6ad7SLin Ling * You may not use this file except in compliance with the License. 73f9d6ad7SLin Ling * 83f9d6ad7SLin Ling * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 93f9d6ad7SLin Ling * or http://www.opensolaris.org/os/licensing. 103f9d6ad7SLin Ling * See the License for the specific language governing permissions 113f9d6ad7SLin Ling * and limitations under the License. 123f9d6ad7SLin Ling * 133f9d6ad7SLin Ling * When distributing Covered Code, include this CDDL HEADER in each 143f9d6ad7SLin Ling * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 153f9d6ad7SLin Ling * If applicable, add the following below this CDDL HEADER, with the 163f9d6ad7SLin Ling * fields enclosed by brackets "[]" replaced with your own identifying 173f9d6ad7SLin Ling * information: Portions Copyright [yyyy] [name of copyright owner] 183f9d6ad7SLin Ling * 193f9d6ad7SLin Ling * CDDL HEADER END 203f9d6ad7SLin Ling */ 213f9d6ad7SLin Ling /* 223f9d6ad7SLin Ling * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23*bb1f4245SMatthew Ahrens * Copyright (c) 2011, 2018 by Delphix. All rights reserved. 248c04a1faSGary Mills * Copyright 2016 Gary Mills 255cabbc6bSPrashanth Sreenivasa * Copyright (c) 2011, 2017 by Delphix. All rights reserved. 26ce1577b0SDave Eddy * Copyright 2017 Joyent, Inc. 271702cce7SAlek Pinchuk * Copyright (c) 2017 Datto Inc. 283f9d6ad7SLin Ling */ 293f9d6ad7SLin Ling 303f9d6ad7SLin Ling #include <sys/dsl_scan.h> 313f9d6ad7SLin Ling #include <sys/dsl_pool.h> 323f9d6ad7SLin Ling #include <sys/dsl_dataset.h> 333f9d6ad7SLin Ling #include <sys/dsl_prop.h> 343f9d6ad7SLin Ling #include <sys/dsl_dir.h> 353f9d6ad7SLin Ling #include <sys/dsl_synctask.h> 363f9d6ad7SLin Ling #include <sys/dnode.h> 373f9d6ad7SLin Ling #include <sys/dmu_tx.h> 383f9d6ad7SLin Ling #include <sys/dmu_objset.h> 393f9d6ad7SLin Ling #include <sys/arc.h> 403f9d6ad7SLin Ling #include <sys/zap.h> 413f9d6ad7SLin Ling #include <sys/zio.h> 423f9d6ad7SLin Ling #include <sys/zfs_context.h> 433f9d6ad7SLin Ling #include <sys/fs/zfs.h> 443f9d6ad7SLin Ling #include <sys/zfs_znode.h> 453f9d6ad7SLin Ling #include <sys/spa_impl.h> 463f9d6ad7SLin Ling #include <sys/vdev_impl.h> 473f9d6ad7SLin Ling #include <sys/zil_impl.h> 483f9d6ad7SLin Ling #include <sys/zio_checksum.h> 493f9d6ad7SLin Ling #include <sys/ddt.h> 503f9d6ad7SLin Ling #include <sys/sa.h> 513f9d6ad7SLin Ling #include <sys/sa_impl.h> 52ad135b5dSChristopher Siden #include <sys/zfeature.h> 53770499e1SDan Kimmel #include <sys/abd.h> 543f9d6ad7SLin Ling #ifdef _KERNEL 553f9d6ad7SLin Ling #include <sys/zfs_vfsops.h> 563f9d6ad7SLin Ling #endif 573f9d6ad7SLin Ling 587802d7bfSMatthew Ahrens typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, 597802d7bfSMatthew Ahrens const zbookmark_phys_t *); 603f9d6ad7SLin Ling 613f9d6ad7SLin Ling static scan_cb_t dsl_scan_scrub_cb; 623b2aab18SMatthew Ahrens static void dsl_scan_cancel_sync(void *, dmu_tx_t *); 631825bc56SNav Ravindranath static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *); 641825bc56SNav Ravindranath static boolean_t dsl_scan_restarting(dsl_scan_t *, dmu_tx_t *); 653f9d6ad7SLin Ling 6644ecc532SGeorge Wilson int zfs_top_maxinflight = 32; /* maximum I/Os per top-level */ 6744ecc532SGeorge Wilson int zfs_resilver_delay = 2; /* number of ticks to delay resilver */ 6844ecc532SGeorge Wilson int zfs_scrub_delay = 4; /* number of ticks to delay scrub */ 6944ecc532SGeorge Wilson int zfs_scan_idle = 50; /* idle window in clock ticks */ 7044ecc532SGeorge Wilson 713f9d6ad7SLin Ling int zfs_scan_min_time_ms = 1000; /* min millisecs to scrub per txg */ 72cde58dbcSMatthew Ahrens int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */ 735cabbc6bSPrashanth Sreenivasa int zfs_obsolete_min_time_ms = 500; /* min millisecs to obsolete per txg */ 743f9d6ad7SLin Ling int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver per txg */ 753f9d6ad7SLin Ling boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */ 767fd05ac4SMatthew Ahrens boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */ 773f9d6ad7SLin Ling enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE; 783f9d6ad7SLin Ling int dsl_scan_delay_completion = B_FALSE; /* set to delay scan completion */ 79af3465daSMax Grossman /* max number of blocks to free in a single TXG */ 805cabbc6bSPrashanth Sreenivasa uint64_t zfs_async_block_max_blocks = UINT64_MAX; 813f9d6ad7SLin Ling 823f9d6ad7SLin Ling #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \ 833f9d6ad7SLin Ling ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \ 843f9d6ad7SLin Ling (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER) 853f9d6ad7SLin Ling 863f9d6ad7SLin Ling extern int zfs_txg_timeout; 873f9d6ad7SLin Ling 88139510fbSGeorge Wilson /* 89139510fbSGeorge Wilson * Enable/disable the processing of the free_bpobj object. 90139510fbSGeorge Wilson */ 91139510fbSGeorge Wilson boolean_t zfs_free_bpobj_enabled = B_TRUE; 92139510fbSGeorge Wilson 933f9d6ad7SLin Ling /* the order has to match pool_scan_type */ 943f9d6ad7SLin Ling static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = { 953f9d6ad7SLin Ling NULL, 963f9d6ad7SLin Ling dsl_scan_scrub_cb, /* POOL_SCAN_SCRUB */ 973f9d6ad7SLin Ling dsl_scan_scrub_cb, /* POOL_SCAN_RESILVER */ 983f9d6ad7SLin Ling }; 993f9d6ad7SLin Ling 1003f9d6ad7SLin Ling int 1013f9d6ad7SLin Ling dsl_scan_init(dsl_pool_t *dp, uint64_t txg) 1023f9d6ad7SLin Ling { 1033f9d6ad7SLin Ling int err; 1043f9d6ad7SLin Ling dsl_scan_t *scn; 1053f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 1063f9d6ad7SLin Ling uint64_t f; 1073f9d6ad7SLin Ling 1083f9d6ad7SLin Ling scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP); 1093f9d6ad7SLin Ling scn->scn_dp = dp; 1103f9d6ad7SLin Ling 1114a923759SGeorge Wilson /* 1124a923759SGeorge Wilson * It's possible that we're resuming a scan after a reboot so 1134a923759SGeorge Wilson * make sure that the scan_async_destroying flag is initialized 1144a923759SGeorge Wilson * appropriately. 1154a923759SGeorge Wilson */ 1164a923759SGeorge Wilson ASSERT(!scn->scn_async_destroying); 1174a923759SGeorge Wilson scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, 1182acef22dSMatthew Ahrens SPA_FEATURE_ASYNC_DESTROY); 1194a923759SGeorge Wilson 1203f9d6ad7SLin Ling err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 1213f9d6ad7SLin Ling "scrub_func", sizeof (uint64_t), 1, &f); 1223f9d6ad7SLin Ling if (err == 0) { 1233f9d6ad7SLin Ling /* 1243f9d6ad7SLin Ling * There was an old-style scrub in progress. Restart a 1253f9d6ad7SLin Ling * new-style scrub from the beginning. 1263f9d6ad7SLin Ling */ 1273f9d6ad7SLin Ling scn->scn_restart_txg = txg; 1283f9d6ad7SLin Ling zfs_dbgmsg("old-style scrub was in progress; " 1293f9d6ad7SLin Ling "restarting new-style scrub in txg %llu", 1303f9d6ad7SLin Ling scn->scn_restart_txg); 1313f9d6ad7SLin Ling 1323f9d6ad7SLin Ling /* 1333f9d6ad7SLin Ling * Load the queue obj from the old location so that it 1343f9d6ad7SLin Ling * can be freed by dsl_scan_done(). 1353f9d6ad7SLin Ling */ 1363f9d6ad7SLin Ling (void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 1373f9d6ad7SLin Ling "scrub_queue", sizeof (uint64_t), 1, 1383f9d6ad7SLin Ling &scn->scn_phys.scn_queue_obj); 1393f9d6ad7SLin Ling } else { 1403f9d6ad7SLin Ling err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 1413f9d6ad7SLin Ling DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 1423f9d6ad7SLin Ling &scn->scn_phys); 1433f9d6ad7SLin Ling if (err == ENOENT) 1443f9d6ad7SLin Ling return (0); 1453f9d6ad7SLin Ling else if (err) 1463f9d6ad7SLin Ling return (err); 1473f9d6ad7SLin Ling 1483f9d6ad7SLin Ling if (scn->scn_phys.scn_state == DSS_SCANNING && 1493f9d6ad7SLin Ling spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) { 1503f9d6ad7SLin Ling /* 1513f9d6ad7SLin Ling * A new-type scrub was in progress on an old 1523f9d6ad7SLin Ling * pool, and the pool was accessed by old 1533f9d6ad7SLin Ling * software. Restart from the beginning, since 1543f9d6ad7SLin Ling * the old software may have changed the pool in 1553f9d6ad7SLin Ling * the meantime. 1563f9d6ad7SLin Ling */ 1573f9d6ad7SLin Ling scn->scn_restart_txg = txg; 1583f9d6ad7SLin Ling zfs_dbgmsg("new-style scrub was modified " 1593f9d6ad7SLin Ling "by old software; restarting in txg %llu", 1603f9d6ad7SLin Ling scn->scn_restart_txg); 1613f9d6ad7SLin Ling } 1623f9d6ad7SLin Ling } 1633f9d6ad7SLin Ling 1643f9d6ad7SLin Ling spa_scan_stat_init(spa); 1653f9d6ad7SLin Ling return (0); 1663f9d6ad7SLin Ling } 1673f9d6ad7SLin Ling 1683f9d6ad7SLin Ling void 1693f9d6ad7SLin Ling dsl_scan_fini(dsl_pool_t *dp) 1703f9d6ad7SLin Ling { 1713f9d6ad7SLin Ling if (dp->dp_scan) { 1723f9d6ad7SLin Ling kmem_free(dp->dp_scan, sizeof (dsl_scan_t)); 1733f9d6ad7SLin Ling dp->dp_scan = NULL; 1743f9d6ad7SLin Ling } 1753f9d6ad7SLin Ling } 1763f9d6ad7SLin Ling 1773f9d6ad7SLin Ling /* ARGSUSED */ 1783f9d6ad7SLin Ling static int 1793b2aab18SMatthew Ahrens dsl_scan_setup_check(void *arg, dmu_tx_t *tx) 1803f9d6ad7SLin Ling { 1813b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 1823f9d6ad7SLin Ling 1833f9d6ad7SLin Ling if (scn->scn_phys.scn_state == DSS_SCANNING) 184be6fd75aSMatthew Ahrens return (SET_ERROR(EBUSY)); 1853f9d6ad7SLin Ling 1863f9d6ad7SLin Ling return (0); 1873f9d6ad7SLin Ling } 1883f9d6ad7SLin Ling 1893f9d6ad7SLin Ling static void 1903b2aab18SMatthew Ahrens dsl_scan_setup_sync(void *arg, dmu_tx_t *tx) 1913f9d6ad7SLin Ling { 1923b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 1933b2aab18SMatthew Ahrens pool_scan_func_t *funcp = arg; 1943f9d6ad7SLin Ling dmu_object_type_t ot = 0; 1953f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 1963f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 1973f9d6ad7SLin Ling 1983f9d6ad7SLin Ling ASSERT(scn->scn_phys.scn_state != DSS_SCANNING); 1993f9d6ad7SLin Ling ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS); 2003f9d6ad7SLin Ling bzero(&scn->scn_phys, sizeof (scn->scn_phys)); 2013f9d6ad7SLin Ling scn->scn_phys.scn_func = *funcp; 2023f9d6ad7SLin Ling scn->scn_phys.scn_state = DSS_SCANNING; 2033f9d6ad7SLin Ling scn->scn_phys.scn_min_txg = 0; 2043f9d6ad7SLin Ling scn->scn_phys.scn_max_txg = tx->tx_txg; 2053f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */ 2063f9d6ad7SLin Ling scn->scn_phys.scn_start_time = gethrestime_sec(); 2073f9d6ad7SLin Ling scn->scn_phys.scn_errors = 0; 2083f9d6ad7SLin Ling scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc; 2093f9d6ad7SLin Ling scn->scn_restart_txg = 0; 210b4952e17SGeorge Wilson scn->scn_done_txg = 0; 2113f9d6ad7SLin Ling spa_scan_stat_init(spa); 2123f9d6ad7SLin Ling 2133f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 2143f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max; 2153f9d6ad7SLin Ling 2163f9d6ad7SLin Ling /* rewrite all disk labels */ 2173f9d6ad7SLin Ling vdev_config_dirty(spa->spa_root_vdev); 2183f9d6ad7SLin Ling 2193f9d6ad7SLin Ling if (vdev_resilver_needed(spa->spa_root_vdev, 2203f9d6ad7SLin Ling &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) { 221ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, 222ce1577b0SDave Eddy ESC_ZFS_RESILVER_START); 2233f9d6ad7SLin Ling } else { 224ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_START); 2253f9d6ad7SLin Ling } 2263f9d6ad7SLin Ling 2273f9d6ad7SLin Ling spa->spa_scrub_started = B_TRUE; 2283f9d6ad7SLin Ling /* 2293f9d6ad7SLin Ling * If this is an incremental scrub, limit the DDT scrub phase 2303f9d6ad7SLin Ling * to just the auto-ditto class (for correctness); the rest 2313f9d6ad7SLin Ling * of the scrub should go faster using top-down pruning. 2323f9d6ad7SLin Ling */ 2333f9d6ad7SLin Ling if (scn->scn_phys.scn_min_txg > TXG_INITIAL) 2343f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO; 2353f9d6ad7SLin Ling 2363f9d6ad7SLin Ling } 2373f9d6ad7SLin Ling 2383f9d6ad7SLin Ling /* back to the generic stuff */ 2393f9d6ad7SLin Ling 2403f9d6ad7SLin Ling if (dp->dp_blkstats == NULL) { 2413f9d6ad7SLin Ling dp->dp_blkstats = 2423f9d6ad7SLin Ling kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP); 2433f9d6ad7SLin Ling } 2443f9d6ad7SLin Ling bzero(dp->dp_blkstats, sizeof (zfs_all_blkstats_t)); 2453f9d6ad7SLin Ling 2463f9d6ad7SLin Ling if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) 2473f9d6ad7SLin Ling ot = DMU_OT_ZAP_OTHER; 2483f9d6ad7SLin Ling 2493f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, 2503f9d6ad7SLin Ling ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx); 2513f9d6ad7SLin Ling 2523f9d6ad7SLin Ling dsl_scan_sync_state(scn, tx); 2533f9d6ad7SLin Ling 2544445fffbSMatthew Ahrens spa_history_log_internal(spa, "scan setup", tx, 2553f9d6ad7SLin Ling "func=%u mintxg=%llu maxtxg=%llu", 2563f9d6ad7SLin Ling *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg); 2573f9d6ad7SLin Ling } 2583f9d6ad7SLin Ling 2593f9d6ad7SLin Ling /* ARGSUSED */ 2603f9d6ad7SLin Ling static void 2613f9d6ad7SLin Ling dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) 2623f9d6ad7SLin Ling { 2633f9d6ad7SLin Ling static const char *old_names[] = { 2643f9d6ad7SLin Ling "scrub_bookmark", 2653f9d6ad7SLin Ling "scrub_ddt_bookmark", 2663f9d6ad7SLin Ling "scrub_ddt_class_max", 2673f9d6ad7SLin Ling "scrub_queue", 2683f9d6ad7SLin Ling "scrub_min_txg", 2693f9d6ad7SLin Ling "scrub_max_txg", 2703f9d6ad7SLin Ling "scrub_func", 2713f9d6ad7SLin Ling "scrub_errors", 2723f9d6ad7SLin Ling NULL 2733f9d6ad7SLin Ling }; 2743f9d6ad7SLin Ling 2753f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 2763f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 2773f9d6ad7SLin Ling int i; 2783f9d6ad7SLin Ling 2793f9d6ad7SLin Ling /* Remove any remnants of an old-style scrub. */ 2803f9d6ad7SLin Ling for (i = 0; old_names[i]; i++) { 2813f9d6ad7SLin Ling (void) zap_remove(dp->dp_meta_objset, 2823f9d6ad7SLin Ling DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx); 2833f9d6ad7SLin Ling } 2843f9d6ad7SLin Ling 2853f9d6ad7SLin Ling if (scn->scn_phys.scn_queue_obj != 0) { 2863f9d6ad7SLin Ling VERIFY(0 == dmu_object_free(dp->dp_meta_objset, 2873f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, tx)); 2883f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj = 0; 2893f9d6ad7SLin Ling } 2903f9d6ad7SLin Ling 2911702cce7SAlek Pinchuk scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; 2921702cce7SAlek Pinchuk 2933f9d6ad7SLin Ling /* 2943f9d6ad7SLin Ling * If we were "restarted" from a stopped state, don't bother 2953f9d6ad7SLin Ling * with anything else. 2963f9d6ad7SLin Ling */ 2973f9d6ad7SLin Ling if (scn->scn_phys.scn_state != DSS_SCANNING) 2983f9d6ad7SLin Ling return; 2993f9d6ad7SLin Ling 3003f9d6ad7SLin Ling if (complete) 3013f9d6ad7SLin Ling scn->scn_phys.scn_state = DSS_FINISHED; 3023f9d6ad7SLin Ling else 3033f9d6ad7SLin Ling scn->scn_phys.scn_state = DSS_CANCELED; 3043f9d6ad7SLin Ling 3051825bc56SNav Ravindranath if (dsl_scan_restarting(scn, tx)) 3061825bc56SNav Ravindranath spa_history_log_internal(spa, "scan aborted, restarting", tx, 3071825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 3081825bc56SNav Ravindranath else if (!complete) 3091825bc56SNav Ravindranath spa_history_log_internal(spa, "scan cancelled", tx, 3101825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 3111825bc56SNav Ravindranath else 3121825bc56SNav Ravindranath spa_history_log_internal(spa, "scan done", tx, 3131825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 3143f9d6ad7SLin Ling 3153f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 3163f9d6ad7SLin Ling mutex_enter(&spa->spa_scrub_lock); 3173f9d6ad7SLin Ling while (spa->spa_scrub_inflight > 0) { 3183f9d6ad7SLin Ling cv_wait(&spa->spa_scrub_io_cv, 3193f9d6ad7SLin Ling &spa->spa_scrub_lock); 3203f9d6ad7SLin Ling } 3213f9d6ad7SLin Ling mutex_exit(&spa->spa_scrub_lock); 3223f9d6ad7SLin Ling spa->spa_scrub_started = B_FALSE; 3233f9d6ad7SLin Ling spa->spa_scrub_active = B_FALSE; 3243f9d6ad7SLin Ling 3253f9d6ad7SLin Ling /* 3263f9d6ad7SLin Ling * If the scrub/resilver completed, update all DTLs to 3273f9d6ad7SLin Ling * reflect this. Whether it succeeded or not, vacate 3283f9d6ad7SLin Ling * all temporary scrub DTLs. 32986714001SSerapheim Dimitropoulos * 33086714001SSerapheim Dimitropoulos * As the scrub does not currently support traversing 33186714001SSerapheim Dimitropoulos * data that have been freed but are part of a checkpoint, 33286714001SSerapheim Dimitropoulos * we don't mark the scrub as done in the DTLs as faults 33386714001SSerapheim Dimitropoulos * may still exist in those vdevs. 3343f9d6ad7SLin Ling */ 33586714001SSerapheim Dimitropoulos if (complete && 33686714001SSerapheim Dimitropoulos !spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 33786714001SSerapheim Dimitropoulos vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 33886714001SSerapheim Dimitropoulos scn->scn_phys.scn_max_txg, B_TRUE); 33986714001SSerapheim Dimitropoulos 340ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, 341ce1577b0SDave Eddy scn->scn_phys.scn_min_txg ? 3423f9d6ad7SLin Ling ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH); 34386714001SSerapheim Dimitropoulos } else { 34486714001SSerapheim Dimitropoulos vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 34586714001SSerapheim Dimitropoulos 0, B_TRUE); 3463f9d6ad7SLin Ling } 3473f9d6ad7SLin Ling spa_errlog_rotate(spa); 3483f9d6ad7SLin Ling 3493f9d6ad7SLin Ling /* 3503f9d6ad7SLin Ling * We may have finished replacing a device. 3513f9d6ad7SLin Ling * Let the async thread assess this and handle the detach. 3523f9d6ad7SLin Ling */ 3533f9d6ad7SLin Ling spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 3543f9d6ad7SLin Ling } 3553f9d6ad7SLin Ling 3563f9d6ad7SLin Ling scn->scn_phys.scn_end_time = gethrestime_sec(); 3573f9d6ad7SLin Ling } 3583f9d6ad7SLin Ling 3593f9d6ad7SLin Ling /* ARGSUSED */ 3603f9d6ad7SLin Ling static int 3613b2aab18SMatthew Ahrens dsl_scan_cancel_check(void *arg, dmu_tx_t *tx) 3623f9d6ad7SLin Ling { 3633b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 3643f9d6ad7SLin Ling 3653f9d6ad7SLin Ling if (scn->scn_phys.scn_state != DSS_SCANNING) 366be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 3673f9d6ad7SLin Ling return (0); 3683f9d6ad7SLin Ling } 3693f9d6ad7SLin Ling 3703f9d6ad7SLin Ling /* ARGSUSED */ 3713f9d6ad7SLin Ling static void 3723b2aab18SMatthew Ahrens dsl_scan_cancel_sync(void *arg, dmu_tx_t *tx) 3733f9d6ad7SLin Ling { 3743b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 3753f9d6ad7SLin Ling 3763f9d6ad7SLin Ling dsl_scan_done(scn, B_FALSE, tx); 3773f9d6ad7SLin Ling dsl_scan_sync_state(scn, tx); 378301fd1d6SSean Eric Fagan spa_event_notify(scn->scn_dp->dp_spa, NULL, NULL, ESC_ZFS_SCRUB_ABORT); 3793f9d6ad7SLin Ling } 3803f9d6ad7SLin Ling 3813f9d6ad7SLin Ling int 3823f9d6ad7SLin Ling dsl_scan_cancel(dsl_pool_t *dp) 3833f9d6ad7SLin Ling { 3843b2aab18SMatthew Ahrens return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check, 3857d46dc6cSMatthew Ahrens dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED)); 3863f9d6ad7SLin Ling } 3873f9d6ad7SLin Ling 3881702cce7SAlek Pinchuk boolean_t 3891702cce7SAlek Pinchuk dsl_scan_is_paused_scrub(const dsl_scan_t *scn) 3901702cce7SAlek Pinchuk { 3911702cce7SAlek Pinchuk if (dsl_scan_scrubbing(scn->scn_dp) && 3921702cce7SAlek Pinchuk scn->scn_phys.scn_flags & DSF_SCRUB_PAUSED) 3931702cce7SAlek Pinchuk return (B_TRUE); 3941702cce7SAlek Pinchuk 3951702cce7SAlek Pinchuk return (B_FALSE); 3961702cce7SAlek Pinchuk } 3971702cce7SAlek Pinchuk 3981702cce7SAlek Pinchuk static int 3991702cce7SAlek Pinchuk dsl_scrub_pause_resume_check(void *arg, dmu_tx_t *tx) 4001702cce7SAlek Pinchuk { 4011702cce7SAlek Pinchuk pool_scrub_cmd_t *cmd = arg; 4021702cce7SAlek Pinchuk dsl_pool_t *dp = dmu_tx_pool(tx); 4031702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 4041702cce7SAlek Pinchuk 4051702cce7SAlek Pinchuk if (*cmd == POOL_SCRUB_PAUSE) { 4061702cce7SAlek Pinchuk /* can't pause a scrub when there is no in-progress scrub */ 4071702cce7SAlek Pinchuk if (!dsl_scan_scrubbing(dp)) 4081702cce7SAlek Pinchuk return (SET_ERROR(ENOENT)); 4091702cce7SAlek Pinchuk 4101702cce7SAlek Pinchuk /* can't pause a paused scrub */ 4111702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) 4121702cce7SAlek Pinchuk return (SET_ERROR(EBUSY)); 4131702cce7SAlek Pinchuk } else if (*cmd != POOL_SCRUB_NORMAL) { 4141702cce7SAlek Pinchuk return (SET_ERROR(ENOTSUP)); 4151702cce7SAlek Pinchuk } 4161702cce7SAlek Pinchuk 4171702cce7SAlek Pinchuk return (0); 4181702cce7SAlek Pinchuk } 4191702cce7SAlek Pinchuk 4201702cce7SAlek Pinchuk static void 4211702cce7SAlek Pinchuk dsl_scrub_pause_resume_sync(void *arg, dmu_tx_t *tx) 4221702cce7SAlek Pinchuk { 4231702cce7SAlek Pinchuk pool_scrub_cmd_t *cmd = arg; 4241702cce7SAlek Pinchuk dsl_pool_t *dp = dmu_tx_pool(tx); 4251702cce7SAlek Pinchuk spa_t *spa = dp->dp_spa; 4261702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 4271702cce7SAlek Pinchuk 4281702cce7SAlek Pinchuk if (*cmd == POOL_SCRUB_PAUSE) { 4291702cce7SAlek Pinchuk /* can't pause a scrub when there is no in-progress scrub */ 4301702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_pause = gethrestime_sec(); 4311702cce7SAlek Pinchuk scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED; 4321702cce7SAlek Pinchuk dsl_scan_sync_state(scn, tx); 433301fd1d6SSean Eric Fagan spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_PAUSED); 4341702cce7SAlek Pinchuk } else { 4351702cce7SAlek Pinchuk ASSERT3U(*cmd, ==, POOL_SCRUB_NORMAL); 4361702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) { 4371702cce7SAlek Pinchuk /* 4381702cce7SAlek Pinchuk * We need to keep track of how much time we spend 4391702cce7SAlek Pinchuk * paused per pass so that we can adjust the scrub rate 4401702cce7SAlek Pinchuk * shown in the output of 'zpool status' 4411702cce7SAlek Pinchuk */ 4421702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_spent_paused += 4431702cce7SAlek Pinchuk gethrestime_sec() - spa->spa_scan_pass_scrub_pause; 4441702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_pause = 0; 4451702cce7SAlek Pinchuk scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; 4461702cce7SAlek Pinchuk dsl_scan_sync_state(scn, tx); 4471702cce7SAlek Pinchuk } 4481702cce7SAlek Pinchuk } 4491702cce7SAlek Pinchuk } 4501702cce7SAlek Pinchuk 4511702cce7SAlek Pinchuk /* 4521702cce7SAlek Pinchuk * Set scrub pause/resume state if it makes sense to do so 4531702cce7SAlek Pinchuk */ 4541702cce7SAlek Pinchuk int 4551702cce7SAlek Pinchuk dsl_scrub_set_pause_resume(const dsl_pool_t *dp, pool_scrub_cmd_t cmd) 4561702cce7SAlek Pinchuk { 4571702cce7SAlek Pinchuk return (dsl_sync_task(spa_name(dp->dp_spa), 4581702cce7SAlek Pinchuk dsl_scrub_pause_resume_check, dsl_scrub_pause_resume_sync, &cmd, 3, 4591702cce7SAlek Pinchuk ZFS_SPACE_CHECK_RESERVED)); 4601702cce7SAlek Pinchuk } 4611702cce7SAlek Pinchuk 4621702cce7SAlek Pinchuk boolean_t 4631702cce7SAlek Pinchuk dsl_scan_scrubbing(const dsl_pool_t *dp) 4641702cce7SAlek Pinchuk { 4651702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 4661702cce7SAlek Pinchuk 4671702cce7SAlek Pinchuk if (scn->scn_phys.scn_state == DSS_SCANNING && 4681702cce7SAlek Pinchuk scn->scn_phys.scn_func == POOL_SCAN_SCRUB) 4691702cce7SAlek Pinchuk return (B_TRUE); 4701702cce7SAlek Pinchuk 4711702cce7SAlek Pinchuk return (B_FALSE); 4721702cce7SAlek Pinchuk } 4731702cce7SAlek Pinchuk 4745f37736aSMatthew Ahrens static void dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 4755f37736aSMatthew Ahrens dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, 4765f37736aSMatthew Ahrens dmu_objset_type_t ostype, dmu_tx_t *tx); 4773f9d6ad7SLin Ling static void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds, 4783f9d6ad7SLin Ling dmu_objset_type_t ostype, 4795f37736aSMatthew Ahrens dnode_phys_t *dnp, uint64_t object, dmu_tx_t *tx); 4803f9d6ad7SLin Ling 4813f9d6ad7SLin Ling void 4823f9d6ad7SLin Ling dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp) 4833f9d6ad7SLin Ling { 4843f9d6ad7SLin Ling zio_free(dp->dp_spa, txg, bp); 4853f9d6ad7SLin Ling } 4863f9d6ad7SLin Ling 4873f9d6ad7SLin Ling void 4883f9d6ad7SLin Ling dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp) 4893f9d6ad7SLin Ling { 4903f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dp)); 4913f9d6ad7SLin Ling zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, pio->io_flags)); 4923f9d6ad7SLin Ling } 4933f9d6ad7SLin Ling 4943f9d6ad7SLin Ling static uint64_t 4953f9d6ad7SLin Ling dsl_scan_ds_maxtxg(dsl_dataset_t *ds) 4963f9d6ad7SLin Ling { 4973f9d6ad7SLin Ling uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg; 498bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) 499c1379625SJustin T. Gibbs return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg)); 5003f9d6ad7SLin Ling return (smt); 5013f9d6ad7SLin Ling } 5023f9d6ad7SLin Ling 5033f9d6ad7SLin Ling static void 5043f9d6ad7SLin Ling dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx) 5053f9d6ad7SLin Ling { 5063b2aab18SMatthew Ahrens VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 5073f9d6ad7SLin Ling DMU_POOL_DIRECTORY_OBJECT, 5083f9d6ad7SLin Ling DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 5093f9d6ad7SLin Ling &scn->scn_phys, tx)); 5103f9d6ad7SLin Ling } 5113f9d6ad7SLin Ling 5126f6a76adSMatthew Ahrens extern int zfs_vdev_async_write_active_min_dirty_percent; 5136f6a76adSMatthew Ahrens 5143f9d6ad7SLin Ling static boolean_t 5151702cce7SAlek Pinchuk dsl_scan_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb) 5163f9d6ad7SLin Ling { 5173f9d6ad7SLin Ling /* we never skip user/group accounting objects */ 5183f9d6ad7SLin Ling if (zb && (int64_t)zb->zb_object < 0) 5193f9d6ad7SLin Ling return (B_FALSE); 5203f9d6ad7SLin Ling 5211702cce7SAlek Pinchuk if (scn->scn_suspending) 5221702cce7SAlek Pinchuk return (B_TRUE); /* we're already suspending */ 5233f9d6ad7SLin Ling 524ad135b5dSChristopher Siden if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) 5253f9d6ad7SLin Ling return (B_FALSE); /* we're resuming */ 5263f9d6ad7SLin Ling 5273f9d6ad7SLin Ling /* We only know how to resume from level-0 blocks. */ 5283f9d6ad7SLin Ling if (zb && zb->zb_level != 0) 5293f9d6ad7SLin Ling return (B_FALSE); 5303f9d6ad7SLin Ling 5316f6a76adSMatthew Ahrens /* 5321702cce7SAlek Pinchuk * We suspend if: 5336f6a76adSMatthew Ahrens * - we have scanned for the maximum time: an entire txg 5346f6a76adSMatthew Ahrens * timeout (default 5 sec) 5356f6a76adSMatthew Ahrens * or 5366f6a76adSMatthew Ahrens * - we have scanned for at least the minimum time (default 1 sec 5376f6a76adSMatthew Ahrens * for scrub, 3 sec for resilver), and either we have sufficient 5386f6a76adSMatthew Ahrens * dirty data that we are starting to write more quickly 5396f6a76adSMatthew Ahrens * (default 30%), or someone is explicitly waiting for this txg 5406f6a76adSMatthew Ahrens * to complete. 5416f6a76adSMatthew Ahrens * or 5426f6a76adSMatthew Ahrens * - the spa is shutting down because this pool is being exported 5436f6a76adSMatthew Ahrens * or the machine is rebooting. 5446f6a76adSMatthew Ahrens */ 5456f6a76adSMatthew Ahrens int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 5463f9d6ad7SLin Ling zfs_resilver_min_time_ms : zfs_scan_min_time_ms; 5476f6a76adSMatthew Ahrens uint64_t elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 5486f6a76adSMatthew Ahrens int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; 5496f6a76adSMatthew Ahrens if (elapsed_nanosecs / NANOSEC >= zfs_txg_timeout || 5500689f76cSAdam Leventhal (NSEC2MSEC(elapsed_nanosecs) > mintime && 5516f6a76adSMatthew Ahrens (txg_sync_waiting(scn->scn_dp) || 5526f6a76adSMatthew Ahrens dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent)) || 5533f9d6ad7SLin Ling spa_shutting_down(scn->scn_dp->dp_spa)) { 5543f9d6ad7SLin Ling if (zb) { 5551702cce7SAlek Pinchuk dprintf("suspending at bookmark %llx/%llx/%llx/%llx\n", 5563f9d6ad7SLin Ling (longlong_t)zb->zb_objset, 5573f9d6ad7SLin Ling (longlong_t)zb->zb_object, 5583f9d6ad7SLin Ling (longlong_t)zb->zb_level, 5593f9d6ad7SLin Ling (longlong_t)zb->zb_blkid); 5603f9d6ad7SLin Ling scn->scn_phys.scn_bookmark = *zb; 5613f9d6ad7SLin Ling } 5621702cce7SAlek Pinchuk dprintf("suspending at DDT bookmark %llx/%llx/%llx/%llx\n", 5633f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class, 5643f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type, 5653f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum, 5663f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor); 5671702cce7SAlek Pinchuk scn->scn_suspending = B_TRUE; 5683f9d6ad7SLin Ling return (B_TRUE); 5693f9d6ad7SLin Ling } 5703f9d6ad7SLin Ling return (B_FALSE); 5713f9d6ad7SLin Ling } 5723f9d6ad7SLin Ling 5733f9d6ad7SLin Ling typedef struct zil_scan_arg { 5743f9d6ad7SLin Ling dsl_pool_t *zsa_dp; 5753f9d6ad7SLin Ling zil_header_t *zsa_zh; 5763f9d6ad7SLin Ling } zil_scan_arg_t; 5773f9d6ad7SLin Ling 5783f9d6ad7SLin Ling /* ARGSUSED */ 5793f9d6ad7SLin Ling static int 5803f9d6ad7SLin Ling dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 5813f9d6ad7SLin Ling { 5823f9d6ad7SLin Ling zil_scan_arg_t *zsa = arg; 5833f9d6ad7SLin Ling dsl_pool_t *dp = zsa->zsa_dp; 5843f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 5853f9d6ad7SLin Ling zil_header_t *zh = zsa->zsa_zh; 5867802d7bfSMatthew Ahrens zbookmark_phys_t zb; 5873f9d6ad7SLin Ling 58843466aaeSMax Grossman if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 5893f9d6ad7SLin Ling return (0); 5903f9d6ad7SLin Ling 5913f9d6ad7SLin Ling /* 5923f9d6ad7SLin Ling * One block ("stubby") can be allocated a long time ago; we 5933f9d6ad7SLin Ling * want to visit that one because it has been allocated 5943f9d6ad7SLin Ling * (on-disk) even if it hasn't been claimed (even though for 5953f9d6ad7SLin Ling * scrub there's nothing to do to it). 5963f9d6ad7SLin Ling */ 59786714001SSerapheim Dimitropoulos if (claim_txg == 0 && bp->blk_birth >= spa_min_claim_txg(dp->dp_spa)) 5983f9d6ad7SLin Ling return (0); 5993f9d6ad7SLin Ling 6003f9d6ad7SLin Ling SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 6013f9d6ad7SLin Ling ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 6023f9d6ad7SLin Ling 6033f9d6ad7SLin Ling VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 6043f9d6ad7SLin Ling return (0); 6053f9d6ad7SLin Ling } 6063f9d6ad7SLin Ling 6073f9d6ad7SLin Ling /* ARGSUSED */ 6083f9d6ad7SLin Ling static int 6093f9d6ad7SLin Ling dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg) 6103f9d6ad7SLin Ling { 6113f9d6ad7SLin Ling if (lrc->lrc_txtype == TX_WRITE) { 6123f9d6ad7SLin Ling zil_scan_arg_t *zsa = arg; 6133f9d6ad7SLin Ling dsl_pool_t *dp = zsa->zsa_dp; 6143f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 6153f9d6ad7SLin Ling zil_header_t *zh = zsa->zsa_zh; 6163f9d6ad7SLin Ling lr_write_t *lr = (lr_write_t *)lrc; 6173f9d6ad7SLin Ling blkptr_t *bp = &lr->lr_blkptr; 6187802d7bfSMatthew Ahrens zbookmark_phys_t zb; 6193f9d6ad7SLin Ling 62043466aaeSMax Grossman if (BP_IS_HOLE(bp) || 62143466aaeSMax Grossman bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 6223f9d6ad7SLin Ling return (0); 6233f9d6ad7SLin Ling 6243f9d6ad7SLin Ling /* 6253f9d6ad7SLin Ling * birth can be < claim_txg if this record's txg is 6263f9d6ad7SLin Ling * already txg sync'ed (but this log block contains 6273f9d6ad7SLin Ling * other records that are not synced) 6283f9d6ad7SLin Ling */ 6293f9d6ad7SLin Ling if (claim_txg == 0 || bp->blk_birth < claim_txg) 6303f9d6ad7SLin Ling return (0); 6313f9d6ad7SLin Ling 6323f9d6ad7SLin Ling SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 6333f9d6ad7SLin Ling lr->lr_foid, ZB_ZIL_LEVEL, 6343f9d6ad7SLin Ling lr->lr_offset / BP_GET_LSIZE(bp)); 6353f9d6ad7SLin Ling 6363f9d6ad7SLin Ling VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 6373f9d6ad7SLin Ling } 6383f9d6ad7SLin Ling return (0); 6393f9d6ad7SLin Ling } 6403f9d6ad7SLin Ling 6413f9d6ad7SLin Ling static void 6423f9d6ad7SLin Ling dsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh) 6433f9d6ad7SLin Ling { 6443f9d6ad7SLin Ling uint64_t claim_txg = zh->zh_claim_txg; 6453f9d6ad7SLin Ling zil_scan_arg_t zsa = { dp, zh }; 6463f9d6ad7SLin Ling zilog_t *zilog; 6473f9d6ad7SLin Ling 64886714001SSerapheim Dimitropoulos ASSERT(spa_writeable(dp->dp_spa)); 64986714001SSerapheim Dimitropoulos 6503f9d6ad7SLin Ling /* 65186714001SSerapheim Dimitropoulos * We only want to visit blocks that have been claimed 65286714001SSerapheim Dimitropoulos * but not yet replayed. 6533f9d6ad7SLin Ling */ 65486714001SSerapheim Dimitropoulos if (claim_txg == 0) 6553f9d6ad7SLin Ling return; 6563f9d6ad7SLin Ling 6573f9d6ad7SLin Ling zilog = zil_alloc(dp->dp_meta_objset, zh); 6583f9d6ad7SLin Ling 6593f9d6ad7SLin Ling (void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa, 6603f9d6ad7SLin Ling claim_txg); 6613f9d6ad7SLin Ling 6623f9d6ad7SLin Ling zil_free(zilog); 6633f9d6ad7SLin Ling } 6643f9d6ad7SLin Ling 6653f9d6ad7SLin Ling /* ARGSUSED */ 6663f9d6ad7SLin Ling static void 6673f9d6ad7SLin Ling dsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp, 6683f9d6ad7SLin Ling uint64_t objset, uint64_t object, uint64_t blkid) 6693f9d6ad7SLin Ling { 6707802d7bfSMatthew Ahrens zbookmark_phys_t czb; 6717adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_NOWAIT | ARC_FLAG_PREFETCH; 6723f9d6ad7SLin Ling 6733f9d6ad7SLin Ling if (zfs_no_scrub_prefetch) 6743f9d6ad7SLin Ling return; 6753f9d6ad7SLin Ling 6763f9d6ad7SLin Ling if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_min_txg || 6773f9d6ad7SLin Ling (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE)) 6783f9d6ad7SLin Ling return; 6793f9d6ad7SLin Ling 6803f9d6ad7SLin Ling SET_BOOKMARK(&czb, objset, object, BP_GET_LEVEL(bp), blkid); 6813f9d6ad7SLin Ling 682cde58dbcSMatthew Ahrens (void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, bp, 6831b912ec7SGeorge Wilson NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 68444ecc532SGeorge Wilson ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD, &flags, &czb); 6853f9d6ad7SLin Ling } 6863f9d6ad7SLin Ling 6873f9d6ad7SLin Ling static boolean_t 6883f9d6ad7SLin Ling dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp, 6897802d7bfSMatthew Ahrens const zbookmark_phys_t *zb) 6903f9d6ad7SLin Ling { 6913f9d6ad7SLin Ling /* 6923f9d6ad7SLin Ling * We never skip over user/group accounting objects (obj<0) 6933f9d6ad7SLin Ling */ 694ad135b5dSChristopher Siden if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) && 6953f9d6ad7SLin Ling (int64_t)zb->zb_object >= 0) { 6963f9d6ad7SLin Ling /* 6973f9d6ad7SLin Ling * If we already visited this bp & everything below (in 6983f9d6ad7SLin Ling * a prior txg sync), don't bother doing it again. 6993f9d6ad7SLin Ling */ 700a2cdcdd2SPaul Dagnelie if (zbookmark_subtree_completed(dnp, zb, 701a2cdcdd2SPaul Dagnelie &scn->scn_phys.scn_bookmark)) 7023f9d6ad7SLin Ling return (B_TRUE); 7033f9d6ad7SLin Ling 7043f9d6ad7SLin Ling /* 7053f9d6ad7SLin Ling * If we found the block we're trying to resume from, or 7063f9d6ad7SLin Ling * we went past it to a different object, zero it out to 7071702cce7SAlek Pinchuk * indicate that it's OK to start checking for suspending 7083f9d6ad7SLin Ling * again. 7093f9d6ad7SLin Ling */ 7103f9d6ad7SLin Ling if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 || 7113f9d6ad7SLin Ling zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) { 7123f9d6ad7SLin Ling dprintf("resuming at %llx/%llx/%llx/%llx\n", 7133f9d6ad7SLin Ling (longlong_t)zb->zb_objset, 7143f9d6ad7SLin Ling (longlong_t)zb->zb_object, 7153f9d6ad7SLin Ling (longlong_t)zb->zb_level, 7163f9d6ad7SLin Ling (longlong_t)zb->zb_blkid); 7173f9d6ad7SLin Ling bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb)); 7183f9d6ad7SLin Ling } 7193f9d6ad7SLin Ling } 7203f9d6ad7SLin Ling return (B_FALSE); 7213f9d6ad7SLin Ling } 7223f9d6ad7SLin Ling 7233f9d6ad7SLin Ling /* 7243f9d6ad7SLin Ling * Return nonzero on i/o error. 7253f9d6ad7SLin Ling * Return new buf to write out in *bufp. 7263f9d6ad7SLin Ling */ 7273f9d6ad7SLin Ling static int 7283f9d6ad7SLin Ling dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, 7293f9d6ad7SLin Ling dnode_phys_t *dnp, const blkptr_t *bp, 7305f37736aSMatthew Ahrens const zbookmark_phys_t *zb, dmu_tx_t *tx) 7313f9d6ad7SLin Ling { 7323f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 73344ecc532SGeorge Wilson int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 7343f9d6ad7SLin Ling int err; 7353f9d6ad7SLin Ling 7363f9d6ad7SLin Ling if (BP_GET_LEVEL(bp) > 0) { 7377adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 7383f9d6ad7SLin Ling int i; 7393f9d6ad7SLin Ling blkptr_t *cbp; 7403f9d6ad7SLin Ling int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 7415f37736aSMatthew Ahrens arc_buf_t *buf; 7423f9d6ad7SLin Ling 7435f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 74444ecc532SGeorge Wilson ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 7453f9d6ad7SLin Ling if (err) { 7463f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 7473f9d6ad7SLin Ling return (err); 7483f9d6ad7SLin Ling } 7495f37736aSMatthew Ahrens for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) { 7505f37736aSMatthew Ahrens dsl_scan_prefetch(scn, buf, cbp, zb->zb_objset, 7513f9d6ad7SLin Ling zb->zb_object, zb->zb_blkid * epb + i); 7523f9d6ad7SLin Ling } 7535f37736aSMatthew Ahrens for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) { 7547802d7bfSMatthew Ahrens zbookmark_phys_t czb; 7553f9d6ad7SLin Ling 7563f9d6ad7SLin Ling SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 7573f9d6ad7SLin Ling zb->zb_level - 1, 7583f9d6ad7SLin Ling zb->zb_blkid * epb + i); 7593f9d6ad7SLin Ling dsl_scan_visitbp(cbp, &czb, dnp, 7605f37736aSMatthew Ahrens ds, scn, ostype, tx); 7613f9d6ad7SLin Ling } 762dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 7633f9d6ad7SLin Ling } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 7647adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 7653f9d6ad7SLin Ling dnode_phys_t *cdnp; 7663f9d6ad7SLin Ling int i, j; 7673f9d6ad7SLin Ling int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 7685f37736aSMatthew Ahrens arc_buf_t *buf; 7693f9d6ad7SLin Ling 7705f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 77144ecc532SGeorge Wilson ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 7723f9d6ad7SLin Ling if (err) { 7733f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 7743f9d6ad7SLin Ling return (err); 7753f9d6ad7SLin Ling } 7765f37736aSMatthew Ahrens for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) { 7773f9d6ad7SLin Ling for (j = 0; j < cdnp->dn_nblkptr; j++) { 7783f9d6ad7SLin Ling blkptr_t *cbp = &cdnp->dn_blkptr[j]; 7795f37736aSMatthew Ahrens dsl_scan_prefetch(scn, buf, cbp, 7803f9d6ad7SLin Ling zb->zb_objset, zb->zb_blkid * epb + i, j); 7813f9d6ad7SLin Ling } 7823f9d6ad7SLin Ling } 7835f37736aSMatthew Ahrens for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) { 7843f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, ostype, 7855f37736aSMatthew Ahrens cdnp, zb->zb_blkid * epb + i, tx); 7863f9d6ad7SLin Ling } 7873f9d6ad7SLin Ling 788dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 7893f9d6ad7SLin Ling } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 7907adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 7913f9d6ad7SLin Ling objset_phys_t *osp; 7925f37736aSMatthew Ahrens arc_buf_t *buf; 7933f9d6ad7SLin Ling 7945f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 79544ecc532SGeorge Wilson ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 7963f9d6ad7SLin Ling if (err) { 7973f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 7983f9d6ad7SLin Ling return (err); 7993f9d6ad7SLin Ling } 8003f9d6ad7SLin Ling 8015f37736aSMatthew Ahrens osp = buf->b_data; 8023f9d6ad7SLin Ling 8033f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 8045f37736aSMatthew Ahrens &osp->os_meta_dnode, DMU_META_DNODE_OBJECT, tx); 8053f9d6ad7SLin Ling 8065f37736aSMatthew Ahrens if (OBJSET_BUF_HAS_USERUSED(buf)) { 8073f9d6ad7SLin Ling /* 8083f9d6ad7SLin Ling * We also always visit user/group accounting 8093f9d6ad7SLin Ling * objects, and never skip them, even if we are 8101702cce7SAlek Pinchuk * suspending. This is necessary so that the space 8113f9d6ad7SLin Ling * deltas from this txg get integrated. 8123f9d6ad7SLin Ling */ 8133f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 8145f37736aSMatthew Ahrens &osp->os_groupused_dnode, 8153f9d6ad7SLin Ling DMU_GROUPUSED_OBJECT, tx); 8163f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 8175f37736aSMatthew Ahrens &osp->os_userused_dnode, 8183f9d6ad7SLin Ling DMU_USERUSED_OBJECT, tx); 8193f9d6ad7SLin Ling } 820dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 8213f9d6ad7SLin Ling } 8223f9d6ad7SLin Ling 8233f9d6ad7SLin Ling return (0); 8243f9d6ad7SLin Ling } 8253f9d6ad7SLin Ling 8263f9d6ad7SLin Ling static void 8273f9d6ad7SLin Ling dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, 8285f37736aSMatthew Ahrens dmu_objset_type_t ostype, dnode_phys_t *dnp, 8293f9d6ad7SLin Ling uint64_t object, dmu_tx_t *tx) 8303f9d6ad7SLin Ling { 8313f9d6ad7SLin Ling int j; 8323f9d6ad7SLin Ling 8333f9d6ad7SLin Ling for (j = 0; j < dnp->dn_nblkptr; j++) { 8347802d7bfSMatthew Ahrens zbookmark_phys_t czb; 8353f9d6ad7SLin Ling 8363f9d6ad7SLin Ling SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 8373f9d6ad7SLin Ling dnp->dn_nlevels - 1, j); 8383f9d6ad7SLin Ling dsl_scan_visitbp(&dnp->dn_blkptr[j], 8395f37736aSMatthew Ahrens &czb, dnp, ds, scn, ostype, tx); 8403f9d6ad7SLin Ling } 8413f9d6ad7SLin Ling 8423f9d6ad7SLin Ling if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 8437802d7bfSMatthew Ahrens zbookmark_phys_t czb; 8443f9d6ad7SLin Ling SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 8453f9d6ad7SLin Ling 0, DMU_SPILL_BLKID); 8463f9d6ad7SLin Ling dsl_scan_visitbp(&dnp->dn_spill, 8475f37736aSMatthew Ahrens &czb, dnp, ds, scn, ostype, tx); 8483f9d6ad7SLin Ling } 8493f9d6ad7SLin Ling } 8503f9d6ad7SLin Ling 8513f9d6ad7SLin Ling /* 8523f9d6ad7SLin Ling * The arguments are in this order because mdb can only print the 8533f9d6ad7SLin Ling * first 5; we want them to be useful. 8543f9d6ad7SLin Ling */ 8553f9d6ad7SLin Ling static void 8567802d7bfSMatthew Ahrens dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 8575f37736aSMatthew Ahrens dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, 8585f37736aSMatthew Ahrens dmu_objset_type_t ostype, dmu_tx_t *tx) 8593f9d6ad7SLin Ling { 8603f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 8613f9d6ad7SLin Ling arc_buf_t *buf = NULL; 8623f9d6ad7SLin Ling blkptr_t bp_toread = *bp; 8633f9d6ad7SLin Ling 8643f9d6ad7SLin Ling /* ASSERT(pbuf == NULL || arc_released(pbuf)); */ 8653f9d6ad7SLin Ling 8661702cce7SAlek Pinchuk if (dsl_scan_check_suspend(scn, zb)) 8673f9d6ad7SLin Ling return; 8683f9d6ad7SLin Ling 8693f9d6ad7SLin Ling if (dsl_scan_check_resume(scn, dnp, zb)) 8703f9d6ad7SLin Ling return; 8713f9d6ad7SLin Ling 87243466aaeSMax Grossman if (BP_IS_HOLE(bp)) 8733f9d6ad7SLin Ling return; 8743f9d6ad7SLin Ling 8753f9d6ad7SLin Ling scn->scn_visited_this_txg++; 8763f9d6ad7SLin Ling 8773f9d6ad7SLin Ling dprintf_bp(bp, 8785f37736aSMatthew Ahrens "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx bp=%p", 8793f9d6ad7SLin Ling ds, ds ? ds->ds_object : 0, 8803f9d6ad7SLin Ling zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid, 8815f37736aSMatthew Ahrens bp); 8823f9d6ad7SLin Ling 8833f9d6ad7SLin Ling if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 8843f9d6ad7SLin Ling return; 8853f9d6ad7SLin Ling 8865f37736aSMatthew Ahrens if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx) != 0) 8873f9d6ad7SLin Ling return; 8883f9d6ad7SLin Ling 8893f9d6ad7SLin Ling /* 89040713f2bSAlan Somers * If dsl_scan_ddt() has already visited this block, it will have 8913f9d6ad7SLin Ling * already done any translations or scrubbing, so don't call the 8923f9d6ad7SLin Ling * callback again. 8933f9d6ad7SLin Ling */ 8943f9d6ad7SLin Ling if (ddt_class_contains(dp->dp_spa, 8953f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max, bp)) { 8963f9d6ad7SLin Ling ASSERT(buf == NULL); 8973f9d6ad7SLin Ling return; 8983f9d6ad7SLin Ling } 8993f9d6ad7SLin Ling 9003f9d6ad7SLin Ling /* 9013f9d6ad7SLin Ling * If this block is from the future (after cur_max_txg), then we 9023f9d6ad7SLin Ling * are doing this on behalf of a deleted snapshot, and we will 9033f9d6ad7SLin Ling * revisit the future block on the next pass of this dataset. 9043f9d6ad7SLin Ling * Don't scan it now unless we need to because something 9053f9d6ad7SLin Ling * under it was modified. 9063f9d6ad7SLin Ling */ 907b4952e17SGeorge Wilson if (BP_PHYSICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_max_txg) { 9083f9d6ad7SLin Ling scan_funcs[scn->scn_phys.scn_func](dp, bp, zb); 9093f9d6ad7SLin Ling } 9103f9d6ad7SLin Ling } 9113f9d6ad7SLin Ling 9123f9d6ad7SLin Ling static void 9133f9d6ad7SLin Ling dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp, 9143f9d6ad7SLin Ling dmu_tx_t *tx) 9153f9d6ad7SLin Ling { 9167802d7bfSMatthew Ahrens zbookmark_phys_t zb; 9173f9d6ad7SLin Ling 9183f9d6ad7SLin Ling SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 9193f9d6ad7SLin Ling ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID); 9205f37736aSMatthew Ahrens dsl_scan_visitbp(bp, &zb, NULL, 9213f9d6ad7SLin Ling ds, scn, DMU_OST_NONE, tx); 9223f9d6ad7SLin Ling 9233f9d6ad7SLin Ling dprintf_ds(ds, "finished scan%s", ""); 9243f9d6ad7SLin Ling } 9253f9d6ad7SLin Ling 9263f9d6ad7SLin Ling void 9273f9d6ad7SLin Ling dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx) 9283f9d6ad7SLin Ling { 9293f9d6ad7SLin Ling dsl_pool_t *dp = ds->ds_dir->dd_pool; 9303f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 9313f9d6ad7SLin Ling uint64_t mintxg; 9323f9d6ad7SLin Ling 9333f9d6ad7SLin Ling if (scn->scn_phys.scn_state != DSS_SCANNING) 9343f9d6ad7SLin Ling return; 9353f9d6ad7SLin Ling 9363f9d6ad7SLin Ling if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) { 937bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) { 93838d61036SMatthew Ahrens /* 93938d61036SMatthew Ahrens * Note: 94038d61036SMatthew Ahrens * - scn_cur_{min,max}_txg stays the same. 94138d61036SMatthew Ahrens * - Setting the flag is not really necessary if 94238d61036SMatthew Ahrens * scn_cur_max_txg == scn_max_txg, because there 94338d61036SMatthew Ahrens * is nothing after this snapshot that we care 94438d61036SMatthew Ahrens * about. However, we set it anyway and then 94538d61036SMatthew Ahrens * ignore it when we retraverse it in 94638d61036SMatthew Ahrens * dsl_scan_visitds(). 94738d61036SMatthew Ahrens */ 9483f9d6ad7SLin Ling scn->scn_phys.scn_bookmark.zb_objset = 949c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj; 9503f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; currently traversing; " 9513f9d6ad7SLin Ling "reset zb_objset to %llu", 9523f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 953c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)-> 954c1379625SJustin T. Gibbs ds_next_snap_obj); 9553f9d6ad7SLin Ling scn->scn_phys.scn_flags |= DSF_VISIT_DS_AGAIN; 9563f9d6ad7SLin Ling } else { 9573f9d6ad7SLin Ling SET_BOOKMARK(&scn->scn_phys.scn_bookmark, 9583f9d6ad7SLin Ling ZB_DESTROYED_OBJSET, 0, 0, 0); 9593f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; currently traversing; " 9603f9d6ad7SLin Ling "reset bookmark to -1,0,0,0", 9613f9d6ad7SLin Ling (u_longlong_t)ds->ds_object); 9623f9d6ad7SLin Ling } 9633f9d6ad7SLin Ling } else if (zap_lookup_int_key(dp->dp_meta_objset, 9643f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) { 965c1379625SJustin T. Gibbs ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1); 966b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 9673f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 968bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) { 9693f9d6ad7SLin Ling /* 9703f9d6ad7SLin Ling * We keep the same mintxg; it could be > 9713f9d6ad7SLin Ling * ds_creation_txg if the previous snapshot was 9723f9d6ad7SLin Ling * deleted too. 9733f9d6ad7SLin Ling */ 9743f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 9753f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 976c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj, 977c1379625SJustin T. Gibbs mintxg, tx) == 0); 9783f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; in queue; " 9793f9d6ad7SLin Ling "replacing with %llu", 9803f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 981c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)-> 982c1379625SJustin T. Gibbs ds_next_snap_obj); 9833f9d6ad7SLin Ling } else { 9843f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; in queue; removing", 9853f9d6ad7SLin Ling (u_longlong_t)ds->ds_object); 9863f9d6ad7SLin Ling } 9873f9d6ad7SLin Ling } 9883f9d6ad7SLin Ling 9893f9d6ad7SLin Ling /* 9903f9d6ad7SLin Ling * dsl_scan_sync() should be called after this, and should sync 9913f9d6ad7SLin Ling * out our changed state, but just to be safe, do it here. 9923f9d6ad7SLin Ling */ 9933f9d6ad7SLin Ling dsl_scan_sync_state(scn, tx); 9943f9d6ad7SLin Ling } 9953f9d6ad7SLin Ling 9963f9d6ad7SLin Ling void 9973f9d6ad7SLin Ling dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx) 9983f9d6ad7SLin Ling { 9993f9d6ad7SLin Ling dsl_pool_t *dp = ds->ds_dir->dd_pool; 10003f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 10013f9d6ad7SLin Ling uint64_t mintxg; 10023f9d6ad7SLin Ling 10033f9d6ad7SLin Ling if (scn->scn_phys.scn_state != DSS_SCANNING) 10043f9d6ad7SLin Ling return; 10053f9d6ad7SLin Ling 1006c1379625SJustin T. Gibbs ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0); 10073f9d6ad7SLin Ling 10083f9d6ad7SLin Ling if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) { 10093f9d6ad7SLin Ling scn->scn_phys.scn_bookmark.zb_objset = 1010c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj; 10113f9d6ad7SLin Ling zfs_dbgmsg("snapshotting ds %llu; currently traversing; " 10123f9d6ad7SLin Ling "reset zb_objset to %llu", 10133f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 1014c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj); 10153f9d6ad7SLin Ling } else if (zap_lookup_int_key(dp->dp_meta_objset, 10163f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) { 1017b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 10183f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 10193f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 10203f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 1021c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg, tx) == 0); 10223f9d6ad7SLin Ling zfs_dbgmsg("snapshotting ds %llu; in queue; " 10233f9d6ad7SLin Ling "replacing with %llu", 10243f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 1025c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj); 10263f9d6ad7SLin Ling } 10273f9d6ad7SLin Ling dsl_scan_sync_state(scn, tx); 10283f9d6ad7SLin Ling } 10293f9d6ad7SLin Ling 10303f9d6ad7SLin Ling void 10313f9d6ad7SLin Ling dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx) 10323f9d6ad7SLin Ling { 10333f9d6ad7SLin Ling dsl_pool_t *dp = ds1->ds_dir->dd_pool; 10343f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 10353f9d6ad7SLin Ling uint64_t mintxg; 10363f9d6ad7SLin Ling 10373f9d6ad7SLin Ling if (scn->scn_phys.scn_state != DSS_SCANNING) 10383f9d6ad7SLin Ling return; 10393f9d6ad7SLin Ling 10403f9d6ad7SLin Ling if (scn->scn_phys.scn_bookmark.zb_objset == ds1->ds_object) { 10413f9d6ad7SLin Ling scn->scn_phys.scn_bookmark.zb_objset = ds2->ds_object; 10423f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 10433f9d6ad7SLin Ling "reset zb_objset to %llu", 10443f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object, 10453f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object); 10463f9d6ad7SLin Ling } else if (scn->scn_phys.scn_bookmark.zb_objset == ds2->ds_object) { 10473f9d6ad7SLin Ling scn->scn_phys.scn_bookmark.zb_objset = ds1->ds_object; 10483f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 10493f9d6ad7SLin Ling "reset zb_objset to %llu", 10503f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object, 10513f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object); 10523f9d6ad7SLin Ling } 10533f9d6ad7SLin Ling 10543f9d6ad7SLin Ling if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 10553f9d6ad7SLin Ling ds1->ds_object, &mintxg) == 0) { 10563f9d6ad7SLin Ling int err; 10573f9d6ad7SLin Ling 1058c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg); 1059c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg); 1060b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 10613f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds1->ds_object, tx)); 10623f9d6ad7SLin Ling err = zap_add_int_key(dp->dp_meta_objset, 10633f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx); 10643f9d6ad7SLin Ling VERIFY(err == 0 || err == EEXIST); 10653f9d6ad7SLin Ling if (err == EEXIST) { 10663f9d6ad7SLin Ling /* Both were there to begin with */ 10673f9d6ad7SLin Ling VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 10683f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 10693f9d6ad7SLin Ling ds1->ds_object, mintxg, tx)); 10703f9d6ad7SLin Ling } 10713f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; in queue; " 10723f9d6ad7SLin Ling "replacing with %llu", 10733f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object, 10743f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object); 10753f9d6ad7SLin Ling } else if (zap_lookup_int_key(dp->dp_meta_objset, 10763f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg) == 0) { 1077c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg); 1078c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg); 1079b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 10803f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, tx)); 10813f9d6ad7SLin Ling VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 10823f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx)); 10833f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; in queue; " 10843f9d6ad7SLin Ling "replacing with %llu", 10853f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object, 10863f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object); 10873f9d6ad7SLin Ling } 10883f9d6ad7SLin Ling 10893f9d6ad7SLin Ling dsl_scan_sync_state(scn, tx); 10903f9d6ad7SLin Ling } 10913f9d6ad7SLin Ling 10923f9d6ad7SLin Ling struct enqueue_clones_arg { 10933f9d6ad7SLin Ling dmu_tx_t *tx; 10943f9d6ad7SLin Ling uint64_t originobj; 10953f9d6ad7SLin Ling }; 10963f9d6ad7SLin Ling 10973f9d6ad7SLin Ling /* ARGSUSED */ 10983f9d6ad7SLin Ling static int 10993b2aab18SMatthew Ahrens enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 11003f9d6ad7SLin Ling { 11013f9d6ad7SLin Ling struct enqueue_clones_arg *eca = arg; 11023f9d6ad7SLin Ling dsl_dataset_t *ds; 11033f9d6ad7SLin Ling int err; 11043f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 11053f9d6ad7SLin Ling 1106c1379625SJustin T. Gibbs if (dsl_dir_phys(hds->ds_dir)->dd_origin_obj != eca->originobj) 11073b2aab18SMatthew Ahrens return (0); 11083b2aab18SMatthew Ahrens 11093b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 11103f9d6ad7SLin Ling if (err) 11113f9d6ad7SLin Ling return (err); 11123f9d6ad7SLin Ling 1113c1379625SJustin T. Gibbs while (dsl_dataset_phys(ds)->ds_prev_snap_obj != eca->originobj) { 11143b2aab18SMatthew Ahrens dsl_dataset_t *prev; 11153b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, 1116c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev); 11173f9d6ad7SLin Ling 11183b2aab18SMatthew Ahrens dsl_dataset_rele(ds, FTAG); 11193b2aab18SMatthew Ahrens if (err) 11203b2aab18SMatthew Ahrens return (err); 11213b2aab18SMatthew Ahrens ds = prev; 11223f9d6ad7SLin Ling } 11233b2aab18SMatthew Ahrens VERIFY(zap_add_int_key(dp->dp_meta_objset, 11243b2aab18SMatthew Ahrens scn->scn_phys.scn_queue_obj, ds->ds_object, 1125c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_txg, eca->tx) == 0); 11263f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 11273f9d6ad7SLin Ling return (0); 11283f9d6ad7SLin Ling } 11293f9d6ad7SLin Ling 11303f9d6ad7SLin Ling static void 11313f9d6ad7SLin Ling dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) 11323f9d6ad7SLin Ling { 11333f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 11343f9d6ad7SLin Ling dsl_dataset_t *ds; 11353f9d6ad7SLin Ling 1136b420f3adSRichard Lowe VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 11373f9d6ad7SLin Ling 113838d61036SMatthew Ahrens if (scn->scn_phys.scn_cur_min_txg >= 113938d61036SMatthew Ahrens scn->scn_phys.scn_max_txg) { 114038d61036SMatthew Ahrens /* 114138d61036SMatthew Ahrens * This can happen if this snapshot was created after the 114238d61036SMatthew Ahrens * scan started, and we already completed a previous snapshot 114338d61036SMatthew Ahrens * that was created after the scan started. This snapshot 114438d61036SMatthew Ahrens * only references blocks with: 114538d61036SMatthew Ahrens * 114638d61036SMatthew Ahrens * birth < our ds_creation_txg 114738d61036SMatthew Ahrens * cur_min_txg is no less than ds_creation_txg. 114838d61036SMatthew Ahrens * We have already visited these blocks. 114938d61036SMatthew Ahrens * or 115038d61036SMatthew Ahrens * birth > scn_max_txg 115138d61036SMatthew Ahrens * The scan requested not to visit these blocks. 115238d61036SMatthew Ahrens * 115338d61036SMatthew Ahrens * Subsequent snapshots (and clones) can reference our 115438d61036SMatthew Ahrens * blocks, or blocks with even higher birth times. 115538d61036SMatthew Ahrens * Therefore we do not need to visit them either, 115638d61036SMatthew Ahrens * so we do not add them to the work queue. 115738d61036SMatthew Ahrens * 115838d61036SMatthew Ahrens * Note that checking for cur_min_txg >= cur_max_txg 115938d61036SMatthew Ahrens * is not sufficient, because in that case we may need to 116038d61036SMatthew Ahrens * visit subsequent snapshots. This happens when min_txg > 0, 116138d61036SMatthew Ahrens * which raises cur_min_txg. In this case we will visit 116238d61036SMatthew Ahrens * this dataset but skip all of its blocks, because the 116338d61036SMatthew Ahrens * rootbp's birth time is < cur_min_txg. Then we will 116438d61036SMatthew Ahrens * add the next snapshots/clones to the work queue. 116538d61036SMatthew Ahrens */ 116638d61036SMatthew Ahrens char *dsname = kmem_alloc(MAXNAMELEN, KM_SLEEP); 116738d61036SMatthew Ahrens dsl_dataset_name(ds, dsname); 116838d61036SMatthew Ahrens zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because " 116938d61036SMatthew Ahrens "cur_min_txg (%llu) >= max_txg (%llu)", 117038d61036SMatthew Ahrens dsobj, dsname, 117138d61036SMatthew Ahrens scn->scn_phys.scn_cur_min_txg, 117238d61036SMatthew Ahrens scn->scn_phys.scn_max_txg); 117338d61036SMatthew Ahrens kmem_free(dsname, MAXNAMELEN); 117438d61036SMatthew Ahrens 117538d61036SMatthew Ahrens goto out; 117638d61036SMatthew Ahrens } 117738d61036SMatthew Ahrens 11786e0cbcaaSMatthew Ahrens /* 11795cabbc6bSPrashanth Sreenivasa * Only the ZIL in the head (non-snapshot) is valid. Even though 11806e0cbcaaSMatthew Ahrens * snapshots can have ZIL block pointers (which may be the same 11815cabbc6bSPrashanth Sreenivasa * BP as in the head), they must be ignored. In addition, $ORIGIN 11825cabbc6bSPrashanth Sreenivasa * doesn't have a objset (i.e. its ds_bp is a hole) so we don't 11835cabbc6bSPrashanth Sreenivasa * need to look for a ZIL in it either. So we traverse the ZIL here, 11845cabbc6bSPrashanth Sreenivasa * rather than in scan_recurse(), because the regular snapshot 11855cabbc6bSPrashanth Sreenivasa * block-sharing rules don't apply to it. 11866e0cbcaaSMatthew Ahrens */ 11875cabbc6bSPrashanth Sreenivasa if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds) && 1188*bb1f4245SMatthew Ahrens (dp->dp_origin_snap == NULL || 1189*bb1f4245SMatthew Ahrens ds->ds_dir != dp->dp_origin_snap->ds_dir)) { 11905cabbc6bSPrashanth Sreenivasa objset_t *os; 11915cabbc6bSPrashanth Sreenivasa if (dmu_objset_from_ds(ds, &os) != 0) { 11925cabbc6bSPrashanth Sreenivasa goto out; 11935cabbc6bSPrashanth Sreenivasa } 11946e0cbcaaSMatthew Ahrens dsl_scan_zil(dp, &os->os_zil_header); 11955cabbc6bSPrashanth Sreenivasa } 11966e0cbcaaSMatthew Ahrens 11973f9d6ad7SLin Ling /* 11983f9d6ad7SLin Ling * Iterate over the bps in this ds. 11993f9d6ad7SLin Ling */ 12003f9d6ad7SLin Ling dmu_buf_will_dirty(ds->ds_dbuf, tx); 1201c166b69dSPaul Dagnelie rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 1202c1379625SJustin T. Gibbs dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx); 1203c166b69dSPaul Dagnelie rrw_exit(&ds->ds_bp_rwlock, FTAG); 12043f9d6ad7SLin Ling 12059adfa60dSMatthew Ahrens char *dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); 12063f9d6ad7SLin Ling dsl_dataset_name(ds, dsname); 12073f9d6ad7SLin Ling zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; " 12081702cce7SAlek Pinchuk "suspending=%u", 12093f9d6ad7SLin Ling (longlong_t)dsobj, dsname, 12103f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_cur_min_txg, 12113f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_cur_max_txg, 12121702cce7SAlek Pinchuk (int)scn->scn_suspending); 12139adfa60dSMatthew Ahrens kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN); 12143f9d6ad7SLin Ling 12151702cce7SAlek Pinchuk if (scn->scn_suspending) 12163f9d6ad7SLin Ling goto out; 12173f9d6ad7SLin Ling 12183f9d6ad7SLin Ling /* 12193f9d6ad7SLin Ling * We've finished this pass over this dataset. 12203f9d6ad7SLin Ling */ 12213f9d6ad7SLin Ling 12223f9d6ad7SLin Ling /* 12233f9d6ad7SLin Ling * If we did not completely visit this dataset, do another pass. 12243f9d6ad7SLin Ling */ 12253f9d6ad7SLin Ling if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) { 12263f9d6ad7SLin Ling zfs_dbgmsg("incomplete pass; visiting again"); 12273f9d6ad7SLin Ling scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN; 12283f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 12293f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, 12303f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg, tx) == 0); 12313f9d6ad7SLin Ling goto out; 12323f9d6ad7SLin Ling } 12333f9d6ad7SLin Ling 12343f9d6ad7SLin Ling /* 12353f9d6ad7SLin Ling * Add descendent datasets to work queue. 12363f9d6ad7SLin Ling */ 1237c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) { 12383f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 1239c1379625SJustin T. Gibbs scn->scn_phys.scn_queue_obj, 1240c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj, 1241c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_creation_txg, tx) == 0); 12423f9d6ad7SLin Ling } 1243c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_num_children > 1) { 12443f9d6ad7SLin Ling boolean_t usenext = B_FALSE; 1245c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) { 12463f9d6ad7SLin Ling uint64_t count; 12473f9d6ad7SLin Ling /* 12483f9d6ad7SLin Ling * A bug in a previous version of the code could 12493f9d6ad7SLin Ling * cause upgrade_clones_cb() to not set 12503f9d6ad7SLin Ling * ds_next_snap_obj when it should, leading to a 12513f9d6ad7SLin Ling * missing entry. Therefore we can only use the 12523f9d6ad7SLin Ling * next_clones_obj when its count is correct. 12533f9d6ad7SLin Ling */ 12543f9d6ad7SLin Ling int err = zap_count(dp->dp_meta_objset, 1255c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_clones_obj, &count); 12563f9d6ad7SLin Ling if (err == 0 && 1257c1379625SJustin T. Gibbs count == dsl_dataset_phys(ds)->ds_num_children - 1) 12583f9d6ad7SLin Ling usenext = B_TRUE; 12593f9d6ad7SLin Ling } 12603f9d6ad7SLin Ling 12613f9d6ad7SLin Ling if (usenext) { 12623b2aab18SMatthew Ahrens VERIFY0(zap_join_key(dp->dp_meta_objset, 1263c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_clones_obj, 12643f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 1265c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_creation_txg, tx)); 12663f9d6ad7SLin Ling } else { 12673f9d6ad7SLin Ling struct enqueue_clones_arg eca; 12683f9d6ad7SLin Ling eca.tx = tx; 12693f9d6ad7SLin Ling eca.originobj = ds->ds_object; 12703f9d6ad7SLin Ling 12713b2aab18SMatthew Ahrens VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 12723b2aab18SMatthew Ahrens enqueue_clones_cb, &eca, DS_FIND_CHILDREN)); 12733f9d6ad7SLin Ling } 12743f9d6ad7SLin Ling } 12753f9d6ad7SLin Ling 12763f9d6ad7SLin Ling out: 12773f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 12783f9d6ad7SLin Ling } 12793f9d6ad7SLin Ling 12803f9d6ad7SLin Ling /* ARGSUSED */ 12813f9d6ad7SLin Ling static int 12823b2aab18SMatthew Ahrens enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 12833f9d6ad7SLin Ling { 12843f9d6ad7SLin Ling dmu_tx_t *tx = arg; 12853f9d6ad7SLin Ling dsl_dataset_t *ds; 12863f9d6ad7SLin Ling int err; 12873f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 12883f9d6ad7SLin Ling 12893b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 12903f9d6ad7SLin Ling if (err) 12913f9d6ad7SLin Ling return (err); 12923f9d6ad7SLin Ling 1293c1379625SJustin T. Gibbs while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) { 12943f9d6ad7SLin Ling dsl_dataset_t *prev; 1295c1379625SJustin T. Gibbs err = dsl_dataset_hold_obj(dp, 1296c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev); 12973f9d6ad7SLin Ling if (err) { 12983f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 12993f9d6ad7SLin Ling return (err); 13003f9d6ad7SLin Ling } 13013f9d6ad7SLin Ling 13023f9d6ad7SLin Ling /* 13033f9d6ad7SLin Ling * If this is a clone, we don't need to worry about it for now. 13043f9d6ad7SLin Ling */ 1305c1379625SJustin T. Gibbs if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object) { 13063f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 13073f9d6ad7SLin Ling dsl_dataset_rele(prev, FTAG); 13083f9d6ad7SLin Ling return (0); 13093f9d6ad7SLin Ling } 13103f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 13113f9d6ad7SLin Ling ds = prev; 13123f9d6ad7SLin Ling } 13133f9d6ad7SLin Ling 13143f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 1315c1379625SJustin T. Gibbs ds->ds_object, dsl_dataset_phys(ds)->ds_prev_snap_txg, tx) == 0); 13163f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 13173f9d6ad7SLin Ling return (0); 13183f9d6ad7SLin Ling } 13193f9d6ad7SLin Ling 13203f9d6ad7SLin Ling /* 13213f9d6ad7SLin Ling * Scrub/dedup interaction. 13223f9d6ad7SLin Ling * 13233f9d6ad7SLin Ling * If there are N references to a deduped block, we don't want to scrub it 13243f9d6ad7SLin Ling * N times -- ideally, we should scrub it exactly once. 13253f9d6ad7SLin Ling * 13263f9d6ad7SLin Ling * We leverage the fact that the dde's replication class (enum ddt_class) 13273f9d6ad7SLin Ling * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest 13283f9d6ad7SLin Ling * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order. 13293f9d6ad7SLin Ling * 13303f9d6ad7SLin Ling * To prevent excess scrubbing, the scrub begins by walking the DDT 13313f9d6ad7SLin Ling * to find all blocks with refcnt > 1, and scrubs each of these once. 13323f9d6ad7SLin Ling * Since there are two replication classes which contain blocks with 13333f9d6ad7SLin Ling * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first. 13343f9d6ad7SLin Ling * Finally the top-down scrub begins, only visiting blocks with refcnt == 1. 13353f9d6ad7SLin Ling * 13363f9d6ad7SLin Ling * There would be nothing more to say if a block's refcnt couldn't change 13373f9d6ad7SLin Ling * during a scrub, but of course it can so we must account for changes 13383f9d6ad7SLin Ling * in a block's replication class. 13393f9d6ad7SLin Ling * 13403f9d6ad7SLin Ling * Here's an example of what can occur: 13413f9d6ad7SLin Ling * 13423f9d6ad7SLin Ling * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1 13433f9d6ad7SLin Ling * when visited during the top-down scrub phase, it will be scrubbed twice. 13443f9d6ad7SLin Ling * This negates our scrub optimization, but is otherwise harmless. 13453f9d6ad7SLin Ling * 13463f9d6ad7SLin Ling * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1 13473f9d6ad7SLin Ling * on each visit during the top-down scrub phase, it will never be scrubbed. 13483f9d6ad7SLin Ling * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's 13493f9d6ad7SLin Ling * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to 13503f9d6ad7SLin Ling * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1 13513f9d6ad7SLin Ling * while a scrub is in progress, it scrubs the block right then. 13523f9d6ad7SLin Ling */ 13533f9d6ad7SLin Ling static void 13543f9d6ad7SLin Ling dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx) 13553f9d6ad7SLin Ling { 13563f9d6ad7SLin Ling ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark; 13573f9d6ad7SLin Ling ddt_entry_t dde = { 0 }; 13583f9d6ad7SLin Ling int error; 13593f9d6ad7SLin Ling uint64_t n = 0; 13603f9d6ad7SLin Ling 13613f9d6ad7SLin Ling while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) { 13623f9d6ad7SLin Ling ddt_t *ddt; 13633f9d6ad7SLin Ling 13643f9d6ad7SLin Ling if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max) 13653f9d6ad7SLin Ling break; 13663f9d6ad7SLin Ling dprintf("visiting ddb=%llu/%llu/%llu/%llx\n", 13673f9d6ad7SLin Ling (longlong_t)ddb->ddb_class, 13683f9d6ad7SLin Ling (longlong_t)ddb->ddb_type, 13693f9d6ad7SLin Ling (longlong_t)ddb->ddb_checksum, 13703f9d6ad7SLin Ling (longlong_t)ddb->ddb_cursor); 13713f9d6ad7SLin Ling 13723f9d6ad7SLin Ling /* There should be no pending changes to the dedup table */ 13733f9d6ad7SLin Ling ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum]; 13743f9d6ad7SLin Ling ASSERT(avl_first(&ddt->ddt_tree) == NULL); 13753f9d6ad7SLin Ling 13763f9d6ad7SLin Ling dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx); 13773f9d6ad7SLin Ling n++; 13783f9d6ad7SLin Ling 13791702cce7SAlek Pinchuk if (dsl_scan_check_suspend(scn, NULL)) 13803f9d6ad7SLin Ling break; 13813f9d6ad7SLin Ling } 13823f9d6ad7SLin Ling 13831702cce7SAlek Pinchuk zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; " 13841702cce7SAlek Pinchuk "suspending=%u", (longlong_t)n, 13851702cce7SAlek Pinchuk (int)scn->scn_phys.scn_ddt_class_max, (int)scn->scn_suspending); 13863f9d6ad7SLin Ling 13873f9d6ad7SLin Ling ASSERT(error == 0 || error == ENOENT); 13883f9d6ad7SLin Ling ASSERT(error != ENOENT || 13893f9d6ad7SLin Ling ddb->ddb_class > scn->scn_phys.scn_ddt_class_max); 13903f9d6ad7SLin Ling } 13913f9d6ad7SLin Ling 13923f9d6ad7SLin Ling /* ARGSUSED */ 13933f9d6ad7SLin Ling void 13943f9d6ad7SLin Ling dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, 13953f9d6ad7SLin Ling ddt_entry_t *dde, dmu_tx_t *tx) 13963f9d6ad7SLin Ling { 13973f9d6ad7SLin Ling const ddt_key_t *ddk = &dde->dde_key; 13983f9d6ad7SLin Ling ddt_phys_t *ddp = dde->dde_phys; 13993f9d6ad7SLin Ling blkptr_t bp; 14007802d7bfSMatthew Ahrens zbookmark_phys_t zb = { 0 }; 14013f9d6ad7SLin Ling 14023f9d6ad7SLin Ling if (scn->scn_phys.scn_state != DSS_SCANNING) 14033f9d6ad7SLin Ling return; 14043f9d6ad7SLin Ling 14053f9d6ad7SLin Ling for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 14063f9d6ad7SLin Ling if (ddp->ddp_phys_birth == 0 || 1407b4952e17SGeorge Wilson ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) 14083f9d6ad7SLin Ling continue; 14093f9d6ad7SLin Ling ddt_bp_create(checksum, ddk, ddp, &bp); 14103f9d6ad7SLin Ling 14113f9d6ad7SLin Ling scn->scn_visited_this_txg++; 14123f9d6ad7SLin Ling scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb); 14133f9d6ad7SLin Ling } 14143f9d6ad7SLin Ling } 14153f9d6ad7SLin Ling 14163f9d6ad7SLin Ling static void 14173f9d6ad7SLin Ling dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx) 14183f9d6ad7SLin Ling { 14193f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 14203f9d6ad7SLin Ling zap_cursor_t zc; 14213f9d6ad7SLin Ling zap_attribute_t za; 14223f9d6ad7SLin Ling 14233f9d6ad7SLin Ling if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 14243f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max) { 14253f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 14263f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 14273f9d6ad7SLin Ling dsl_scan_ddt(scn, tx); 14281702cce7SAlek Pinchuk if (scn->scn_suspending) 14293f9d6ad7SLin Ling return; 14303f9d6ad7SLin Ling } 14313f9d6ad7SLin Ling 14323f9d6ad7SLin Ling if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) { 14333f9d6ad7SLin Ling /* First do the MOS & ORIGIN */ 14343f9d6ad7SLin Ling 14353f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 14363f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 14373f9d6ad7SLin Ling dsl_scan_visit_rootbp(scn, NULL, 14383f9d6ad7SLin Ling &dp->dp_meta_rootbp, tx); 14393f9d6ad7SLin Ling spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp); 14401702cce7SAlek Pinchuk if (scn->scn_suspending) 14413f9d6ad7SLin Ling return; 14423f9d6ad7SLin Ling 14433f9d6ad7SLin Ling if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) { 14443b2aab18SMatthew Ahrens VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 14453b2aab18SMatthew Ahrens enqueue_cb, tx, DS_FIND_CHILDREN)); 14463f9d6ad7SLin Ling } else { 14473f9d6ad7SLin Ling dsl_scan_visitds(scn, 14483f9d6ad7SLin Ling dp->dp_origin_snap->ds_object, tx); 14493f9d6ad7SLin Ling } 14501702cce7SAlek Pinchuk ASSERT(!scn->scn_suspending); 14513f9d6ad7SLin Ling } else if (scn->scn_phys.scn_bookmark.zb_objset != 14523f9d6ad7SLin Ling ZB_DESTROYED_OBJSET) { 14533f9d6ad7SLin Ling /* 14541702cce7SAlek Pinchuk * If we were suspended, continue from here. Note if the 14551702cce7SAlek Pinchuk * ds we were suspended on was deleted, the zb_objset may 14563f9d6ad7SLin Ling * be -1, so we will skip this and find a new objset 14573f9d6ad7SLin Ling * below. 14583f9d6ad7SLin Ling */ 14593f9d6ad7SLin Ling dsl_scan_visitds(scn, scn->scn_phys.scn_bookmark.zb_objset, tx); 14601702cce7SAlek Pinchuk if (scn->scn_suspending) 14613f9d6ad7SLin Ling return; 14623f9d6ad7SLin Ling } 14633f9d6ad7SLin Ling 14643f9d6ad7SLin Ling /* 14651702cce7SAlek Pinchuk * In case we were suspended right at the end of the ds, zero the 14663f9d6ad7SLin Ling * bookmark so we don't think that we're still trying to resume. 14673f9d6ad7SLin Ling */ 14687802d7bfSMatthew Ahrens bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t)); 14693f9d6ad7SLin Ling 14703f9d6ad7SLin Ling /* keep pulling things out of the zap-object-as-queue */ 14713f9d6ad7SLin Ling while (zap_cursor_init(&zc, dp->dp_meta_objset, 14723f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj), 14733f9d6ad7SLin Ling zap_cursor_retrieve(&zc, &za) == 0) { 14743f9d6ad7SLin Ling dsl_dataset_t *ds; 14753f9d6ad7SLin Ling uint64_t dsobj; 14763f9d6ad7SLin Ling 14774585130bSYuri Pankov dsobj = zfs_strtonum(za.za_name, NULL); 1478b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 14793f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, dsobj, tx)); 14803f9d6ad7SLin Ling 14813f9d6ad7SLin Ling /* Set up min/max txg */ 1482b420f3adSRichard Lowe VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 14833f9d6ad7SLin Ling if (za.za_first_integer != 0) { 14843f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = 14853f9d6ad7SLin Ling MAX(scn->scn_phys.scn_min_txg, 14863f9d6ad7SLin Ling za.za_first_integer); 14873f9d6ad7SLin Ling } else { 14883f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = 14893f9d6ad7SLin Ling MAX(scn->scn_phys.scn_min_txg, 1490c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_txg); 14913f9d6ad7SLin Ling } 14923f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds); 14933f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 14943f9d6ad7SLin Ling 14953f9d6ad7SLin Ling dsl_scan_visitds(scn, dsobj, tx); 14963f9d6ad7SLin Ling zap_cursor_fini(&zc); 14971702cce7SAlek Pinchuk if (scn->scn_suspending) 14983f9d6ad7SLin Ling return; 14993f9d6ad7SLin Ling } 15003f9d6ad7SLin Ling zap_cursor_fini(&zc); 15013f9d6ad7SLin Ling } 15023f9d6ad7SLin Ling 1503ad135b5dSChristopher Siden static boolean_t 15045cabbc6bSPrashanth Sreenivasa dsl_scan_async_block_should_pause(dsl_scan_t *scn) 1505cde58dbcSMatthew Ahrens { 1506cde58dbcSMatthew Ahrens uint64_t elapsed_nanosecs; 1507cde58dbcSMatthew Ahrens 15088b36997aSMatthew Ahrens if (zfs_recover) 15098b36997aSMatthew Ahrens return (B_FALSE); 15108b36997aSMatthew Ahrens 15115cabbc6bSPrashanth Sreenivasa if (scn->scn_visited_this_txg >= zfs_async_block_max_blocks) 1512af3465daSMax Grossman return (B_TRUE); 1513af3465daSMax Grossman 1514cde58dbcSMatthew Ahrens elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 1515ad135b5dSChristopher Siden return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 15165cabbc6bSPrashanth Sreenivasa (NSEC2MSEC(elapsed_nanosecs) > scn->scn_async_block_min_time_ms && 1517cde58dbcSMatthew Ahrens txg_sync_waiting(scn->scn_dp)) || 1518ad135b5dSChristopher Siden spa_shutting_down(scn->scn_dp->dp_spa)); 1519ad135b5dSChristopher Siden } 1520ad135b5dSChristopher Siden 1521ad135b5dSChristopher Siden static int 1522ad135b5dSChristopher Siden dsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 1523ad135b5dSChristopher Siden { 1524ad135b5dSChristopher Siden dsl_scan_t *scn = arg; 1525ad135b5dSChristopher Siden 1526ad135b5dSChristopher Siden if (!scn->scn_is_bptree || 1527ad135b5dSChristopher Siden (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)) { 15285cabbc6bSPrashanth Sreenivasa if (dsl_scan_async_block_should_pause(scn)) 1529be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 1530ad135b5dSChristopher Siden } 1531cde58dbcSMatthew Ahrens 1532cde58dbcSMatthew Ahrens zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa, 1533cde58dbcSMatthew Ahrens dmu_tx_get_txg(tx), bp, 0)); 1534cde58dbcSMatthew Ahrens dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 1535cde58dbcSMatthew Ahrens -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp), 1536cde58dbcSMatthew Ahrens -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 1537cde58dbcSMatthew Ahrens scn->scn_visited_this_txg++; 1538cde58dbcSMatthew Ahrens return (0); 1539cde58dbcSMatthew Ahrens } 1540cde58dbcSMatthew Ahrens 15415cabbc6bSPrashanth Sreenivasa static int 15425cabbc6bSPrashanth Sreenivasa dsl_scan_obsolete_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 15435cabbc6bSPrashanth Sreenivasa { 15445cabbc6bSPrashanth Sreenivasa dsl_scan_t *scn = arg; 15455cabbc6bSPrashanth Sreenivasa const dva_t *dva = &bp->blk_dva[0]; 15465cabbc6bSPrashanth Sreenivasa 15475cabbc6bSPrashanth Sreenivasa if (dsl_scan_async_block_should_pause(scn)) 15485cabbc6bSPrashanth Sreenivasa return (SET_ERROR(ERESTART)); 15495cabbc6bSPrashanth Sreenivasa 15505cabbc6bSPrashanth Sreenivasa spa_vdev_indirect_mark_obsolete(scn->scn_dp->dp_spa, 15515cabbc6bSPrashanth Sreenivasa DVA_GET_VDEV(dva), DVA_GET_OFFSET(dva), 15525cabbc6bSPrashanth Sreenivasa DVA_GET_ASIZE(dva), tx); 15535cabbc6bSPrashanth Sreenivasa scn->scn_visited_this_txg++; 15545cabbc6bSPrashanth Sreenivasa return (0); 15555cabbc6bSPrashanth Sreenivasa } 15565cabbc6bSPrashanth Sreenivasa 1557cde58dbcSMatthew Ahrens boolean_t 1558cde58dbcSMatthew Ahrens dsl_scan_active(dsl_scan_t *scn) 1559cde58dbcSMatthew Ahrens { 1560cde58dbcSMatthew Ahrens spa_t *spa = scn->scn_dp->dp_spa; 1561cde58dbcSMatthew Ahrens uint64_t used = 0, comp, uncomp; 1562cde58dbcSMatthew Ahrens 1563cde58dbcSMatthew Ahrens if (spa->spa_load_state != SPA_LOAD_NONE) 1564cde58dbcSMatthew Ahrens return (B_FALSE); 1565cde58dbcSMatthew Ahrens if (spa_shutting_down(spa)) 1566cde58dbcSMatthew Ahrens return (B_FALSE); 15671702cce7SAlek Pinchuk if ((scn->scn_phys.scn_state == DSS_SCANNING && 15681702cce7SAlek Pinchuk !dsl_scan_is_paused_scrub(scn)) || 15697fd05ac4SMatthew Ahrens (scn->scn_async_destroying && !scn->scn_async_stalled)) 1570cde58dbcSMatthew Ahrens return (B_TRUE); 1571cde58dbcSMatthew Ahrens 1572cde58dbcSMatthew Ahrens if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 1573cde58dbcSMatthew Ahrens (void) bpobj_space(&scn->scn_dp->dp_free_bpobj, 1574cde58dbcSMatthew Ahrens &used, &comp, &uncomp); 1575cde58dbcSMatthew Ahrens } 1576cde58dbcSMatthew Ahrens return (used != 0); 1577cde58dbcSMatthew Ahrens } 1578cde58dbcSMatthew Ahrens 157986714001SSerapheim Dimitropoulos static int 158086714001SSerapheim Dimitropoulos dsl_process_async_destroys(dsl_pool_t *dp, dmu_tx_t *tx) 15813f9d6ad7SLin Ling { 15823f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 15833f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 15847fd05ac4SMatthew Ahrens int err = 0; 15853f9d6ad7SLin Ling 158686714001SSerapheim Dimitropoulos if (spa_suspend_async_destroy(spa)) 158786714001SSerapheim Dimitropoulos return (0); 15883f9d6ad7SLin Ling 1589139510fbSGeorge Wilson if (zfs_free_bpobj_enabled && 1590139510fbSGeorge Wilson spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 1591ad135b5dSChristopher Siden scn->scn_is_bptree = B_FALSE; 15925cabbc6bSPrashanth Sreenivasa scn->scn_async_block_min_time_ms = zfs_free_min_time_ms; 1593cde58dbcSMatthew Ahrens scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1594cde58dbcSMatthew Ahrens NULL, ZIO_FLAG_MUSTSUCCEED); 1595cde58dbcSMatthew Ahrens err = bpobj_iterate(&dp->dp_free_bpobj, 1596ad135b5dSChristopher Siden dsl_scan_free_block_cb, scn, tx); 1597b420f3adSRichard Lowe VERIFY3U(0, ==, zio_wait(scn->scn_zio_root)); 1598ad135b5dSChristopher Siden 15997fd05ac4SMatthew Ahrens if (err != 0 && err != ERESTART) 16007fd05ac4SMatthew Ahrens zfs_panic_recover("error %u from bpobj_iterate()", err); 16017fd05ac4SMatthew Ahrens } 16027fd05ac4SMatthew Ahrens 16037fd05ac4SMatthew Ahrens if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) { 16047fd05ac4SMatthew Ahrens ASSERT(scn->scn_async_destroying); 16057fd05ac4SMatthew Ahrens scn->scn_is_bptree = B_TRUE; 16067fd05ac4SMatthew Ahrens scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 16077fd05ac4SMatthew Ahrens NULL, ZIO_FLAG_MUSTSUCCEED); 16087fd05ac4SMatthew Ahrens err = bptree_iterate(dp->dp_meta_objset, 16097fd05ac4SMatthew Ahrens dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx); 16107fd05ac4SMatthew Ahrens VERIFY0(zio_wait(scn->scn_zio_root)); 16117fd05ac4SMatthew Ahrens 16127fd05ac4SMatthew Ahrens if (err == EIO || err == ECKSUM) { 16137fd05ac4SMatthew Ahrens err = 0; 16147fd05ac4SMatthew Ahrens } else if (err != 0 && err != ERESTART) { 16157fd05ac4SMatthew Ahrens zfs_panic_recover("error %u from " 16167fd05ac4SMatthew Ahrens "traverse_dataset_destroyed()", err); 1617ad135b5dSChristopher Siden } 16187fd05ac4SMatthew Ahrens 16197fd05ac4SMatthew Ahrens if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) { 16207fd05ac4SMatthew Ahrens /* finished; deactivate async destroy feature */ 16217fd05ac4SMatthew Ahrens spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx); 16227fd05ac4SMatthew Ahrens ASSERT(!spa_feature_is_active(spa, 16237fd05ac4SMatthew Ahrens SPA_FEATURE_ASYNC_DESTROY)); 16247fd05ac4SMatthew Ahrens VERIFY0(zap_remove(dp->dp_meta_objset, 16257fd05ac4SMatthew Ahrens DMU_POOL_DIRECTORY_OBJECT, 16267fd05ac4SMatthew Ahrens DMU_POOL_BPTREE_OBJ, tx)); 16277fd05ac4SMatthew Ahrens VERIFY0(bptree_free(dp->dp_meta_objset, 16287fd05ac4SMatthew Ahrens dp->dp_bptree_obj, tx)); 16297fd05ac4SMatthew Ahrens dp->dp_bptree_obj = 0; 16307fd05ac4SMatthew Ahrens scn->scn_async_destroying = B_FALSE; 1631231aab85SMatthew Ahrens scn->scn_async_stalled = B_FALSE; 1632231aab85SMatthew Ahrens } else { 1633231aab85SMatthew Ahrens /* 1634231aab85SMatthew Ahrens * If we didn't make progress, mark the async 1635231aab85SMatthew Ahrens * destroy as stalled, so that we will not initiate 1636231aab85SMatthew Ahrens * a spa_sync() on its behalf. Note that we only 1637231aab85SMatthew Ahrens * check this if we are not finished, because if the 1638231aab85SMatthew Ahrens * bptree had no blocks for us to visit, we can 1639231aab85SMatthew Ahrens * finish without "making progress". 1640231aab85SMatthew Ahrens */ 1641231aab85SMatthew Ahrens scn->scn_async_stalled = 1642231aab85SMatthew Ahrens (scn->scn_visited_this_txg == 0); 1643cde58dbcSMatthew Ahrens } 16447fd05ac4SMatthew Ahrens } 16457fd05ac4SMatthew Ahrens if (scn->scn_visited_this_txg) { 16467fd05ac4SMatthew Ahrens zfs_dbgmsg("freed %llu blocks in %llums from " 16477fd05ac4SMatthew Ahrens "free_bpobj/bptree txg %llu; err=%u", 16487fd05ac4SMatthew Ahrens (longlong_t)scn->scn_visited_this_txg, 16497fd05ac4SMatthew Ahrens (longlong_t) 16507fd05ac4SMatthew Ahrens NSEC2MSEC(gethrtime() - scn->scn_sync_start_time), 16517fd05ac4SMatthew Ahrens (longlong_t)tx->tx_txg, err); 16527fd05ac4SMatthew Ahrens scn->scn_visited_this_txg = 0; 16537fd05ac4SMatthew Ahrens 16547fd05ac4SMatthew Ahrens /* 16557fd05ac4SMatthew Ahrens * Write out changes to the DDT that may be required as a 16567fd05ac4SMatthew Ahrens * result of the blocks freed. This ensures that the DDT 16577fd05ac4SMatthew Ahrens * is clean when a scrub/resilver runs. 16587fd05ac4SMatthew Ahrens */ 16597fd05ac4SMatthew Ahrens ddt_sync(spa, tx->tx_txg); 16607fd05ac4SMatthew Ahrens } 16617fd05ac4SMatthew Ahrens if (err != 0) 166286714001SSerapheim Dimitropoulos return (err); 16638c04a1faSGary Mills if (dp->dp_free_dir != NULL && !scn->scn_async_destroying && 16648c04a1faSGary Mills zfs_free_leak_on_eio && 1665c1379625SJustin T. Gibbs (dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes != 0 || 1666c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes != 0 || 1667c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes != 0)) { 16687fd05ac4SMatthew Ahrens /* 16697fd05ac4SMatthew Ahrens * We have finished background destroying, but there is still 16707fd05ac4SMatthew Ahrens * some space left in the dp_free_dir. Transfer this leaked 16717fd05ac4SMatthew Ahrens * space to the dp_leak_dir. 16727fd05ac4SMatthew Ahrens */ 16737fd05ac4SMatthew Ahrens if (dp->dp_leak_dir == NULL) { 16747fd05ac4SMatthew Ahrens rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 16757fd05ac4SMatthew Ahrens (void) dsl_dir_create_sync(dp, dp->dp_root_dir, 16767fd05ac4SMatthew Ahrens LEAK_DIR_NAME, tx); 16777fd05ac4SMatthew Ahrens VERIFY0(dsl_pool_open_special_dir(dp, 16787fd05ac4SMatthew Ahrens LEAK_DIR_NAME, &dp->dp_leak_dir)); 16797fd05ac4SMatthew Ahrens rrw_exit(&dp->dp_config_rwlock, FTAG); 16807fd05ac4SMatthew Ahrens } 16817fd05ac4SMatthew Ahrens dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD, 1682c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes, 1683c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes, 1684c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx); 16857fd05ac4SMatthew Ahrens dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD, 1686c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes, 1687c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes, 1688c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx); 16897fd05ac4SMatthew Ahrens } 16905cabbc6bSPrashanth Sreenivasa 16918c04a1faSGary Mills if (dp->dp_free_dir != NULL && !scn->scn_async_destroying) { 16925d7b4d43SMatthew Ahrens /* finished; verify that space accounting went to zero */ 1693c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes); 1694c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes); 1695c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes); 1696cde58dbcSMatthew Ahrens } 1697cde58dbcSMatthew Ahrens 16985cabbc6bSPrashanth Sreenivasa EQUIV(bpobj_is_open(&dp->dp_obsolete_bpobj), 16995cabbc6bSPrashanth Sreenivasa 0 == zap_contains(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 17005cabbc6bSPrashanth Sreenivasa DMU_POOL_OBSOLETE_BPOBJ)); 17015cabbc6bSPrashanth Sreenivasa if (err == 0 && bpobj_is_open(&dp->dp_obsolete_bpobj)) { 17025cabbc6bSPrashanth Sreenivasa ASSERT(spa_feature_is_active(dp->dp_spa, 17035cabbc6bSPrashanth Sreenivasa SPA_FEATURE_OBSOLETE_COUNTS)); 17045cabbc6bSPrashanth Sreenivasa 17055cabbc6bSPrashanth Sreenivasa scn->scn_is_bptree = B_FALSE; 17065cabbc6bSPrashanth Sreenivasa scn->scn_async_block_min_time_ms = zfs_obsolete_min_time_ms; 17075cabbc6bSPrashanth Sreenivasa err = bpobj_iterate(&dp->dp_obsolete_bpobj, 17085cabbc6bSPrashanth Sreenivasa dsl_scan_obsolete_block_cb, scn, tx); 17095cabbc6bSPrashanth Sreenivasa if (err != 0 && err != ERESTART) 17105cabbc6bSPrashanth Sreenivasa zfs_panic_recover("error %u from bpobj_iterate()", err); 17115cabbc6bSPrashanth Sreenivasa 17125cabbc6bSPrashanth Sreenivasa if (bpobj_is_empty(&dp->dp_obsolete_bpobj)) 17135cabbc6bSPrashanth Sreenivasa dsl_pool_destroy_obsolete_bpobj(dp, tx); 17145cabbc6bSPrashanth Sreenivasa } 17155cabbc6bSPrashanth Sreenivasa 171686714001SSerapheim Dimitropoulos return (0); 171786714001SSerapheim Dimitropoulos } 171886714001SSerapheim Dimitropoulos 171986714001SSerapheim Dimitropoulos void 172086714001SSerapheim Dimitropoulos dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx) 172186714001SSerapheim Dimitropoulos { 172286714001SSerapheim Dimitropoulos dsl_scan_t *scn = dp->dp_scan; 172386714001SSerapheim Dimitropoulos spa_t *spa = dp->dp_spa; 172486714001SSerapheim Dimitropoulos int err = 0; 172586714001SSerapheim Dimitropoulos 172686714001SSerapheim Dimitropoulos /* 172786714001SSerapheim Dimitropoulos * Check for scn_restart_txg before checking spa_load_state, so 172886714001SSerapheim Dimitropoulos * that we can restart an old-style scan while the pool is being 172986714001SSerapheim Dimitropoulos * imported (see dsl_scan_init). 173086714001SSerapheim Dimitropoulos */ 173186714001SSerapheim Dimitropoulos if (dsl_scan_restarting(scn, tx)) { 173286714001SSerapheim Dimitropoulos pool_scan_func_t func = POOL_SCAN_SCRUB; 173386714001SSerapheim Dimitropoulos dsl_scan_done(scn, B_FALSE, tx); 173486714001SSerapheim Dimitropoulos if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) 173586714001SSerapheim Dimitropoulos func = POOL_SCAN_RESILVER; 173686714001SSerapheim Dimitropoulos zfs_dbgmsg("restarting scan func=%u txg=%llu", 173786714001SSerapheim Dimitropoulos func, tx->tx_txg); 173886714001SSerapheim Dimitropoulos dsl_scan_setup_sync(&func, tx); 173986714001SSerapheim Dimitropoulos } 174086714001SSerapheim Dimitropoulos 174186714001SSerapheim Dimitropoulos /* 174286714001SSerapheim Dimitropoulos * Only process scans in sync pass 1. 174386714001SSerapheim Dimitropoulos */ 174486714001SSerapheim Dimitropoulos if (spa_sync_pass(dp->dp_spa) > 1) 174586714001SSerapheim Dimitropoulos return; 174686714001SSerapheim Dimitropoulos 174786714001SSerapheim Dimitropoulos /* 174886714001SSerapheim Dimitropoulos * If the spa is shutting down, then stop scanning. This will 174986714001SSerapheim Dimitropoulos * ensure that the scan does not dirty any new data during the 175086714001SSerapheim Dimitropoulos * shutdown phase. 175186714001SSerapheim Dimitropoulos */ 175286714001SSerapheim Dimitropoulos if (spa_shutting_down(spa)) 175386714001SSerapheim Dimitropoulos return; 175486714001SSerapheim Dimitropoulos 175586714001SSerapheim Dimitropoulos /* 175686714001SSerapheim Dimitropoulos * If the scan is inactive due to a stalled async destroy, try again. 175786714001SSerapheim Dimitropoulos */ 175886714001SSerapheim Dimitropoulos if (!scn->scn_async_stalled && !dsl_scan_active(scn)) 175986714001SSerapheim Dimitropoulos return; 176086714001SSerapheim Dimitropoulos 176186714001SSerapheim Dimitropoulos scn->scn_visited_this_txg = 0; 176286714001SSerapheim Dimitropoulos scn->scn_suspending = B_FALSE; 176386714001SSerapheim Dimitropoulos scn->scn_sync_start_time = gethrtime(); 176486714001SSerapheim Dimitropoulos spa->spa_scrub_active = B_TRUE; 176586714001SSerapheim Dimitropoulos 176686714001SSerapheim Dimitropoulos /* 176786714001SSerapheim Dimitropoulos * First process the async destroys. If we pause, don't do 176886714001SSerapheim Dimitropoulos * any scrubbing or resilvering. This ensures that there are no 176986714001SSerapheim Dimitropoulos * async destroys while we are scanning, so the scan code doesn't 177086714001SSerapheim Dimitropoulos * have to worry about traversing it. It is also faster to free the 177186714001SSerapheim Dimitropoulos * blocks than to scrub them. 177286714001SSerapheim Dimitropoulos */ 177386714001SSerapheim Dimitropoulos err = dsl_process_async_destroys(dp, tx); 177486714001SSerapheim Dimitropoulos if (err != 0) 177586714001SSerapheim Dimitropoulos return; 177686714001SSerapheim Dimitropoulos 1777cde58dbcSMatthew Ahrens if (scn->scn_phys.scn_state != DSS_SCANNING) 1778cde58dbcSMatthew Ahrens return; 1779cde58dbcSMatthew Ahrens 1780b4952e17SGeorge Wilson if (scn->scn_done_txg == tx->tx_txg) { 17811702cce7SAlek Pinchuk ASSERT(!scn->scn_suspending); 1782b4952e17SGeorge Wilson /* finished with scan. */ 1783b4952e17SGeorge Wilson zfs_dbgmsg("txg %llu scan complete", tx->tx_txg); 1784b4952e17SGeorge Wilson dsl_scan_done(scn, B_TRUE, tx); 1785b4952e17SGeorge Wilson ASSERT3U(spa->spa_scrub_inflight, ==, 0); 1786b4952e17SGeorge Wilson dsl_scan_sync_state(scn, tx); 1787b4952e17SGeorge Wilson return; 1788b4952e17SGeorge Wilson } 1789b4952e17SGeorge Wilson 17901702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) 17911702cce7SAlek Pinchuk return; 17921702cce7SAlek Pinchuk 17933f9d6ad7SLin Ling if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 17943f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max) { 17953f9d6ad7SLin Ling zfs_dbgmsg("doing scan sync txg %llu; " 17963f9d6ad7SLin Ling "ddt bm=%llu/%llu/%llu/%llx", 17973f9d6ad7SLin Ling (longlong_t)tx->tx_txg, 17983f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class, 17993f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type, 18003f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum, 18013f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor); 18023f9d6ad7SLin Ling ASSERT(scn->scn_phys.scn_bookmark.zb_objset == 0); 18033f9d6ad7SLin Ling ASSERT(scn->scn_phys.scn_bookmark.zb_object == 0); 18043f9d6ad7SLin Ling ASSERT(scn->scn_phys.scn_bookmark.zb_level == 0); 18053f9d6ad7SLin Ling ASSERT(scn->scn_phys.scn_bookmark.zb_blkid == 0); 18063f9d6ad7SLin Ling } else { 18073f9d6ad7SLin Ling zfs_dbgmsg("doing scan sync txg %llu; bm=%llu/%llu/%llu/%llu", 18083f9d6ad7SLin Ling (longlong_t)tx->tx_txg, 18093f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_bookmark.zb_objset, 18103f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_bookmark.zb_object, 18113f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_bookmark.zb_level, 18123f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_bookmark.zb_blkid); 18133f9d6ad7SLin Ling } 18143f9d6ad7SLin Ling 1815cde58dbcSMatthew Ahrens scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 18163f9d6ad7SLin Ling NULL, ZIO_FLAG_CANFAIL); 18173b2aab18SMatthew Ahrens dsl_pool_config_enter(dp, FTAG); 18183f9d6ad7SLin Ling dsl_scan_visit(scn, tx); 18193b2aab18SMatthew Ahrens dsl_pool_config_exit(dp, FTAG); 1820cde58dbcSMatthew Ahrens (void) zio_wait(scn->scn_zio_root); 1821cde58dbcSMatthew Ahrens scn->scn_zio_root = NULL; 18223f9d6ad7SLin Ling 18233f9d6ad7SLin Ling zfs_dbgmsg("visited %llu blocks in %llums", 18243f9d6ad7SLin Ling (longlong_t)scn->scn_visited_this_txg, 18250689f76cSAdam Leventhal (longlong_t)NSEC2MSEC(gethrtime() - scn->scn_sync_start_time)); 18263f9d6ad7SLin Ling 18271702cce7SAlek Pinchuk if (!scn->scn_suspending) { 1828b4952e17SGeorge Wilson scn->scn_done_txg = tx->tx_txg + 1; 1829b4952e17SGeorge Wilson zfs_dbgmsg("txg %llu traversal complete, waiting till txg %llu", 1830b4952e17SGeorge Wilson tx->tx_txg, scn->scn_done_txg); 18313f9d6ad7SLin Ling } 18323f9d6ad7SLin Ling 18333f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 18343f9d6ad7SLin Ling mutex_enter(&spa->spa_scrub_lock); 18353f9d6ad7SLin Ling while (spa->spa_scrub_inflight > 0) { 18363f9d6ad7SLin Ling cv_wait(&spa->spa_scrub_io_cv, 18373f9d6ad7SLin Ling &spa->spa_scrub_lock); 18383f9d6ad7SLin Ling } 18393f9d6ad7SLin Ling mutex_exit(&spa->spa_scrub_lock); 18403f9d6ad7SLin Ling } 18413f9d6ad7SLin Ling 18423f9d6ad7SLin Ling dsl_scan_sync_state(scn, tx); 18433f9d6ad7SLin Ling } 18443f9d6ad7SLin Ling 18453f9d6ad7SLin Ling /* 18463f9d6ad7SLin Ling * This will start a new scan, or restart an existing one. 18473f9d6ad7SLin Ling */ 18483f9d6ad7SLin Ling void 18493f9d6ad7SLin Ling dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg) 18503f9d6ad7SLin Ling { 18513f9d6ad7SLin Ling if (txg == 0) { 18523f9d6ad7SLin Ling dmu_tx_t *tx; 18533f9d6ad7SLin Ling tx = dmu_tx_create_dd(dp->dp_mos_dir); 18543f9d6ad7SLin Ling VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT)); 18553f9d6ad7SLin Ling 18563f9d6ad7SLin Ling txg = dmu_tx_get_txg(tx); 18573f9d6ad7SLin Ling dp->dp_scan->scn_restart_txg = txg; 18583f9d6ad7SLin Ling dmu_tx_commit(tx); 18593f9d6ad7SLin Ling } else { 18603f9d6ad7SLin Ling dp->dp_scan->scn_restart_txg = txg; 18613f9d6ad7SLin Ling } 18623f9d6ad7SLin Ling zfs_dbgmsg("restarting resilver txg=%llu", txg); 18633f9d6ad7SLin Ling } 18643f9d6ad7SLin Ling 18653f9d6ad7SLin Ling boolean_t 18663f9d6ad7SLin Ling dsl_scan_resilvering(dsl_pool_t *dp) 18673f9d6ad7SLin Ling { 18683f9d6ad7SLin Ling return (dp->dp_scan->scn_phys.scn_state == DSS_SCANNING && 18693f9d6ad7SLin Ling dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER); 18703f9d6ad7SLin Ling } 18713f9d6ad7SLin Ling 18723f9d6ad7SLin Ling /* 18733f9d6ad7SLin Ling * scrub consumers 18743f9d6ad7SLin Ling */ 18753f9d6ad7SLin Ling 18763f9d6ad7SLin Ling static void 18773f9d6ad7SLin Ling count_block(zfs_all_blkstats_t *zab, const blkptr_t *bp) 18783f9d6ad7SLin Ling { 18793f9d6ad7SLin Ling int i; 18803f9d6ad7SLin Ling 18813f9d6ad7SLin Ling /* 18823f9d6ad7SLin Ling * If we resume after a reboot, zab will be NULL; don't record 18833f9d6ad7SLin Ling * incomplete stats in that case. 18843f9d6ad7SLin Ling */ 18853f9d6ad7SLin Ling if (zab == NULL) 18863f9d6ad7SLin Ling return; 18873f9d6ad7SLin Ling 18883f9d6ad7SLin Ling for (i = 0; i < 4; i++) { 18893f9d6ad7SLin Ling int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS; 18903f9d6ad7SLin Ling int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL; 1891ad135b5dSChristopher Siden if (t & DMU_OT_NEWTYPE) 1892ad135b5dSChristopher Siden t = DMU_OT_OTHER; 18933f9d6ad7SLin Ling zfs_blkstat_t *zb = &zab->zab_type[l][t]; 18943f9d6ad7SLin Ling int equal; 18953f9d6ad7SLin Ling 18963f9d6ad7SLin Ling zb->zb_count++; 18973f9d6ad7SLin Ling zb->zb_asize += BP_GET_ASIZE(bp); 18983f9d6ad7SLin Ling zb->zb_lsize += BP_GET_LSIZE(bp); 18993f9d6ad7SLin Ling zb->zb_psize += BP_GET_PSIZE(bp); 19003f9d6ad7SLin Ling zb->zb_gangs += BP_COUNT_GANG(bp); 19013f9d6ad7SLin Ling 19023f9d6ad7SLin Ling switch (BP_GET_NDVAS(bp)) { 19033f9d6ad7SLin Ling case 2: 19043f9d6ad7SLin Ling if (DVA_GET_VDEV(&bp->blk_dva[0]) == 19053f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[1])) 19063f9d6ad7SLin Ling zb->zb_ditto_2_of_2_samevdev++; 19073f9d6ad7SLin Ling break; 19083f9d6ad7SLin Ling case 3: 19093f9d6ad7SLin Ling equal = (DVA_GET_VDEV(&bp->blk_dva[0]) == 19103f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[1])) + 19113f9d6ad7SLin Ling (DVA_GET_VDEV(&bp->blk_dva[0]) == 19123f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[2])) + 19133f9d6ad7SLin Ling (DVA_GET_VDEV(&bp->blk_dva[1]) == 19143f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[2])); 19153f9d6ad7SLin Ling if (equal == 1) 19163f9d6ad7SLin Ling zb->zb_ditto_2_of_3_samevdev++; 19173f9d6ad7SLin Ling else if (equal == 3) 19183f9d6ad7SLin Ling zb->zb_ditto_3_of_3_samevdev++; 19193f9d6ad7SLin Ling break; 19203f9d6ad7SLin Ling } 19213f9d6ad7SLin Ling } 19223f9d6ad7SLin Ling } 19233f9d6ad7SLin Ling 19243f9d6ad7SLin Ling static void 19253f9d6ad7SLin Ling dsl_scan_scrub_done(zio_t *zio) 19263f9d6ad7SLin Ling { 19273f9d6ad7SLin Ling spa_t *spa = zio->io_spa; 19283f9d6ad7SLin Ling 1929770499e1SDan Kimmel abd_free(zio->io_abd); 19303f9d6ad7SLin Ling 19313f9d6ad7SLin Ling mutex_enter(&spa->spa_scrub_lock); 19323f9d6ad7SLin Ling spa->spa_scrub_inflight--; 19333f9d6ad7SLin Ling cv_broadcast(&spa->spa_scrub_io_cv); 19343f9d6ad7SLin Ling 19353f9d6ad7SLin Ling if (zio->io_error && (zio->io_error != ECKSUM || 19363f9d6ad7SLin Ling !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) { 19373f9d6ad7SLin Ling spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors++; 19383f9d6ad7SLin Ling } 19393f9d6ad7SLin Ling mutex_exit(&spa->spa_scrub_lock); 19403f9d6ad7SLin Ling } 19413f9d6ad7SLin Ling 19423f9d6ad7SLin Ling static int 19433f9d6ad7SLin Ling dsl_scan_scrub_cb(dsl_pool_t *dp, 19447802d7bfSMatthew Ahrens const blkptr_t *bp, const zbookmark_phys_t *zb) 19453f9d6ad7SLin Ling { 19463f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 19473f9d6ad7SLin Ling size_t size = BP_GET_PSIZE(bp); 19483f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 19493f9d6ad7SLin Ling uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp); 19503f9d6ad7SLin Ling boolean_t needs_io; 195144ecc532SGeorge Wilson int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL; 195244ecc532SGeorge Wilson int scan_delay = 0; 19533f9d6ad7SLin Ling 19543f9d6ad7SLin Ling if (phys_birth <= scn->scn_phys.scn_min_txg || 19553f9d6ad7SLin Ling phys_birth >= scn->scn_phys.scn_max_txg) 19563f9d6ad7SLin Ling return (0); 19573f9d6ad7SLin Ling 19583f9d6ad7SLin Ling count_block(dp->dp_blkstats, bp); 19593f9d6ad7SLin Ling 19605d7b4d43SMatthew Ahrens if (BP_IS_EMBEDDED(bp)) 19615d7b4d43SMatthew Ahrens return (0); 19625d7b4d43SMatthew Ahrens 19633f9d6ad7SLin Ling ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn)); 19643f9d6ad7SLin Ling if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) { 19653f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_SCRUB; 19663f9d6ad7SLin Ling needs_io = B_TRUE; 196744ecc532SGeorge Wilson scan_delay = zfs_scrub_delay; 1968d5285caeSGeorge Wilson } else { 1969d5285caeSGeorge Wilson ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER); 19703f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_RESILVER; 19713f9d6ad7SLin Ling needs_io = B_FALSE; 197244ecc532SGeorge Wilson scan_delay = zfs_resilver_delay; 19733f9d6ad7SLin Ling } 19743f9d6ad7SLin Ling 19753f9d6ad7SLin Ling /* If it's an intent log block, failure is expected. */ 19763f9d6ad7SLin Ling if (zb->zb_level == ZB_ZIL_LEVEL) 19773f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_SPECULATIVE; 19783f9d6ad7SLin Ling 19793f9d6ad7SLin Ling for (int d = 0; d < BP_GET_NDVAS(bp); d++) { 19803f9d6ad7SLin Ling vdev_t *vd = vdev_lookup_top(spa, 19813f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[d])); 19823f9d6ad7SLin Ling 19833f9d6ad7SLin Ling /* 19843f9d6ad7SLin Ling * Keep track of how much data we've examined so that 19853f9d6ad7SLin Ling * zpool(1M) status can make useful progress reports. 19863f9d6ad7SLin Ling */ 19873f9d6ad7SLin Ling scn->scn_phys.scn_examined += DVA_GET_ASIZE(&bp->blk_dva[d]); 19883f9d6ad7SLin Ling spa->spa_scan_pass_exam += DVA_GET_ASIZE(&bp->blk_dva[d]); 19893f9d6ad7SLin Ling 19903f9d6ad7SLin Ling /* if it's a resilver, this may not be in the target range */ 19913f9d6ad7SLin Ling if (!needs_io) { 19923a4b1be9SMatthew Ahrens if (vd->vdev_ops == &vdev_indirect_ops) { 19933a4b1be9SMatthew Ahrens /* 19943a4b1be9SMatthew Ahrens * The indirect vdev can point to multiple 19953a4b1be9SMatthew Ahrens * vdevs. For simplicity, always create 19963a4b1be9SMatthew Ahrens * the resilver zio_t. zio_vdev_io_start() 19973a4b1be9SMatthew Ahrens * will bypass the child resilver i/o's if 19983a4b1be9SMatthew Ahrens * they are on vdevs that don't have DTL's. 19993a4b1be9SMatthew Ahrens */ 20003a4b1be9SMatthew Ahrens needs_io = B_TRUE; 20013a4b1be9SMatthew Ahrens } else if (DVA_GET_GANG(&bp->blk_dva[d])) { 20023f9d6ad7SLin Ling /* 20033f9d6ad7SLin Ling * Gang members may be spread across multiple 20043f9d6ad7SLin Ling * vdevs, so the best estimate we have is the 20053f9d6ad7SLin Ling * scrub range, which has already been checked. 20063f9d6ad7SLin Ling * XXX -- it would be better to change our 20073f9d6ad7SLin Ling * allocation policy to ensure that all 20083f9d6ad7SLin Ling * gang members reside on the same vdev. 20093f9d6ad7SLin Ling */ 20103f9d6ad7SLin Ling needs_io = B_TRUE; 20113f9d6ad7SLin Ling } else { 20123f9d6ad7SLin Ling needs_io = vdev_dtl_contains(vd, DTL_PARTIAL, 20133f9d6ad7SLin Ling phys_birth, 1); 20143f9d6ad7SLin Ling } 20153f9d6ad7SLin Ling } 20163f9d6ad7SLin Ling } 20173f9d6ad7SLin Ling 20183f9d6ad7SLin Ling if (needs_io && !zfs_no_scrub_io) { 201944ecc532SGeorge Wilson vdev_t *rvd = spa->spa_root_vdev; 202044ecc532SGeorge Wilson uint64_t maxinflight = rvd->vdev_children * zfs_top_maxinflight; 20213f9d6ad7SLin Ling 20223f9d6ad7SLin Ling mutex_enter(&spa->spa_scrub_lock); 202344ecc532SGeorge Wilson while (spa->spa_scrub_inflight >= maxinflight) 20243f9d6ad7SLin Ling cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 20253f9d6ad7SLin Ling spa->spa_scrub_inflight++; 20263f9d6ad7SLin Ling mutex_exit(&spa->spa_scrub_lock); 20273f9d6ad7SLin Ling 202844ecc532SGeorge Wilson /* 202944ecc532SGeorge Wilson * If we're seeing recent (zfs_scan_idle) "important" I/Os 203044ecc532SGeorge Wilson * then throttle our workload to limit the impact of a scan. 203144ecc532SGeorge Wilson */ 203244ecc532SGeorge Wilson if (ddi_get_lbolt64() - spa->spa_last_io <= zfs_scan_idle) 203344ecc532SGeorge Wilson delay(scan_delay); 203444ecc532SGeorge Wilson 2035770499e1SDan Kimmel zio_nowait(zio_read(NULL, spa, bp, 2036770499e1SDan Kimmel abd_alloc_for_io(size, B_FALSE), size, dsl_scan_scrub_done, 2037770499e1SDan Kimmel NULL, ZIO_PRIORITY_SCRUB, zio_flags, zb)); 20383f9d6ad7SLin Ling } 20393f9d6ad7SLin Ling 20403f9d6ad7SLin Ling /* do not relocate this block */ 20413f9d6ad7SLin Ling return (0); 20423f9d6ad7SLin Ling } 20433f9d6ad7SLin Ling 20441702cce7SAlek Pinchuk /* 20451702cce7SAlek Pinchuk * Called by the ZFS_IOC_POOL_SCAN ioctl to start a scrub or resilver. 20461702cce7SAlek Pinchuk * Can also be called to resume a paused scrub. 20471702cce7SAlek Pinchuk */ 20483f9d6ad7SLin Ling int 20493f9d6ad7SLin Ling dsl_scan(dsl_pool_t *dp, pool_scan_func_t func) 20503f9d6ad7SLin Ling { 20513f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 20521702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 20533f9d6ad7SLin Ling 20543f9d6ad7SLin Ling /* 20553f9d6ad7SLin Ling * Purge all vdev caches and probe all devices. We do this here 20563f9d6ad7SLin Ling * rather than in sync context because this requires a writer lock 20573f9d6ad7SLin Ling * on the spa_config lock, which we can't do from sync context. The 20583f9d6ad7SLin Ling * spa_scrub_reopen flag indicates that vdev_open() should not 20593f9d6ad7SLin Ling * attempt to start another scrub. 20603f9d6ad7SLin Ling */ 20613f9d6ad7SLin Ling spa_vdev_state_enter(spa, SCL_NONE); 20623f9d6ad7SLin Ling spa->spa_scrub_reopen = B_TRUE; 20633f9d6ad7SLin Ling vdev_reopen(spa->spa_root_vdev); 20643f9d6ad7SLin Ling spa->spa_scrub_reopen = B_FALSE; 20653f9d6ad7SLin Ling (void) spa_vdev_state_exit(spa, NULL, 0); 20663f9d6ad7SLin Ling 20671702cce7SAlek Pinchuk if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) { 20681702cce7SAlek Pinchuk /* got scrub start cmd, resume paused scrub */ 20691702cce7SAlek Pinchuk int err = dsl_scrub_set_pause_resume(scn->scn_dp, 20701702cce7SAlek Pinchuk POOL_SCRUB_NORMAL); 2071301fd1d6SSean Eric Fagan if (err == 0) { 2072301fd1d6SSean Eric Fagan spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_RESUME); 20731702cce7SAlek Pinchuk return (ECANCELED); 2074301fd1d6SSean Eric Fagan } 20751702cce7SAlek Pinchuk 20761702cce7SAlek Pinchuk return (SET_ERROR(err)); 20771702cce7SAlek Pinchuk } 20781702cce7SAlek Pinchuk 20793b2aab18SMatthew Ahrens return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check, 208086714001SSerapheim Dimitropoulos dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED)); 20813f9d6ad7SLin Ling } 20821825bc56SNav Ravindranath 20831825bc56SNav Ravindranath static boolean_t 20841825bc56SNav Ravindranath dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx) 20851825bc56SNav Ravindranath { 20861825bc56SNav Ravindranath return (scn->scn_restart_txg != 0 && 20871825bc56SNav Ravindranath scn->scn_restart_txg <= tx->tx_txg); 20881825bc56SNav Ravindranath } 2089