13f9d6ad7SLin Ling /* 23f9d6ad7SLin Ling * CDDL HEADER START 33f9d6ad7SLin Ling * 43f9d6ad7SLin Ling * The contents of this file are subject to the terms of the 53f9d6ad7SLin Ling * Common Development and Distribution License (the "License"). 63f9d6ad7SLin Ling * You may not use this file except in compliance with the License. 73f9d6ad7SLin Ling * 83f9d6ad7SLin Ling * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 93f9d6ad7SLin Ling * or http://www.opensolaris.org/os/licensing. 103f9d6ad7SLin Ling * See the License for the specific language governing permissions 113f9d6ad7SLin Ling * and limitations under the License. 123f9d6ad7SLin Ling * 133f9d6ad7SLin Ling * When distributing Covered Code, include this CDDL HEADER in each 143f9d6ad7SLin Ling * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 153f9d6ad7SLin Ling * If applicable, add the following below this CDDL HEADER, with the 163f9d6ad7SLin Ling * fields enclosed by brackets "[]" replaced with your own identifying 173f9d6ad7SLin Ling * information: Portions Copyright [yyyy] [name of copyright owner] 183f9d6ad7SLin Ling * 193f9d6ad7SLin Ling * CDDL HEADER END 203f9d6ad7SLin Ling */ 213f9d6ad7SLin Ling /* 223f9d6ad7SLin Ling * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23bb1f4245SMatthew Ahrens * Copyright (c) 2011, 2018 by Delphix. All rights reserved. 248c04a1faSGary Mills * Copyright 2016 Gary Mills 255cabbc6bSPrashanth Sreenivasa * Copyright (c) 2011, 2017 by Delphix. All rights reserved. 26*233f6c49SKody Kantor * Copyright 2019 Joyent, Inc. 271702cce7SAlek Pinchuk * Copyright (c) 2017 Datto Inc. 283f9d6ad7SLin Ling */ 293f9d6ad7SLin Ling 303f9d6ad7SLin Ling #include <sys/dsl_scan.h> 313f9d6ad7SLin Ling #include <sys/dsl_pool.h> 323f9d6ad7SLin Ling #include <sys/dsl_dataset.h> 333f9d6ad7SLin Ling #include <sys/dsl_prop.h> 343f9d6ad7SLin Ling #include <sys/dsl_dir.h> 353f9d6ad7SLin Ling #include <sys/dsl_synctask.h> 363f9d6ad7SLin Ling #include <sys/dnode.h> 373f9d6ad7SLin Ling #include <sys/dmu_tx.h> 383f9d6ad7SLin Ling #include <sys/dmu_objset.h> 393f9d6ad7SLin Ling #include <sys/arc.h> 403f9d6ad7SLin Ling #include <sys/zap.h> 413f9d6ad7SLin Ling #include <sys/zio.h> 423f9d6ad7SLin Ling #include <sys/zfs_context.h> 433f9d6ad7SLin Ling #include <sys/fs/zfs.h> 443f9d6ad7SLin Ling #include <sys/zfs_znode.h> 453f9d6ad7SLin Ling #include <sys/spa_impl.h> 463f9d6ad7SLin Ling #include <sys/vdev_impl.h> 473f9d6ad7SLin Ling #include <sys/zil_impl.h> 483f9d6ad7SLin Ling #include <sys/zio_checksum.h> 493f9d6ad7SLin Ling #include <sys/ddt.h> 503f9d6ad7SLin Ling #include <sys/sa.h> 513f9d6ad7SLin Ling #include <sys/sa_impl.h> 52ad135b5dSChristopher Siden #include <sys/zfeature.h> 53770499e1SDan Kimmel #include <sys/abd.h> 54a3874b8bSToomas Soome #include <sys/range_tree.h> 553f9d6ad7SLin Ling #ifdef _KERNEL 563f9d6ad7SLin Ling #include <sys/zfs_vfsops.h> 573f9d6ad7SLin Ling #endif 583f9d6ad7SLin Ling 59a3874b8bSToomas Soome /* 60a3874b8bSToomas Soome * Grand theory statement on scan queue sorting 61a3874b8bSToomas Soome * 62a3874b8bSToomas Soome * Scanning is implemented by recursively traversing all indirection levels 63a3874b8bSToomas Soome * in an object and reading all blocks referenced from said objects. This 64a3874b8bSToomas Soome * results in us approximately traversing the object from lowest logical 65a3874b8bSToomas Soome * offset to the highest. For best performance, we would want the logical 66a3874b8bSToomas Soome * blocks to be physically contiguous. However, this is frequently not the 67a3874b8bSToomas Soome * case with pools given the allocation patterns of copy-on-write filesystems. 68a3874b8bSToomas Soome * So instead, we put the I/Os into a reordering queue and issue them in a 69a3874b8bSToomas Soome * way that will most benefit physical disks (LBA-order). 70a3874b8bSToomas Soome * 71a3874b8bSToomas Soome * Queue management: 72a3874b8bSToomas Soome * 73a3874b8bSToomas Soome * Ideally, we would want to scan all metadata and queue up all block I/O 74a3874b8bSToomas Soome * prior to starting to issue it, because that allows us to do an optimal 75a3874b8bSToomas Soome * sorting job. This can however consume large amounts of memory. Therefore 76a3874b8bSToomas Soome * we continuously monitor the size of the queues and constrain them to 5% 77a3874b8bSToomas Soome * (zfs_scan_mem_lim_fact) of physmem. If the queues grow larger than this 78a3874b8bSToomas Soome * limit, we clear out a few of the largest extents at the head of the queues 79a3874b8bSToomas Soome * to make room for more scanning. Hopefully, these extents will be fairly 80a3874b8bSToomas Soome * large and contiguous, allowing us to approach sequential I/O throughput 81a3874b8bSToomas Soome * even without a fully sorted tree. 82a3874b8bSToomas Soome * 83a3874b8bSToomas Soome * Metadata scanning takes place in dsl_scan_visit(), which is called from 84a3874b8bSToomas Soome * dsl_scan_sync() every spa_sync(). If we have either fully scanned all 85a3874b8bSToomas Soome * metadata on the pool, or we need to make room in memory because our 86a3874b8bSToomas Soome * queues are too large, dsl_scan_visit() is postponed and 87a3874b8bSToomas Soome * scan_io_queues_run() is called from dsl_scan_sync() instead. This implies 88a3874b8bSToomas Soome * that metadata scanning and queued I/O issuing are mutually exclusive. This 89a3874b8bSToomas Soome * allows us to provide maximum sequential I/O throughput for the majority of 90a3874b8bSToomas Soome * I/O's issued since sequential I/O performance is significantly negatively 91a3874b8bSToomas Soome * impacted if it is interleaved with random I/O. 92a3874b8bSToomas Soome * 93a3874b8bSToomas Soome * Implementation Notes 94a3874b8bSToomas Soome * 95a3874b8bSToomas Soome * One side effect of the queued scanning algorithm is that the scanning code 96a3874b8bSToomas Soome * needs to be notified whenever a block is freed. This is needed to allow 97a3874b8bSToomas Soome * the scanning code to remove these I/Os from the issuing queue. Additionally, 98a3874b8bSToomas Soome * we do not attempt to queue gang blocks to be issued sequentially since this 99a3874b8bSToomas Soome * is very hard to do and would have an extremely limited performance benefit. 100a3874b8bSToomas Soome * Instead, we simply issue gang I/Os as soon as we find them using the legacy 101a3874b8bSToomas Soome * algorithm. 102a3874b8bSToomas Soome * 103a3874b8bSToomas Soome * Backwards compatibility 104a3874b8bSToomas Soome * 105a3874b8bSToomas Soome * This new algorithm is backwards compatible with the legacy on-disk data 106a3874b8bSToomas Soome * structures (and therefore does not require a new feature flag). 107a3874b8bSToomas Soome * Periodically during scanning (see zfs_scan_checkpoint_intval), the scan 108a3874b8bSToomas Soome * will stop scanning metadata (in logical order) and wait for all outstanding 109a3874b8bSToomas Soome * sorted I/O to complete. Once this is done, we write out a checkpoint 110a3874b8bSToomas Soome * bookmark, indicating that we have scanned everything logically before it. 111a3874b8bSToomas Soome * If the pool is imported on a machine without the new sorting algorithm, 112a3874b8bSToomas Soome * the scan simply resumes from the last checkpoint using the legacy algorithm. 113a3874b8bSToomas Soome */ 114a3874b8bSToomas Soome 1157802d7bfSMatthew Ahrens typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, 1167802d7bfSMatthew Ahrens const zbookmark_phys_t *); 1173f9d6ad7SLin Ling 1183f9d6ad7SLin Ling static scan_cb_t dsl_scan_scrub_cb; 1193f9d6ad7SLin Ling 120a3874b8bSToomas Soome static int scan_ds_queue_compare(const void *a, const void *b); 121a3874b8bSToomas Soome static int scan_prefetch_queue_compare(const void *a, const void *b); 122a3874b8bSToomas Soome static void scan_ds_queue_clear(dsl_scan_t *scn); 123a3874b8bSToomas Soome static boolean_t scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj, 124a3874b8bSToomas Soome uint64_t *txg); 125a3874b8bSToomas Soome static void scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg); 126a3874b8bSToomas Soome static void scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj); 127a3874b8bSToomas Soome static void scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx); 128a3874b8bSToomas Soome 129a3874b8bSToomas Soome extern int zfs_vdev_async_write_active_min_dirty_percent; 130a3874b8bSToomas Soome 131a3874b8bSToomas Soome /* 132a3874b8bSToomas Soome * By default zfs will check to ensure it is not over the hard memory 133a3874b8bSToomas Soome * limit before each txg. If finer-grained control of this is needed 134a3874b8bSToomas Soome * this value can be set to 1 to enable checking before scanning each 135a3874b8bSToomas Soome * block. 136a3874b8bSToomas Soome */ 137a3874b8bSToomas Soome int zfs_scan_strict_mem_lim = B_FALSE; 138a3874b8bSToomas Soome 139a3874b8bSToomas Soome /* 140a3874b8bSToomas Soome * Maximum number of parallelly executing I/Os per top-level vdev. 141a3874b8bSToomas Soome * Tune with care. Very high settings (hundreds) are known to trigger 142a3874b8bSToomas Soome * some firmware bugs and resets on certain SSDs. 143a3874b8bSToomas Soome */ 14444ecc532SGeorge Wilson int zfs_top_maxinflight = 32; /* maximum I/Os per top-level */ 145a3874b8bSToomas Soome unsigned int zfs_resilver_delay = 2; /* number of ticks to delay resilver */ 146a3874b8bSToomas Soome unsigned int zfs_scrub_delay = 4; /* number of ticks to delay scrub */ 147a3874b8bSToomas Soome unsigned int zfs_scan_idle = 50; /* idle window in clock ticks */ 148a3874b8bSToomas Soome 149a3874b8bSToomas Soome /* 150a3874b8bSToomas Soome * Maximum number of parallelly executed bytes per leaf vdev. We attempt 151a3874b8bSToomas Soome * to strike a balance here between keeping the vdev queues full of I/Os 152a3874b8bSToomas Soome * at all times and not overflowing the queues to cause long latency, 153a3874b8bSToomas Soome * which would cause long txg sync times. No matter what, we will not 154a3874b8bSToomas Soome * overload the drives with I/O, since that is protected by 155a3874b8bSToomas Soome * zfs_vdev_scrub_max_active. 156a3874b8bSToomas Soome */ 157a3874b8bSToomas Soome unsigned long zfs_scan_vdev_limit = 4 << 20; 158a3874b8bSToomas Soome 159a3874b8bSToomas Soome int zfs_scan_issue_strategy = 0; 160a3874b8bSToomas Soome int zfs_scan_legacy = B_FALSE; /* don't queue & sort zios, go direct */ 161a3874b8bSToomas Soome uint64_t zfs_scan_max_ext_gap = 2 << 20; /* in bytes */ 162a3874b8bSToomas Soome 163a3874b8bSToomas Soome unsigned int zfs_scan_checkpoint_intval = 7200; /* seconds */ 164a3874b8bSToomas Soome #define ZFS_SCAN_CHECKPOINT_INTVAL SEC_TO_TICK(zfs_scan_checkpoint_intval) 165a3874b8bSToomas Soome 166a3874b8bSToomas Soome /* 167a3874b8bSToomas Soome * fill_weight is non-tunable at runtime, so we copy it at module init from 168a3874b8bSToomas Soome * zfs_scan_fill_weight. Runtime adjustments to zfs_scan_fill_weight would 169a3874b8bSToomas Soome * break queue sorting. 170a3874b8bSToomas Soome */ 171a3874b8bSToomas Soome uint64_t zfs_scan_fill_weight = 3; 172a3874b8bSToomas Soome static uint64_t fill_weight; 173a3874b8bSToomas Soome 174a3874b8bSToomas Soome /* See dsl_scan_should_clear() for details on the memory limit tunables */ 175a3874b8bSToomas Soome uint64_t zfs_scan_mem_lim_min = 16 << 20; /* bytes */ 176a3874b8bSToomas Soome uint64_t zfs_scan_mem_lim_soft_max = 128 << 20; /* bytes */ 177a3874b8bSToomas Soome int zfs_scan_mem_lim_fact = 20; /* fraction of physmem */ 178a3874b8bSToomas Soome int zfs_scan_mem_lim_soft_fact = 20; /* fraction of mem lim above */ 179a3874b8bSToomas Soome 180a3874b8bSToomas Soome unsigned int zfs_scrub_min_time_ms = 1000; /* min millisecs to scrub per txg */ 181a3874b8bSToomas Soome unsigned int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */ 182a3874b8bSToomas Soome /* min millisecs to obsolete per txg */ 183a3874b8bSToomas Soome unsigned int zfs_obsolete_min_time_ms = 500; 184a3874b8bSToomas Soome /* min millisecs to resilver per txg */ 185a3874b8bSToomas Soome unsigned int zfs_resilver_min_time_ms = 3000; 186e4c795beSTom Caputi int zfs_scan_suspend_progress = 0; /* set to prevent scans from progressing */ 1873f9d6ad7SLin Ling boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */ 1887fd05ac4SMatthew Ahrens boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */ 1893f9d6ad7SLin Ling enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE; 190af3465daSMax Grossman /* max number of blocks to free in a single TXG */ 1915cabbc6bSPrashanth Sreenivasa uint64_t zfs_async_block_max_blocks = UINT64_MAX; 1923f9d6ad7SLin Ling 193e4c795beSTom Caputi int zfs_resilver_disable_defer = 0; /* set to disable resilver deferring */ 194e4c795beSTom Caputi 195a3874b8bSToomas Soome /* 196a3874b8bSToomas Soome * We wait a few txgs after importing a pool to begin scanning so that 197a3874b8bSToomas Soome * the import / mounting code isn't held up by scrub / resilver IO. 198a3874b8bSToomas Soome * Unfortunately, it is a bit difficult to determine exactly how long 199a3874b8bSToomas Soome * this will take since userspace will trigger fs mounts asynchronously 200a3874b8bSToomas Soome * and the kernel will create zvol minors asynchronously. As a result, 201a3874b8bSToomas Soome * the value provided here is a bit arbitrary, but represents a 202a3874b8bSToomas Soome * reasonable estimate of how many txgs it will take to finish fully 203a3874b8bSToomas Soome * importing a pool 204a3874b8bSToomas Soome */ 205a3874b8bSToomas Soome #define SCAN_IMPORT_WAIT_TXGS 5 206a3874b8bSToomas Soome 207a3874b8bSToomas Soome 2083f9d6ad7SLin Ling #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \ 2093f9d6ad7SLin Ling ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \ 2103f9d6ad7SLin Ling (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER) 2113f9d6ad7SLin Ling 2123f9d6ad7SLin Ling extern int zfs_txg_timeout; 2133f9d6ad7SLin Ling 214139510fbSGeorge Wilson /* 215139510fbSGeorge Wilson * Enable/disable the processing of the free_bpobj object. 216139510fbSGeorge Wilson */ 217139510fbSGeorge Wilson boolean_t zfs_free_bpobj_enabled = B_TRUE; 218139510fbSGeorge Wilson 2193f9d6ad7SLin Ling /* the order has to match pool_scan_type */ 2203f9d6ad7SLin Ling static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = { 2213f9d6ad7SLin Ling NULL, 2223f9d6ad7SLin Ling dsl_scan_scrub_cb, /* POOL_SCAN_SCRUB */ 2233f9d6ad7SLin Ling dsl_scan_scrub_cb, /* POOL_SCAN_RESILVER */ 2243f9d6ad7SLin Ling }; 2253f9d6ad7SLin Ling 226a3874b8bSToomas Soome /* In core node for the scn->scn_queue. Represents a dataset to be scanned */ 227a3874b8bSToomas Soome typedef struct { 228a3874b8bSToomas Soome uint64_t sds_dsobj; 229a3874b8bSToomas Soome uint64_t sds_txg; 230a3874b8bSToomas Soome avl_node_t sds_node; 231a3874b8bSToomas Soome } scan_ds_t; 232a3874b8bSToomas Soome 233a3874b8bSToomas Soome /* 234a3874b8bSToomas Soome * This controls what conditions are placed on dsl_scan_sync_state(): 235a3874b8bSToomas Soome * SYNC_OPTIONAL) write out scn_phys iff scn_bytes_pending == 0 236a3874b8bSToomas Soome * SYNC_MANDATORY) write out scn_phys always. scn_bytes_pending must be 0. 237a3874b8bSToomas Soome * SYNC_CACHED) if scn_bytes_pending == 0, write out scn_phys. Otherwise 238a3874b8bSToomas Soome * write out the scn_phys_cached version. 239a3874b8bSToomas Soome * See dsl_scan_sync_state for details. 240a3874b8bSToomas Soome */ 241a3874b8bSToomas Soome typedef enum { 242a3874b8bSToomas Soome SYNC_OPTIONAL, 243a3874b8bSToomas Soome SYNC_MANDATORY, 244a3874b8bSToomas Soome SYNC_CACHED 245a3874b8bSToomas Soome } state_sync_type_t; 246a3874b8bSToomas Soome 247a3874b8bSToomas Soome /* 248a3874b8bSToomas Soome * This struct represents the minimum information needed to reconstruct a 249a3874b8bSToomas Soome * zio for sequential scanning. This is useful because many of these will 250a3874b8bSToomas Soome * accumulate in the sequential IO queues before being issued, so saving 251a3874b8bSToomas Soome * memory matters here. 252a3874b8bSToomas Soome */ 253a3874b8bSToomas Soome typedef struct scan_io { 254a3874b8bSToomas Soome /* fields from blkptr_t */ 255a3874b8bSToomas Soome uint64_t sio_blk_prop; 256a3874b8bSToomas Soome uint64_t sio_phys_birth; 257a3874b8bSToomas Soome uint64_t sio_birth; 258a3874b8bSToomas Soome zio_cksum_t sio_cksum; 25912a8814cSTom Caputi uint32_t sio_nr_dvas; 260a3874b8bSToomas Soome 261a3874b8bSToomas Soome /* fields from zio_t */ 26212a8814cSTom Caputi uint32_t sio_flags; 263a3874b8bSToomas Soome zbookmark_phys_t sio_zb; 264a3874b8bSToomas Soome 265a3874b8bSToomas Soome /* members for queue sorting */ 266a3874b8bSToomas Soome union { 26712a8814cSTom Caputi avl_node_t sio_addr_node; /* link into issuing queue */ 268a3874b8bSToomas Soome list_node_t sio_list_node; /* link for issuing to disk */ 269a3874b8bSToomas Soome } sio_nodes; 27012a8814cSTom Caputi 27112a8814cSTom Caputi /* 27212a8814cSTom Caputi * There may be up to SPA_DVAS_PER_BP DVAs here from the bp, 27312a8814cSTom Caputi * depending on how many were in the original bp. Only the 27412a8814cSTom Caputi * first DVA is really used for sorting and issuing purposes. 27512a8814cSTom Caputi * The other DVAs (if provided) simply exist so that the zio 27612a8814cSTom Caputi * layer can find additional copies to repair from in the 27712a8814cSTom Caputi * event of an error. This array must go at the end of the 27812a8814cSTom Caputi * struct to allow this for the variable number of elements. 27912a8814cSTom Caputi */ 28012a8814cSTom Caputi dva_t sio_dva[0]; 281a3874b8bSToomas Soome } scan_io_t; 282a3874b8bSToomas Soome 28312a8814cSTom Caputi #define SIO_SET_OFFSET(sio, x) DVA_SET_OFFSET(&(sio)->sio_dva[0], x) 28412a8814cSTom Caputi #define SIO_SET_ASIZE(sio, x) DVA_SET_ASIZE(&(sio)->sio_dva[0], x) 28512a8814cSTom Caputi #define SIO_GET_OFFSET(sio) DVA_GET_OFFSET(&(sio)->sio_dva[0]) 28612a8814cSTom Caputi #define SIO_GET_ASIZE(sio) DVA_GET_ASIZE(&(sio)->sio_dva[0]) 28712a8814cSTom Caputi #define SIO_GET_END_OFFSET(sio) \ 28812a8814cSTom Caputi (SIO_GET_OFFSET(sio) + SIO_GET_ASIZE(sio)) 28912a8814cSTom Caputi #define SIO_GET_MUSED(sio) \ 29012a8814cSTom Caputi (sizeof (scan_io_t) + ((sio)->sio_nr_dvas * sizeof (dva_t))) 29112a8814cSTom Caputi 292a3874b8bSToomas Soome struct dsl_scan_io_queue { 293a3874b8bSToomas Soome dsl_scan_t *q_scn; /* associated dsl_scan_t */ 294a3874b8bSToomas Soome vdev_t *q_vd; /* top-level vdev that this queue represents */ 295a3874b8bSToomas Soome 296a3874b8bSToomas Soome /* trees used for sorting I/Os and extents of I/Os */ 297a3874b8bSToomas Soome range_tree_t *q_exts_by_addr; 298a3874b8bSToomas Soome avl_tree_t q_exts_by_size; 299a3874b8bSToomas Soome avl_tree_t q_sios_by_addr; 30012a8814cSTom Caputi uint64_t q_sio_memused; 301a3874b8bSToomas Soome 302a3874b8bSToomas Soome /* members for zio rate limiting */ 303a3874b8bSToomas Soome uint64_t q_maxinflight_bytes; 304a3874b8bSToomas Soome uint64_t q_inflight_bytes; 305a3874b8bSToomas Soome kcondvar_t q_zio_cv; /* used under vd->vdev_scan_io_queue_lock */ 306a3874b8bSToomas Soome 307a3874b8bSToomas Soome /* per txg statistics */ 308a3874b8bSToomas Soome uint64_t q_total_seg_size_this_txg; 309a3874b8bSToomas Soome uint64_t q_segs_this_txg; 310a3874b8bSToomas Soome uint64_t q_total_zio_size_this_txg; 311a3874b8bSToomas Soome uint64_t q_zios_this_txg; 312a3874b8bSToomas Soome }; 313a3874b8bSToomas Soome 314a3874b8bSToomas Soome /* private data for dsl_scan_prefetch_cb() */ 315a3874b8bSToomas Soome typedef struct scan_prefetch_ctx { 316a3874b8bSToomas Soome zfs_refcount_t spc_refcnt; /* refcount for memory management */ 317a3874b8bSToomas Soome dsl_scan_t *spc_scn; /* dsl_scan_t for the pool */ 318a3874b8bSToomas Soome boolean_t spc_root; /* is this prefetch for an objset? */ 319a3874b8bSToomas Soome uint8_t spc_indblkshift; /* dn_indblkshift of current dnode */ 320a3874b8bSToomas Soome uint16_t spc_datablkszsec; /* dn_idatablkszsec of current dnode */ 321a3874b8bSToomas Soome } scan_prefetch_ctx_t; 322a3874b8bSToomas Soome 323a3874b8bSToomas Soome /* private data for dsl_scan_prefetch() */ 324a3874b8bSToomas Soome typedef struct scan_prefetch_issue_ctx { 325a3874b8bSToomas Soome avl_node_t spic_avl_node; /* link into scn->scn_prefetch_queue */ 326a3874b8bSToomas Soome scan_prefetch_ctx_t *spic_spc; /* spc for the callback */ 327a3874b8bSToomas Soome blkptr_t spic_bp; /* bp to prefetch */ 328a3874b8bSToomas Soome zbookmark_phys_t spic_zb; /* bookmark to prefetch */ 329a3874b8bSToomas Soome } scan_prefetch_issue_ctx_t; 330a3874b8bSToomas Soome 331a3874b8bSToomas Soome static void scan_exec_io(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags, 332a3874b8bSToomas Soome const zbookmark_phys_t *zb, dsl_scan_io_queue_t *queue); 333a3874b8bSToomas Soome static void scan_io_queue_insert_impl(dsl_scan_io_queue_t *queue, 334a3874b8bSToomas Soome scan_io_t *sio); 335a3874b8bSToomas Soome 336a3874b8bSToomas Soome static dsl_scan_io_queue_t *scan_io_queue_create(vdev_t *vd); 337a3874b8bSToomas Soome static void scan_io_queues_destroy(dsl_scan_t *scn); 338a3874b8bSToomas Soome 33912a8814cSTom Caputi static kmem_cache_t *sio_cache[SPA_DVAS_PER_BP]; 34012a8814cSTom Caputi 34112a8814cSTom Caputi /* sio->sio_nr_dvas must be set so we know which cache to free from */ 34212a8814cSTom Caputi static void 34312a8814cSTom Caputi sio_free(scan_io_t *sio) 34412a8814cSTom Caputi { 34512a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, >, 0); 34612a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, <=, SPA_DVAS_PER_BP); 34712a8814cSTom Caputi 34812a8814cSTom Caputi kmem_cache_free(sio_cache[sio->sio_nr_dvas - 1], sio); 34912a8814cSTom Caputi } 35012a8814cSTom Caputi 35112a8814cSTom Caputi /* It is up to the caller to set sio->sio_nr_dvas for freeing */ 35212a8814cSTom Caputi static scan_io_t * 35312a8814cSTom Caputi sio_alloc(unsigned short nr_dvas) 35412a8814cSTom Caputi { 35512a8814cSTom Caputi ASSERT3U(nr_dvas, >, 0); 35612a8814cSTom Caputi ASSERT3U(nr_dvas, <=, SPA_DVAS_PER_BP); 35712a8814cSTom Caputi 35812a8814cSTom Caputi return (kmem_cache_alloc(sio_cache[nr_dvas - 1], KM_SLEEP)); 35912a8814cSTom Caputi } 360a3874b8bSToomas Soome 361a3874b8bSToomas Soome void 362a3874b8bSToomas Soome scan_init(void) 363a3874b8bSToomas Soome { 364a3874b8bSToomas Soome /* 365a3874b8bSToomas Soome * This is used in ext_size_compare() to weight segments 366a3874b8bSToomas Soome * based on how sparse they are. This cannot be changed 367a3874b8bSToomas Soome * mid-scan and the tree comparison functions don't currently 368a3874b8bSToomas Soome * have a mechansim for passing additional context to the 369a3874b8bSToomas Soome * compare functions. Thus we store this value globally and 370a3874b8bSToomas Soome * we only allow it to be set at module intiailization time 371a3874b8bSToomas Soome */ 372a3874b8bSToomas Soome fill_weight = zfs_scan_fill_weight; 373a3874b8bSToomas Soome 37412a8814cSTom Caputi for (int i = 0; i < SPA_DVAS_PER_BP; i++) { 37512a8814cSTom Caputi char name[36]; 37612a8814cSTom Caputi 37712a8814cSTom Caputi (void) sprintf(name, "sio_cache_%d", i); 37812a8814cSTom Caputi sio_cache[i] = kmem_cache_create(name, 37912a8814cSTom Caputi (sizeof (scan_io_t) + ((i + 1) * sizeof (dva_t))), 38012a8814cSTom Caputi 0, NULL, NULL, NULL, NULL, NULL, 0); 38112a8814cSTom Caputi } 382a3874b8bSToomas Soome } 383a3874b8bSToomas Soome 384a3874b8bSToomas Soome void 385a3874b8bSToomas Soome scan_fini(void) 386a3874b8bSToomas Soome { 38712a8814cSTom Caputi for (int i = 0; i < SPA_DVAS_PER_BP; i++) { 38812a8814cSTom Caputi kmem_cache_destroy(sio_cache[i]); 38912a8814cSTom Caputi } 390a3874b8bSToomas Soome } 391a3874b8bSToomas Soome 392a3874b8bSToomas Soome static inline boolean_t 393a3874b8bSToomas Soome dsl_scan_is_running(const dsl_scan_t *scn) 394a3874b8bSToomas Soome { 395a3874b8bSToomas Soome return (scn->scn_phys.scn_state == DSS_SCANNING); 396a3874b8bSToomas Soome } 397a3874b8bSToomas Soome 398a3874b8bSToomas Soome boolean_t 399a3874b8bSToomas Soome dsl_scan_resilvering(dsl_pool_t *dp) 400a3874b8bSToomas Soome { 401a3874b8bSToomas Soome return (dsl_scan_is_running(dp->dp_scan) && 402a3874b8bSToomas Soome dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER); 403a3874b8bSToomas Soome } 404a3874b8bSToomas Soome 405a3874b8bSToomas Soome static inline void 40612a8814cSTom Caputi sio2bp(const scan_io_t *sio, blkptr_t *bp) 407a3874b8bSToomas Soome { 408a3874b8bSToomas Soome bzero(bp, sizeof (*bp)); 409a3874b8bSToomas Soome bp->blk_prop = sio->sio_blk_prop; 410a3874b8bSToomas Soome bp->blk_phys_birth = sio->sio_phys_birth; 411a3874b8bSToomas Soome bp->blk_birth = sio->sio_birth; 412a3874b8bSToomas Soome bp->blk_fill = 1; /* we always only work with data pointers */ 413a3874b8bSToomas Soome bp->blk_cksum = sio->sio_cksum; 41412a8814cSTom Caputi 41512a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, >, 0); 41612a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, <=, SPA_DVAS_PER_BP); 41712a8814cSTom Caputi 41812a8814cSTom Caputi bcopy(sio->sio_dva, bp->blk_dva, sio->sio_nr_dvas * sizeof (dva_t)); 419a3874b8bSToomas Soome } 420a3874b8bSToomas Soome 421a3874b8bSToomas Soome static inline void 422a3874b8bSToomas Soome bp2sio(const blkptr_t *bp, scan_io_t *sio, int dva_i) 423a3874b8bSToomas Soome { 424a3874b8bSToomas Soome sio->sio_blk_prop = bp->blk_prop; 425a3874b8bSToomas Soome sio->sio_phys_birth = bp->blk_phys_birth; 426a3874b8bSToomas Soome sio->sio_birth = bp->blk_birth; 427a3874b8bSToomas Soome sio->sio_cksum = bp->blk_cksum; 42812a8814cSTom Caputi sio->sio_nr_dvas = BP_GET_NDVAS(bp); 42912a8814cSTom Caputi 43012a8814cSTom Caputi /* 43112a8814cSTom Caputi * Copy the DVAs to the sio. We need all copies of the block so 43212a8814cSTom Caputi * that the self healing code can use the alternate copies if the 43312a8814cSTom Caputi * first is corrupted. We want the DVA at index dva_i to be first 43412a8814cSTom Caputi * in the sio since this is the primary one that we want to issue. 43512a8814cSTom Caputi */ 43612a8814cSTom Caputi for (int i = 0, j = dva_i; i < sio->sio_nr_dvas; i++, j++) { 43712a8814cSTom Caputi sio->sio_dva[i] = bp->blk_dva[j % sio->sio_nr_dvas]; 43812a8814cSTom Caputi } 439a3874b8bSToomas Soome } 440a3874b8bSToomas Soome 4413f9d6ad7SLin Ling int 4423f9d6ad7SLin Ling dsl_scan_init(dsl_pool_t *dp, uint64_t txg) 4433f9d6ad7SLin Ling { 4443f9d6ad7SLin Ling int err; 4453f9d6ad7SLin Ling dsl_scan_t *scn; 4463f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 4473f9d6ad7SLin Ling uint64_t f; 4483f9d6ad7SLin Ling 4493f9d6ad7SLin Ling scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP); 4503f9d6ad7SLin Ling scn->scn_dp = dp; 4513f9d6ad7SLin Ling 4524a923759SGeorge Wilson /* 4534a923759SGeorge Wilson * It's possible that we're resuming a scan after a reboot so 4544a923759SGeorge Wilson * make sure that the scan_async_destroying flag is initialized 4554a923759SGeorge Wilson * appropriately. 4564a923759SGeorge Wilson */ 4574a923759SGeorge Wilson ASSERT(!scn->scn_async_destroying); 4584a923759SGeorge Wilson scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, 4592acef22dSMatthew Ahrens SPA_FEATURE_ASYNC_DESTROY); 4604a923759SGeorge Wilson 461a3874b8bSToomas Soome avl_create(&scn->scn_queue, scan_ds_queue_compare, sizeof (scan_ds_t), 462a3874b8bSToomas Soome offsetof(scan_ds_t, sds_node)); 463a3874b8bSToomas Soome avl_create(&scn->scn_prefetch_queue, scan_prefetch_queue_compare, 464a3874b8bSToomas Soome sizeof (scan_prefetch_issue_ctx_t), 465a3874b8bSToomas Soome offsetof(scan_prefetch_issue_ctx_t, spic_avl_node)); 466a3874b8bSToomas Soome 4673f9d6ad7SLin Ling err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4683f9d6ad7SLin Ling "scrub_func", sizeof (uint64_t), 1, &f); 4693f9d6ad7SLin Ling if (err == 0) { 4703f9d6ad7SLin Ling /* 4713f9d6ad7SLin Ling * There was an old-style scrub in progress. Restart a 4723f9d6ad7SLin Ling * new-style scrub from the beginning. 4733f9d6ad7SLin Ling */ 4743f9d6ad7SLin Ling scn->scn_restart_txg = txg; 4753f9d6ad7SLin Ling zfs_dbgmsg("old-style scrub was in progress; " 4763f9d6ad7SLin Ling "restarting new-style scrub in txg %llu", 477a3874b8bSToomas Soome (longlong_t)scn->scn_restart_txg); 4783f9d6ad7SLin Ling 4793f9d6ad7SLin Ling /* 4803f9d6ad7SLin Ling * Load the queue obj from the old location so that it 4813f9d6ad7SLin Ling * can be freed by dsl_scan_done(). 4823f9d6ad7SLin Ling */ 4833f9d6ad7SLin Ling (void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4843f9d6ad7SLin Ling "scrub_queue", sizeof (uint64_t), 1, 4853f9d6ad7SLin Ling &scn->scn_phys.scn_queue_obj); 4863f9d6ad7SLin Ling } else { 4873f9d6ad7SLin Ling err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4883f9d6ad7SLin Ling DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 4893f9d6ad7SLin Ling &scn->scn_phys); 490eb633035STom Caputi 491eb633035STom Caputi /* 492eb633035STom Caputi * Detect if the pool contains the signature of #2094. If it 493eb633035STom Caputi * does properly update the scn->scn_phys structure and notify 494eb633035STom Caputi * the administrator by setting an errata for the pool. 495eb633035STom Caputi */ 496eb633035STom Caputi if (err == EOVERFLOW) { 497eb633035STom Caputi uint64_t zaptmp[SCAN_PHYS_NUMINTS + 1]; 498eb633035STom Caputi VERIFY3S(SCAN_PHYS_NUMINTS, ==, 24); 499eb633035STom Caputi VERIFY3S(offsetof(dsl_scan_phys_t, scn_flags), ==, 500eb633035STom Caputi (23 * sizeof (uint64_t))); 501eb633035STom Caputi 502eb633035STom Caputi err = zap_lookup(dp->dp_meta_objset, 503eb633035STom Caputi DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SCAN, 504eb633035STom Caputi sizeof (uint64_t), SCAN_PHYS_NUMINTS + 1, &zaptmp); 505eb633035STom Caputi if (err == 0) { 506eb633035STom Caputi uint64_t overflow = zaptmp[SCAN_PHYS_NUMINTS]; 507eb633035STom Caputi 508eb633035STom Caputi if (overflow & ~DSF_VISIT_DS_AGAIN || 509eb633035STom Caputi scn->scn_async_destroying) { 510eb633035STom Caputi spa->spa_errata = 511eb633035STom Caputi ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY; 512eb633035STom Caputi return (EOVERFLOW); 513eb633035STom Caputi } 514eb633035STom Caputi 515eb633035STom Caputi bcopy(zaptmp, &scn->scn_phys, 516eb633035STom Caputi SCAN_PHYS_NUMINTS * sizeof (uint64_t)); 517eb633035STom Caputi scn->scn_phys.scn_flags = overflow; 518eb633035STom Caputi 519eb633035STom Caputi /* Required scrub already in progress. */ 520eb633035STom Caputi if (scn->scn_phys.scn_state == DSS_FINISHED || 521eb633035STom Caputi scn->scn_phys.scn_state == DSS_CANCELED) 522eb633035STom Caputi spa->spa_errata = 523eb633035STom Caputi ZPOOL_ERRATA_ZOL_2094_SCRUB; 524eb633035STom Caputi } 525eb633035STom Caputi } 526eb633035STom Caputi 5273f9d6ad7SLin Ling if (err == ENOENT) 5283f9d6ad7SLin Ling return (0); 5293f9d6ad7SLin Ling else if (err) 5303f9d6ad7SLin Ling return (err); 5313f9d6ad7SLin Ling 532a3874b8bSToomas Soome /* 533a3874b8bSToomas Soome * We might be restarting after a reboot, so jump the issued 534a3874b8bSToomas Soome * counter to how far we've scanned. We know we're consistent 535a3874b8bSToomas Soome * up to here. 536a3874b8bSToomas Soome */ 537a3874b8bSToomas Soome scn->scn_issued_before_pass = scn->scn_phys.scn_examined; 538a3874b8bSToomas Soome 539a3874b8bSToomas Soome if (dsl_scan_is_running(scn) && 5403f9d6ad7SLin Ling spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) { 5413f9d6ad7SLin Ling /* 5423f9d6ad7SLin Ling * A new-type scrub was in progress on an old 5433f9d6ad7SLin Ling * pool, and the pool was accessed by old 5443f9d6ad7SLin Ling * software. Restart from the beginning, since 5453f9d6ad7SLin Ling * the old software may have changed the pool in 5463f9d6ad7SLin Ling * the meantime. 5473f9d6ad7SLin Ling */ 5483f9d6ad7SLin Ling scn->scn_restart_txg = txg; 5493f9d6ad7SLin Ling zfs_dbgmsg("new-style scrub was modified " 5503f9d6ad7SLin Ling "by old software; restarting in txg %llu", 551a3874b8bSToomas Soome (longlong_t)scn->scn_restart_txg); 552a3874b8bSToomas Soome } 553a3874b8bSToomas Soome } 554a3874b8bSToomas Soome 555e4c795beSTom Caputi bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); 556e4c795beSTom Caputi 557a3874b8bSToomas Soome /* reload the queue into the in-core state */ 558a3874b8bSToomas Soome if (scn->scn_phys.scn_queue_obj != 0) { 559a3874b8bSToomas Soome zap_cursor_t zc; 560a3874b8bSToomas Soome zap_attribute_t za; 561a3874b8bSToomas Soome 562a3874b8bSToomas Soome for (zap_cursor_init(&zc, dp->dp_meta_objset, 563a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj); 564a3874b8bSToomas Soome zap_cursor_retrieve(&zc, &za) == 0; 565a3874b8bSToomas Soome (void) zap_cursor_advance(&zc)) { 566a3874b8bSToomas Soome scan_ds_queue_insert(scn, 567a3874b8bSToomas Soome zfs_strtonum(za.za_name, NULL), 568a3874b8bSToomas Soome za.za_first_integer); 5693f9d6ad7SLin Ling } 570a3874b8bSToomas Soome zap_cursor_fini(&zc); 5713f9d6ad7SLin Ling } 5723f9d6ad7SLin Ling 5733f9d6ad7SLin Ling spa_scan_stat_init(spa); 5743f9d6ad7SLin Ling return (0); 5753f9d6ad7SLin Ling } 5763f9d6ad7SLin Ling 5773f9d6ad7SLin Ling void 5783f9d6ad7SLin Ling dsl_scan_fini(dsl_pool_t *dp) 5793f9d6ad7SLin Ling { 580a3874b8bSToomas Soome if (dp->dp_scan != NULL) { 581a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 582a3874b8bSToomas Soome 583a3874b8bSToomas Soome if (scn->scn_taskq != NULL) 584a3874b8bSToomas Soome taskq_destroy(scn->scn_taskq); 585a3874b8bSToomas Soome scan_ds_queue_clear(scn); 586a3874b8bSToomas Soome avl_destroy(&scn->scn_queue); 587a3874b8bSToomas Soome avl_destroy(&scn->scn_prefetch_queue); 588a3874b8bSToomas Soome 5893f9d6ad7SLin Ling kmem_free(dp->dp_scan, sizeof (dsl_scan_t)); 5903f9d6ad7SLin Ling dp->dp_scan = NULL; 5913f9d6ad7SLin Ling } 5923f9d6ad7SLin Ling } 5933f9d6ad7SLin Ling 594a3874b8bSToomas Soome static boolean_t 595a3874b8bSToomas Soome dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx) 596a3874b8bSToomas Soome { 597a3874b8bSToomas Soome return (scn->scn_restart_txg != 0 && 598a3874b8bSToomas Soome scn->scn_restart_txg <= tx->tx_txg); 599a3874b8bSToomas Soome } 600a3874b8bSToomas Soome 601a3874b8bSToomas Soome boolean_t 602a3874b8bSToomas Soome dsl_scan_scrubbing(const dsl_pool_t *dp) 603a3874b8bSToomas Soome { 604a3874b8bSToomas Soome dsl_scan_phys_t *scn_phys = &dp->dp_scan->scn_phys; 605a3874b8bSToomas Soome 606a3874b8bSToomas Soome return (scn_phys->scn_state == DSS_SCANNING && 607a3874b8bSToomas Soome scn_phys->scn_func == POOL_SCAN_SCRUB); 608a3874b8bSToomas Soome } 609a3874b8bSToomas Soome 610a3874b8bSToomas Soome boolean_t 611a3874b8bSToomas Soome dsl_scan_is_paused_scrub(const dsl_scan_t *scn) 612a3874b8bSToomas Soome { 613a3874b8bSToomas Soome return (dsl_scan_scrubbing(scn->scn_dp) && 614a3874b8bSToomas Soome scn->scn_phys.scn_flags & DSF_SCRUB_PAUSED); 615a3874b8bSToomas Soome } 616a3874b8bSToomas Soome 617a3874b8bSToomas Soome /* 618a3874b8bSToomas Soome * Writes out a persistent dsl_scan_phys_t record to the pool directory. 619a3874b8bSToomas Soome * Because we can be running in the block sorting algorithm, we do not always 620a3874b8bSToomas Soome * want to write out the record, only when it is "safe" to do so. This safety 621a3874b8bSToomas Soome * condition is achieved by making sure that the sorting queues are empty 622a3874b8bSToomas Soome * (scn_bytes_pending == 0). When this condition is not true, the sync'd state 623a3874b8bSToomas Soome * is inconsistent with how much actual scanning progress has been made. The 624a3874b8bSToomas Soome * kind of sync to be performed is specified by the sync_type argument. If the 625a3874b8bSToomas Soome * sync is optional, we only sync if the queues are empty. If the sync is 626a3874b8bSToomas Soome * mandatory, we do a hard ASSERT to make sure that the queues are empty. The 627a3874b8bSToomas Soome * third possible state is a "cached" sync. This is done in response to: 628a3874b8bSToomas Soome * 1) The dataset that was in the last sync'd dsl_scan_phys_t having been 629a3874b8bSToomas Soome * destroyed, so we wouldn't be able to restart scanning from it. 630a3874b8bSToomas Soome * 2) The snapshot that was in the last sync'd dsl_scan_phys_t having been 631a3874b8bSToomas Soome * superseded by a newer snapshot. 632a3874b8bSToomas Soome * 3) The dataset that was in the last sync'd dsl_scan_phys_t having been 633a3874b8bSToomas Soome * swapped with its clone. 634a3874b8bSToomas Soome * In all cases, a cached sync simply rewrites the last record we've written, 635a3874b8bSToomas Soome * just slightly modified. For the modifications that are performed to the 636a3874b8bSToomas Soome * last written dsl_scan_phys_t, see dsl_scan_ds_destroyed, 637a3874b8bSToomas Soome * dsl_scan_ds_snapshotted and dsl_scan_ds_clone_swapped. 638a3874b8bSToomas Soome */ 639a3874b8bSToomas Soome static void 640a3874b8bSToomas Soome dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx, state_sync_type_t sync_type) 641a3874b8bSToomas Soome { 642a3874b8bSToomas Soome int i; 643a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 644a3874b8bSToomas Soome 645a3874b8bSToomas Soome ASSERT(sync_type != SYNC_MANDATORY || scn->scn_bytes_pending == 0); 646a3874b8bSToomas Soome if (scn->scn_bytes_pending == 0) { 647a3874b8bSToomas Soome for (i = 0; i < spa->spa_root_vdev->vdev_children; i++) { 648a3874b8bSToomas Soome vdev_t *vd = spa->spa_root_vdev->vdev_child[i]; 649a3874b8bSToomas Soome dsl_scan_io_queue_t *q = vd->vdev_scan_io_queue; 650a3874b8bSToomas Soome 651a3874b8bSToomas Soome if (q == NULL) 652a3874b8bSToomas Soome continue; 653a3874b8bSToomas Soome 654a3874b8bSToomas Soome mutex_enter(&vd->vdev_scan_io_queue_lock); 655a3874b8bSToomas Soome ASSERT3P(avl_first(&q->q_sios_by_addr), ==, NULL); 656a3874b8bSToomas Soome ASSERT3P(avl_first(&q->q_exts_by_size), ==, NULL); 657a3874b8bSToomas Soome ASSERT3P(range_tree_first(q->q_exts_by_addr), ==, NULL); 658a3874b8bSToomas Soome mutex_exit(&vd->vdev_scan_io_queue_lock); 659a3874b8bSToomas Soome } 660a3874b8bSToomas Soome 661a3874b8bSToomas Soome if (scn->scn_phys.scn_queue_obj != 0) 662a3874b8bSToomas Soome scan_ds_queue_sync(scn, tx); 663a3874b8bSToomas Soome VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 664a3874b8bSToomas Soome DMU_POOL_DIRECTORY_OBJECT, 665a3874b8bSToomas Soome DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 666a3874b8bSToomas Soome &scn->scn_phys, tx)); 667a3874b8bSToomas Soome bcopy(&scn->scn_phys, &scn->scn_phys_cached, 668a3874b8bSToomas Soome sizeof (scn->scn_phys)); 669a3874b8bSToomas Soome 670a3874b8bSToomas Soome if (scn->scn_checkpointing) 671a3874b8bSToomas Soome zfs_dbgmsg("finish scan checkpoint"); 672a3874b8bSToomas Soome 673a3874b8bSToomas Soome scn->scn_checkpointing = B_FALSE; 674a3874b8bSToomas Soome scn->scn_last_checkpoint = ddi_get_lbolt(); 675a3874b8bSToomas Soome } else if (sync_type == SYNC_CACHED) { 676a3874b8bSToomas Soome VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 677a3874b8bSToomas Soome DMU_POOL_DIRECTORY_OBJECT, 678a3874b8bSToomas Soome DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 679a3874b8bSToomas Soome &scn->scn_phys_cached, tx)); 680a3874b8bSToomas Soome } 681a3874b8bSToomas Soome } 682a3874b8bSToomas Soome 6833f9d6ad7SLin Ling /* ARGSUSED */ 6843f9d6ad7SLin Ling static int 6853b2aab18SMatthew Ahrens dsl_scan_setup_check(void *arg, dmu_tx_t *tx) 6863f9d6ad7SLin Ling { 6873b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 6883f9d6ad7SLin Ling 689a3874b8bSToomas Soome if (dsl_scan_is_running(scn)) 690be6fd75aSMatthew Ahrens return (SET_ERROR(EBUSY)); 6913f9d6ad7SLin Ling 6923f9d6ad7SLin Ling return (0); 6933f9d6ad7SLin Ling } 6943f9d6ad7SLin Ling 6953f9d6ad7SLin Ling static void 6963b2aab18SMatthew Ahrens dsl_scan_setup_sync(void *arg, dmu_tx_t *tx) 6973f9d6ad7SLin Ling { 6983b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 6993b2aab18SMatthew Ahrens pool_scan_func_t *funcp = arg; 7003f9d6ad7SLin Ling dmu_object_type_t ot = 0; 7013f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 7023f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 7033f9d6ad7SLin Ling 704a3874b8bSToomas Soome ASSERT(!dsl_scan_is_running(scn)); 7053f9d6ad7SLin Ling ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS); 7063f9d6ad7SLin Ling bzero(&scn->scn_phys, sizeof (scn->scn_phys)); 7073f9d6ad7SLin Ling scn->scn_phys.scn_func = *funcp; 7083f9d6ad7SLin Ling scn->scn_phys.scn_state = DSS_SCANNING; 7093f9d6ad7SLin Ling scn->scn_phys.scn_min_txg = 0; 7103f9d6ad7SLin Ling scn->scn_phys.scn_max_txg = tx->tx_txg; 7113f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */ 7123f9d6ad7SLin Ling scn->scn_phys.scn_start_time = gethrestime_sec(); 7133f9d6ad7SLin Ling scn->scn_phys.scn_errors = 0; 7143f9d6ad7SLin Ling scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc; 715a3874b8bSToomas Soome scn->scn_issued_before_pass = 0; 7163f9d6ad7SLin Ling scn->scn_restart_txg = 0; 717b4952e17SGeorge Wilson scn->scn_done_txg = 0; 718a3874b8bSToomas Soome scn->scn_last_checkpoint = 0; 719a3874b8bSToomas Soome scn->scn_checkpointing = B_FALSE; 7203f9d6ad7SLin Ling spa_scan_stat_init(spa); 7213f9d6ad7SLin Ling 7223f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 7233f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max; 7243f9d6ad7SLin Ling 7253f9d6ad7SLin Ling /* rewrite all disk labels */ 7263f9d6ad7SLin Ling vdev_config_dirty(spa->spa_root_vdev); 7273f9d6ad7SLin Ling 7283f9d6ad7SLin Ling if (vdev_resilver_needed(spa->spa_root_vdev, 7293f9d6ad7SLin Ling &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) { 730ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, 731ce1577b0SDave Eddy ESC_ZFS_RESILVER_START); 7323f9d6ad7SLin Ling } else { 733ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_START); 7343f9d6ad7SLin Ling } 7353f9d6ad7SLin Ling 7363f9d6ad7SLin Ling spa->spa_scrub_started = B_TRUE; 7373f9d6ad7SLin Ling /* 7383f9d6ad7SLin Ling * If this is an incremental scrub, limit the DDT scrub phase 7393f9d6ad7SLin Ling * to just the auto-ditto class (for correctness); the rest 7403f9d6ad7SLin Ling * of the scrub should go faster using top-down pruning. 7413f9d6ad7SLin Ling */ 7423f9d6ad7SLin Ling if (scn->scn_phys.scn_min_txg > TXG_INITIAL) 7433f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO; 7443f9d6ad7SLin Ling 7453f9d6ad7SLin Ling } 7463f9d6ad7SLin Ling 7473f9d6ad7SLin Ling /* back to the generic stuff */ 7483f9d6ad7SLin Ling 7493f9d6ad7SLin Ling if (dp->dp_blkstats == NULL) { 7503f9d6ad7SLin Ling dp->dp_blkstats = 7513f9d6ad7SLin Ling kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP); 752a3874b8bSToomas Soome mutex_init(&dp->dp_blkstats->zab_lock, NULL, 753a3874b8bSToomas Soome MUTEX_DEFAULT, NULL); 7543f9d6ad7SLin Ling } 755a3874b8bSToomas Soome bzero(&dp->dp_blkstats->zab_type, sizeof (dp->dp_blkstats->zab_type)); 7563f9d6ad7SLin Ling 7573f9d6ad7SLin Ling if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) 7583f9d6ad7SLin Ling ot = DMU_OT_ZAP_OTHER; 7593f9d6ad7SLin Ling 7603f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, 7613f9d6ad7SLin Ling ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx); 7623f9d6ad7SLin Ling 763a3874b8bSToomas Soome bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); 764a3874b8bSToomas Soome 765a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_MANDATORY); 7663f9d6ad7SLin Ling 7674445fffbSMatthew Ahrens spa_history_log_internal(spa, "scan setup", tx, 7683f9d6ad7SLin Ling "func=%u mintxg=%llu maxtxg=%llu", 7693f9d6ad7SLin Ling *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg); 7703f9d6ad7SLin Ling } 7713f9d6ad7SLin Ling 772a3874b8bSToomas Soome /* 773a3874b8bSToomas Soome * Called by the ZFS_IOC_POOL_SCAN ioctl to start a scrub or resilver. 774a3874b8bSToomas Soome * Can also be called to resume a paused scrub. 775a3874b8bSToomas Soome */ 776a3874b8bSToomas Soome int 777a3874b8bSToomas Soome dsl_scan(dsl_pool_t *dp, pool_scan_func_t func) 778a3874b8bSToomas Soome { 779a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 780a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 781a3874b8bSToomas Soome 782a3874b8bSToomas Soome /* 783a3874b8bSToomas Soome * Purge all vdev caches and probe all devices. We do this here 784a3874b8bSToomas Soome * rather than in sync context because this requires a writer lock 785a3874b8bSToomas Soome * on the spa_config lock, which we can't do from sync context. The 786a3874b8bSToomas Soome * spa_scrub_reopen flag indicates that vdev_open() should not 787a3874b8bSToomas Soome * attempt to start another scrub. 788a3874b8bSToomas Soome */ 789a3874b8bSToomas Soome spa_vdev_state_enter(spa, SCL_NONE); 790a3874b8bSToomas Soome spa->spa_scrub_reopen = B_TRUE; 791a3874b8bSToomas Soome vdev_reopen(spa->spa_root_vdev); 792a3874b8bSToomas Soome spa->spa_scrub_reopen = B_FALSE; 793a3874b8bSToomas Soome (void) spa_vdev_state_exit(spa, NULL, 0); 794a3874b8bSToomas Soome 795e4c795beSTom Caputi if (func == POOL_SCAN_RESILVER) { 796e4c795beSTom Caputi dsl_resilver_restart(spa->spa_dsl_pool, 0); 797e4c795beSTom Caputi return (0); 798e4c795beSTom Caputi } 799e4c795beSTom Caputi 800a3874b8bSToomas Soome if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) { 801a3874b8bSToomas Soome /* got scrub start cmd, resume paused scrub */ 802a3874b8bSToomas Soome int err = dsl_scrub_set_pause_resume(scn->scn_dp, 803a3874b8bSToomas Soome POOL_SCRUB_NORMAL); 804a3874b8bSToomas Soome if (err == 0) { 805a3874b8bSToomas Soome spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_RESUME); 806a3874b8bSToomas Soome return (ECANCELED); 807a3874b8bSToomas Soome } 808a3874b8bSToomas Soome return (SET_ERROR(err)); 809a3874b8bSToomas Soome } 810a3874b8bSToomas Soome 811a3874b8bSToomas Soome return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check, 812a3874b8bSToomas Soome dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED)); 813a3874b8bSToomas Soome } 814a3874b8bSToomas Soome 815e4c795beSTom Caputi /* 816e4c795beSTom Caputi * Sets the resilver defer flag to B_FALSE on all leaf devs under vd. Returns 817e4c795beSTom Caputi * B_TRUE if we have devices that need to be resilvered and are available to 818e4c795beSTom Caputi * accept resilver I/Os. 819e4c795beSTom Caputi */ 820e4c795beSTom Caputi static boolean_t 821e4c795beSTom Caputi dsl_scan_clear_deferred(vdev_t *vd, dmu_tx_t *tx) 822e4c795beSTom Caputi { 823e4c795beSTom Caputi boolean_t resilver_needed = B_FALSE; 824e4c795beSTom Caputi spa_t *spa = vd->vdev_spa; 825e4c795beSTom Caputi 826e4c795beSTom Caputi for (int c = 0; c < vd->vdev_children; c++) { 827e4c795beSTom Caputi resilver_needed |= 828e4c795beSTom Caputi dsl_scan_clear_deferred(vd->vdev_child[c], tx); 829e4c795beSTom Caputi } 830e4c795beSTom Caputi 831e4c795beSTom Caputi if (vd == spa->spa_root_vdev && 832e4c795beSTom Caputi spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) { 833e4c795beSTom Caputi spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx); 834e4c795beSTom Caputi vdev_config_dirty(vd); 835e4c795beSTom Caputi spa->spa_resilver_deferred = B_FALSE; 836e4c795beSTom Caputi return (resilver_needed); 837e4c795beSTom Caputi } 838e4c795beSTom Caputi 839e4c795beSTom Caputi if (!vdev_is_concrete(vd) || vd->vdev_aux || 840e4c795beSTom Caputi !vd->vdev_ops->vdev_op_leaf) 841e4c795beSTom Caputi return (resilver_needed); 842e4c795beSTom Caputi 843e4c795beSTom Caputi if (vd->vdev_resilver_deferred) 844e4c795beSTom Caputi vd->vdev_resilver_deferred = B_FALSE; 845e4c795beSTom Caputi 846e4c795beSTom Caputi return (!vdev_is_dead(vd) && !vd->vdev_offline && 847e4c795beSTom Caputi vdev_resilver_needed(vd, NULL, NULL)); 848e4c795beSTom Caputi } 849e4c795beSTom Caputi 8503f9d6ad7SLin Ling /* ARGSUSED */ 8513f9d6ad7SLin Ling static void 8523f9d6ad7SLin Ling dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) 8533f9d6ad7SLin Ling { 8543f9d6ad7SLin Ling static const char *old_names[] = { 8553f9d6ad7SLin Ling "scrub_bookmark", 8563f9d6ad7SLin Ling "scrub_ddt_bookmark", 8573f9d6ad7SLin Ling "scrub_ddt_class_max", 8583f9d6ad7SLin Ling "scrub_queue", 8593f9d6ad7SLin Ling "scrub_min_txg", 8603f9d6ad7SLin Ling "scrub_max_txg", 8613f9d6ad7SLin Ling "scrub_func", 8623f9d6ad7SLin Ling "scrub_errors", 8633f9d6ad7SLin Ling NULL 8643f9d6ad7SLin Ling }; 8653f9d6ad7SLin Ling 8663f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 8673f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 8683f9d6ad7SLin Ling int i; 8693f9d6ad7SLin Ling 8703f9d6ad7SLin Ling /* Remove any remnants of an old-style scrub. */ 8713f9d6ad7SLin Ling for (i = 0; old_names[i]; i++) { 8723f9d6ad7SLin Ling (void) zap_remove(dp->dp_meta_objset, 8733f9d6ad7SLin Ling DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx); 8743f9d6ad7SLin Ling } 8753f9d6ad7SLin Ling 8763f9d6ad7SLin Ling if (scn->scn_phys.scn_queue_obj != 0) { 877a3874b8bSToomas Soome VERIFY0(dmu_object_free(dp->dp_meta_objset, 8783f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, tx)); 8793f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj = 0; 8803f9d6ad7SLin Ling } 881a3874b8bSToomas Soome scan_ds_queue_clear(scn); 8823f9d6ad7SLin Ling 8831702cce7SAlek Pinchuk scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; 8841702cce7SAlek Pinchuk 8853f9d6ad7SLin Ling /* 8863f9d6ad7SLin Ling * If we were "restarted" from a stopped state, don't bother 8873f9d6ad7SLin Ling * with anything else. 8883f9d6ad7SLin Ling */ 889a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) { 890a3874b8bSToomas Soome ASSERT(!scn->scn_is_sorted); 8913f9d6ad7SLin Ling return; 892a3874b8bSToomas Soome } 8933f9d6ad7SLin Ling 894a3874b8bSToomas Soome if (scn->scn_is_sorted) { 895a3874b8bSToomas Soome scan_io_queues_destroy(scn); 896a3874b8bSToomas Soome scn->scn_is_sorted = B_FALSE; 897a3874b8bSToomas Soome 898a3874b8bSToomas Soome if (scn->scn_taskq != NULL) { 899a3874b8bSToomas Soome taskq_destroy(scn->scn_taskq); 900a3874b8bSToomas Soome scn->scn_taskq = NULL; 901a3874b8bSToomas Soome } 902a3874b8bSToomas Soome } 903a3874b8bSToomas Soome 904a3874b8bSToomas Soome scn->scn_phys.scn_state = complete ? DSS_FINISHED : DSS_CANCELED; 9053f9d6ad7SLin Ling 9061825bc56SNav Ravindranath if (dsl_scan_restarting(scn, tx)) 9071825bc56SNav Ravindranath spa_history_log_internal(spa, "scan aborted, restarting", tx, 9081825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 9091825bc56SNav Ravindranath else if (!complete) 9101825bc56SNav Ravindranath spa_history_log_internal(spa, "scan cancelled", tx, 9111825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 9121825bc56SNav Ravindranath else 9131825bc56SNav Ravindranath spa_history_log_internal(spa, "scan done", tx, 9141825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 9153f9d6ad7SLin Ling 9163f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 9173f9d6ad7SLin Ling spa->spa_scrub_started = B_FALSE; 9183f9d6ad7SLin Ling spa->spa_scrub_active = B_FALSE; 9193f9d6ad7SLin Ling 9203f9d6ad7SLin Ling /* 9213f9d6ad7SLin Ling * If the scrub/resilver completed, update all DTLs to 9223f9d6ad7SLin Ling * reflect this. Whether it succeeded or not, vacate 9233f9d6ad7SLin Ling * all temporary scrub DTLs. 92486714001SSerapheim Dimitropoulos * 92586714001SSerapheim Dimitropoulos * As the scrub does not currently support traversing 92686714001SSerapheim Dimitropoulos * data that have been freed but are part of a checkpoint, 92786714001SSerapheim Dimitropoulos * we don't mark the scrub as done in the DTLs as faults 92886714001SSerapheim Dimitropoulos * may still exist in those vdevs. 9293f9d6ad7SLin Ling */ 93086714001SSerapheim Dimitropoulos if (complete && 93186714001SSerapheim Dimitropoulos !spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 93286714001SSerapheim Dimitropoulos vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 93386714001SSerapheim Dimitropoulos scn->scn_phys.scn_max_txg, B_TRUE); 93486714001SSerapheim Dimitropoulos 935ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, 936ce1577b0SDave Eddy scn->scn_phys.scn_min_txg ? 9373f9d6ad7SLin Ling ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH); 93886714001SSerapheim Dimitropoulos } else { 93986714001SSerapheim Dimitropoulos vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 94086714001SSerapheim Dimitropoulos 0, B_TRUE); 9413f9d6ad7SLin Ling } 9423f9d6ad7SLin Ling spa_errlog_rotate(spa); 9433f9d6ad7SLin Ling 9443f9d6ad7SLin Ling /* 9453f9d6ad7SLin Ling * We may have finished replacing a device. 9463f9d6ad7SLin Ling * Let the async thread assess this and handle the detach. 9473f9d6ad7SLin Ling */ 9483f9d6ad7SLin Ling spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 949e4c795beSTom Caputi 950e4c795beSTom Caputi /* 951e4c795beSTom Caputi * Clear any deferred_resilver flags in the config. 952e4c795beSTom Caputi * If there are drives that need resilvering, kick 953e4c795beSTom Caputi * off an asynchronous request to start resilver. 954e4c795beSTom Caputi * dsl_scan_clear_deferred() may update the config 955e4c795beSTom Caputi * before the resilver can restart. In the event of 956e4c795beSTom Caputi * a crash during this period, the spa loading code 957e4c795beSTom Caputi * will find the drives that need to be resilvered 958e4c795beSTom Caputi * when the machine reboots and start the resilver then. 959e4c795beSTom Caputi */ 960*233f6c49SKody Kantor if (spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 961*233f6c49SKody Kantor boolean_t resilver_needed = 962*233f6c49SKody Kantor dsl_scan_clear_deferred(spa->spa_root_vdev, tx); 963*233f6c49SKody Kantor if (resilver_needed) { 964*233f6c49SKody Kantor spa_history_log_internal(spa, 965*233f6c49SKody Kantor "starting deferred resilver", tx, 966*233f6c49SKody Kantor "errors=%llu", spa_get_errlog_size(spa)); 967*233f6c49SKody Kantor spa_async_request(spa, SPA_ASYNC_RESILVER); 968*233f6c49SKody Kantor } 969e4c795beSTom Caputi } 9703f9d6ad7SLin Ling } 9713f9d6ad7SLin Ling 9723f9d6ad7SLin Ling scn->scn_phys.scn_end_time = gethrestime_sec(); 973a3874b8bSToomas Soome 974a3874b8bSToomas Soome ASSERT(!dsl_scan_is_running(scn)); 9753f9d6ad7SLin Ling } 9763f9d6ad7SLin Ling 9773f9d6ad7SLin Ling /* ARGSUSED */ 9783f9d6ad7SLin Ling static int 9793b2aab18SMatthew Ahrens dsl_scan_cancel_check(void *arg, dmu_tx_t *tx) 9803f9d6ad7SLin Ling { 9813b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 9823f9d6ad7SLin Ling 983a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 984be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 9853f9d6ad7SLin Ling return (0); 9863f9d6ad7SLin Ling } 9873f9d6ad7SLin Ling 9883f9d6ad7SLin Ling /* ARGSUSED */ 9893f9d6ad7SLin Ling static void 9903b2aab18SMatthew Ahrens dsl_scan_cancel_sync(void *arg, dmu_tx_t *tx) 9913f9d6ad7SLin Ling { 9923b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 9933f9d6ad7SLin Ling 9943f9d6ad7SLin Ling dsl_scan_done(scn, B_FALSE, tx); 995a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_MANDATORY); 996301fd1d6SSean Eric Fagan spa_event_notify(scn->scn_dp->dp_spa, NULL, NULL, ESC_ZFS_SCRUB_ABORT); 9973f9d6ad7SLin Ling } 9983f9d6ad7SLin Ling 9993f9d6ad7SLin Ling int 10003f9d6ad7SLin Ling dsl_scan_cancel(dsl_pool_t *dp) 10013f9d6ad7SLin Ling { 10023b2aab18SMatthew Ahrens return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check, 10037d46dc6cSMatthew Ahrens dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED)); 10043f9d6ad7SLin Ling } 10053f9d6ad7SLin Ling 10061702cce7SAlek Pinchuk static int 10071702cce7SAlek Pinchuk dsl_scrub_pause_resume_check(void *arg, dmu_tx_t *tx) 10081702cce7SAlek Pinchuk { 10091702cce7SAlek Pinchuk pool_scrub_cmd_t *cmd = arg; 10101702cce7SAlek Pinchuk dsl_pool_t *dp = dmu_tx_pool(tx); 10111702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 10121702cce7SAlek Pinchuk 10131702cce7SAlek Pinchuk if (*cmd == POOL_SCRUB_PAUSE) { 10141702cce7SAlek Pinchuk /* can't pause a scrub when there is no in-progress scrub */ 10151702cce7SAlek Pinchuk if (!dsl_scan_scrubbing(dp)) 10161702cce7SAlek Pinchuk return (SET_ERROR(ENOENT)); 10171702cce7SAlek Pinchuk 10181702cce7SAlek Pinchuk /* can't pause a paused scrub */ 10191702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) 10201702cce7SAlek Pinchuk return (SET_ERROR(EBUSY)); 10211702cce7SAlek Pinchuk } else if (*cmd != POOL_SCRUB_NORMAL) { 10221702cce7SAlek Pinchuk return (SET_ERROR(ENOTSUP)); 10231702cce7SAlek Pinchuk } 10241702cce7SAlek Pinchuk 10251702cce7SAlek Pinchuk return (0); 10261702cce7SAlek Pinchuk } 10271702cce7SAlek Pinchuk 10281702cce7SAlek Pinchuk static void 10291702cce7SAlek Pinchuk dsl_scrub_pause_resume_sync(void *arg, dmu_tx_t *tx) 10301702cce7SAlek Pinchuk { 10311702cce7SAlek Pinchuk pool_scrub_cmd_t *cmd = arg; 10321702cce7SAlek Pinchuk dsl_pool_t *dp = dmu_tx_pool(tx); 10331702cce7SAlek Pinchuk spa_t *spa = dp->dp_spa; 10341702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 10351702cce7SAlek Pinchuk 10361702cce7SAlek Pinchuk if (*cmd == POOL_SCRUB_PAUSE) { 10371702cce7SAlek Pinchuk /* can't pause a scrub when there is no in-progress scrub */ 10381702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_pause = gethrestime_sec(); 10391702cce7SAlek Pinchuk scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED; 1040e4c795beSTom Caputi scn->scn_phys_cached.scn_flags |= DSF_SCRUB_PAUSED; 1041a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 1042301fd1d6SSean Eric Fagan spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_PAUSED); 10431702cce7SAlek Pinchuk } else { 10441702cce7SAlek Pinchuk ASSERT3U(*cmd, ==, POOL_SCRUB_NORMAL); 10451702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) { 10461702cce7SAlek Pinchuk /* 10471702cce7SAlek Pinchuk * We need to keep track of how much time we spend 10481702cce7SAlek Pinchuk * paused per pass so that we can adjust the scrub rate 10491702cce7SAlek Pinchuk * shown in the output of 'zpool status' 10501702cce7SAlek Pinchuk */ 10511702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_spent_paused += 10521702cce7SAlek Pinchuk gethrestime_sec() - spa->spa_scan_pass_scrub_pause; 10531702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_pause = 0; 10541702cce7SAlek Pinchuk scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; 1055e4c795beSTom Caputi scn->scn_phys_cached.scn_flags &= ~DSF_SCRUB_PAUSED; 1056a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 10571702cce7SAlek Pinchuk } 10581702cce7SAlek Pinchuk } 10591702cce7SAlek Pinchuk } 10601702cce7SAlek Pinchuk 10611702cce7SAlek Pinchuk /* 10621702cce7SAlek Pinchuk * Set scrub pause/resume state if it makes sense to do so 10631702cce7SAlek Pinchuk */ 10641702cce7SAlek Pinchuk int 10651702cce7SAlek Pinchuk dsl_scrub_set_pause_resume(const dsl_pool_t *dp, pool_scrub_cmd_t cmd) 10661702cce7SAlek Pinchuk { 10671702cce7SAlek Pinchuk return (dsl_sync_task(spa_name(dp->dp_spa), 10681702cce7SAlek Pinchuk dsl_scrub_pause_resume_check, dsl_scrub_pause_resume_sync, &cmd, 3, 10691702cce7SAlek Pinchuk ZFS_SPACE_CHECK_RESERVED)); 10701702cce7SAlek Pinchuk } 10711702cce7SAlek Pinchuk 10721702cce7SAlek Pinchuk 1073a3874b8bSToomas Soome /* start a new scan, or restart an existing one. */ 1074a3874b8bSToomas Soome void 1075a3874b8bSToomas Soome dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg) 1076a3874b8bSToomas Soome { 1077a3874b8bSToomas Soome if (txg == 0) { 1078a3874b8bSToomas Soome dmu_tx_t *tx; 1079a3874b8bSToomas Soome tx = dmu_tx_create_dd(dp->dp_mos_dir); 1080a3874b8bSToomas Soome VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT)); 10811702cce7SAlek Pinchuk 1082a3874b8bSToomas Soome txg = dmu_tx_get_txg(tx); 1083a3874b8bSToomas Soome dp->dp_scan->scn_restart_txg = txg; 1084a3874b8bSToomas Soome dmu_tx_commit(tx); 1085a3874b8bSToomas Soome } else { 1086a3874b8bSToomas Soome dp->dp_scan->scn_restart_txg = txg; 1087a3874b8bSToomas Soome } 1088a3874b8bSToomas Soome zfs_dbgmsg("restarting resilver txg=%llu", txg); 10891702cce7SAlek Pinchuk } 10901702cce7SAlek Pinchuk 10913f9d6ad7SLin Ling void 10923f9d6ad7SLin Ling dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp) 10933f9d6ad7SLin Ling { 10943f9d6ad7SLin Ling zio_free(dp->dp_spa, txg, bp); 10953f9d6ad7SLin Ling } 10963f9d6ad7SLin Ling 10973f9d6ad7SLin Ling void 10983f9d6ad7SLin Ling dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp) 10993f9d6ad7SLin Ling { 11003f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dp)); 11013f9d6ad7SLin Ling zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, pio->io_flags)); 11023f9d6ad7SLin Ling } 11033f9d6ad7SLin Ling 1104a3874b8bSToomas Soome static int 1105a3874b8bSToomas Soome scan_ds_queue_compare(const void *a, const void *b) 11063f9d6ad7SLin Ling { 1107a3874b8bSToomas Soome const scan_ds_t *sds_a = a, *sds_b = b; 1108a3874b8bSToomas Soome 1109a3874b8bSToomas Soome if (sds_a->sds_dsobj < sds_b->sds_dsobj) 1110a3874b8bSToomas Soome return (-1); 1111a3874b8bSToomas Soome if (sds_a->sds_dsobj == sds_b->sds_dsobj) 1112a3874b8bSToomas Soome return (0); 1113a3874b8bSToomas Soome return (1); 11143f9d6ad7SLin Ling } 11153f9d6ad7SLin Ling 11163f9d6ad7SLin Ling static void 1117a3874b8bSToomas Soome scan_ds_queue_clear(dsl_scan_t *scn) 11183f9d6ad7SLin Ling { 1119a3874b8bSToomas Soome void *cookie = NULL; 1120a3874b8bSToomas Soome scan_ds_t *sds; 1121a3874b8bSToomas Soome while ((sds = avl_destroy_nodes(&scn->scn_queue, &cookie)) != NULL) { 1122a3874b8bSToomas Soome kmem_free(sds, sizeof (*sds)); 1123a3874b8bSToomas Soome } 11243f9d6ad7SLin Ling } 11253f9d6ad7SLin Ling 1126a3874b8bSToomas Soome static boolean_t 1127a3874b8bSToomas Soome scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj, uint64_t *txg) 1128a3874b8bSToomas Soome { 1129a3874b8bSToomas Soome scan_ds_t srch, *sds; 1130a3874b8bSToomas Soome 1131a3874b8bSToomas Soome srch.sds_dsobj = dsobj; 1132a3874b8bSToomas Soome sds = avl_find(&scn->scn_queue, &srch, NULL); 1133a3874b8bSToomas Soome if (sds != NULL && txg != NULL) 1134a3874b8bSToomas Soome *txg = sds->sds_txg; 1135a3874b8bSToomas Soome return (sds != NULL); 1136a3874b8bSToomas Soome } 1137a3874b8bSToomas Soome 1138a3874b8bSToomas Soome static void 1139a3874b8bSToomas Soome scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg) 1140a3874b8bSToomas Soome { 1141a3874b8bSToomas Soome scan_ds_t *sds; 1142a3874b8bSToomas Soome avl_index_t where; 1143a3874b8bSToomas Soome 1144a3874b8bSToomas Soome sds = kmem_zalloc(sizeof (*sds), KM_SLEEP); 1145a3874b8bSToomas Soome sds->sds_dsobj = dsobj; 1146a3874b8bSToomas Soome sds->sds_txg = txg; 1147a3874b8bSToomas Soome 1148a3874b8bSToomas Soome VERIFY3P(avl_find(&scn->scn_queue, sds, &where), ==, NULL); 1149a3874b8bSToomas Soome avl_insert(&scn->scn_queue, sds, where); 1150a3874b8bSToomas Soome } 1151a3874b8bSToomas Soome 1152a3874b8bSToomas Soome static void 1153a3874b8bSToomas Soome scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj) 1154a3874b8bSToomas Soome { 1155a3874b8bSToomas Soome scan_ds_t srch, *sds; 1156a3874b8bSToomas Soome 1157a3874b8bSToomas Soome srch.sds_dsobj = dsobj; 1158a3874b8bSToomas Soome 1159a3874b8bSToomas Soome sds = avl_find(&scn->scn_queue, &srch, NULL); 1160a3874b8bSToomas Soome VERIFY(sds != NULL); 1161a3874b8bSToomas Soome avl_remove(&scn->scn_queue, sds); 1162a3874b8bSToomas Soome kmem_free(sds, sizeof (*sds)); 1163a3874b8bSToomas Soome } 1164a3874b8bSToomas Soome 1165a3874b8bSToomas Soome static void 1166a3874b8bSToomas Soome scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx) 1167a3874b8bSToomas Soome { 1168a3874b8bSToomas Soome dsl_pool_t *dp = scn->scn_dp; 1169a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 1170a3874b8bSToomas Soome dmu_object_type_t ot = (spa_version(spa) >= SPA_VERSION_DSL_SCRUB) ? 1171a3874b8bSToomas Soome DMU_OT_SCAN_QUEUE : DMU_OT_ZAP_OTHER; 1172a3874b8bSToomas Soome 1173a3874b8bSToomas Soome ASSERT0(scn->scn_bytes_pending); 1174a3874b8bSToomas Soome ASSERT(scn->scn_phys.scn_queue_obj != 0); 1175a3874b8bSToomas Soome 1176a3874b8bSToomas Soome VERIFY0(dmu_object_free(dp->dp_meta_objset, 1177a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj, tx)); 1178a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, ot, 1179a3874b8bSToomas Soome DMU_OT_NONE, 0, tx); 1180a3874b8bSToomas Soome for (scan_ds_t *sds = avl_first(&scn->scn_queue); 1181a3874b8bSToomas Soome sds != NULL; sds = AVL_NEXT(&scn->scn_queue, sds)) { 1182a3874b8bSToomas Soome VERIFY0(zap_add_int_key(dp->dp_meta_objset, 1183a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj, sds->sds_dsobj, 1184a3874b8bSToomas Soome sds->sds_txg, tx)); 1185a3874b8bSToomas Soome } 1186a3874b8bSToomas Soome } 1187a3874b8bSToomas Soome 1188a3874b8bSToomas Soome /* 1189a3874b8bSToomas Soome * Computes the memory limit state that we're currently in. A sorted scan 1190a3874b8bSToomas Soome * needs quite a bit of memory to hold the sorting queue, so we need to 1191a3874b8bSToomas Soome * reasonably constrain the size so it doesn't impact overall system 1192a3874b8bSToomas Soome * performance. We compute two limits: 1193a3874b8bSToomas Soome * 1) Hard memory limit: if the amount of memory used by the sorting 1194a3874b8bSToomas Soome * queues on a pool gets above this value, we stop the metadata 1195a3874b8bSToomas Soome * scanning portion and start issuing the queued up and sorted 1196a3874b8bSToomas Soome * I/Os to reduce memory usage. 1197a3874b8bSToomas Soome * This limit is calculated as a fraction of physmem (by default 5%). 1198a3874b8bSToomas Soome * We constrain the lower bound of the hard limit to an absolute 1199a3874b8bSToomas Soome * minimum of zfs_scan_mem_lim_min (default: 16 MiB). We also constrain 1200a3874b8bSToomas Soome * the upper bound to 5% of the total pool size - no chance we'll 1201a3874b8bSToomas Soome * ever need that much memory, but just to keep the value in check. 1202a3874b8bSToomas Soome * 2) Soft memory limit: once we hit the hard memory limit, we start 1203a3874b8bSToomas Soome * issuing I/O to reduce queue memory usage, but we don't want to 1204a3874b8bSToomas Soome * completely empty out the queues, since we might be able to find I/Os 1205a3874b8bSToomas Soome * that will fill in the gaps of our non-sequential IOs at some point 1206a3874b8bSToomas Soome * in the future. So we stop the issuing of I/Os once the amount of 1207a3874b8bSToomas Soome * memory used drops below the soft limit (at which point we stop issuing 1208a3874b8bSToomas Soome * I/O and start scanning metadata again). 1209a3874b8bSToomas Soome * 1210a3874b8bSToomas Soome * This limit is calculated by subtracting a fraction of the hard 1211a3874b8bSToomas Soome * limit from the hard limit. By default this fraction is 5%, so 1212a3874b8bSToomas Soome * the soft limit is 95% of the hard limit. We cap the size of the 1213a3874b8bSToomas Soome * difference between the hard and soft limits at an absolute 1214a3874b8bSToomas Soome * maximum of zfs_scan_mem_lim_soft_max (default: 128 MiB) - this is 1215a3874b8bSToomas Soome * sufficient to not cause too frequent switching between the 1216a3874b8bSToomas Soome * metadata scan and I/O issue (even at 2k recordsize, 128 MiB's 1217a3874b8bSToomas Soome * worth of queues is about 1.2 GiB of on-pool data, so scanning 1218a3874b8bSToomas Soome * that should take at least a decent fraction of a second). 1219a3874b8bSToomas Soome */ 1220a3874b8bSToomas Soome static boolean_t 1221a3874b8bSToomas Soome dsl_scan_should_clear(dsl_scan_t *scn) 1222a3874b8bSToomas Soome { 1223a3874b8bSToomas Soome vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; 1224a3874b8bSToomas Soome uint64_t mlim_hard, mlim_soft, mused; 1225a3874b8bSToomas Soome uint64_t alloc = metaslab_class_get_alloc(spa_normal_class( 1226a3874b8bSToomas Soome scn->scn_dp->dp_spa)); 1227a3874b8bSToomas Soome 1228a3874b8bSToomas Soome mlim_hard = MAX((physmem / zfs_scan_mem_lim_fact) * PAGESIZE, 1229a3874b8bSToomas Soome zfs_scan_mem_lim_min); 1230a3874b8bSToomas Soome mlim_hard = MIN(mlim_hard, alloc / 20); 1231a3874b8bSToomas Soome mlim_soft = mlim_hard - MIN(mlim_hard / zfs_scan_mem_lim_soft_fact, 1232a3874b8bSToomas Soome zfs_scan_mem_lim_soft_max); 1233a3874b8bSToomas Soome mused = 0; 1234a3874b8bSToomas Soome for (uint64_t i = 0; i < rvd->vdev_children; i++) { 1235a3874b8bSToomas Soome vdev_t *tvd = rvd->vdev_child[i]; 1236a3874b8bSToomas Soome dsl_scan_io_queue_t *queue; 1237a3874b8bSToomas Soome 1238a3874b8bSToomas Soome mutex_enter(&tvd->vdev_scan_io_queue_lock); 1239a3874b8bSToomas Soome queue = tvd->vdev_scan_io_queue; 1240a3874b8bSToomas Soome if (queue != NULL) { 124112a8814cSTom Caputi /* # extents in exts_by_size = # in exts_by_addr */ 1242a3874b8bSToomas Soome mused += avl_numnodes(&queue->q_exts_by_size) * 124312a8814cSTom Caputi sizeof (range_seg_t) + queue->q_sio_memused; 1244a3874b8bSToomas Soome } 1245a3874b8bSToomas Soome mutex_exit(&tvd->vdev_scan_io_queue_lock); 1246a3874b8bSToomas Soome } 1247a3874b8bSToomas Soome 1248a3874b8bSToomas Soome dprintf("current scan memory usage: %llu bytes\n", (longlong_t)mused); 1249a3874b8bSToomas Soome 1250a3874b8bSToomas Soome if (mused == 0) 1251a3874b8bSToomas Soome ASSERT0(scn->scn_bytes_pending); 1252a3874b8bSToomas Soome 1253a3874b8bSToomas Soome /* 1254a3874b8bSToomas Soome * If we are above our hard limit, we need to clear out memory. 1255a3874b8bSToomas Soome * If we are below our soft limit, we need to accumulate sequential IOs. 1256a3874b8bSToomas Soome * Otherwise, we should keep doing whatever we are currently doing. 1257a3874b8bSToomas Soome */ 1258a3874b8bSToomas Soome if (mused >= mlim_hard) 1259a3874b8bSToomas Soome return (B_TRUE); 1260a3874b8bSToomas Soome else if (mused < mlim_soft) 1261a3874b8bSToomas Soome return (B_FALSE); 1262a3874b8bSToomas Soome else 1263a3874b8bSToomas Soome return (scn->scn_clearing); 1264a3874b8bSToomas Soome } 12656f6a76adSMatthew Ahrens 12663f9d6ad7SLin Ling static boolean_t 12671702cce7SAlek Pinchuk dsl_scan_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb) 12683f9d6ad7SLin Ling { 12693f9d6ad7SLin Ling /* we never skip user/group accounting objects */ 12703f9d6ad7SLin Ling if (zb && (int64_t)zb->zb_object < 0) 12713f9d6ad7SLin Ling return (B_FALSE); 12723f9d6ad7SLin Ling 12731702cce7SAlek Pinchuk if (scn->scn_suspending) 12741702cce7SAlek Pinchuk return (B_TRUE); /* we're already suspending */ 12753f9d6ad7SLin Ling 1276ad135b5dSChristopher Siden if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) 12773f9d6ad7SLin Ling return (B_FALSE); /* we're resuming */ 12783f9d6ad7SLin Ling 12793f9d6ad7SLin Ling /* We only know how to resume from level-0 blocks. */ 12803f9d6ad7SLin Ling if (zb && zb->zb_level != 0) 12813f9d6ad7SLin Ling return (B_FALSE); 12823f9d6ad7SLin Ling 12836f6a76adSMatthew Ahrens /* 12841702cce7SAlek Pinchuk * We suspend if: 12856f6a76adSMatthew Ahrens * - we have scanned for at least the minimum time (default 1 sec 12866f6a76adSMatthew Ahrens * for scrub, 3 sec for resilver), and either we have sufficient 12876f6a76adSMatthew Ahrens * dirty data that we are starting to write more quickly 12886f6a76adSMatthew Ahrens * (default 30%), or someone is explicitly waiting for this txg 12896f6a76adSMatthew Ahrens * to complete. 12906f6a76adSMatthew Ahrens * or 12916f6a76adSMatthew Ahrens * - the spa is shutting down because this pool is being exported 12926f6a76adSMatthew Ahrens * or the machine is rebooting. 1293a3874b8bSToomas Soome * or 1294a3874b8bSToomas Soome * - the scan queue has reached its memory use limit 12956f6a76adSMatthew Ahrens */ 1296a3874b8bSToomas Soome hrtime_t curr_time_ns = gethrtime(); 1297a3874b8bSToomas Soome uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time; 1298a3874b8bSToomas Soome uint64_t sync_time_ns = curr_time_ns - 1299a3874b8bSToomas Soome scn->scn_dp->dp_spa->spa_sync_starttime; 1300a3874b8bSToomas Soome 13016f6a76adSMatthew Ahrens int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; 1302a3874b8bSToomas Soome int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 1303a3874b8bSToomas Soome zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; 1304a3874b8bSToomas Soome 1305a3874b8bSToomas Soome if ((NSEC2MSEC(scan_time_ns) > mintime && 1306a3874b8bSToomas Soome (dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent || 1307a3874b8bSToomas Soome txg_sync_waiting(scn->scn_dp) || 1308a3874b8bSToomas Soome NSEC2SEC(sync_time_ns) >= zfs_txg_timeout)) || 1309a3874b8bSToomas Soome spa_shutting_down(scn->scn_dp->dp_spa) || 1310a3874b8bSToomas Soome (zfs_scan_strict_mem_lim && dsl_scan_should_clear(scn))) { 13113f9d6ad7SLin Ling if (zb) { 13121702cce7SAlek Pinchuk dprintf("suspending at bookmark %llx/%llx/%llx/%llx\n", 13133f9d6ad7SLin Ling (longlong_t)zb->zb_objset, 13143f9d6ad7SLin Ling (longlong_t)zb->zb_object, 13153f9d6ad7SLin Ling (longlong_t)zb->zb_level, 13163f9d6ad7SLin Ling (longlong_t)zb->zb_blkid); 13173f9d6ad7SLin Ling scn->scn_phys.scn_bookmark = *zb; 1318a3874b8bSToomas Soome } else { 1319a3874b8bSToomas Soome dsl_scan_phys_t *scnp = &scn->scn_phys; 1320a3874b8bSToomas Soome 1321a3874b8bSToomas Soome dprintf("suspending at DDT bookmark " 1322a3874b8bSToomas Soome "%llx/%llx/%llx/%llx\n", 1323a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_class, 1324a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_type, 1325a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_checksum, 1326a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_cursor); 13273f9d6ad7SLin Ling } 13281702cce7SAlek Pinchuk scn->scn_suspending = B_TRUE; 13293f9d6ad7SLin Ling return (B_TRUE); 13303f9d6ad7SLin Ling } 13313f9d6ad7SLin Ling return (B_FALSE); 13323f9d6ad7SLin Ling } 13333f9d6ad7SLin Ling 13343f9d6ad7SLin Ling typedef struct zil_scan_arg { 13353f9d6ad7SLin Ling dsl_pool_t *zsa_dp; 13363f9d6ad7SLin Ling zil_header_t *zsa_zh; 13373f9d6ad7SLin Ling } zil_scan_arg_t; 13383f9d6ad7SLin Ling 13393f9d6ad7SLin Ling /* ARGSUSED */ 13403f9d6ad7SLin Ling static int 13413f9d6ad7SLin Ling dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 13423f9d6ad7SLin Ling { 13433f9d6ad7SLin Ling zil_scan_arg_t *zsa = arg; 13443f9d6ad7SLin Ling dsl_pool_t *dp = zsa->zsa_dp; 13453f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 13463f9d6ad7SLin Ling zil_header_t *zh = zsa->zsa_zh; 13477802d7bfSMatthew Ahrens zbookmark_phys_t zb; 13483f9d6ad7SLin Ling 134943466aaeSMax Grossman if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 13503f9d6ad7SLin Ling return (0); 13513f9d6ad7SLin Ling 13523f9d6ad7SLin Ling /* 13533f9d6ad7SLin Ling * One block ("stubby") can be allocated a long time ago; we 13543f9d6ad7SLin Ling * want to visit that one because it has been allocated 13553f9d6ad7SLin Ling * (on-disk) even if it hasn't been claimed (even though for 13563f9d6ad7SLin Ling * scrub there's nothing to do to it). 13573f9d6ad7SLin Ling */ 135886714001SSerapheim Dimitropoulos if (claim_txg == 0 && bp->blk_birth >= spa_min_claim_txg(dp->dp_spa)) 13593f9d6ad7SLin Ling return (0); 13603f9d6ad7SLin Ling 13613f9d6ad7SLin Ling SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 13623f9d6ad7SLin Ling ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 13633f9d6ad7SLin Ling 13643f9d6ad7SLin Ling VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 13653f9d6ad7SLin Ling return (0); 13663f9d6ad7SLin Ling } 13673f9d6ad7SLin Ling 13683f9d6ad7SLin Ling /* ARGSUSED */ 13693f9d6ad7SLin Ling static int 13703f9d6ad7SLin Ling dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg) 13713f9d6ad7SLin Ling { 13723f9d6ad7SLin Ling if (lrc->lrc_txtype == TX_WRITE) { 13733f9d6ad7SLin Ling zil_scan_arg_t *zsa = arg; 13743f9d6ad7SLin Ling dsl_pool_t *dp = zsa->zsa_dp; 13753f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 13763f9d6ad7SLin Ling zil_header_t *zh = zsa->zsa_zh; 13773f9d6ad7SLin Ling lr_write_t *lr = (lr_write_t *)lrc; 13783f9d6ad7SLin Ling blkptr_t *bp = &lr->lr_blkptr; 13797802d7bfSMatthew Ahrens zbookmark_phys_t zb; 13803f9d6ad7SLin Ling 138143466aaeSMax Grossman if (BP_IS_HOLE(bp) || 138243466aaeSMax Grossman bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 13833f9d6ad7SLin Ling return (0); 13843f9d6ad7SLin Ling 13853f9d6ad7SLin Ling /* 13863f9d6ad7SLin Ling * birth can be < claim_txg if this record's txg is 13873f9d6ad7SLin Ling * already txg sync'ed (but this log block contains 13883f9d6ad7SLin Ling * other records that are not synced) 13893f9d6ad7SLin Ling */ 13903f9d6ad7SLin Ling if (claim_txg == 0 || bp->blk_birth < claim_txg) 13913f9d6ad7SLin Ling return (0); 13923f9d6ad7SLin Ling 13933f9d6ad7SLin Ling SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 13943f9d6ad7SLin Ling lr->lr_foid, ZB_ZIL_LEVEL, 13953f9d6ad7SLin Ling lr->lr_offset / BP_GET_LSIZE(bp)); 13963f9d6ad7SLin Ling 13973f9d6ad7SLin Ling VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 13983f9d6ad7SLin Ling } 13993f9d6ad7SLin Ling return (0); 14003f9d6ad7SLin Ling } 14013f9d6ad7SLin Ling 14023f9d6ad7SLin Ling static void 14033f9d6ad7SLin Ling dsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh) 14043f9d6ad7SLin Ling { 14053f9d6ad7SLin Ling uint64_t claim_txg = zh->zh_claim_txg; 14063f9d6ad7SLin Ling zil_scan_arg_t zsa = { dp, zh }; 14073f9d6ad7SLin Ling zilog_t *zilog; 14083f9d6ad7SLin Ling 140986714001SSerapheim Dimitropoulos ASSERT(spa_writeable(dp->dp_spa)); 141086714001SSerapheim Dimitropoulos 14113f9d6ad7SLin Ling /* 141286714001SSerapheim Dimitropoulos * We only want to visit blocks that have been claimed 141386714001SSerapheim Dimitropoulos * but not yet replayed. 14143f9d6ad7SLin Ling */ 141586714001SSerapheim Dimitropoulos if (claim_txg == 0) 14163f9d6ad7SLin Ling return; 14173f9d6ad7SLin Ling 14183f9d6ad7SLin Ling zilog = zil_alloc(dp->dp_meta_objset, zh); 14193f9d6ad7SLin Ling 14203f9d6ad7SLin Ling (void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa, 1421eb633035STom Caputi claim_txg, B_FALSE); 14223f9d6ad7SLin Ling 14233f9d6ad7SLin Ling zil_free(zilog); 14243f9d6ad7SLin Ling } 14253f9d6ad7SLin Ling 1426a3874b8bSToomas Soome /* 1427a3874b8bSToomas Soome * We compare scan_prefetch_issue_ctx_t's based on their bookmarks. The idea 1428a3874b8bSToomas Soome * here is to sort the AVL tree by the order each block will be needed. 1429a3874b8bSToomas Soome */ 1430a3874b8bSToomas Soome static int 1431a3874b8bSToomas Soome scan_prefetch_queue_compare(const void *a, const void *b) 1432a3874b8bSToomas Soome { 1433a3874b8bSToomas Soome const scan_prefetch_issue_ctx_t *spic_a = a, *spic_b = b; 1434a3874b8bSToomas Soome const scan_prefetch_ctx_t *spc_a = spic_a->spic_spc; 1435a3874b8bSToomas Soome const scan_prefetch_ctx_t *spc_b = spic_b->spic_spc; 1436a3874b8bSToomas Soome 1437a3874b8bSToomas Soome return (zbookmark_compare(spc_a->spc_datablkszsec, 1438a3874b8bSToomas Soome spc_a->spc_indblkshift, spc_b->spc_datablkszsec, 1439a3874b8bSToomas Soome spc_b->spc_indblkshift, &spic_a->spic_zb, &spic_b->spic_zb)); 1440a3874b8bSToomas Soome } 1441a3874b8bSToomas Soome 1442a3874b8bSToomas Soome static void 1443a3874b8bSToomas Soome scan_prefetch_ctx_rele(scan_prefetch_ctx_t *spc, void *tag) 1444a3874b8bSToomas Soome { 1445a3874b8bSToomas Soome if (zfs_refcount_remove(&spc->spc_refcnt, tag) == 0) { 1446a3874b8bSToomas Soome zfs_refcount_destroy(&spc->spc_refcnt); 1447a3874b8bSToomas Soome kmem_free(spc, sizeof (scan_prefetch_ctx_t)); 1448a3874b8bSToomas Soome } 1449a3874b8bSToomas Soome } 1450a3874b8bSToomas Soome 1451a3874b8bSToomas Soome static scan_prefetch_ctx_t * 1452a3874b8bSToomas Soome scan_prefetch_ctx_create(dsl_scan_t *scn, dnode_phys_t *dnp, void *tag) 1453a3874b8bSToomas Soome { 1454a3874b8bSToomas Soome scan_prefetch_ctx_t *spc; 1455a3874b8bSToomas Soome 1456a3874b8bSToomas Soome spc = kmem_alloc(sizeof (scan_prefetch_ctx_t), KM_SLEEP); 1457a3874b8bSToomas Soome zfs_refcount_create(&spc->spc_refcnt); 1458a3874b8bSToomas Soome zfs_refcount_add(&spc->spc_refcnt, tag); 1459a3874b8bSToomas Soome spc->spc_scn = scn; 1460a3874b8bSToomas Soome if (dnp != NULL) { 1461a3874b8bSToomas Soome spc->spc_datablkszsec = dnp->dn_datablkszsec; 1462a3874b8bSToomas Soome spc->spc_indblkshift = dnp->dn_indblkshift; 1463a3874b8bSToomas Soome spc->spc_root = B_FALSE; 1464a3874b8bSToomas Soome } else { 1465a3874b8bSToomas Soome spc->spc_datablkszsec = 0; 1466a3874b8bSToomas Soome spc->spc_indblkshift = 0; 1467a3874b8bSToomas Soome spc->spc_root = B_TRUE; 1468a3874b8bSToomas Soome } 1469a3874b8bSToomas Soome 1470a3874b8bSToomas Soome return (spc); 1471a3874b8bSToomas Soome } 1472a3874b8bSToomas Soome 1473a3874b8bSToomas Soome static void 1474a3874b8bSToomas Soome scan_prefetch_ctx_add_ref(scan_prefetch_ctx_t *spc, void *tag) 1475a3874b8bSToomas Soome { 1476a3874b8bSToomas Soome zfs_refcount_add(&spc->spc_refcnt, tag); 1477a3874b8bSToomas Soome } 1478a3874b8bSToomas Soome 1479a3874b8bSToomas Soome static boolean_t 1480a3874b8bSToomas Soome dsl_scan_check_prefetch_resume(scan_prefetch_ctx_t *spc, 1481a3874b8bSToomas Soome const zbookmark_phys_t *zb) 1482a3874b8bSToomas Soome { 1483a3874b8bSToomas Soome zbookmark_phys_t *last_zb = &spc->spc_scn->scn_prefetch_bookmark; 1484a3874b8bSToomas Soome dnode_phys_t tmp_dnp; 1485a3874b8bSToomas Soome dnode_phys_t *dnp = (spc->spc_root) ? NULL : &tmp_dnp; 1486a3874b8bSToomas Soome 1487a3874b8bSToomas Soome if (zb->zb_objset != last_zb->zb_objset) 1488a3874b8bSToomas Soome return (B_TRUE); 1489a3874b8bSToomas Soome if ((int64_t)zb->zb_object < 0) 1490a3874b8bSToomas Soome return (B_FALSE); 1491a3874b8bSToomas Soome 1492a3874b8bSToomas Soome tmp_dnp.dn_datablkszsec = spc->spc_datablkszsec; 1493a3874b8bSToomas Soome tmp_dnp.dn_indblkshift = spc->spc_indblkshift; 1494a3874b8bSToomas Soome 1495a3874b8bSToomas Soome if (zbookmark_subtree_completed(dnp, zb, last_zb)) 1496a3874b8bSToomas Soome return (B_TRUE); 1497a3874b8bSToomas Soome 1498a3874b8bSToomas Soome return (B_FALSE); 1499a3874b8bSToomas Soome } 1500a3874b8bSToomas Soome 1501a3874b8bSToomas Soome static void 1502a3874b8bSToomas Soome dsl_scan_prefetch(scan_prefetch_ctx_t *spc, blkptr_t *bp, zbookmark_phys_t *zb) 15033f9d6ad7SLin Ling { 1504a3874b8bSToomas Soome avl_index_t idx; 1505a3874b8bSToomas Soome dsl_scan_t *scn = spc->spc_scn; 1506a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 1507a3874b8bSToomas Soome scan_prefetch_issue_ctx_t *spic; 15083f9d6ad7SLin Ling 15093f9d6ad7SLin Ling if (zfs_no_scrub_prefetch) 15103f9d6ad7SLin Ling return; 15113f9d6ad7SLin Ling 1512a3874b8bSToomas Soome if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg || 1513a3874b8bSToomas Soome (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE && 1514a3874b8bSToomas Soome BP_GET_TYPE(bp) != DMU_OT_OBJSET)) 1515a3874b8bSToomas Soome return; 1516a3874b8bSToomas Soome 1517a3874b8bSToomas Soome if (dsl_scan_check_prefetch_resume(spc, zb)) 1518a3874b8bSToomas Soome return; 1519a3874b8bSToomas Soome 1520a3874b8bSToomas Soome scan_prefetch_ctx_add_ref(spc, scn); 1521a3874b8bSToomas Soome spic = kmem_alloc(sizeof (scan_prefetch_issue_ctx_t), KM_SLEEP); 1522a3874b8bSToomas Soome spic->spic_spc = spc; 1523a3874b8bSToomas Soome spic->spic_bp = *bp; 1524a3874b8bSToomas Soome spic->spic_zb = *zb; 1525a3874b8bSToomas Soome 1526a3874b8bSToomas Soome /* 1527a3874b8bSToomas Soome * Add the IO to the queue of blocks to prefetch. This allows us to 1528a3874b8bSToomas Soome * prioritize blocks that we will need first for the main traversal 1529a3874b8bSToomas Soome * thread. 1530a3874b8bSToomas Soome */ 1531a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1532a3874b8bSToomas Soome if (avl_find(&scn->scn_prefetch_queue, spic, &idx) != NULL) { 1533a3874b8bSToomas Soome /* this block is already queued for prefetch */ 1534a3874b8bSToomas Soome kmem_free(spic, sizeof (scan_prefetch_issue_ctx_t)); 1535a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, scn); 1536a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1537a3874b8bSToomas Soome return; 1538a3874b8bSToomas Soome } 1539a3874b8bSToomas Soome 1540a3874b8bSToomas Soome avl_insert(&scn->scn_prefetch_queue, spic, idx); 1541a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 1542a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1543a3874b8bSToomas Soome } 1544a3874b8bSToomas Soome 1545a3874b8bSToomas Soome static void 1546a3874b8bSToomas Soome dsl_scan_prefetch_dnode(dsl_scan_t *scn, dnode_phys_t *dnp, 1547a3874b8bSToomas Soome uint64_t objset, uint64_t object) 1548a3874b8bSToomas Soome { 1549a3874b8bSToomas Soome int i; 1550a3874b8bSToomas Soome zbookmark_phys_t zb; 1551a3874b8bSToomas Soome scan_prefetch_ctx_t *spc; 1552a3874b8bSToomas Soome 1553a3874b8bSToomas Soome if (dnp->dn_nblkptr == 0 && !(dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 15543f9d6ad7SLin Ling return; 15553f9d6ad7SLin Ling 1556a3874b8bSToomas Soome SET_BOOKMARK(&zb, objset, object, 0, 0); 1557a3874b8bSToomas Soome 1558a3874b8bSToomas Soome spc = scan_prefetch_ctx_create(scn, dnp, FTAG); 1559a3874b8bSToomas Soome 1560a3874b8bSToomas Soome for (i = 0; i < dnp->dn_nblkptr; i++) { 1561a3874b8bSToomas Soome zb.zb_level = BP_GET_LEVEL(&dnp->dn_blkptr[i]); 1562a3874b8bSToomas Soome zb.zb_blkid = i; 1563a3874b8bSToomas Soome dsl_scan_prefetch(spc, &dnp->dn_blkptr[i], &zb); 1564a3874b8bSToomas Soome } 1565a3874b8bSToomas Soome 1566a3874b8bSToomas Soome if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1567a3874b8bSToomas Soome zb.zb_level = 0; 1568a3874b8bSToomas Soome zb.zb_blkid = DMU_SPILL_BLKID; 1569a3874b8bSToomas Soome dsl_scan_prefetch(spc, &dnp->dn_spill, &zb); 1570a3874b8bSToomas Soome } 1571a3874b8bSToomas Soome 1572a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, FTAG); 1573a3874b8bSToomas Soome } 1574a3874b8bSToomas Soome 1575a3874b8bSToomas Soome void 1576a3874b8bSToomas Soome dsl_scan_prefetch_cb(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, 1577a3874b8bSToomas Soome arc_buf_t *buf, void *private) 1578a3874b8bSToomas Soome { 1579a3874b8bSToomas Soome scan_prefetch_ctx_t *spc = private; 1580a3874b8bSToomas Soome dsl_scan_t *scn = spc->spc_scn; 1581a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 1582a3874b8bSToomas Soome 1583a3874b8bSToomas Soome /* broadcast that the IO has completed for rate limitting purposes */ 1584a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1585a3874b8bSToomas Soome ASSERT3U(spa->spa_scrub_inflight, >=, BP_GET_PSIZE(bp)); 1586a3874b8bSToomas Soome spa->spa_scrub_inflight -= BP_GET_PSIZE(bp); 1587a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 1588a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1589a3874b8bSToomas Soome 1590a3874b8bSToomas Soome /* if there was an error or we are done prefetching, just cleanup */ 1591a3874b8bSToomas Soome if (buf == NULL || scn->scn_suspending) 1592a3874b8bSToomas Soome goto out; 1593a3874b8bSToomas Soome 1594a3874b8bSToomas Soome if (BP_GET_LEVEL(bp) > 0) { 1595a3874b8bSToomas Soome int i; 1596a3874b8bSToomas Soome blkptr_t *cbp; 1597a3874b8bSToomas Soome int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 1598a3874b8bSToomas Soome zbookmark_phys_t czb; 1599a3874b8bSToomas Soome 1600a3874b8bSToomas Soome for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) { 1601a3874b8bSToomas Soome SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 1602a3874b8bSToomas Soome zb->zb_level - 1, zb->zb_blkid * epb + i); 1603a3874b8bSToomas Soome dsl_scan_prefetch(spc, cbp, &czb); 1604a3874b8bSToomas Soome } 1605a3874b8bSToomas Soome } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 1606a3874b8bSToomas Soome dnode_phys_t *cdnp = buf->b_data; 1607a3874b8bSToomas Soome int i; 1608a3874b8bSToomas Soome int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 1609a3874b8bSToomas Soome 1610a3874b8bSToomas Soome for (i = 0, cdnp = buf->b_data; i < epb; 1611a3874b8bSToomas Soome i += cdnp->dn_extra_slots + 1, 1612a3874b8bSToomas Soome cdnp += cdnp->dn_extra_slots + 1) { 1613a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, cdnp, 1614a3874b8bSToomas Soome zb->zb_objset, zb->zb_blkid * epb + i); 1615a3874b8bSToomas Soome } 1616a3874b8bSToomas Soome } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 1617a3874b8bSToomas Soome objset_phys_t *osp = buf->b_data; 1618a3874b8bSToomas Soome 1619a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, &osp->os_meta_dnode, 1620a3874b8bSToomas Soome zb->zb_objset, DMU_META_DNODE_OBJECT); 16213f9d6ad7SLin Ling 1622a3874b8bSToomas Soome if (OBJSET_BUF_HAS_USERUSED(buf)) { 1623a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, 1624a3874b8bSToomas Soome &osp->os_groupused_dnode, zb->zb_objset, 1625a3874b8bSToomas Soome DMU_GROUPUSED_OBJECT); 1626a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, 1627a3874b8bSToomas Soome &osp->os_userused_dnode, zb->zb_objset, 1628a3874b8bSToomas Soome DMU_USERUSED_OBJECT); 1629a3874b8bSToomas Soome } 1630a3874b8bSToomas Soome } 1631a3874b8bSToomas Soome 1632a3874b8bSToomas Soome out: 1633a3874b8bSToomas Soome if (buf != NULL) 1634a3874b8bSToomas Soome arc_buf_destroy(buf, private); 1635a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, scn); 1636a3874b8bSToomas Soome } 1637a3874b8bSToomas Soome 1638a3874b8bSToomas Soome /* ARGSUSED */ 1639a3874b8bSToomas Soome static void 1640a3874b8bSToomas Soome dsl_scan_prefetch_thread(void *arg) 1641a3874b8bSToomas Soome { 1642a3874b8bSToomas Soome dsl_scan_t *scn = arg; 1643a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 1644a3874b8bSToomas Soome vdev_t *rvd = spa->spa_root_vdev; 1645a3874b8bSToomas Soome uint64_t maxinflight = rvd->vdev_children * zfs_top_maxinflight; 1646a3874b8bSToomas Soome scan_prefetch_issue_ctx_t *spic; 1647a3874b8bSToomas Soome 1648a3874b8bSToomas Soome /* loop until we are told to stop */ 1649a3874b8bSToomas Soome while (!scn->scn_prefetch_stop) { 1650a3874b8bSToomas Soome arc_flags_t flags = ARC_FLAG_NOWAIT | 1651a3874b8bSToomas Soome ARC_FLAG_PRESCIENT_PREFETCH | ARC_FLAG_PREFETCH; 1652a3874b8bSToomas Soome int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 1653a3874b8bSToomas Soome 1654a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1655a3874b8bSToomas Soome 1656a3874b8bSToomas Soome /* 1657a3874b8bSToomas Soome * Wait until we have an IO to issue and are not above our 1658a3874b8bSToomas Soome * maximum in flight limit. 1659a3874b8bSToomas Soome */ 1660a3874b8bSToomas Soome while (!scn->scn_prefetch_stop && 1661a3874b8bSToomas Soome (avl_numnodes(&scn->scn_prefetch_queue) == 0 || 1662a3874b8bSToomas Soome spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes)) { 1663a3874b8bSToomas Soome cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 1664a3874b8bSToomas Soome } 1665a3874b8bSToomas Soome 1666a3874b8bSToomas Soome /* recheck if we should stop since we waited for the cv */ 1667a3874b8bSToomas Soome if (scn->scn_prefetch_stop) { 1668a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1669a3874b8bSToomas Soome break; 1670a3874b8bSToomas Soome } 1671a3874b8bSToomas Soome 1672a3874b8bSToomas Soome /* remove the prefetch IO from the tree */ 1673a3874b8bSToomas Soome spic = avl_first(&scn->scn_prefetch_queue); 1674a3874b8bSToomas Soome spa->spa_scrub_inflight += BP_GET_PSIZE(&spic->spic_bp); 1675a3874b8bSToomas Soome avl_remove(&scn->scn_prefetch_queue, spic); 1676a3874b8bSToomas Soome 1677a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1678a3874b8bSToomas Soome 1679eb633035STom Caputi if (BP_IS_PROTECTED(&spic->spic_bp)) { 1680eb633035STom Caputi ASSERT(BP_GET_TYPE(&spic->spic_bp) == DMU_OT_DNODE || 1681eb633035STom Caputi BP_GET_TYPE(&spic->spic_bp) == DMU_OT_OBJSET); 1682eb633035STom Caputi ASSERT3U(BP_GET_LEVEL(&spic->spic_bp), ==, 0); 1683eb633035STom Caputi zio_flags |= ZIO_FLAG_RAW; 1684eb633035STom Caputi } 1685eb633035STom Caputi 1686a3874b8bSToomas Soome /* issue the prefetch asynchronously */ 1687a3874b8bSToomas Soome (void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, 1688a3874b8bSToomas Soome &spic->spic_bp, dsl_scan_prefetch_cb, spic->spic_spc, 1689a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, &spic->spic_zb); 1690a3874b8bSToomas Soome 1691a3874b8bSToomas Soome kmem_free(spic, sizeof (scan_prefetch_issue_ctx_t)); 1692a3874b8bSToomas Soome } 1693a3874b8bSToomas Soome 1694a3874b8bSToomas Soome ASSERT(scn->scn_prefetch_stop); 1695a3874b8bSToomas Soome 1696a3874b8bSToomas Soome /* free any prefetches we didn't get to complete */ 1697a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1698a3874b8bSToomas Soome while ((spic = avl_first(&scn->scn_prefetch_queue)) != NULL) { 1699a3874b8bSToomas Soome avl_remove(&scn->scn_prefetch_queue, spic); 1700a3874b8bSToomas Soome scan_prefetch_ctx_rele(spic->spic_spc, scn); 1701a3874b8bSToomas Soome kmem_free(spic, sizeof (scan_prefetch_issue_ctx_t)); 1702a3874b8bSToomas Soome } 1703a3874b8bSToomas Soome ASSERT0(avl_numnodes(&scn->scn_prefetch_queue)); 1704a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 17053f9d6ad7SLin Ling } 17063f9d6ad7SLin Ling 17073f9d6ad7SLin Ling static boolean_t 17083f9d6ad7SLin Ling dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp, 17097802d7bfSMatthew Ahrens const zbookmark_phys_t *zb) 17103f9d6ad7SLin Ling { 17113f9d6ad7SLin Ling /* 17123f9d6ad7SLin Ling * We never skip over user/group accounting objects (obj<0) 17133f9d6ad7SLin Ling */ 1714ad135b5dSChristopher Siden if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) && 17153f9d6ad7SLin Ling (int64_t)zb->zb_object >= 0) { 17163f9d6ad7SLin Ling /* 17173f9d6ad7SLin Ling * If we already visited this bp & everything below (in 17183f9d6ad7SLin Ling * a prior txg sync), don't bother doing it again. 17193f9d6ad7SLin Ling */ 1720a2cdcdd2SPaul Dagnelie if (zbookmark_subtree_completed(dnp, zb, 1721a2cdcdd2SPaul Dagnelie &scn->scn_phys.scn_bookmark)) 17223f9d6ad7SLin Ling return (B_TRUE); 17233f9d6ad7SLin Ling 17243f9d6ad7SLin Ling /* 17253f9d6ad7SLin Ling * If we found the block we're trying to resume from, or 17263f9d6ad7SLin Ling * we went past it to a different object, zero it out to 17271702cce7SAlek Pinchuk * indicate that it's OK to start checking for suspending 17283f9d6ad7SLin Ling * again. 17293f9d6ad7SLin Ling */ 17303f9d6ad7SLin Ling if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 || 17313f9d6ad7SLin Ling zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) { 17323f9d6ad7SLin Ling dprintf("resuming at %llx/%llx/%llx/%llx\n", 17333f9d6ad7SLin Ling (longlong_t)zb->zb_objset, 17343f9d6ad7SLin Ling (longlong_t)zb->zb_object, 17353f9d6ad7SLin Ling (longlong_t)zb->zb_level, 17363f9d6ad7SLin Ling (longlong_t)zb->zb_blkid); 17373f9d6ad7SLin Ling bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb)); 17383f9d6ad7SLin Ling } 17393f9d6ad7SLin Ling } 17403f9d6ad7SLin Ling return (B_FALSE); 17413f9d6ad7SLin Ling } 17423f9d6ad7SLin Ling 1743a3874b8bSToomas Soome static void dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 1744a3874b8bSToomas Soome dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, 1745a3874b8bSToomas Soome dmu_objset_type_t ostype, dmu_tx_t *tx); 1746a3874b8bSToomas Soome static void dsl_scan_visitdnode( 1747a3874b8bSToomas Soome dsl_scan_t *, dsl_dataset_t *ds, dmu_objset_type_t ostype, 1748a3874b8bSToomas Soome dnode_phys_t *dnp, uint64_t object, dmu_tx_t *tx); 1749a3874b8bSToomas Soome 17503f9d6ad7SLin Ling /* 17513f9d6ad7SLin Ling * Return nonzero on i/o error. 17523f9d6ad7SLin Ling * Return new buf to write out in *bufp. 17533f9d6ad7SLin Ling */ 17543f9d6ad7SLin Ling static int 17553f9d6ad7SLin Ling dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, 17563f9d6ad7SLin Ling dnode_phys_t *dnp, const blkptr_t *bp, 17575f37736aSMatthew Ahrens const zbookmark_phys_t *zb, dmu_tx_t *tx) 17583f9d6ad7SLin Ling { 17593f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 176044ecc532SGeorge Wilson int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 17613f9d6ad7SLin Ling int err; 17623f9d6ad7SLin Ling 17633f9d6ad7SLin Ling if (BP_GET_LEVEL(bp) > 0) { 17647adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 17653f9d6ad7SLin Ling int i; 17663f9d6ad7SLin Ling blkptr_t *cbp; 17673f9d6ad7SLin Ling int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 17685f37736aSMatthew Ahrens arc_buf_t *buf; 17693f9d6ad7SLin Ling 17705f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 1771a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, zb); 17723f9d6ad7SLin Ling if (err) { 17733f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 17743f9d6ad7SLin Ling return (err); 17753f9d6ad7SLin Ling } 17765f37736aSMatthew Ahrens for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) { 17777802d7bfSMatthew Ahrens zbookmark_phys_t czb; 17783f9d6ad7SLin Ling 17793f9d6ad7SLin Ling SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 17803f9d6ad7SLin Ling zb->zb_level - 1, 17813f9d6ad7SLin Ling zb->zb_blkid * epb + i); 17823f9d6ad7SLin Ling dsl_scan_visitbp(cbp, &czb, dnp, 17835f37736aSMatthew Ahrens ds, scn, ostype, tx); 17843f9d6ad7SLin Ling } 1785dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 17863f9d6ad7SLin Ling } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 17877adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 17883f9d6ad7SLin Ling dnode_phys_t *cdnp; 1789a3874b8bSToomas Soome int i; 17903f9d6ad7SLin Ling int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 17915f37736aSMatthew Ahrens arc_buf_t *buf; 17923f9d6ad7SLin Ling 1793eb633035STom Caputi if (BP_IS_PROTECTED(bp)) { 1794eb633035STom Caputi ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF); 1795eb633035STom Caputi zio_flags |= ZIO_FLAG_RAW; 1796eb633035STom Caputi } 1797eb633035STom Caputi 17985f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 1799a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, zb); 18003f9d6ad7SLin Ling if (err) { 18013f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 18023f9d6ad7SLin Ling return (err); 18033f9d6ad7SLin Ling } 180454811da5SToomas Soome for (i = 0, cdnp = buf->b_data; i < epb; 180554811da5SToomas Soome i += cdnp->dn_extra_slots + 1, 180654811da5SToomas Soome cdnp += cdnp->dn_extra_slots + 1) { 18073f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, ostype, 18085f37736aSMatthew Ahrens cdnp, zb->zb_blkid * epb + i, tx); 18093f9d6ad7SLin Ling } 18103f9d6ad7SLin Ling 1811dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 18123f9d6ad7SLin Ling } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 18137adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 18143f9d6ad7SLin Ling objset_phys_t *osp; 18155f37736aSMatthew Ahrens arc_buf_t *buf; 18163f9d6ad7SLin Ling 18175f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 1818a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, zb); 18193f9d6ad7SLin Ling if (err) { 18203f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 18213f9d6ad7SLin Ling return (err); 18223f9d6ad7SLin Ling } 18233f9d6ad7SLin Ling 18245f37736aSMatthew Ahrens osp = buf->b_data; 18253f9d6ad7SLin Ling 18263f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 18275f37736aSMatthew Ahrens &osp->os_meta_dnode, DMU_META_DNODE_OBJECT, tx); 18283f9d6ad7SLin Ling 18295f37736aSMatthew Ahrens if (OBJSET_BUF_HAS_USERUSED(buf)) { 18303f9d6ad7SLin Ling /* 1831f67950b2SNasf-Fan * We also always visit user/group/project accounting 18323f9d6ad7SLin Ling * objects, and never skip them, even if we are 18331702cce7SAlek Pinchuk * suspending. This is necessary so that the space 18343f9d6ad7SLin Ling * deltas from this txg get integrated. 18353f9d6ad7SLin Ling */ 1836f67950b2SNasf-Fan if (OBJSET_BUF_HAS_PROJECTUSED(buf)) 1837f67950b2SNasf-Fan dsl_scan_visitdnode(scn, ds, osp->os_type, 1838f67950b2SNasf-Fan &osp->os_projectused_dnode, 1839f67950b2SNasf-Fan DMU_PROJECTUSED_OBJECT, tx); 18403f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 18415f37736aSMatthew Ahrens &osp->os_groupused_dnode, 18423f9d6ad7SLin Ling DMU_GROUPUSED_OBJECT, tx); 18433f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 18445f37736aSMatthew Ahrens &osp->os_userused_dnode, 18453f9d6ad7SLin Ling DMU_USERUSED_OBJECT, tx); 18463f9d6ad7SLin Ling } 1847dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 18483f9d6ad7SLin Ling } 18493f9d6ad7SLin Ling 18503f9d6ad7SLin Ling return (0); 18513f9d6ad7SLin Ling } 18523f9d6ad7SLin Ling 18533f9d6ad7SLin Ling static void 18543f9d6ad7SLin Ling dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, 18555f37736aSMatthew Ahrens dmu_objset_type_t ostype, dnode_phys_t *dnp, 18563f9d6ad7SLin Ling uint64_t object, dmu_tx_t *tx) 18573f9d6ad7SLin Ling { 18583f9d6ad7SLin Ling int j; 18593f9d6ad7SLin Ling 18603f9d6ad7SLin Ling for (j = 0; j < dnp->dn_nblkptr; j++) { 18617802d7bfSMatthew Ahrens zbookmark_phys_t czb; 18623f9d6ad7SLin Ling 18633f9d6ad7SLin Ling SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 18643f9d6ad7SLin Ling dnp->dn_nlevels - 1, j); 18653f9d6ad7SLin Ling dsl_scan_visitbp(&dnp->dn_blkptr[j], 18665f37736aSMatthew Ahrens &czb, dnp, ds, scn, ostype, tx); 18673f9d6ad7SLin Ling } 18683f9d6ad7SLin Ling 18693f9d6ad7SLin Ling if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 18707802d7bfSMatthew Ahrens zbookmark_phys_t czb; 18713f9d6ad7SLin Ling SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 18723f9d6ad7SLin Ling 0, DMU_SPILL_BLKID); 187354811da5SToomas Soome dsl_scan_visitbp(DN_SPILL_BLKPTR(dnp), 18745f37736aSMatthew Ahrens &czb, dnp, ds, scn, ostype, tx); 18753f9d6ad7SLin Ling } 18763f9d6ad7SLin Ling } 18773f9d6ad7SLin Ling 18783f9d6ad7SLin Ling /* 18793f9d6ad7SLin Ling * The arguments are in this order because mdb can only print the 18803f9d6ad7SLin Ling * first 5; we want them to be useful. 18813f9d6ad7SLin Ling */ 18823f9d6ad7SLin Ling static void 18837802d7bfSMatthew Ahrens dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 18845f37736aSMatthew Ahrens dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, 18855f37736aSMatthew Ahrens dmu_objset_type_t ostype, dmu_tx_t *tx) 18863f9d6ad7SLin Ling { 18873f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 1888a3874b8bSToomas Soome blkptr_t *bp_toread = NULL; 18893f9d6ad7SLin Ling 18901702cce7SAlek Pinchuk if (dsl_scan_check_suspend(scn, zb)) 18913f9d6ad7SLin Ling return; 18923f9d6ad7SLin Ling 18933f9d6ad7SLin Ling if (dsl_scan_check_resume(scn, dnp, zb)) 18943f9d6ad7SLin Ling return; 18953f9d6ad7SLin Ling 18963f9d6ad7SLin Ling scn->scn_visited_this_txg++; 18973f9d6ad7SLin Ling 1898a3874b8bSToomas Soome /* 1899a3874b8bSToomas Soome * This debugging is commented out to conserve stack space. This 1900a3874b8bSToomas Soome * function is called recursively and the debugging addes several 1901a3874b8bSToomas Soome * bytes to the stack for each call. It can be commented back in 1902a3874b8bSToomas Soome * if required to debug an issue in dsl_scan_visitbp(). 1903a3874b8bSToomas Soome * 1904a3874b8bSToomas Soome * dprintf_bp(bp, 1905a3874b8bSToomas Soome * "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx bp=%p", 1906a3874b8bSToomas Soome * ds, ds ? ds->ds_object : 0, 1907a3874b8bSToomas Soome * zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid, 1908a3874b8bSToomas Soome * bp); 1909a3874b8bSToomas Soome */ 19103f9d6ad7SLin Ling 1911a3874b8bSToomas Soome if (BP_IS_HOLE(bp)) { 1912a3874b8bSToomas Soome scn->scn_holes_this_txg++; 19133f9d6ad7SLin Ling return; 1914a3874b8bSToomas Soome } 19153f9d6ad7SLin Ling 1916a3874b8bSToomas Soome if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) { 1917a3874b8bSToomas Soome scn->scn_lt_min_this_txg++; 19183f9d6ad7SLin Ling return; 1919a3874b8bSToomas Soome } 1920a3874b8bSToomas Soome 1921a3874b8bSToomas Soome bp_toread = kmem_alloc(sizeof (blkptr_t), KM_SLEEP); 1922a3874b8bSToomas Soome *bp_toread = *bp; 1923a3874b8bSToomas Soome 1924a3874b8bSToomas Soome if (dsl_scan_recurse(scn, ds, ostype, dnp, bp_toread, zb, tx) != 0) 1925a3874b8bSToomas Soome goto out; 19263f9d6ad7SLin Ling 19273f9d6ad7SLin Ling /* 192840713f2bSAlan Somers * If dsl_scan_ddt() has already visited this block, it will have 19293f9d6ad7SLin Ling * already done any translations or scrubbing, so don't call the 19303f9d6ad7SLin Ling * callback again. 19313f9d6ad7SLin Ling */ 19323f9d6ad7SLin Ling if (ddt_class_contains(dp->dp_spa, 19333f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max, bp)) { 1934a3874b8bSToomas Soome scn->scn_ddt_contained_this_txg++; 1935a3874b8bSToomas Soome goto out; 19363f9d6ad7SLin Ling } 19373f9d6ad7SLin Ling 19383f9d6ad7SLin Ling /* 19393f9d6ad7SLin Ling * If this block is from the future (after cur_max_txg), then we 19403f9d6ad7SLin Ling * are doing this on behalf of a deleted snapshot, and we will 19413f9d6ad7SLin Ling * revisit the future block on the next pass of this dataset. 19423f9d6ad7SLin Ling * Don't scan it now unless we need to because something 19433f9d6ad7SLin Ling * under it was modified. 19443f9d6ad7SLin Ling */ 1945a3874b8bSToomas Soome if (BP_PHYSICAL_BIRTH(bp) > scn->scn_phys.scn_cur_max_txg) { 1946a3874b8bSToomas Soome scn->scn_gt_max_this_txg++; 1947a3874b8bSToomas Soome goto out; 19483f9d6ad7SLin Ling } 1949a3874b8bSToomas Soome 1950a3874b8bSToomas Soome scan_funcs[scn->scn_phys.scn_func](dp, bp, zb); 1951a3874b8bSToomas Soome 1952a3874b8bSToomas Soome out: 1953a3874b8bSToomas Soome kmem_free(bp_toread, sizeof (blkptr_t)); 19543f9d6ad7SLin Ling } 19553f9d6ad7SLin Ling 19563f9d6ad7SLin Ling static void 19573f9d6ad7SLin Ling dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp, 19583f9d6ad7SLin Ling dmu_tx_t *tx) 19593f9d6ad7SLin Ling { 19607802d7bfSMatthew Ahrens zbookmark_phys_t zb; 1961a3874b8bSToomas Soome scan_prefetch_ctx_t *spc; 19623f9d6ad7SLin Ling 19633f9d6ad7SLin Ling SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 19643f9d6ad7SLin Ling ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID); 1965a3874b8bSToomas Soome 1966a3874b8bSToomas Soome if (ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) { 1967a3874b8bSToomas Soome SET_BOOKMARK(&scn->scn_prefetch_bookmark, 1968a3874b8bSToomas Soome zb.zb_objset, 0, 0, 0); 1969a3874b8bSToomas Soome } else { 1970a3874b8bSToomas Soome scn->scn_prefetch_bookmark = scn->scn_phys.scn_bookmark; 1971a3874b8bSToomas Soome } 1972a3874b8bSToomas Soome 1973a3874b8bSToomas Soome scn->scn_objsets_visited_this_txg++; 1974a3874b8bSToomas Soome 1975a3874b8bSToomas Soome spc = scan_prefetch_ctx_create(scn, NULL, FTAG); 1976a3874b8bSToomas Soome dsl_scan_prefetch(spc, bp, &zb); 1977a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, FTAG); 1978a3874b8bSToomas Soome 1979a3874b8bSToomas Soome dsl_scan_visitbp(bp, &zb, NULL, ds, scn, DMU_OST_NONE, tx); 19803f9d6ad7SLin Ling 19813f9d6ad7SLin Ling dprintf_ds(ds, "finished scan%s", ""); 19823f9d6ad7SLin Ling } 19833f9d6ad7SLin Ling 1984a3874b8bSToomas Soome static void 1985a3874b8bSToomas Soome ds_destroyed_scn_phys(dsl_dataset_t *ds, dsl_scan_phys_t *scn_phys) 19863f9d6ad7SLin Ling { 1987a3874b8bSToomas Soome if (scn_phys->scn_bookmark.zb_objset == ds->ds_object) { 1988bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) { 198938d61036SMatthew Ahrens /* 199038d61036SMatthew Ahrens * Note: 199138d61036SMatthew Ahrens * - scn_cur_{min,max}_txg stays the same. 199238d61036SMatthew Ahrens * - Setting the flag is not really necessary if 199338d61036SMatthew Ahrens * scn_cur_max_txg == scn_max_txg, because there 199438d61036SMatthew Ahrens * is nothing after this snapshot that we care 199538d61036SMatthew Ahrens * about. However, we set it anyway and then 199638d61036SMatthew Ahrens * ignore it when we retraverse it in 199738d61036SMatthew Ahrens * dsl_scan_visitds(). 199838d61036SMatthew Ahrens */ 1999a3874b8bSToomas Soome scn_phys->scn_bookmark.zb_objset = 2000c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj; 20013f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; currently traversing; " 20023f9d6ad7SLin Ling "reset zb_objset to %llu", 20033f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 2004c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)-> 2005c1379625SJustin T. Gibbs ds_next_snap_obj); 2006a3874b8bSToomas Soome scn_phys->scn_flags |= DSF_VISIT_DS_AGAIN; 20073f9d6ad7SLin Ling } else { 2008a3874b8bSToomas Soome SET_BOOKMARK(&scn_phys->scn_bookmark, 20093f9d6ad7SLin Ling ZB_DESTROYED_OBJSET, 0, 0, 0); 20103f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; currently traversing; " 20113f9d6ad7SLin Ling "reset bookmark to -1,0,0,0", 20123f9d6ad7SLin Ling (u_longlong_t)ds->ds_object); 20133f9d6ad7SLin Ling } 2014a3874b8bSToomas Soome } 2015a3874b8bSToomas Soome } 2016a3874b8bSToomas Soome 2017a3874b8bSToomas Soome /* 2018a3874b8bSToomas Soome * Invoked when a dataset is destroyed. We need to make sure that: 2019a3874b8bSToomas Soome * 2020a3874b8bSToomas Soome * 1) If it is the dataset that was currently being scanned, we write 2021a3874b8bSToomas Soome * a new dsl_scan_phys_t and marking the objset reference in it 2022a3874b8bSToomas Soome * as destroyed. 2023a3874b8bSToomas Soome * 2) Remove it from the work queue, if it was present. 2024a3874b8bSToomas Soome * 2025a3874b8bSToomas Soome * If the dataset was actually a snapshot, instead of marking the dataset 2026a3874b8bSToomas Soome * as destroyed, we instead substitute the next snapshot in line. 2027a3874b8bSToomas Soome */ 2028a3874b8bSToomas Soome void 2029a3874b8bSToomas Soome dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx) 2030a3874b8bSToomas Soome { 2031a3874b8bSToomas Soome dsl_pool_t *dp = ds->ds_dir->dd_pool; 2032a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 2033a3874b8bSToomas Soome uint64_t mintxg; 2034a3874b8bSToomas Soome 2035a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 2036a3874b8bSToomas Soome return; 2037a3874b8bSToomas Soome 2038a3874b8bSToomas Soome ds_destroyed_scn_phys(ds, &scn->scn_phys); 2039a3874b8bSToomas Soome ds_destroyed_scn_phys(ds, &scn->scn_phys_cached); 2040a3874b8bSToomas Soome 2041a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) { 2042a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds->ds_object); 2043a3874b8bSToomas Soome if (ds->ds_is_snapshot) 2044a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2045a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_next_snap_obj, mintxg); 2046a3874b8bSToomas Soome } 2047a3874b8bSToomas Soome 2048a3874b8bSToomas Soome if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 2049a3874b8bSToomas Soome ds->ds_object, &mintxg) == 0) { 2050c1379625SJustin T. Gibbs ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1); 2051b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 20523f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 2053bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) { 20543f9d6ad7SLin Ling /* 20553f9d6ad7SLin Ling * We keep the same mintxg; it could be > 20563f9d6ad7SLin Ling * ds_creation_txg if the previous snapshot was 20573f9d6ad7SLin Ling * deleted too. 20583f9d6ad7SLin Ling */ 20593f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 20603f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 2061c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj, 2062c1379625SJustin T. Gibbs mintxg, tx) == 0); 20633f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; in queue; " 20643f9d6ad7SLin Ling "replacing with %llu", 20653f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 2066c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)-> 2067c1379625SJustin T. Gibbs ds_next_snap_obj); 20683f9d6ad7SLin Ling } else { 20693f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; in queue; removing", 20703f9d6ad7SLin Ling (u_longlong_t)ds->ds_object); 20713f9d6ad7SLin Ling } 20723f9d6ad7SLin Ling } 20733f9d6ad7SLin Ling 20743f9d6ad7SLin Ling /* 20753f9d6ad7SLin Ling * dsl_scan_sync() should be called after this, and should sync 20763f9d6ad7SLin Ling * out our changed state, but just to be safe, do it here. 20773f9d6ad7SLin Ling */ 2078a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 2079a3874b8bSToomas Soome } 2080a3874b8bSToomas Soome 2081a3874b8bSToomas Soome static void 2082a3874b8bSToomas Soome ds_snapshotted_bookmark(dsl_dataset_t *ds, zbookmark_phys_t *scn_bookmark) 2083a3874b8bSToomas Soome { 2084a3874b8bSToomas Soome if (scn_bookmark->zb_objset == ds->ds_object) { 2085a3874b8bSToomas Soome scn_bookmark->zb_objset = 2086a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_obj; 2087a3874b8bSToomas Soome zfs_dbgmsg("snapshotting ds %llu; currently traversing; " 2088a3874b8bSToomas Soome "reset zb_objset to %llu", 2089a3874b8bSToomas Soome (u_longlong_t)ds->ds_object, 2090a3874b8bSToomas Soome (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj); 2091a3874b8bSToomas Soome } 20923f9d6ad7SLin Ling } 20933f9d6ad7SLin Ling 2094a3874b8bSToomas Soome /* 2095a3874b8bSToomas Soome * Called when a dataset is snapshotted. If we were currently traversing 2096a3874b8bSToomas Soome * this snapshot, we reset our bookmark to point at the newly created 2097a3874b8bSToomas Soome * snapshot. We also modify our work queue to remove the old snapshot and 2098a3874b8bSToomas Soome * replace with the new one. 2099a3874b8bSToomas Soome */ 21003f9d6ad7SLin Ling void 21013f9d6ad7SLin Ling dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx) 21023f9d6ad7SLin Ling { 21033f9d6ad7SLin Ling dsl_pool_t *dp = ds->ds_dir->dd_pool; 21043f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 21053f9d6ad7SLin Ling uint64_t mintxg; 21063f9d6ad7SLin Ling 2107a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 21083f9d6ad7SLin Ling return; 21093f9d6ad7SLin Ling 2110c1379625SJustin T. Gibbs ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0); 21113f9d6ad7SLin Ling 2112a3874b8bSToomas Soome ds_snapshotted_bookmark(ds, &scn->scn_phys.scn_bookmark); 2113a3874b8bSToomas Soome ds_snapshotted_bookmark(ds, &scn->scn_phys_cached.scn_bookmark); 2114a3874b8bSToomas Soome 2115a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) { 2116a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds->ds_object); 2117a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2118a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg); 2119a3874b8bSToomas Soome } 2120a3874b8bSToomas Soome 2121a3874b8bSToomas Soome if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 2122a3874b8bSToomas Soome ds->ds_object, &mintxg) == 0) { 2123b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 21243f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 21253f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 21263f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 2127c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg, tx) == 0); 21283f9d6ad7SLin Ling zfs_dbgmsg("snapshotting ds %llu; in queue; " 21293f9d6ad7SLin Ling "replacing with %llu", 21303f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 2131c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj); 21323f9d6ad7SLin Ling } 2133a3874b8bSToomas Soome 2134a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 21353f9d6ad7SLin Ling } 21363f9d6ad7SLin Ling 2137a3874b8bSToomas Soome static void 2138a3874b8bSToomas Soome ds_clone_swapped_bookmark(dsl_dataset_t *ds1, dsl_dataset_t *ds2, 2139a3874b8bSToomas Soome zbookmark_phys_t *scn_bookmark) 21403f9d6ad7SLin Ling { 2141a3874b8bSToomas Soome if (scn_bookmark->zb_objset == ds1->ds_object) { 2142a3874b8bSToomas Soome scn_bookmark->zb_objset = ds2->ds_object; 21433f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 21443f9d6ad7SLin Ling "reset zb_objset to %llu", 21453f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object, 21463f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object); 2147a3874b8bSToomas Soome } else if (scn_bookmark->zb_objset == ds2->ds_object) { 2148a3874b8bSToomas Soome scn_bookmark->zb_objset = ds1->ds_object; 21493f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 21503f9d6ad7SLin Ling "reset zb_objset to %llu", 21513f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object, 21523f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object); 21533f9d6ad7SLin Ling } 2154a3874b8bSToomas Soome } 2155a3874b8bSToomas Soome 2156a3874b8bSToomas Soome /* 2157a3874b8bSToomas Soome * Called when a parent dataset and its clone are swapped. If we were 2158a3874b8bSToomas Soome * currently traversing the dataset, we need to switch to traversing the 2159a3874b8bSToomas Soome * newly promoted parent. 2160a3874b8bSToomas Soome */ 2161a3874b8bSToomas Soome void 2162a3874b8bSToomas Soome dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx) 2163a3874b8bSToomas Soome { 2164a3874b8bSToomas Soome dsl_pool_t *dp = ds1->ds_dir->dd_pool; 2165a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 2166a3874b8bSToomas Soome uint64_t mintxg; 2167a3874b8bSToomas Soome 2168a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 2169a3874b8bSToomas Soome return; 2170a3874b8bSToomas Soome 2171a3874b8bSToomas Soome ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys.scn_bookmark); 2172a3874b8bSToomas Soome ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys_cached.scn_bookmark); 2173a3874b8bSToomas Soome 2174a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds1->ds_object, &mintxg)) { 2175a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds1->ds_object); 2176a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds2->ds_object, mintxg); 2177a3874b8bSToomas Soome } 2178a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds2->ds_object, &mintxg)) { 2179a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds2->ds_object); 2180a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds1->ds_object, mintxg); 2181a3874b8bSToomas Soome } 21823f9d6ad7SLin Ling 21833f9d6ad7SLin Ling if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 21843f9d6ad7SLin Ling ds1->ds_object, &mintxg) == 0) { 21853f9d6ad7SLin Ling int err; 2186c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg); 2187c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg); 2188b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 21893f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds1->ds_object, tx)); 21903f9d6ad7SLin Ling err = zap_add_int_key(dp->dp_meta_objset, 21913f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx); 21923f9d6ad7SLin Ling VERIFY(err == 0 || err == EEXIST); 21933f9d6ad7SLin Ling if (err == EEXIST) { 21943f9d6ad7SLin Ling /* Both were there to begin with */ 21953f9d6ad7SLin Ling VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 21963f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 21973f9d6ad7SLin Ling ds1->ds_object, mintxg, tx)); 21983f9d6ad7SLin Ling } 21993f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; in queue; " 22003f9d6ad7SLin Ling "replacing with %llu", 22013f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object, 22023f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object); 2203a3874b8bSToomas Soome } 2204a3874b8bSToomas Soome if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 2205a3874b8bSToomas Soome ds2->ds_object, &mintxg) == 0) { 2206c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg); 2207c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg); 2208b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 22093f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, tx)); 22103f9d6ad7SLin Ling VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 22113f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx)); 22123f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; in queue; " 22133f9d6ad7SLin Ling "replacing with %llu", 22143f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object, 22153f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object); 22163f9d6ad7SLin Ling } 22173f9d6ad7SLin Ling 2218a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 22193f9d6ad7SLin Ling } 22203f9d6ad7SLin Ling 22213f9d6ad7SLin Ling /* ARGSUSED */ 22223f9d6ad7SLin Ling static int 22233b2aab18SMatthew Ahrens enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 22243f9d6ad7SLin Ling { 2225a3874b8bSToomas Soome uint64_t originobj = *(uint64_t *)arg; 22263f9d6ad7SLin Ling dsl_dataset_t *ds; 22273f9d6ad7SLin Ling int err; 22283f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 22293f9d6ad7SLin Ling 2230a3874b8bSToomas Soome if (dsl_dir_phys(hds->ds_dir)->dd_origin_obj != originobj) 22313b2aab18SMatthew Ahrens return (0); 22323b2aab18SMatthew Ahrens 22333b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 22343f9d6ad7SLin Ling if (err) 22353f9d6ad7SLin Ling return (err); 22363f9d6ad7SLin Ling 2237a3874b8bSToomas Soome while (dsl_dataset_phys(ds)->ds_prev_snap_obj != originobj) { 22383b2aab18SMatthew Ahrens dsl_dataset_t *prev; 22393b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, 2240c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev); 22413f9d6ad7SLin Ling 22423b2aab18SMatthew Ahrens dsl_dataset_rele(ds, FTAG); 22433b2aab18SMatthew Ahrens if (err) 22443b2aab18SMatthew Ahrens return (err); 22453b2aab18SMatthew Ahrens ds = prev; 22463f9d6ad7SLin Ling } 2247a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds->ds_object, 2248a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_txg); 22493f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 22503f9d6ad7SLin Ling return (0); 22513f9d6ad7SLin Ling } 22523f9d6ad7SLin Ling 22533f9d6ad7SLin Ling static void 22543f9d6ad7SLin Ling dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) 22553f9d6ad7SLin Ling { 22563f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 22573f9d6ad7SLin Ling dsl_dataset_t *ds; 22583f9d6ad7SLin Ling 2259b420f3adSRichard Lowe VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 22603f9d6ad7SLin Ling 226138d61036SMatthew Ahrens if (scn->scn_phys.scn_cur_min_txg >= 226238d61036SMatthew Ahrens scn->scn_phys.scn_max_txg) { 226338d61036SMatthew Ahrens /* 226438d61036SMatthew Ahrens * This can happen if this snapshot was created after the 226538d61036SMatthew Ahrens * scan started, and we already completed a previous snapshot 226638d61036SMatthew Ahrens * that was created after the scan started. This snapshot 226738d61036SMatthew Ahrens * only references blocks with: 226838d61036SMatthew Ahrens * 226938d61036SMatthew Ahrens * birth < our ds_creation_txg 227038d61036SMatthew Ahrens * cur_min_txg is no less than ds_creation_txg. 227138d61036SMatthew Ahrens * We have already visited these blocks. 227238d61036SMatthew Ahrens * or 227338d61036SMatthew Ahrens * birth > scn_max_txg 227438d61036SMatthew Ahrens * The scan requested not to visit these blocks. 227538d61036SMatthew Ahrens * 227638d61036SMatthew Ahrens * Subsequent snapshots (and clones) can reference our 227738d61036SMatthew Ahrens * blocks, or blocks with even higher birth times. 227838d61036SMatthew Ahrens * Therefore we do not need to visit them either, 227938d61036SMatthew Ahrens * so we do not add them to the work queue. 228038d61036SMatthew Ahrens * 228138d61036SMatthew Ahrens * Note that checking for cur_min_txg >= cur_max_txg 228238d61036SMatthew Ahrens * is not sufficient, because in that case we may need to 228338d61036SMatthew Ahrens * visit subsequent snapshots. This happens when min_txg > 0, 228438d61036SMatthew Ahrens * which raises cur_min_txg. In this case we will visit 228538d61036SMatthew Ahrens * this dataset but skip all of its blocks, because the 228638d61036SMatthew Ahrens * rootbp's birth time is < cur_min_txg. Then we will 228738d61036SMatthew Ahrens * add the next snapshots/clones to the work queue. 228838d61036SMatthew Ahrens */ 228938d61036SMatthew Ahrens char *dsname = kmem_alloc(MAXNAMELEN, KM_SLEEP); 229038d61036SMatthew Ahrens dsl_dataset_name(ds, dsname); 229138d61036SMatthew Ahrens zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because " 229238d61036SMatthew Ahrens "cur_min_txg (%llu) >= max_txg (%llu)", 2293a3874b8bSToomas Soome (longlong_t)dsobj, dsname, 2294a3874b8bSToomas Soome (longlong_t)scn->scn_phys.scn_cur_min_txg, 2295a3874b8bSToomas Soome (longlong_t)scn->scn_phys.scn_max_txg); 229638d61036SMatthew Ahrens kmem_free(dsname, MAXNAMELEN); 229738d61036SMatthew Ahrens 229838d61036SMatthew Ahrens goto out; 229938d61036SMatthew Ahrens } 230038d61036SMatthew Ahrens 23016e0cbcaaSMatthew Ahrens /* 23025cabbc6bSPrashanth Sreenivasa * Only the ZIL in the head (non-snapshot) is valid. Even though 23036e0cbcaaSMatthew Ahrens * snapshots can have ZIL block pointers (which may be the same 23045cabbc6bSPrashanth Sreenivasa * BP as in the head), they must be ignored. In addition, $ORIGIN 23055cabbc6bSPrashanth Sreenivasa * doesn't have a objset (i.e. its ds_bp is a hole) so we don't 23065cabbc6bSPrashanth Sreenivasa * need to look for a ZIL in it either. So we traverse the ZIL here, 23075cabbc6bSPrashanth Sreenivasa * rather than in scan_recurse(), because the regular snapshot 23085cabbc6bSPrashanth Sreenivasa * block-sharing rules don't apply to it. 23096e0cbcaaSMatthew Ahrens */ 23105cabbc6bSPrashanth Sreenivasa if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds) && 2311bb1f4245SMatthew Ahrens (dp->dp_origin_snap == NULL || 2312bb1f4245SMatthew Ahrens ds->ds_dir != dp->dp_origin_snap->ds_dir)) { 23135cabbc6bSPrashanth Sreenivasa objset_t *os; 23145cabbc6bSPrashanth Sreenivasa if (dmu_objset_from_ds(ds, &os) != 0) { 23155cabbc6bSPrashanth Sreenivasa goto out; 23165cabbc6bSPrashanth Sreenivasa } 23176e0cbcaaSMatthew Ahrens dsl_scan_zil(dp, &os->os_zil_header); 23185cabbc6bSPrashanth Sreenivasa } 23196e0cbcaaSMatthew Ahrens 23203f9d6ad7SLin Ling /* 23213f9d6ad7SLin Ling * Iterate over the bps in this ds. 23223f9d6ad7SLin Ling */ 23233f9d6ad7SLin Ling dmu_buf_will_dirty(ds->ds_dbuf, tx); 2324c166b69dSPaul Dagnelie rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2325c1379625SJustin T. Gibbs dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx); 2326c166b69dSPaul Dagnelie rrw_exit(&ds->ds_bp_rwlock, FTAG); 23273f9d6ad7SLin Ling 23289adfa60dSMatthew Ahrens char *dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); 23293f9d6ad7SLin Ling dsl_dataset_name(ds, dsname); 23303f9d6ad7SLin Ling zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; " 23311702cce7SAlek Pinchuk "suspending=%u", 23323f9d6ad7SLin Ling (longlong_t)dsobj, dsname, 23333f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_cur_min_txg, 23343f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_cur_max_txg, 23351702cce7SAlek Pinchuk (int)scn->scn_suspending); 23369adfa60dSMatthew Ahrens kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN); 23373f9d6ad7SLin Ling 23381702cce7SAlek Pinchuk if (scn->scn_suspending) 23393f9d6ad7SLin Ling goto out; 23403f9d6ad7SLin Ling 23413f9d6ad7SLin Ling /* 23423f9d6ad7SLin Ling * We've finished this pass over this dataset. 23433f9d6ad7SLin Ling */ 23443f9d6ad7SLin Ling 23453f9d6ad7SLin Ling /* 23463f9d6ad7SLin Ling * If we did not completely visit this dataset, do another pass. 23473f9d6ad7SLin Ling */ 23483f9d6ad7SLin Ling if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) { 23493f9d6ad7SLin Ling zfs_dbgmsg("incomplete pass; visiting again"); 23503f9d6ad7SLin Ling scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN; 2351a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds->ds_object, 2352a3874b8bSToomas Soome scn->scn_phys.scn_cur_max_txg); 23533f9d6ad7SLin Ling goto out; 23543f9d6ad7SLin Ling } 23553f9d6ad7SLin Ling 23563f9d6ad7SLin Ling /* 23573f9d6ad7SLin Ling * Add descendent datasets to work queue. 23583f9d6ad7SLin Ling */ 2359c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) { 2360a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2361c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj, 2362a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_creation_txg); 23633f9d6ad7SLin Ling } 2364c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_num_children > 1) { 23653f9d6ad7SLin Ling boolean_t usenext = B_FALSE; 2366c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) { 23673f9d6ad7SLin Ling uint64_t count; 23683f9d6ad7SLin Ling /* 23693f9d6ad7SLin Ling * A bug in a previous version of the code could 23703f9d6ad7SLin Ling * cause upgrade_clones_cb() to not set 23713f9d6ad7SLin Ling * ds_next_snap_obj when it should, leading to a 23723f9d6ad7SLin Ling * missing entry. Therefore we can only use the 23733f9d6ad7SLin Ling * next_clones_obj when its count is correct. 23743f9d6ad7SLin Ling */ 23753f9d6ad7SLin Ling int err = zap_count(dp->dp_meta_objset, 2376c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_clones_obj, &count); 23773f9d6ad7SLin Ling if (err == 0 && 2378c1379625SJustin T. Gibbs count == dsl_dataset_phys(ds)->ds_num_children - 1) 23793f9d6ad7SLin Ling usenext = B_TRUE; 23803f9d6ad7SLin Ling } 23813f9d6ad7SLin Ling 23823f9d6ad7SLin Ling if (usenext) { 2383a3874b8bSToomas Soome zap_cursor_t zc; 2384a3874b8bSToomas Soome zap_attribute_t za; 2385a3874b8bSToomas Soome for (zap_cursor_init(&zc, dp->dp_meta_objset, 2386a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_next_clones_obj); 2387a3874b8bSToomas Soome zap_cursor_retrieve(&zc, &za) == 0; 2388a3874b8bSToomas Soome (void) zap_cursor_advance(&zc)) { 2389a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2390a3874b8bSToomas Soome zfs_strtonum(za.za_name, NULL), 2391a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_creation_txg); 2392a3874b8bSToomas Soome } 2393a3874b8bSToomas Soome zap_cursor_fini(&zc); 23943f9d6ad7SLin Ling } else { 23953b2aab18SMatthew Ahrens VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2396a3874b8bSToomas Soome enqueue_clones_cb, &ds->ds_object, 2397a3874b8bSToomas Soome DS_FIND_CHILDREN)); 23983f9d6ad7SLin Ling } 23993f9d6ad7SLin Ling } 24003f9d6ad7SLin Ling 24013f9d6ad7SLin Ling out: 24023f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24033f9d6ad7SLin Ling } 24043f9d6ad7SLin Ling 24053f9d6ad7SLin Ling /* ARGSUSED */ 24063f9d6ad7SLin Ling static int 24073b2aab18SMatthew Ahrens enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 24083f9d6ad7SLin Ling { 24093f9d6ad7SLin Ling dsl_dataset_t *ds; 24103f9d6ad7SLin Ling int err; 24113f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 24123f9d6ad7SLin Ling 24133b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 24143f9d6ad7SLin Ling if (err) 24153f9d6ad7SLin Ling return (err); 24163f9d6ad7SLin Ling 2417c1379625SJustin T. Gibbs while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) { 24183f9d6ad7SLin Ling dsl_dataset_t *prev; 2419c1379625SJustin T. Gibbs err = dsl_dataset_hold_obj(dp, 2420c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev); 24213f9d6ad7SLin Ling if (err) { 24223f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24233f9d6ad7SLin Ling return (err); 24243f9d6ad7SLin Ling } 24253f9d6ad7SLin Ling 24263f9d6ad7SLin Ling /* 24273f9d6ad7SLin Ling * If this is a clone, we don't need to worry about it for now. 24283f9d6ad7SLin Ling */ 2429c1379625SJustin T. Gibbs if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object) { 24303f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24313f9d6ad7SLin Ling dsl_dataset_rele(prev, FTAG); 24323f9d6ad7SLin Ling return (0); 24333f9d6ad7SLin Ling } 24343f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24353f9d6ad7SLin Ling ds = prev; 24363f9d6ad7SLin Ling } 24373f9d6ad7SLin Ling 2438a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds->ds_object, 2439a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_txg); 24403f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24413f9d6ad7SLin Ling return (0); 24423f9d6ad7SLin Ling } 24433f9d6ad7SLin Ling 2444a3874b8bSToomas Soome /* ARGSUSED */ 2445a3874b8bSToomas Soome void 2446a3874b8bSToomas Soome dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, 2447a3874b8bSToomas Soome ddt_entry_t *dde, dmu_tx_t *tx) 2448a3874b8bSToomas Soome { 2449a3874b8bSToomas Soome const ddt_key_t *ddk = &dde->dde_key; 2450a3874b8bSToomas Soome ddt_phys_t *ddp = dde->dde_phys; 2451a3874b8bSToomas Soome blkptr_t bp; 2452a3874b8bSToomas Soome zbookmark_phys_t zb = { 0 }; 2453a3874b8bSToomas Soome int p; 2454a3874b8bSToomas Soome 2455a3874b8bSToomas Soome if (scn->scn_phys.scn_state != DSS_SCANNING) 2456a3874b8bSToomas Soome return; 2457a3874b8bSToomas Soome 2458e4c795beSTom Caputi /* 2459e4c795beSTom Caputi * This function is special because it is the only thing 2460e4c795beSTom Caputi * that can add scan_io_t's to the vdev scan queues from 2461e4c795beSTom Caputi * outside dsl_scan_sync(). For the most part this is ok 2462e4c795beSTom Caputi * as long as it is called from within syncing context. 2463e4c795beSTom Caputi * However, dsl_scan_sync() expects that no new sio's will 2464e4c795beSTom Caputi * be added between when all the work for a scan is done 2465e4c795beSTom Caputi * and the next txg when the scan is actually marked as 2466e4c795beSTom Caputi * completed. This check ensures we do not issue new sio's 2467e4c795beSTom Caputi * during this period. 2468e4c795beSTom Caputi */ 2469e4c795beSTom Caputi if (scn->scn_done_txg != 0) 2470e4c795beSTom Caputi return; 2471e4c795beSTom Caputi 2472a3874b8bSToomas Soome for (p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 2473a3874b8bSToomas Soome if (ddp->ddp_phys_birth == 0 || 2474a3874b8bSToomas Soome ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) 2475a3874b8bSToomas Soome continue; 2476a3874b8bSToomas Soome ddt_bp_create(checksum, ddk, ddp, &bp); 2477a3874b8bSToomas Soome 2478a3874b8bSToomas Soome scn->scn_visited_this_txg++; 2479a3874b8bSToomas Soome scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb); 2480a3874b8bSToomas Soome } 2481a3874b8bSToomas Soome } 2482a3874b8bSToomas Soome 2483a3874b8bSToomas Soome /* 24843f9d6ad7SLin Ling * Scrub/dedup interaction. 24853f9d6ad7SLin Ling * 24863f9d6ad7SLin Ling * If there are N references to a deduped block, we don't want to scrub it 24873f9d6ad7SLin Ling * N times -- ideally, we should scrub it exactly once. 24883f9d6ad7SLin Ling * 24893f9d6ad7SLin Ling * We leverage the fact that the dde's replication class (enum ddt_class) 24903f9d6ad7SLin Ling * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest 24913f9d6ad7SLin Ling * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order. 24923f9d6ad7SLin Ling * 24933f9d6ad7SLin Ling * To prevent excess scrubbing, the scrub begins by walking the DDT 24943f9d6ad7SLin Ling * to find all blocks with refcnt > 1, and scrubs each of these once. 24953f9d6ad7SLin Ling * Since there are two replication classes which contain blocks with 24963f9d6ad7SLin Ling * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first. 24973f9d6ad7SLin Ling * Finally the top-down scrub begins, only visiting blocks with refcnt == 1. 24983f9d6ad7SLin Ling * 24993f9d6ad7SLin Ling * There would be nothing more to say if a block's refcnt couldn't change 25003f9d6ad7SLin Ling * during a scrub, but of course it can so we must account for changes 25013f9d6ad7SLin Ling * in a block's replication class. 25023f9d6ad7SLin Ling * 25033f9d6ad7SLin Ling * Here's an example of what can occur: 25043f9d6ad7SLin Ling * 25053f9d6ad7SLin Ling * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1 25063f9d6ad7SLin Ling * when visited during the top-down scrub phase, it will be scrubbed twice. 25073f9d6ad7SLin Ling * This negates our scrub optimization, but is otherwise harmless. 25083f9d6ad7SLin Ling * 25093f9d6ad7SLin Ling * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1 25103f9d6ad7SLin Ling * on each visit during the top-down scrub phase, it will never be scrubbed. 25113f9d6ad7SLin Ling * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's 25123f9d6ad7SLin Ling * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to 25133f9d6ad7SLin Ling * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1 25143f9d6ad7SLin Ling * while a scrub is in progress, it scrubs the block right then. 25153f9d6ad7SLin Ling */ 25163f9d6ad7SLin Ling static void 25173f9d6ad7SLin Ling dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx) 25183f9d6ad7SLin Ling { 25193f9d6ad7SLin Ling ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark; 25203f9d6ad7SLin Ling ddt_entry_t dde = { 0 }; 25213f9d6ad7SLin Ling int error; 25223f9d6ad7SLin Ling uint64_t n = 0; 25233f9d6ad7SLin Ling 25243f9d6ad7SLin Ling while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) { 25253f9d6ad7SLin Ling ddt_t *ddt; 25263f9d6ad7SLin Ling 25273f9d6ad7SLin Ling if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max) 25283f9d6ad7SLin Ling break; 25293f9d6ad7SLin Ling dprintf("visiting ddb=%llu/%llu/%llu/%llx\n", 25303f9d6ad7SLin Ling (longlong_t)ddb->ddb_class, 25313f9d6ad7SLin Ling (longlong_t)ddb->ddb_type, 25323f9d6ad7SLin Ling (longlong_t)ddb->ddb_checksum, 25333f9d6ad7SLin Ling (longlong_t)ddb->ddb_cursor); 25343f9d6ad7SLin Ling 25353f9d6ad7SLin Ling /* There should be no pending changes to the dedup table */ 25363f9d6ad7SLin Ling ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum]; 25373f9d6ad7SLin Ling ASSERT(avl_first(&ddt->ddt_tree) == NULL); 25383f9d6ad7SLin Ling 25393f9d6ad7SLin Ling dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx); 25403f9d6ad7SLin Ling n++; 25413f9d6ad7SLin Ling 25421702cce7SAlek Pinchuk if (dsl_scan_check_suspend(scn, NULL)) 25433f9d6ad7SLin Ling break; 25443f9d6ad7SLin Ling } 25453f9d6ad7SLin Ling 25461702cce7SAlek Pinchuk zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; " 25471702cce7SAlek Pinchuk "suspending=%u", (longlong_t)n, 25481702cce7SAlek Pinchuk (int)scn->scn_phys.scn_ddt_class_max, (int)scn->scn_suspending); 25493f9d6ad7SLin Ling 25503f9d6ad7SLin Ling ASSERT(error == 0 || error == ENOENT); 25513f9d6ad7SLin Ling ASSERT(error != ENOENT || 25523f9d6ad7SLin Ling ddb->ddb_class > scn->scn_phys.scn_ddt_class_max); 25533f9d6ad7SLin Ling } 25543f9d6ad7SLin Ling 2555a3874b8bSToomas Soome static uint64_t 2556a3874b8bSToomas Soome dsl_scan_ds_maxtxg(dsl_dataset_t *ds) 25573f9d6ad7SLin Ling { 2558a3874b8bSToomas Soome uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg; 2559a3874b8bSToomas Soome if (ds->ds_is_snapshot) 2560a3874b8bSToomas Soome return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg)); 2561a3874b8bSToomas Soome return (smt); 25623f9d6ad7SLin Ling } 25633f9d6ad7SLin Ling 25643f9d6ad7SLin Ling static void 25653f9d6ad7SLin Ling dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx) 25663f9d6ad7SLin Ling { 2567a3874b8bSToomas Soome scan_ds_t *sds; 25683f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 25693f9d6ad7SLin Ling 25703f9d6ad7SLin Ling if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 25713f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max) { 25723f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 25733f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 25743f9d6ad7SLin Ling dsl_scan_ddt(scn, tx); 25751702cce7SAlek Pinchuk if (scn->scn_suspending) 25763f9d6ad7SLin Ling return; 25773f9d6ad7SLin Ling } 25783f9d6ad7SLin Ling 25793f9d6ad7SLin Ling if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) { 25803f9d6ad7SLin Ling /* First do the MOS & ORIGIN */ 25813f9d6ad7SLin Ling 25823f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 25833f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 25843f9d6ad7SLin Ling dsl_scan_visit_rootbp(scn, NULL, 25853f9d6ad7SLin Ling &dp->dp_meta_rootbp, tx); 25863f9d6ad7SLin Ling spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp); 25871702cce7SAlek Pinchuk if (scn->scn_suspending) 25883f9d6ad7SLin Ling return; 25893f9d6ad7SLin Ling 25903f9d6ad7SLin Ling if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) { 25913b2aab18SMatthew Ahrens VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2592a3874b8bSToomas Soome enqueue_cb, NULL, DS_FIND_CHILDREN)); 25933f9d6ad7SLin Ling } else { 25943f9d6ad7SLin Ling dsl_scan_visitds(scn, 25953f9d6ad7SLin Ling dp->dp_origin_snap->ds_object, tx); 25963f9d6ad7SLin Ling } 25971702cce7SAlek Pinchuk ASSERT(!scn->scn_suspending); 25983f9d6ad7SLin Ling } else if (scn->scn_phys.scn_bookmark.zb_objset != 25993f9d6ad7SLin Ling ZB_DESTROYED_OBJSET) { 2600a3874b8bSToomas Soome uint64_t dsobj = scn->scn_phys.scn_bookmark.zb_objset; 26013f9d6ad7SLin Ling /* 2602a3874b8bSToomas Soome * If we were suspended, continue from here. Note if the 26031702cce7SAlek Pinchuk * ds we were suspended on was deleted, the zb_objset may 26043f9d6ad7SLin Ling * be -1, so we will skip this and find a new objset 26053f9d6ad7SLin Ling * below. 26063f9d6ad7SLin Ling */ 2607a3874b8bSToomas Soome dsl_scan_visitds(scn, dsobj, tx); 26081702cce7SAlek Pinchuk if (scn->scn_suspending) 26093f9d6ad7SLin Ling return; 26103f9d6ad7SLin Ling } 26113f9d6ad7SLin Ling 26123f9d6ad7SLin Ling /* 2613a3874b8bSToomas Soome * In case we suspended right at the end of the ds, zero the 26143f9d6ad7SLin Ling * bookmark so we don't think that we're still trying to resume. 26153f9d6ad7SLin Ling */ 26167802d7bfSMatthew Ahrens bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t)); 26173f9d6ad7SLin Ling 2618a3874b8bSToomas Soome /* 2619a3874b8bSToomas Soome * Keep pulling things out of the dataset avl queue. Updates to the 2620a3874b8bSToomas Soome * persistent zap-object-as-queue happen only at checkpoints. 2621a3874b8bSToomas Soome */ 2622a3874b8bSToomas Soome while ((sds = avl_first(&scn->scn_queue)) != NULL) { 26233f9d6ad7SLin Ling dsl_dataset_t *ds; 2624a3874b8bSToomas Soome uint64_t dsobj = sds->sds_dsobj; 2625a3874b8bSToomas Soome uint64_t txg = sds->sds_txg; 26263f9d6ad7SLin Ling 2627a3874b8bSToomas Soome /* dequeue and free the ds from the queue */ 2628a3874b8bSToomas Soome scan_ds_queue_remove(scn, dsobj); 2629a3874b8bSToomas Soome sds = NULL; /* must not be touched after removal */ 26303f9d6ad7SLin Ling 2631a3874b8bSToomas Soome /* Set up min / max txg */ 2632b420f3adSRichard Lowe VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 2633a3874b8bSToomas Soome if (txg != 0) { 26343f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = 2635a3874b8bSToomas Soome MAX(scn->scn_phys.scn_min_txg, txg); 26363f9d6ad7SLin Ling } else { 26373f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = 26383f9d6ad7SLin Ling MAX(scn->scn_phys.scn_min_txg, 2639c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_txg); 26403f9d6ad7SLin Ling } 26413f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds); 26423f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 26433f9d6ad7SLin Ling 26443f9d6ad7SLin Ling dsl_scan_visitds(scn, dsobj, tx); 26451702cce7SAlek Pinchuk if (scn->scn_suspending) 26463f9d6ad7SLin Ling return; 26473f9d6ad7SLin Ling } 2648a3874b8bSToomas Soome /* No more objsets to fetch, we're done */ 2649a3874b8bSToomas Soome scn->scn_phys.scn_bookmark.zb_objset = ZB_DESTROYED_OBJSET; 2650a3874b8bSToomas Soome ASSERT0(scn->scn_suspending); 2651a3874b8bSToomas Soome } 2652a3874b8bSToomas Soome 2653a3874b8bSToomas Soome static uint64_t 2654a3874b8bSToomas Soome dsl_scan_count_leaves(vdev_t *vd) 2655a3874b8bSToomas Soome { 2656a3874b8bSToomas Soome uint64_t i, leaves = 0; 2657a3874b8bSToomas Soome 2658a3874b8bSToomas Soome /* we only count leaves that belong to the main pool and are readable */ 2659a3874b8bSToomas Soome if (vd->vdev_islog || vd->vdev_isspare || 2660a3874b8bSToomas Soome vd->vdev_isl2cache || !vdev_readable(vd)) 2661a3874b8bSToomas Soome return (0); 2662a3874b8bSToomas Soome 2663a3874b8bSToomas Soome if (vd->vdev_ops->vdev_op_leaf) 2664a3874b8bSToomas Soome return (1); 2665a3874b8bSToomas Soome 2666a3874b8bSToomas Soome for (i = 0; i < vd->vdev_children; i++) { 2667a3874b8bSToomas Soome leaves += dsl_scan_count_leaves(vd->vdev_child[i]); 2668a3874b8bSToomas Soome } 2669a3874b8bSToomas Soome 2670a3874b8bSToomas Soome return (leaves); 2671a3874b8bSToomas Soome } 2672a3874b8bSToomas Soome 2673a3874b8bSToomas Soome 2674a3874b8bSToomas Soome static void 2675a3874b8bSToomas Soome scan_io_queues_update_zio_stats(dsl_scan_io_queue_t *q, const blkptr_t *bp) 2676a3874b8bSToomas Soome { 2677a3874b8bSToomas Soome int i; 2678a3874b8bSToomas Soome uint64_t cur_size = 0; 2679a3874b8bSToomas Soome 2680a3874b8bSToomas Soome for (i = 0; i < BP_GET_NDVAS(bp); i++) { 2681a3874b8bSToomas Soome cur_size += DVA_GET_ASIZE(&bp->blk_dva[i]); 2682a3874b8bSToomas Soome } 2683a3874b8bSToomas Soome 2684a3874b8bSToomas Soome q->q_total_zio_size_this_txg += cur_size; 2685a3874b8bSToomas Soome q->q_zios_this_txg++; 2686a3874b8bSToomas Soome } 2687a3874b8bSToomas Soome 2688a3874b8bSToomas Soome static void 2689a3874b8bSToomas Soome scan_io_queues_update_seg_stats(dsl_scan_io_queue_t *q, uint64_t start, 2690a3874b8bSToomas Soome uint64_t end) 2691a3874b8bSToomas Soome { 2692a3874b8bSToomas Soome q->q_total_seg_size_this_txg += end - start; 2693a3874b8bSToomas Soome q->q_segs_this_txg++; 2694a3874b8bSToomas Soome } 2695a3874b8bSToomas Soome 2696a3874b8bSToomas Soome static boolean_t 2697a3874b8bSToomas Soome scan_io_queue_check_suspend(dsl_scan_t *scn) 2698a3874b8bSToomas Soome { 2699a3874b8bSToomas Soome /* See comment in dsl_scan_check_suspend() */ 2700a3874b8bSToomas Soome uint64_t curr_time_ns = gethrtime(); 2701a3874b8bSToomas Soome uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time; 2702a3874b8bSToomas Soome uint64_t sync_time_ns = curr_time_ns - 2703a3874b8bSToomas Soome scn->scn_dp->dp_spa->spa_sync_starttime; 2704a3874b8bSToomas Soome int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; 2705a3874b8bSToomas Soome int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 2706a3874b8bSToomas Soome zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; 2707a3874b8bSToomas Soome 2708a3874b8bSToomas Soome return ((NSEC2MSEC(scan_time_ns) > mintime && 2709a3874b8bSToomas Soome (dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent || 2710a3874b8bSToomas Soome txg_sync_waiting(scn->scn_dp) || 2711a3874b8bSToomas Soome NSEC2SEC(sync_time_ns) >= zfs_txg_timeout)) || 2712a3874b8bSToomas Soome spa_shutting_down(scn->scn_dp->dp_spa)); 2713a3874b8bSToomas Soome } 2714a3874b8bSToomas Soome 2715a3874b8bSToomas Soome /* 2716a3874b8bSToomas Soome * Given a list of scan_io_t's in io_list, this issues the io's out to 2717a3874b8bSToomas Soome * disk. This consumes the io_list and frees the scan_io_t's. This is 2718a3874b8bSToomas Soome * called when emptying queues, either when we're up against the memory 2719a3874b8bSToomas Soome * limit or when we have finished scanning. Returns B_TRUE if we stopped 2720a3874b8bSToomas Soome * processing the list before we finished. Any zios that were not issued 2721a3874b8bSToomas Soome * will remain in the io_list. 2722a3874b8bSToomas Soome */ 2723a3874b8bSToomas Soome static boolean_t 2724a3874b8bSToomas Soome scan_io_queue_issue(dsl_scan_io_queue_t *queue, list_t *io_list) 2725a3874b8bSToomas Soome { 2726a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 2727a3874b8bSToomas Soome scan_io_t *sio; 2728a3874b8bSToomas Soome int64_t bytes_issued = 0; 2729a3874b8bSToomas Soome boolean_t suspended = B_FALSE; 2730a3874b8bSToomas Soome 2731a3874b8bSToomas Soome while ((sio = list_head(io_list)) != NULL) { 2732a3874b8bSToomas Soome blkptr_t bp; 2733a3874b8bSToomas Soome 2734a3874b8bSToomas Soome if (scan_io_queue_check_suspend(scn)) { 2735a3874b8bSToomas Soome suspended = B_TRUE; 2736a3874b8bSToomas Soome break; 2737a3874b8bSToomas Soome } 2738a3874b8bSToomas Soome 273912a8814cSTom Caputi sio2bp(sio, &bp); 274012a8814cSTom Caputi bytes_issued += SIO_GET_ASIZE(sio); 2741a3874b8bSToomas Soome scan_exec_io(scn->scn_dp, &bp, sio->sio_flags, 2742a3874b8bSToomas Soome &sio->sio_zb, queue); 2743a3874b8bSToomas Soome (void) list_remove_head(io_list); 2744a3874b8bSToomas Soome scan_io_queues_update_zio_stats(queue, &bp); 274512a8814cSTom Caputi sio_free(sio); 2746a3874b8bSToomas Soome } 2747a3874b8bSToomas Soome 2748a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -bytes_issued); 2749a3874b8bSToomas Soome 2750a3874b8bSToomas Soome return (suspended); 2751a3874b8bSToomas Soome } 2752a3874b8bSToomas Soome 2753a3874b8bSToomas Soome /* 2754a3874b8bSToomas Soome * Given a range_seg_t (extent) and a list, this function passes over a 2755a3874b8bSToomas Soome * scan queue and gathers up the appropriate ios which fit into that 2756a3874b8bSToomas Soome * scan seg (starting from lowest LBA). At the end, we remove the segment 2757a3874b8bSToomas Soome * from the q_exts_by_addr range tree. 2758a3874b8bSToomas Soome */ 2759a3874b8bSToomas Soome static boolean_t 2760a3874b8bSToomas Soome scan_io_queue_gather(dsl_scan_io_queue_t *queue, range_seg_t *rs, list_t *list) 2761a3874b8bSToomas Soome { 276212a8814cSTom Caputi scan_io_t *srch_sio, *sio, *next_sio; 2763a3874b8bSToomas Soome avl_index_t idx; 2764a3874b8bSToomas Soome uint_t num_sios = 0; 2765a3874b8bSToomas Soome int64_t bytes_issued = 0; 2766a3874b8bSToomas Soome 2767a3874b8bSToomas Soome ASSERT(rs != NULL); 2768a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 2769a3874b8bSToomas Soome 277012a8814cSTom Caputi srch_sio = sio_alloc(1); 277112a8814cSTom Caputi srch_sio->sio_nr_dvas = 1; 277212a8814cSTom Caputi SIO_SET_OFFSET(srch_sio, rs->rs_start); 2773a3874b8bSToomas Soome 2774a3874b8bSToomas Soome /* 2775a3874b8bSToomas Soome * The exact start of the extent might not contain any matching zios, 2776a3874b8bSToomas Soome * so if that's the case, examine the next one in the tree. 2777a3874b8bSToomas Soome */ 277812a8814cSTom Caputi sio = avl_find(&queue->q_sios_by_addr, srch_sio, &idx); 277912a8814cSTom Caputi sio_free(srch_sio); 278012a8814cSTom Caputi 2781a3874b8bSToomas Soome if (sio == NULL) 2782a3874b8bSToomas Soome sio = avl_nearest(&queue->q_sios_by_addr, idx, AVL_AFTER); 2783a3874b8bSToomas Soome 278412a8814cSTom Caputi while (sio != NULL && 278512a8814cSTom Caputi SIO_GET_OFFSET(sio) < rs->rs_end && num_sios <= 32) { 278612a8814cSTom Caputi ASSERT3U(SIO_GET_OFFSET(sio), >=, rs->rs_start); 278712a8814cSTom Caputi ASSERT3U(SIO_GET_END_OFFSET(sio), <=, rs->rs_end); 2788a3874b8bSToomas Soome 2789a3874b8bSToomas Soome next_sio = AVL_NEXT(&queue->q_sios_by_addr, sio); 2790a3874b8bSToomas Soome avl_remove(&queue->q_sios_by_addr, sio); 279112a8814cSTom Caputi queue->q_sio_memused -= SIO_GET_MUSED(sio); 2792a3874b8bSToomas Soome 279312a8814cSTom Caputi bytes_issued += SIO_GET_ASIZE(sio); 2794a3874b8bSToomas Soome num_sios++; 2795a3874b8bSToomas Soome list_insert_tail(list, sio); 2796a3874b8bSToomas Soome sio = next_sio; 2797a3874b8bSToomas Soome } 2798a3874b8bSToomas Soome 2799a3874b8bSToomas Soome /* 2800a3874b8bSToomas Soome * We limit the number of sios we process at once to 32 to avoid 2801a3874b8bSToomas Soome * biting off more than we can chew. If we didn't take everything 2802a3874b8bSToomas Soome * in the segment we update it to reflect the work we were able to 2803a3874b8bSToomas Soome * complete. Otherwise, we remove it from the range tree entirely. 2804a3874b8bSToomas Soome */ 280512a8814cSTom Caputi if (sio != NULL && SIO_GET_OFFSET(sio) < rs->rs_end) { 2806a3874b8bSToomas Soome range_tree_adjust_fill(queue->q_exts_by_addr, rs, 2807a3874b8bSToomas Soome -bytes_issued); 2808a3874b8bSToomas Soome range_tree_resize_segment(queue->q_exts_by_addr, rs, 280912a8814cSTom Caputi SIO_GET_OFFSET(sio), rs->rs_end - SIO_GET_OFFSET(sio)); 2810a3874b8bSToomas Soome 2811a3874b8bSToomas Soome return (B_TRUE); 2812a3874b8bSToomas Soome } else { 2813a3874b8bSToomas Soome range_tree_remove(queue->q_exts_by_addr, rs->rs_start, 2814a3874b8bSToomas Soome rs->rs_end - rs->rs_start); 2815a3874b8bSToomas Soome return (B_FALSE); 2816a3874b8bSToomas Soome } 2817a3874b8bSToomas Soome } 2818a3874b8bSToomas Soome 2819a3874b8bSToomas Soome 2820a3874b8bSToomas Soome /* 2821a3874b8bSToomas Soome * This is called from the queue emptying thread and selects the next 2822a3874b8bSToomas Soome * extent from which we are to issue io's. The behavior of this function 2823a3874b8bSToomas Soome * depends on the state of the scan, the current memory consumption and 2824a3874b8bSToomas Soome * whether or not we are performing a scan shutdown. 2825a3874b8bSToomas Soome * 1) We select extents in an elevator algorithm (LBA-order) if the scan 2826a3874b8bSToomas Soome * needs to perform a checkpoint 2827a3874b8bSToomas Soome * 2) We select the largest available extent if we are up against the 2828a3874b8bSToomas Soome * memory limit. 2829a3874b8bSToomas Soome * 3) Otherwise we don't select any extents. 2830a3874b8bSToomas Soome */ 2831a3874b8bSToomas Soome static const range_seg_t * 2832a3874b8bSToomas Soome scan_io_queue_fetch_ext(dsl_scan_io_queue_t *queue) 2833a3874b8bSToomas Soome { 2834a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 2835a3874b8bSToomas Soome 2836a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 2837a3874b8bSToomas Soome ASSERT(scn->scn_is_sorted); 2838a3874b8bSToomas Soome 2839a3874b8bSToomas Soome /* handle tunable overrides */ 2840a3874b8bSToomas Soome if (scn->scn_checkpointing || scn->scn_clearing) { 2841a3874b8bSToomas Soome if (zfs_scan_issue_strategy == 1) { 2842a3874b8bSToomas Soome return (range_tree_first(queue->q_exts_by_addr)); 2843a3874b8bSToomas Soome } else if (zfs_scan_issue_strategy == 2) { 2844a3874b8bSToomas Soome return (avl_first(&queue->q_exts_by_size)); 2845a3874b8bSToomas Soome } 2846a3874b8bSToomas Soome } 2847a3874b8bSToomas Soome 2848a3874b8bSToomas Soome /* 2849a3874b8bSToomas Soome * During normal clearing, we want to issue our largest segments 2850a3874b8bSToomas Soome * first, keeping IO as sequential as possible, and leaving the 2851a3874b8bSToomas Soome * smaller extents for later with the hope that they might eventually 2852a3874b8bSToomas Soome * grow to larger sequential segments. However, when the scan is 2853a3874b8bSToomas Soome * checkpointing, no new extents will be added to the sorting queue, 2854a3874b8bSToomas Soome * so the way we are sorted now is as good as it will ever get. 2855a3874b8bSToomas Soome * In this case, we instead switch to issuing extents in LBA order. 2856a3874b8bSToomas Soome */ 2857a3874b8bSToomas Soome if (scn->scn_checkpointing) { 2858a3874b8bSToomas Soome return (range_tree_first(queue->q_exts_by_addr)); 2859a3874b8bSToomas Soome } else if (scn->scn_clearing) { 2860a3874b8bSToomas Soome return (avl_first(&queue->q_exts_by_size)); 2861a3874b8bSToomas Soome } else { 2862a3874b8bSToomas Soome return (NULL); 2863a3874b8bSToomas Soome } 2864a3874b8bSToomas Soome } 2865a3874b8bSToomas Soome 2866a3874b8bSToomas Soome static void 2867a3874b8bSToomas Soome scan_io_queues_run_one(void *arg) 2868a3874b8bSToomas Soome { 2869a3874b8bSToomas Soome dsl_scan_io_queue_t *queue = arg; 2870a3874b8bSToomas Soome kmutex_t *q_lock = &queue->q_vd->vdev_scan_io_queue_lock; 2871a3874b8bSToomas Soome boolean_t suspended = B_FALSE; 2872a3874b8bSToomas Soome range_seg_t *rs = NULL; 2873a3874b8bSToomas Soome scan_io_t *sio = NULL; 2874a3874b8bSToomas Soome list_t sio_list; 2875a3874b8bSToomas Soome uint64_t bytes_per_leaf = zfs_scan_vdev_limit; 2876a3874b8bSToomas Soome uint64_t nr_leaves = dsl_scan_count_leaves(queue->q_vd); 2877a3874b8bSToomas Soome 2878a3874b8bSToomas Soome ASSERT(queue->q_scn->scn_is_sorted); 2879a3874b8bSToomas Soome 2880a3874b8bSToomas Soome list_create(&sio_list, sizeof (scan_io_t), 2881a3874b8bSToomas Soome offsetof(scan_io_t, sio_nodes.sio_list_node)); 2882a3874b8bSToomas Soome mutex_enter(q_lock); 2883a3874b8bSToomas Soome 2884a3874b8bSToomas Soome /* calculate maximum in-flight bytes for this txg (min 1MB) */ 2885a3874b8bSToomas Soome queue->q_maxinflight_bytes = 2886a3874b8bSToomas Soome MAX(nr_leaves * bytes_per_leaf, 1ULL << 20); 2887a3874b8bSToomas Soome 2888a3874b8bSToomas Soome /* reset per-queue scan statistics for this txg */ 2889a3874b8bSToomas Soome queue->q_total_seg_size_this_txg = 0; 2890a3874b8bSToomas Soome queue->q_segs_this_txg = 0; 2891a3874b8bSToomas Soome queue->q_total_zio_size_this_txg = 0; 2892a3874b8bSToomas Soome queue->q_zios_this_txg = 0; 2893a3874b8bSToomas Soome 2894a3874b8bSToomas Soome /* loop until we have run out of time or sios */ 2895a3874b8bSToomas Soome while ((rs = (range_seg_t *)scan_io_queue_fetch_ext(queue)) != NULL) { 2896a3874b8bSToomas Soome uint64_t seg_start = 0, seg_end = 0; 2897a3874b8bSToomas Soome boolean_t more_left = B_TRUE; 2898a3874b8bSToomas Soome 2899a3874b8bSToomas Soome ASSERT(list_is_empty(&sio_list)); 2900a3874b8bSToomas Soome 2901a3874b8bSToomas Soome /* loop while we still have sios left to process in this rs */ 2902a3874b8bSToomas Soome while (more_left) { 2903a3874b8bSToomas Soome scan_io_t *first_sio, *last_sio; 2904a3874b8bSToomas Soome 2905a3874b8bSToomas Soome /* 2906a3874b8bSToomas Soome * We have selected which extent needs to be 2907a3874b8bSToomas Soome * processed next. Gather up the corresponding sios. 2908a3874b8bSToomas Soome */ 2909a3874b8bSToomas Soome more_left = scan_io_queue_gather(queue, rs, &sio_list); 2910a3874b8bSToomas Soome ASSERT(!list_is_empty(&sio_list)); 2911a3874b8bSToomas Soome first_sio = list_head(&sio_list); 2912a3874b8bSToomas Soome last_sio = list_tail(&sio_list); 2913a3874b8bSToomas Soome 291412a8814cSTom Caputi seg_end = SIO_GET_END_OFFSET(last_sio); 2915a3874b8bSToomas Soome if (seg_start == 0) 291612a8814cSTom Caputi seg_start = SIO_GET_OFFSET(first_sio); 2917a3874b8bSToomas Soome 2918a3874b8bSToomas Soome /* 2919a3874b8bSToomas Soome * Issuing sios can take a long time so drop the 2920a3874b8bSToomas Soome * queue lock. The sio queue won't be updated by 2921a3874b8bSToomas Soome * other threads since we're in syncing context so 2922a3874b8bSToomas Soome * we can be sure that our trees will remain exactly 2923a3874b8bSToomas Soome * as we left them. 2924a3874b8bSToomas Soome */ 2925a3874b8bSToomas Soome mutex_exit(q_lock); 2926a3874b8bSToomas Soome suspended = scan_io_queue_issue(queue, &sio_list); 2927a3874b8bSToomas Soome mutex_enter(q_lock); 2928a3874b8bSToomas Soome 2929a3874b8bSToomas Soome if (suspended) 2930a3874b8bSToomas Soome break; 2931a3874b8bSToomas Soome } 2932a3874b8bSToomas Soome /* update statistics for debugging purposes */ 2933a3874b8bSToomas Soome scan_io_queues_update_seg_stats(queue, seg_start, seg_end); 2934a3874b8bSToomas Soome 2935a3874b8bSToomas Soome if (suspended) 2936a3874b8bSToomas Soome break; 2937a3874b8bSToomas Soome } 2938a3874b8bSToomas Soome 2939a3874b8bSToomas Soome 2940a3874b8bSToomas Soome /* 2941a3874b8bSToomas Soome * If we were suspended in the middle of processing, 2942a3874b8bSToomas Soome * requeue any unfinished sios and exit. 2943a3874b8bSToomas Soome */ 2944a3874b8bSToomas Soome while ((sio = list_head(&sio_list)) != NULL) { 2945a3874b8bSToomas Soome list_remove(&sio_list, sio); 2946a3874b8bSToomas Soome scan_io_queue_insert_impl(queue, sio); 2947a3874b8bSToomas Soome } 2948a3874b8bSToomas Soome 2949a3874b8bSToomas Soome mutex_exit(q_lock); 2950a3874b8bSToomas Soome list_destroy(&sio_list); 2951a3874b8bSToomas Soome } 2952a3874b8bSToomas Soome 2953a3874b8bSToomas Soome /* 2954a3874b8bSToomas Soome * Performs an emptying run on all scan queues in the pool. This just 2955a3874b8bSToomas Soome * punches out one thread per top-level vdev, each of which processes 2956a3874b8bSToomas Soome * only that vdev's scan queue. We can parallelize the I/O here because 2957a3874b8bSToomas Soome * we know that each queue's io's only affect its own top-level vdev. 2958a3874b8bSToomas Soome * 2959a3874b8bSToomas Soome * This function waits for the queue runs to complete, and must be 2960a3874b8bSToomas Soome * called from dsl_scan_sync (or in general, syncing context). 2961a3874b8bSToomas Soome */ 2962a3874b8bSToomas Soome static void 2963a3874b8bSToomas Soome scan_io_queues_run(dsl_scan_t *scn) 2964a3874b8bSToomas Soome { 2965a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 2966a3874b8bSToomas Soome 2967a3874b8bSToomas Soome ASSERT(scn->scn_is_sorted); 2968a3874b8bSToomas Soome ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 2969a3874b8bSToomas Soome 2970a3874b8bSToomas Soome if (scn->scn_bytes_pending == 0) 2971a3874b8bSToomas Soome return; 2972a3874b8bSToomas Soome 2973a3874b8bSToomas Soome if (scn->scn_taskq == NULL) { 2974a3874b8bSToomas Soome char *tq_name = kmem_zalloc(ZFS_MAX_DATASET_NAME_LEN + 16, 2975a3874b8bSToomas Soome KM_SLEEP); 2976a3874b8bSToomas Soome int nthreads = spa->spa_root_vdev->vdev_children; 2977a3874b8bSToomas Soome 2978a3874b8bSToomas Soome /* 2979a3874b8bSToomas Soome * We need to make this taskq *always* execute as many 2980a3874b8bSToomas Soome * threads in parallel as we have top-level vdevs and no 2981a3874b8bSToomas Soome * less, otherwise strange serialization of the calls to 2982a3874b8bSToomas Soome * scan_io_queues_run_one can occur during spa_sync runs 2983a3874b8bSToomas Soome * and that significantly impacts performance. 2984a3874b8bSToomas Soome */ 2985a3874b8bSToomas Soome (void) snprintf(tq_name, ZFS_MAX_DATASET_NAME_LEN + 16, 2986a3874b8bSToomas Soome "dsl_scan_tq_%s", spa->spa_name); 2987a3874b8bSToomas Soome scn->scn_taskq = taskq_create(tq_name, nthreads, minclsyspri, 2988a3874b8bSToomas Soome nthreads, nthreads, TASKQ_PREPOPULATE); 2989a3874b8bSToomas Soome kmem_free(tq_name, ZFS_MAX_DATASET_NAME_LEN + 16); 2990a3874b8bSToomas Soome } 2991a3874b8bSToomas Soome 2992a3874b8bSToomas Soome for (uint64_t i = 0; i < spa->spa_root_vdev->vdev_children; i++) { 2993a3874b8bSToomas Soome vdev_t *vd = spa->spa_root_vdev->vdev_child[i]; 2994a3874b8bSToomas Soome 2995a3874b8bSToomas Soome mutex_enter(&vd->vdev_scan_io_queue_lock); 2996a3874b8bSToomas Soome if (vd->vdev_scan_io_queue != NULL) { 2997a3874b8bSToomas Soome VERIFY(taskq_dispatch(scn->scn_taskq, 2998a3874b8bSToomas Soome scan_io_queues_run_one, vd->vdev_scan_io_queue, 2999a3874b8bSToomas Soome TQ_SLEEP) != TASKQID_INVALID); 3000a3874b8bSToomas Soome } 3001a3874b8bSToomas Soome mutex_exit(&vd->vdev_scan_io_queue_lock); 3002a3874b8bSToomas Soome } 3003a3874b8bSToomas Soome 3004a3874b8bSToomas Soome /* 3005a3874b8bSToomas Soome * Wait for the queues to finish issuing thir IOs for this run 3006a3874b8bSToomas Soome * before we return. There may still be IOs in flight at this 3007a3874b8bSToomas Soome * point. 3008a3874b8bSToomas Soome */ 3009a3874b8bSToomas Soome taskq_wait(scn->scn_taskq); 30103f9d6ad7SLin Ling } 30113f9d6ad7SLin Ling 3012ad135b5dSChristopher Siden static boolean_t 30135cabbc6bSPrashanth Sreenivasa dsl_scan_async_block_should_pause(dsl_scan_t *scn) 3014cde58dbcSMatthew Ahrens { 3015cde58dbcSMatthew Ahrens uint64_t elapsed_nanosecs; 3016cde58dbcSMatthew Ahrens 30178b36997aSMatthew Ahrens if (zfs_recover) 30188b36997aSMatthew Ahrens return (B_FALSE); 30198b36997aSMatthew Ahrens 30205cabbc6bSPrashanth Sreenivasa if (scn->scn_visited_this_txg >= zfs_async_block_max_blocks) 3021af3465daSMax Grossman return (B_TRUE); 3022af3465daSMax Grossman 3023cde58dbcSMatthew Ahrens elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 3024ad135b5dSChristopher Siden return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 30255cabbc6bSPrashanth Sreenivasa (NSEC2MSEC(elapsed_nanosecs) > scn->scn_async_block_min_time_ms && 3026cde58dbcSMatthew Ahrens txg_sync_waiting(scn->scn_dp)) || 3027ad135b5dSChristopher Siden spa_shutting_down(scn->scn_dp->dp_spa)); 3028ad135b5dSChristopher Siden } 3029ad135b5dSChristopher Siden 3030ad135b5dSChristopher Siden static int 3031ad135b5dSChristopher Siden dsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 3032ad135b5dSChristopher Siden { 3033ad135b5dSChristopher Siden dsl_scan_t *scn = arg; 3034ad135b5dSChristopher Siden 3035ad135b5dSChristopher Siden if (!scn->scn_is_bptree || 3036ad135b5dSChristopher Siden (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)) { 30375cabbc6bSPrashanth Sreenivasa if (dsl_scan_async_block_should_pause(scn)) 3038be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 3039ad135b5dSChristopher Siden } 3040cde58dbcSMatthew Ahrens 3041cde58dbcSMatthew Ahrens zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa, 3042cde58dbcSMatthew Ahrens dmu_tx_get_txg(tx), bp, 0)); 3043cde58dbcSMatthew Ahrens dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 3044cde58dbcSMatthew Ahrens -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp), 3045cde58dbcSMatthew Ahrens -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 3046cde58dbcSMatthew Ahrens scn->scn_visited_this_txg++; 3047cde58dbcSMatthew Ahrens return (0); 3048cde58dbcSMatthew Ahrens } 3049cde58dbcSMatthew Ahrens 3050a3874b8bSToomas Soome static void 3051a3874b8bSToomas Soome dsl_scan_update_stats(dsl_scan_t *scn) 3052a3874b8bSToomas Soome { 3053a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 3054a3874b8bSToomas Soome uint64_t i; 3055a3874b8bSToomas Soome uint64_t seg_size_total = 0, zio_size_total = 0; 3056a3874b8bSToomas Soome uint64_t seg_count_total = 0, zio_count_total = 0; 3057a3874b8bSToomas Soome 3058a3874b8bSToomas Soome for (i = 0; i < spa->spa_root_vdev->vdev_children; i++) { 3059a3874b8bSToomas Soome vdev_t *vd = spa->spa_root_vdev->vdev_child[i]; 3060a3874b8bSToomas Soome dsl_scan_io_queue_t *queue = vd->vdev_scan_io_queue; 3061a3874b8bSToomas Soome 3062a3874b8bSToomas Soome if (queue == NULL) 3063a3874b8bSToomas Soome continue; 3064a3874b8bSToomas Soome 3065a3874b8bSToomas Soome seg_size_total += queue->q_total_seg_size_this_txg; 3066a3874b8bSToomas Soome zio_size_total += queue->q_total_zio_size_this_txg; 3067a3874b8bSToomas Soome seg_count_total += queue->q_segs_this_txg; 3068a3874b8bSToomas Soome zio_count_total += queue->q_zios_this_txg; 3069a3874b8bSToomas Soome } 3070a3874b8bSToomas Soome 3071a3874b8bSToomas Soome if (seg_count_total == 0 || zio_count_total == 0) { 3072a3874b8bSToomas Soome scn->scn_avg_seg_size_this_txg = 0; 3073a3874b8bSToomas Soome scn->scn_avg_zio_size_this_txg = 0; 3074a3874b8bSToomas Soome scn->scn_segs_this_txg = 0; 3075a3874b8bSToomas Soome scn->scn_zios_this_txg = 0; 3076a3874b8bSToomas Soome return; 3077a3874b8bSToomas Soome } 3078a3874b8bSToomas Soome 3079a3874b8bSToomas Soome scn->scn_avg_seg_size_this_txg = seg_size_total / seg_count_total; 3080a3874b8bSToomas Soome scn->scn_avg_zio_size_this_txg = zio_size_total / zio_count_total; 3081a3874b8bSToomas Soome scn->scn_segs_this_txg = seg_count_total; 3082a3874b8bSToomas Soome scn->scn_zios_this_txg = zio_count_total; 3083a3874b8bSToomas Soome } 3084a3874b8bSToomas Soome 30855cabbc6bSPrashanth Sreenivasa static int 30865cabbc6bSPrashanth Sreenivasa dsl_scan_obsolete_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 30875cabbc6bSPrashanth Sreenivasa { 30885cabbc6bSPrashanth Sreenivasa dsl_scan_t *scn = arg; 30895cabbc6bSPrashanth Sreenivasa const dva_t *dva = &bp->blk_dva[0]; 30905cabbc6bSPrashanth Sreenivasa 30915cabbc6bSPrashanth Sreenivasa if (dsl_scan_async_block_should_pause(scn)) 30925cabbc6bSPrashanth Sreenivasa return (SET_ERROR(ERESTART)); 30935cabbc6bSPrashanth Sreenivasa 30945cabbc6bSPrashanth Sreenivasa spa_vdev_indirect_mark_obsolete(scn->scn_dp->dp_spa, 30955cabbc6bSPrashanth Sreenivasa DVA_GET_VDEV(dva), DVA_GET_OFFSET(dva), 30965cabbc6bSPrashanth Sreenivasa DVA_GET_ASIZE(dva), tx); 30975cabbc6bSPrashanth Sreenivasa scn->scn_visited_this_txg++; 30985cabbc6bSPrashanth Sreenivasa return (0); 30995cabbc6bSPrashanth Sreenivasa } 31005cabbc6bSPrashanth Sreenivasa 3101cde58dbcSMatthew Ahrens boolean_t 3102cde58dbcSMatthew Ahrens dsl_scan_active(dsl_scan_t *scn) 3103cde58dbcSMatthew Ahrens { 3104cde58dbcSMatthew Ahrens spa_t *spa = scn->scn_dp->dp_spa; 3105cde58dbcSMatthew Ahrens uint64_t used = 0, comp, uncomp; 3106cde58dbcSMatthew Ahrens 3107cde58dbcSMatthew Ahrens if (spa->spa_load_state != SPA_LOAD_NONE) 3108cde58dbcSMatthew Ahrens return (B_FALSE); 3109cde58dbcSMatthew Ahrens if (spa_shutting_down(spa)) 3110cde58dbcSMatthew Ahrens return (B_FALSE); 3111a3874b8bSToomas Soome if ((dsl_scan_is_running(scn) && !dsl_scan_is_paused_scrub(scn)) || 31127fd05ac4SMatthew Ahrens (scn->scn_async_destroying && !scn->scn_async_stalled)) 3113cde58dbcSMatthew Ahrens return (B_TRUE); 3114cde58dbcSMatthew Ahrens 3115cde58dbcSMatthew Ahrens if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 3116cde58dbcSMatthew Ahrens (void) bpobj_space(&scn->scn_dp->dp_free_bpobj, 3117cde58dbcSMatthew Ahrens &used, &comp, &uncomp); 3118cde58dbcSMatthew Ahrens } 3119cde58dbcSMatthew Ahrens return (used != 0); 3120cde58dbcSMatthew Ahrens } 3121cde58dbcSMatthew Ahrens 3122e4c795beSTom Caputi static boolean_t 3123e4c795beSTom Caputi dsl_scan_check_deferred(vdev_t *vd) 3124e4c795beSTom Caputi { 3125e4c795beSTom Caputi boolean_t need_resilver = B_FALSE; 3126e4c795beSTom Caputi 3127e4c795beSTom Caputi for (int c = 0; c < vd->vdev_children; c++) { 3128e4c795beSTom Caputi need_resilver |= 3129e4c795beSTom Caputi dsl_scan_check_deferred(vd->vdev_child[c]); 3130e4c795beSTom Caputi } 3131e4c795beSTom Caputi 3132e4c795beSTom Caputi if (!vdev_is_concrete(vd) || vd->vdev_aux || 3133e4c795beSTom Caputi !vd->vdev_ops->vdev_op_leaf) 3134e4c795beSTom Caputi return (need_resilver); 3135e4c795beSTom Caputi 3136e4c795beSTom Caputi if (!vd->vdev_resilver_deferred) 3137e4c795beSTom Caputi need_resilver = B_TRUE; 3138e4c795beSTom Caputi 3139e4c795beSTom Caputi return (need_resilver); 3140e4c795beSTom Caputi } 3141e4c795beSTom Caputi 3142a3874b8bSToomas Soome static boolean_t 3143a3874b8bSToomas Soome dsl_scan_need_resilver(spa_t *spa, const dva_t *dva, size_t psize, 3144a3874b8bSToomas Soome uint64_t phys_birth) 3145a3874b8bSToomas Soome { 3146a3874b8bSToomas Soome vdev_t *vd; 3147a3874b8bSToomas Soome 3148a3874b8bSToomas Soome vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva)); 3149a3874b8bSToomas Soome 3150a3874b8bSToomas Soome if (vd->vdev_ops == &vdev_indirect_ops) { 3151a3874b8bSToomas Soome /* 3152a3874b8bSToomas Soome * The indirect vdev can point to multiple 3153a3874b8bSToomas Soome * vdevs. For simplicity, always create 3154a3874b8bSToomas Soome * the resilver zio_t. zio_vdev_io_start() 3155a3874b8bSToomas Soome * will bypass the child resilver i/o's if 3156a3874b8bSToomas Soome * they are on vdevs that don't have DTL's. 3157a3874b8bSToomas Soome */ 3158a3874b8bSToomas Soome return (B_TRUE); 3159a3874b8bSToomas Soome } 3160a3874b8bSToomas Soome 3161a3874b8bSToomas Soome if (DVA_GET_GANG(dva)) { 3162a3874b8bSToomas Soome /* 3163a3874b8bSToomas Soome * Gang members may be spread across multiple 3164a3874b8bSToomas Soome * vdevs, so the best estimate we have is the 3165a3874b8bSToomas Soome * scrub range, which has already been checked. 3166a3874b8bSToomas Soome * XXX -- it would be better to change our 3167a3874b8bSToomas Soome * allocation policy to ensure that all 3168a3874b8bSToomas Soome * gang members reside on the same vdev. 3169a3874b8bSToomas Soome */ 3170a3874b8bSToomas Soome return (B_TRUE); 3171a3874b8bSToomas Soome } 3172a3874b8bSToomas Soome 3173a3874b8bSToomas Soome /* 3174a3874b8bSToomas Soome * Check if the txg falls within the range which must be 3175a3874b8bSToomas Soome * resilvered. DVAs outside this range can always be skipped. 3176a3874b8bSToomas Soome */ 3177a3874b8bSToomas Soome if (!vdev_dtl_contains(vd, DTL_PARTIAL, phys_birth, 1)) 3178a3874b8bSToomas Soome return (B_FALSE); 3179a3874b8bSToomas Soome 3180a3874b8bSToomas Soome /* 3181a3874b8bSToomas Soome * Check if the top-level vdev must resilver this offset. 3182a3874b8bSToomas Soome * When the offset does not intersect with a dirty leaf DTL 3183a3874b8bSToomas Soome * then it may be possible to skip the resilver IO. The psize 3184a3874b8bSToomas Soome * is provided instead of asize to simplify the check for RAIDZ. 3185a3874b8bSToomas Soome */ 3186a3874b8bSToomas Soome if (!vdev_dtl_need_resilver(vd, DVA_GET_OFFSET(dva), psize)) 3187a3874b8bSToomas Soome return (B_FALSE); 3188a3874b8bSToomas Soome 3189e4c795beSTom Caputi /* 3190e4c795beSTom Caputi * Check that this top-level vdev has a device under it which 3191e4c795beSTom Caputi * is resilvering and is not deferred. 3192e4c795beSTom Caputi */ 3193e4c795beSTom Caputi if (!dsl_scan_check_deferred(vd)) 3194e4c795beSTom Caputi return (B_FALSE); 3195e4c795beSTom Caputi 3196a3874b8bSToomas Soome return (B_TRUE); 3197a3874b8bSToomas Soome } 3198a3874b8bSToomas Soome 319986714001SSerapheim Dimitropoulos static int 320086714001SSerapheim Dimitropoulos dsl_process_async_destroys(dsl_pool_t *dp, dmu_tx_t *tx) 32013f9d6ad7SLin Ling { 3202a3874b8bSToomas Soome int err = 0; 32033f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 32043f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 32053f9d6ad7SLin Ling 320686714001SSerapheim Dimitropoulos if (spa_suspend_async_destroy(spa)) 320786714001SSerapheim Dimitropoulos return (0); 32083f9d6ad7SLin Ling 3209139510fbSGeorge Wilson if (zfs_free_bpobj_enabled && 3210a3874b8bSToomas Soome spa_version(spa) >= SPA_VERSION_DEADLISTS) { 3211ad135b5dSChristopher Siden scn->scn_is_bptree = B_FALSE; 32125cabbc6bSPrashanth Sreenivasa scn->scn_async_block_min_time_ms = zfs_free_min_time_ms; 3213a3874b8bSToomas Soome scn->scn_zio_root = zio_root(spa, NULL, 3214cde58dbcSMatthew Ahrens NULL, ZIO_FLAG_MUSTSUCCEED); 3215cde58dbcSMatthew Ahrens err = bpobj_iterate(&dp->dp_free_bpobj, 3216ad135b5dSChristopher Siden dsl_scan_free_block_cb, scn, tx); 3217a3874b8bSToomas Soome VERIFY0(zio_wait(scn->scn_zio_root)); 3218a3874b8bSToomas Soome scn->scn_zio_root = NULL; 3219ad135b5dSChristopher Siden 32207fd05ac4SMatthew Ahrens if (err != 0 && err != ERESTART) 32217fd05ac4SMatthew Ahrens zfs_panic_recover("error %u from bpobj_iterate()", err); 32227fd05ac4SMatthew Ahrens } 32237fd05ac4SMatthew Ahrens 32247fd05ac4SMatthew Ahrens if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) { 32257fd05ac4SMatthew Ahrens ASSERT(scn->scn_async_destroying); 32267fd05ac4SMatthew Ahrens scn->scn_is_bptree = B_TRUE; 3227a3874b8bSToomas Soome scn->scn_zio_root = zio_root(spa, NULL, 32287fd05ac4SMatthew Ahrens NULL, ZIO_FLAG_MUSTSUCCEED); 32297fd05ac4SMatthew Ahrens err = bptree_iterate(dp->dp_meta_objset, 32307fd05ac4SMatthew Ahrens dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx); 32317fd05ac4SMatthew Ahrens VERIFY0(zio_wait(scn->scn_zio_root)); 3232a3874b8bSToomas Soome scn->scn_zio_root = NULL; 32337fd05ac4SMatthew Ahrens 32347fd05ac4SMatthew Ahrens if (err == EIO || err == ECKSUM) { 32357fd05ac4SMatthew Ahrens err = 0; 32367fd05ac4SMatthew Ahrens } else if (err != 0 && err != ERESTART) { 32377fd05ac4SMatthew Ahrens zfs_panic_recover("error %u from " 32387fd05ac4SMatthew Ahrens "traverse_dataset_destroyed()", err); 3239ad135b5dSChristopher Siden } 32407fd05ac4SMatthew Ahrens 32417fd05ac4SMatthew Ahrens if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) { 32427fd05ac4SMatthew Ahrens /* finished; deactivate async destroy feature */ 32437fd05ac4SMatthew Ahrens spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx); 32447fd05ac4SMatthew Ahrens ASSERT(!spa_feature_is_active(spa, 32457fd05ac4SMatthew Ahrens SPA_FEATURE_ASYNC_DESTROY)); 32467fd05ac4SMatthew Ahrens VERIFY0(zap_remove(dp->dp_meta_objset, 32477fd05ac4SMatthew Ahrens DMU_POOL_DIRECTORY_OBJECT, 32487fd05ac4SMatthew Ahrens DMU_POOL_BPTREE_OBJ, tx)); 32497fd05ac4SMatthew Ahrens VERIFY0(bptree_free(dp->dp_meta_objset, 32507fd05ac4SMatthew Ahrens dp->dp_bptree_obj, tx)); 32517fd05ac4SMatthew Ahrens dp->dp_bptree_obj = 0; 32527fd05ac4SMatthew Ahrens scn->scn_async_destroying = B_FALSE; 3253231aab85SMatthew Ahrens scn->scn_async_stalled = B_FALSE; 3254231aab85SMatthew Ahrens } else { 3255231aab85SMatthew Ahrens /* 3256231aab85SMatthew Ahrens * If we didn't make progress, mark the async 3257231aab85SMatthew Ahrens * destroy as stalled, so that we will not initiate 3258231aab85SMatthew Ahrens * a spa_sync() on its behalf. Note that we only 3259231aab85SMatthew Ahrens * check this if we are not finished, because if the 3260231aab85SMatthew Ahrens * bptree had no blocks for us to visit, we can 3261231aab85SMatthew Ahrens * finish without "making progress". 3262231aab85SMatthew Ahrens */ 3263231aab85SMatthew Ahrens scn->scn_async_stalled = 3264231aab85SMatthew Ahrens (scn->scn_visited_this_txg == 0); 3265cde58dbcSMatthew Ahrens } 32667fd05ac4SMatthew Ahrens } 32677fd05ac4SMatthew Ahrens if (scn->scn_visited_this_txg) { 32687fd05ac4SMatthew Ahrens zfs_dbgmsg("freed %llu blocks in %llums from " 3269a3874b8bSToomas Soome "free_bpobj/bptree txg %llu; err=%d", 32707fd05ac4SMatthew Ahrens (longlong_t)scn->scn_visited_this_txg, 32717fd05ac4SMatthew Ahrens (longlong_t) 32727fd05ac4SMatthew Ahrens NSEC2MSEC(gethrtime() - scn->scn_sync_start_time), 32737fd05ac4SMatthew Ahrens (longlong_t)tx->tx_txg, err); 32747fd05ac4SMatthew Ahrens scn->scn_visited_this_txg = 0; 32757fd05ac4SMatthew Ahrens 32767fd05ac4SMatthew Ahrens /* 32777fd05ac4SMatthew Ahrens * Write out changes to the DDT that may be required as a 32787fd05ac4SMatthew Ahrens * result of the blocks freed. This ensures that the DDT 32797fd05ac4SMatthew Ahrens * is clean when a scrub/resilver runs. 32807fd05ac4SMatthew Ahrens */ 32817fd05ac4SMatthew Ahrens ddt_sync(spa, tx->tx_txg); 32827fd05ac4SMatthew Ahrens } 32837fd05ac4SMatthew Ahrens if (err != 0) 328486714001SSerapheim Dimitropoulos return (err); 32858c04a1faSGary Mills if (dp->dp_free_dir != NULL && !scn->scn_async_destroying && 32868c04a1faSGary Mills zfs_free_leak_on_eio && 3287c1379625SJustin T. Gibbs (dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes != 0 || 3288c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes != 0 || 3289c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes != 0)) { 32907fd05ac4SMatthew Ahrens /* 32917fd05ac4SMatthew Ahrens * We have finished background destroying, but there is still 32927fd05ac4SMatthew Ahrens * some space left in the dp_free_dir. Transfer this leaked 32937fd05ac4SMatthew Ahrens * space to the dp_leak_dir. 32947fd05ac4SMatthew Ahrens */ 32957fd05ac4SMatthew Ahrens if (dp->dp_leak_dir == NULL) { 32967fd05ac4SMatthew Ahrens rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 32977fd05ac4SMatthew Ahrens (void) dsl_dir_create_sync(dp, dp->dp_root_dir, 32987fd05ac4SMatthew Ahrens LEAK_DIR_NAME, tx); 32997fd05ac4SMatthew Ahrens VERIFY0(dsl_pool_open_special_dir(dp, 33007fd05ac4SMatthew Ahrens LEAK_DIR_NAME, &dp->dp_leak_dir)); 33017fd05ac4SMatthew Ahrens rrw_exit(&dp->dp_config_rwlock, FTAG); 33027fd05ac4SMatthew Ahrens } 33037fd05ac4SMatthew Ahrens dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD, 3304c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes, 3305c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes, 3306c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx); 33077fd05ac4SMatthew Ahrens dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD, 3308c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes, 3309c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes, 3310c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx); 33117fd05ac4SMatthew Ahrens } 33125cabbc6bSPrashanth Sreenivasa 33138c04a1faSGary Mills if (dp->dp_free_dir != NULL && !scn->scn_async_destroying) { 33145d7b4d43SMatthew Ahrens /* finished; verify that space accounting went to zero */ 3315c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes); 3316c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes); 3317c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes); 3318cde58dbcSMatthew Ahrens } 3319cde58dbcSMatthew Ahrens 33205cabbc6bSPrashanth Sreenivasa EQUIV(bpobj_is_open(&dp->dp_obsolete_bpobj), 33215cabbc6bSPrashanth Sreenivasa 0 == zap_contains(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 33225cabbc6bSPrashanth Sreenivasa DMU_POOL_OBSOLETE_BPOBJ)); 33235cabbc6bSPrashanth Sreenivasa if (err == 0 && bpobj_is_open(&dp->dp_obsolete_bpobj)) { 33245cabbc6bSPrashanth Sreenivasa ASSERT(spa_feature_is_active(dp->dp_spa, 33255cabbc6bSPrashanth Sreenivasa SPA_FEATURE_OBSOLETE_COUNTS)); 33265cabbc6bSPrashanth Sreenivasa 33275cabbc6bSPrashanth Sreenivasa scn->scn_is_bptree = B_FALSE; 33285cabbc6bSPrashanth Sreenivasa scn->scn_async_block_min_time_ms = zfs_obsolete_min_time_ms; 33295cabbc6bSPrashanth Sreenivasa err = bpobj_iterate(&dp->dp_obsolete_bpobj, 33305cabbc6bSPrashanth Sreenivasa dsl_scan_obsolete_block_cb, scn, tx); 33315cabbc6bSPrashanth Sreenivasa if (err != 0 && err != ERESTART) 33325cabbc6bSPrashanth Sreenivasa zfs_panic_recover("error %u from bpobj_iterate()", err); 33335cabbc6bSPrashanth Sreenivasa 33345cabbc6bSPrashanth Sreenivasa if (bpobj_is_empty(&dp->dp_obsolete_bpobj)) 33355cabbc6bSPrashanth Sreenivasa dsl_pool_destroy_obsolete_bpobj(dp, tx); 33365cabbc6bSPrashanth Sreenivasa } 33375cabbc6bSPrashanth Sreenivasa 333886714001SSerapheim Dimitropoulos return (0); 333986714001SSerapheim Dimitropoulos } 334086714001SSerapheim Dimitropoulos 3341a3874b8bSToomas Soome /* 3342a3874b8bSToomas Soome * This is the primary entry point for scans that is called from syncing 3343a3874b8bSToomas Soome * context. Scans must happen entirely during syncing context so that we 3344a3874b8bSToomas Soome * cna guarantee that blocks we are currently scanning will not change out 3345a3874b8bSToomas Soome * from under us. While a scan is active, this funciton controls how quickly 3346a3874b8bSToomas Soome * transaction groups proceed, instead of the normal handling provided by 3347a3874b8bSToomas Soome * txg_sync_thread(). 3348a3874b8bSToomas Soome */ 334986714001SSerapheim Dimitropoulos void 335086714001SSerapheim Dimitropoulos dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx) 335186714001SSerapheim Dimitropoulos { 335286714001SSerapheim Dimitropoulos dsl_scan_t *scn = dp->dp_scan; 335386714001SSerapheim Dimitropoulos spa_t *spa = dp->dp_spa; 335486714001SSerapheim Dimitropoulos int err = 0; 3355a3874b8bSToomas Soome state_sync_type_t sync_type = SYNC_OPTIONAL; 335686714001SSerapheim Dimitropoulos 3357e4c795beSTom Caputi if (spa->spa_resilver_deferred && 3358e4c795beSTom Caputi !spa_feature_is_active(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)) 3359e4c795beSTom Caputi spa_feature_incr(spa, SPA_FEATURE_RESILVER_DEFER, tx); 3360e4c795beSTom Caputi 336186714001SSerapheim Dimitropoulos /* 336286714001SSerapheim Dimitropoulos * Check for scn_restart_txg before checking spa_load_state, so 336386714001SSerapheim Dimitropoulos * that we can restart an old-style scan while the pool is being 3364e4c795beSTom Caputi * imported (see dsl_scan_init). We also restart scans if there 3365e4c795beSTom Caputi * is a deferred resilver and the user has manually disabled 3366e4c795beSTom Caputi * deferred resilvers via the tunable. 336786714001SSerapheim Dimitropoulos */ 3368e4c795beSTom Caputi if (dsl_scan_restarting(scn, tx) || 3369e4c795beSTom Caputi (spa->spa_resilver_deferred && zfs_resilver_disable_defer)) { 337086714001SSerapheim Dimitropoulos pool_scan_func_t func = POOL_SCAN_SCRUB; 337186714001SSerapheim Dimitropoulos dsl_scan_done(scn, B_FALSE, tx); 337286714001SSerapheim Dimitropoulos if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) 337386714001SSerapheim Dimitropoulos func = POOL_SCAN_RESILVER; 337486714001SSerapheim Dimitropoulos zfs_dbgmsg("restarting scan func=%u txg=%llu", 3375a3874b8bSToomas Soome func, (longlong_t)tx->tx_txg); 337686714001SSerapheim Dimitropoulos dsl_scan_setup_sync(&func, tx); 337786714001SSerapheim Dimitropoulos } 337886714001SSerapheim Dimitropoulos 337986714001SSerapheim Dimitropoulos /* 338086714001SSerapheim Dimitropoulos * Only process scans in sync pass 1. 338186714001SSerapheim Dimitropoulos */ 338286714001SSerapheim Dimitropoulos if (spa_sync_pass(dp->dp_spa) > 1) 338386714001SSerapheim Dimitropoulos return; 338486714001SSerapheim Dimitropoulos 338586714001SSerapheim Dimitropoulos /* 338686714001SSerapheim Dimitropoulos * If the spa is shutting down, then stop scanning. This will 338786714001SSerapheim Dimitropoulos * ensure that the scan does not dirty any new data during the 338886714001SSerapheim Dimitropoulos * shutdown phase. 338986714001SSerapheim Dimitropoulos */ 339086714001SSerapheim Dimitropoulos if (spa_shutting_down(spa)) 339186714001SSerapheim Dimitropoulos return; 339286714001SSerapheim Dimitropoulos 339386714001SSerapheim Dimitropoulos /* 339486714001SSerapheim Dimitropoulos * If the scan is inactive due to a stalled async destroy, try again. 339586714001SSerapheim Dimitropoulos */ 339686714001SSerapheim Dimitropoulos if (!scn->scn_async_stalled && !dsl_scan_active(scn)) 339786714001SSerapheim Dimitropoulos return; 339886714001SSerapheim Dimitropoulos 3399a3874b8bSToomas Soome /* reset scan statistics */ 340086714001SSerapheim Dimitropoulos scn->scn_visited_this_txg = 0; 3401a3874b8bSToomas Soome scn->scn_holes_this_txg = 0; 3402a3874b8bSToomas Soome scn->scn_lt_min_this_txg = 0; 3403a3874b8bSToomas Soome scn->scn_gt_max_this_txg = 0; 3404a3874b8bSToomas Soome scn->scn_ddt_contained_this_txg = 0; 3405a3874b8bSToomas Soome scn->scn_objsets_visited_this_txg = 0; 3406a3874b8bSToomas Soome scn->scn_avg_seg_size_this_txg = 0; 3407a3874b8bSToomas Soome scn->scn_segs_this_txg = 0; 3408a3874b8bSToomas Soome scn->scn_avg_zio_size_this_txg = 0; 3409a3874b8bSToomas Soome scn->scn_zios_this_txg = 0; 341086714001SSerapheim Dimitropoulos scn->scn_suspending = B_FALSE; 341186714001SSerapheim Dimitropoulos scn->scn_sync_start_time = gethrtime(); 341286714001SSerapheim Dimitropoulos spa->spa_scrub_active = B_TRUE; 341386714001SSerapheim Dimitropoulos 341486714001SSerapheim Dimitropoulos /* 341586714001SSerapheim Dimitropoulos * First process the async destroys. If we pause, don't do 341686714001SSerapheim Dimitropoulos * any scrubbing or resilvering. This ensures that there are no 341786714001SSerapheim Dimitropoulos * async destroys while we are scanning, so the scan code doesn't 341886714001SSerapheim Dimitropoulos * have to worry about traversing it. It is also faster to free the 341986714001SSerapheim Dimitropoulos * blocks than to scrub them. 342086714001SSerapheim Dimitropoulos */ 342186714001SSerapheim Dimitropoulos err = dsl_process_async_destroys(dp, tx); 342286714001SSerapheim Dimitropoulos if (err != 0) 342386714001SSerapheim Dimitropoulos return; 342486714001SSerapheim Dimitropoulos 3425a3874b8bSToomas Soome if (!dsl_scan_is_running(scn) || dsl_scan_is_paused_scrub(scn)) 3426cde58dbcSMatthew Ahrens return; 3427cde58dbcSMatthew Ahrens 3428a3874b8bSToomas Soome /* 3429a3874b8bSToomas Soome * Wait a few txgs after importing to begin scanning so that 3430a3874b8bSToomas Soome * we can get the pool imported quickly. 3431a3874b8bSToomas Soome */ 3432a3874b8bSToomas Soome if (spa->spa_syncing_txg < spa->spa_first_txg + SCAN_IMPORT_WAIT_TXGS) 3433b4952e17SGeorge Wilson return; 3434b4952e17SGeorge Wilson 3435e4c795beSTom Caputi /* 3436e4c795beSTom Caputi * zfs_scan_suspend_progress can be set to disable scan progress. 3437e4c795beSTom Caputi * We don't want to spin the txg_sync thread, so we add a delay 3438e4c795beSTom Caputi * here to simulate the time spent doing a scan. This is mostly 3439e4c795beSTom Caputi * useful for testing and debugging. 3440e4c795beSTom Caputi */ 3441e4c795beSTom Caputi if (zfs_scan_suspend_progress) { 3442e4c795beSTom Caputi uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time; 3443e4c795beSTom Caputi int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 3444e4c795beSTom Caputi zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; 3445e4c795beSTom Caputi 3446e4c795beSTom Caputi while (zfs_scan_suspend_progress && 3447e4c795beSTom Caputi !txg_sync_waiting(scn->scn_dp) && 3448e4c795beSTom Caputi !spa_shutting_down(scn->scn_dp->dp_spa) && 3449e4c795beSTom Caputi NSEC2MSEC(scan_time_ns) < mintime) { 3450e4c795beSTom Caputi delay(hz); 3451e4c795beSTom Caputi scan_time_ns = gethrtime() - scn->scn_sync_start_time; 3452e4c795beSTom Caputi } 3453e4c795beSTom Caputi return; 3454e4c795beSTom Caputi } 3455e4c795beSTom Caputi 3456a3874b8bSToomas Soome /* 3457a3874b8bSToomas Soome * It is possible to switch from unsorted to sorted at any time, 3458a3874b8bSToomas Soome * but afterwards the scan will remain sorted unless reloaded from 3459a3874b8bSToomas Soome * a checkpoint after a reboot. 3460a3874b8bSToomas Soome */ 3461a3874b8bSToomas Soome if (!zfs_scan_legacy) { 3462a3874b8bSToomas Soome scn->scn_is_sorted = B_TRUE; 3463a3874b8bSToomas Soome if (scn->scn_last_checkpoint == 0) 3464a3874b8bSToomas Soome scn->scn_last_checkpoint = ddi_get_lbolt(); 3465a3874b8bSToomas Soome } 34661702cce7SAlek Pinchuk 3467a3874b8bSToomas Soome /* 3468a3874b8bSToomas Soome * For sorted scans, determine what kind of work we will be doing 3469a3874b8bSToomas Soome * this txg based on our memory limitations and whether or not we 3470a3874b8bSToomas Soome * need to perform a checkpoint. 3471a3874b8bSToomas Soome */ 3472a3874b8bSToomas Soome if (scn->scn_is_sorted) { 3473a3874b8bSToomas Soome /* 3474a3874b8bSToomas Soome * If we are over our checkpoint interval, set scn_clearing 3475a3874b8bSToomas Soome * so that we can begin checkpointing immediately. The 3476a3874b8bSToomas Soome * checkpoint allows us to save a consisent bookmark 3477a3874b8bSToomas Soome * representing how much data we have scrubbed so far. 3478a3874b8bSToomas Soome * Otherwise, use the memory limit to determine if we should 3479a3874b8bSToomas Soome * scan for metadata or start issue scrub IOs. We accumulate 3480a3874b8bSToomas Soome * metadata until we hit our hard memory limit at which point 3481a3874b8bSToomas Soome * we issue scrub IOs until we are at our soft memory limit. 3482a3874b8bSToomas Soome */ 3483a3874b8bSToomas Soome if (scn->scn_checkpointing || 3484a3874b8bSToomas Soome ddi_get_lbolt() - scn->scn_last_checkpoint > 3485a3874b8bSToomas Soome SEC_TO_TICK(zfs_scan_checkpoint_intval)) { 3486a3874b8bSToomas Soome if (!scn->scn_checkpointing) 3487a3874b8bSToomas Soome zfs_dbgmsg("begin scan checkpoint"); 3488a3874b8bSToomas Soome 3489a3874b8bSToomas Soome scn->scn_checkpointing = B_TRUE; 3490a3874b8bSToomas Soome scn->scn_clearing = B_TRUE; 3491a3874b8bSToomas Soome } else { 3492a3874b8bSToomas Soome boolean_t should_clear = dsl_scan_should_clear(scn); 3493a3874b8bSToomas Soome if (should_clear && !scn->scn_clearing) { 3494a3874b8bSToomas Soome zfs_dbgmsg("begin scan clearing"); 3495a3874b8bSToomas Soome scn->scn_clearing = B_TRUE; 3496a3874b8bSToomas Soome } else if (!should_clear && scn->scn_clearing) { 3497a3874b8bSToomas Soome zfs_dbgmsg("finish scan clearing"); 3498a3874b8bSToomas Soome scn->scn_clearing = B_FALSE; 3499a3874b8bSToomas Soome } 3500a3874b8bSToomas Soome } 35013f9d6ad7SLin Ling } else { 3502a3874b8bSToomas Soome ASSERT0(scn->scn_checkpointing); 3503a3874b8bSToomas Soome ASSERT0(scn->scn_clearing); 35043f9d6ad7SLin Ling } 35053f9d6ad7SLin Ling 3506a3874b8bSToomas Soome if (!scn->scn_clearing && scn->scn_done_txg == 0) { 3507a3874b8bSToomas Soome /* Need to scan metadata for more blocks to scrub */ 3508a3874b8bSToomas Soome dsl_scan_phys_t *scnp = &scn->scn_phys; 3509a3874b8bSToomas Soome taskqid_t prefetch_tqid; 3510a3874b8bSToomas Soome uint64_t bytes_per_leaf = zfs_scan_vdev_limit; 3511a3874b8bSToomas Soome uint64_t nr_leaves = dsl_scan_count_leaves(spa->spa_root_vdev); 35123f9d6ad7SLin Ling 3513a3874b8bSToomas Soome /* 3514a3874b8bSToomas Soome * Calculate the max number of in-flight bytes for pool-wide 3515a3874b8bSToomas Soome * scanning operations (minimum 1MB). Limits for the issuing 3516a3874b8bSToomas Soome * phase are done per top-level vdev and are handled separately. 3517a3874b8bSToomas Soome */ 3518a3874b8bSToomas Soome scn->scn_maxinflight_bytes = 3519a3874b8bSToomas Soome MAX(nr_leaves * bytes_per_leaf, 1ULL << 20); 3520a3874b8bSToomas Soome 3521a3874b8bSToomas Soome if (scnp->scn_ddt_bookmark.ddb_class <= 3522a3874b8bSToomas Soome scnp->scn_ddt_class_max) { 3523a3874b8bSToomas Soome ASSERT(ZB_IS_ZERO(&scnp->scn_bookmark)); 3524a3874b8bSToomas Soome zfs_dbgmsg("doing scan sync txg %llu; " 3525a3874b8bSToomas Soome "ddt bm=%llu/%llu/%llu/%llx", 3526a3874b8bSToomas Soome (longlong_t)tx->tx_txg, 3527a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_class, 3528a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_type, 3529a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_checksum, 3530a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_cursor); 3531a3874b8bSToomas Soome } else { 3532a3874b8bSToomas Soome zfs_dbgmsg("doing scan sync txg %llu; " 3533a3874b8bSToomas Soome "bm=%llu/%llu/%llu/%llu", 3534a3874b8bSToomas Soome (longlong_t)tx->tx_txg, 3535a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_objset, 3536a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_object, 3537a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_level, 3538a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_blkid); 3539a3874b8bSToomas Soome } 35403f9d6ad7SLin Ling 3541a3874b8bSToomas Soome scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 3542a3874b8bSToomas Soome NULL, ZIO_FLAG_CANFAIL); 35433f9d6ad7SLin Ling 3544a3874b8bSToomas Soome scn->scn_prefetch_stop = B_FALSE; 3545a3874b8bSToomas Soome prefetch_tqid = taskq_dispatch(dp->dp_sync_taskq, 3546a3874b8bSToomas Soome dsl_scan_prefetch_thread, scn, TQ_SLEEP); 3547a3874b8bSToomas Soome ASSERT(prefetch_tqid != TASKQID_INVALID); 35483f9d6ad7SLin Ling 3549a3874b8bSToomas Soome dsl_pool_config_enter(dp, FTAG); 3550a3874b8bSToomas Soome dsl_scan_visit(scn, tx); 3551a3874b8bSToomas Soome dsl_pool_config_exit(dp, FTAG); 35523f9d6ad7SLin Ling 3553a3874b8bSToomas Soome mutex_enter(&dp->dp_spa->spa_scrub_lock); 3554a3874b8bSToomas Soome scn->scn_prefetch_stop = B_TRUE; 3555a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 3556a3874b8bSToomas Soome mutex_exit(&dp->dp_spa->spa_scrub_lock); 35573f9d6ad7SLin Ling 3558a3874b8bSToomas Soome taskq_wait_id(dp->dp_sync_taskq, prefetch_tqid); 3559a3874b8bSToomas Soome (void) zio_wait(scn->scn_zio_root); 3560a3874b8bSToomas Soome scn->scn_zio_root = NULL; 3561a3874b8bSToomas Soome 3562a3874b8bSToomas Soome zfs_dbgmsg("scan visited %llu blocks in %llums " 3563a3874b8bSToomas Soome "(%llu os's, %llu holes, %llu < mintxg, " 3564a3874b8bSToomas Soome "%llu in ddt, %llu > maxtxg)", 3565a3874b8bSToomas Soome (longlong_t)scn->scn_visited_this_txg, 3566a3874b8bSToomas Soome (longlong_t)NSEC2MSEC(gethrtime() - 3567a3874b8bSToomas Soome scn->scn_sync_start_time), 3568a3874b8bSToomas Soome (longlong_t)scn->scn_objsets_visited_this_txg, 3569a3874b8bSToomas Soome (longlong_t)scn->scn_holes_this_txg, 3570a3874b8bSToomas Soome (longlong_t)scn->scn_lt_min_this_txg, 3571a3874b8bSToomas Soome (longlong_t)scn->scn_ddt_contained_this_txg, 3572a3874b8bSToomas Soome (longlong_t)scn->scn_gt_max_this_txg); 3573a3874b8bSToomas Soome 3574a3874b8bSToomas Soome if (!scn->scn_suspending) { 3575a3874b8bSToomas Soome ASSERT0(avl_numnodes(&scn->scn_queue)); 3576a3874b8bSToomas Soome scn->scn_done_txg = tx->tx_txg + 1; 3577a3874b8bSToomas Soome if (scn->scn_is_sorted) { 3578a3874b8bSToomas Soome scn->scn_checkpointing = B_TRUE; 3579a3874b8bSToomas Soome scn->scn_clearing = B_TRUE; 3580a3874b8bSToomas Soome } 3581a3874b8bSToomas Soome zfs_dbgmsg("scan complete txg %llu", 3582a3874b8bSToomas Soome (longlong_t)tx->tx_txg); 3583a3874b8bSToomas Soome } 3584a3874b8bSToomas Soome } else if (scn->scn_is_sorted && scn->scn_bytes_pending != 0) { 3585e4c795beSTom Caputi ASSERT(scn->scn_clearing); 3586e4c795beSTom Caputi 3587a3874b8bSToomas Soome /* need to issue scrubbing IOs from per-vdev queues */ 3588a3874b8bSToomas Soome scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 3589a3874b8bSToomas Soome NULL, ZIO_FLAG_CANFAIL); 3590a3874b8bSToomas Soome scan_io_queues_run(scn); 3591a3874b8bSToomas Soome (void) zio_wait(scn->scn_zio_root); 3592a3874b8bSToomas Soome scn->scn_zio_root = NULL; 3593a3874b8bSToomas Soome 3594a3874b8bSToomas Soome /* calculate and dprintf the current memory usage */ 3595a3874b8bSToomas Soome (void) dsl_scan_should_clear(scn); 3596a3874b8bSToomas Soome dsl_scan_update_stats(scn); 3597a3874b8bSToomas Soome 3598a3874b8bSToomas Soome zfs_dbgmsg("scrubbed %llu blocks (%llu segs) in %llums " 3599a3874b8bSToomas Soome "(avg_block_size = %llu, avg_seg_size = %llu)", 3600a3874b8bSToomas Soome (longlong_t)scn->scn_zios_this_txg, 3601a3874b8bSToomas Soome (longlong_t)scn->scn_segs_this_txg, 3602a3874b8bSToomas Soome (longlong_t)NSEC2MSEC(gethrtime() - 3603a3874b8bSToomas Soome scn->scn_sync_start_time), 3604a3874b8bSToomas Soome (longlong_t)scn->scn_avg_zio_size_this_txg, 3605a3874b8bSToomas Soome (longlong_t)scn->scn_avg_seg_size_this_txg); 3606a3874b8bSToomas Soome } else if (scn->scn_done_txg != 0 && scn->scn_done_txg <= tx->tx_txg) { 3607a3874b8bSToomas Soome /* Finished with everything. Mark the scrub as complete */ 3608a3874b8bSToomas Soome zfs_dbgmsg("scan issuing complete txg %llu", 3609a3874b8bSToomas Soome (longlong_t)tx->tx_txg); 3610a3874b8bSToomas Soome ASSERT3U(scn->scn_done_txg, !=, 0); 3611a3874b8bSToomas Soome ASSERT0(spa->spa_scrub_inflight); 3612a3874b8bSToomas Soome ASSERT0(scn->scn_bytes_pending); 3613a3874b8bSToomas Soome dsl_scan_done(scn, B_TRUE, tx); 3614a3874b8bSToomas Soome sync_type = SYNC_MANDATORY; 36153f9d6ad7SLin Ling } 36163f9d6ad7SLin Ling 3617a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, sync_type); 36183f9d6ad7SLin Ling } 36193f9d6ad7SLin Ling 36203f9d6ad7SLin Ling static void 3621a3874b8bSToomas Soome count_block(dsl_scan_t *scn, zfs_all_blkstats_t *zab, const blkptr_t *bp) 36223f9d6ad7SLin Ling { 36233f9d6ad7SLin Ling int i; 36243f9d6ad7SLin Ling 3625ee2f9ca4SBill Sommerfeld /* 3626ee2f9ca4SBill Sommerfeld * Don't count embedded bp's, since we already did the work of 3627ee2f9ca4SBill Sommerfeld * scanning these when we scanned the containing block. 3628ee2f9ca4SBill Sommerfeld */ 3629ee2f9ca4SBill Sommerfeld if (BP_IS_EMBEDDED(bp)) 3630ee2f9ca4SBill Sommerfeld return; 3631ee2f9ca4SBill Sommerfeld 363212a8814cSTom Caputi /* 363312a8814cSTom Caputi * Update the spa's stats on how many bytes we have issued. 363412a8814cSTom Caputi * Sequential scrubs create a zio for each DVA of the bp. Each 363512a8814cSTom Caputi * of these will include all DVAs for repair purposes, but the 363612a8814cSTom Caputi * zio code will only try the first one unless there is an issue. 363712a8814cSTom Caputi * Therefore, we should only count the first DVA for these IOs. 363812a8814cSTom Caputi */ 363912a8814cSTom Caputi if (scn->scn_is_sorted) { 3640a3874b8bSToomas Soome atomic_add_64(&scn->scn_dp->dp_spa->spa_scan_pass_issued, 364112a8814cSTom Caputi DVA_GET_ASIZE(&bp->blk_dva[0])); 364212a8814cSTom Caputi } else { 364312a8814cSTom Caputi spa_t *spa = scn->scn_dp->dp_spa; 364412a8814cSTom Caputi 364512a8814cSTom Caputi for (i = 0; i < BP_GET_NDVAS(bp); i++) { 364612a8814cSTom Caputi atomic_add_64(&spa->spa_scan_pass_issued, 364712a8814cSTom Caputi DVA_GET_ASIZE(&bp->blk_dva[i])); 364812a8814cSTom Caputi } 3649a3874b8bSToomas Soome } 3650a3874b8bSToomas Soome 36513f9d6ad7SLin Ling /* 36523f9d6ad7SLin Ling * If we resume after a reboot, zab will be NULL; don't record 36533f9d6ad7SLin Ling * incomplete stats in that case. 36543f9d6ad7SLin Ling */ 36553f9d6ad7SLin Ling if (zab == NULL) 36563f9d6ad7SLin Ling return; 36573f9d6ad7SLin Ling 3658a3874b8bSToomas Soome mutex_enter(&zab->zab_lock); 3659a3874b8bSToomas Soome 36603f9d6ad7SLin Ling for (i = 0; i < 4; i++) { 36613f9d6ad7SLin Ling int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS; 36623f9d6ad7SLin Ling int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL; 3663ad135b5dSChristopher Siden if (t & DMU_OT_NEWTYPE) 3664ad135b5dSChristopher Siden t = DMU_OT_OTHER; 36653f9d6ad7SLin Ling zfs_blkstat_t *zb = &zab->zab_type[l][t]; 36663f9d6ad7SLin Ling int equal; 36673f9d6ad7SLin Ling 36683f9d6ad7SLin Ling zb->zb_count++; 36693f9d6ad7SLin Ling zb->zb_asize += BP_GET_ASIZE(bp); 36703f9d6ad7SLin Ling zb->zb_lsize += BP_GET_LSIZE(bp); 36713f9d6ad7SLin Ling zb->zb_psize += BP_GET_PSIZE(bp); 36723f9d6ad7SLin Ling zb->zb_gangs += BP_COUNT_GANG(bp); 36733f9d6ad7SLin Ling 36743f9d6ad7SLin Ling switch (BP_GET_NDVAS(bp)) { 36753f9d6ad7SLin Ling case 2: 36763f9d6ad7SLin Ling if (DVA_GET_VDEV(&bp->blk_dva[0]) == 36773f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[1])) 36783f9d6ad7SLin Ling zb->zb_ditto_2_of_2_samevdev++; 36793f9d6ad7SLin Ling break; 36803f9d6ad7SLin Ling case 3: 36813f9d6ad7SLin Ling equal = (DVA_GET_VDEV(&bp->blk_dva[0]) == 36823f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[1])) + 36833f9d6ad7SLin Ling (DVA_GET_VDEV(&bp->blk_dva[0]) == 36843f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[2])) + 36853f9d6ad7SLin Ling (DVA_GET_VDEV(&bp->blk_dva[1]) == 36863f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[2])); 36873f9d6ad7SLin Ling if (equal == 1) 36883f9d6ad7SLin Ling zb->zb_ditto_2_of_3_samevdev++; 36893f9d6ad7SLin Ling else if (equal == 3) 36903f9d6ad7SLin Ling zb->zb_ditto_3_of_3_samevdev++; 36913f9d6ad7SLin Ling break; 36923f9d6ad7SLin Ling } 36933f9d6ad7SLin Ling } 3694a3874b8bSToomas Soome 3695a3874b8bSToomas Soome mutex_exit(&zab->zab_lock); 36963f9d6ad7SLin Ling } 36973f9d6ad7SLin Ling 36983f9d6ad7SLin Ling static void 3699a3874b8bSToomas Soome scan_io_queue_insert_impl(dsl_scan_io_queue_t *queue, scan_io_t *sio) 37003f9d6ad7SLin Ling { 3701a3874b8bSToomas Soome avl_index_t idx; 370212a8814cSTom Caputi int64_t asize = SIO_GET_ASIZE(sio); 3703a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 37043f9d6ad7SLin Ling 3705a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 37063f9d6ad7SLin Ling 3707a3874b8bSToomas Soome if (avl_find(&queue->q_sios_by_addr, sio, &idx) != NULL) { 3708a3874b8bSToomas Soome /* block is already scheduled for reading */ 3709a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -asize); 371012a8814cSTom Caputi sio_free(sio); 3711a3874b8bSToomas Soome return; 3712a3874b8bSToomas Soome } 3713a3874b8bSToomas Soome avl_insert(&queue->q_sios_by_addr, sio, idx); 371412a8814cSTom Caputi queue->q_sio_memused += SIO_GET_MUSED(sio); 371512a8814cSTom Caputi range_tree_add(queue->q_exts_by_addr, SIO_GET_OFFSET(sio), asize); 3716a3874b8bSToomas Soome } 37173f9d6ad7SLin Ling 3718a3874b8bSToomas Soome /* 3719a3874b8bSToomas Soome * Given all the info we got from our metadata scanning process, we 3720a3874b8bSToomas Soome * construct a scan_io_t and insert it into the scan sorting queue. The 3721a3874b8bSToomas Soome * I/O must already be suitable for us to process. This is controlled 3722a3874b8bSToomas Soome * by dsl_scan_enqueue(). 3723a3874b8bSToomas Soome */ 3724a3874b8bSToomas Soome static void 3725a3874b8bSToomas Soome scan_io_queue_insert(dsl_scan_io_queue_t *queue, const blkptr_t *bp, int dva_i, 3726a3874b8bSToomas Soome int zio_flags, const zbookmark_phys_t *zb) 3727a3874b8bSToomas Soome { 3728a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 372912a8814cSTom Caputi scan_io_t *sio = sio_alloc(BP_GET_NDVAS(bp)); 3730a3874b8bSToomas Soome 3731a3874b8bSToomas Soome ASSERT0(BP_IS_GANG(bp)); 3732a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 3733a3874b8bSToomas Soome 3734a3874b8bSToomas Soome bp2sio(bp, sio, dva_i); 3735a3874b8bSToomas Soome sio->sio_flags = zio_flags; 3736a3874b8bSToomas Soome sio->sio_zb = *zb; 3737a3874b8bSToomas Soome 3738a3874b8bSToomas Soome /* 3739a3874b8bSToomas Soome * Increment the bytes pending counter now so that we can't 3740a3874b8bSToomas Soome * get an integer underflow in case the worker processes the 3741a3874b8bSToomas Soome * zio before we get to incrementing this counter. 3742a3874b8bSToomas Soome */ 374312a8814cSTom Caputi atomic_add_64(&scn->scn_bytes_pending, SIO_GET_ASIZE(sio)); 3744a3874b8bSToomas Soome 3745a3874b8bSToomas Soome scan_io_queue_insert_impl(queue, sio); 3746a3874b8bSToomas Soome } 3747a3874b8bSToomas Soome 3748a3874b8bSToomas Soome /* 3749a3874b8bSToomas Soome * Given a set of I/O parameters as discovered by the metadata traversal 3750a3874b8bSToomas Soome * process, attempts to place the I/O into the sorted queues (if allowed), 3751a3874b8bSToomas Soome * or immediately executes the I/O. 3752a3874b8bSToomas Soome */ 3753a3874b8bSToomas Soome static void 3754a3874b8bSToomas Soome dsl_scan_enqueue(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags, 3755a3874b8bSToomas Soome const zbookmark_phys_t *zb) 3756a3874b8bSToomas Soome { 3757a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 3758a3874b8bSToomas Soome 3759a3874b8bSToomas Soome ASSERT(!BP_IS_EMBEDDED(bp)); 3760a3874b8bSToomas Soome 3761a3874b8bSToomas Soome /* 3762a3874b8bSToomas Soome * Gang blocks are hard to issue sequentially, so we just issue them 3763a3874b8bSToomas Soome * here immediately instead of queuing them. 3764a3874b8bSToomas Soome */ 3765a3874b8bSToomas Soome if (!dp->dp_scan->scn_is_sorted || BP_IS_GANG(bp)) { 3766a3874b8bSToomas Soome scan_exec_io(dp, bp, zio_flags, zb, NULL); 3767a3874b8bSToomas Soome return; 3768a3874b8bSToomas Soome } 3769a3874b8bSToomas Soome for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 3770a3874b8bSToomas Soome dva_t dva; 3771a3874b8bSToomas Soome vdev_t *vdev; 3772a3874b8bSToomas Soome 3773a3874b8bSToomas Soome dva = bp->blk_dva[i]; 3774a3874b8bSToomas Soome vdev = vdev_lookup_top(spa, DVA_GET_VDEV(&dva)); 3775a3874b8bSToomas Soome ASSERT(vdev != NULL); 3776a3874b8bSToomas Soome 3777a3874b8bSToomas Soome mutex_enter(&vdev->vdev_scan_io_queue_lock); 3778a3874b8bSToomas Soome if (vdev->vdev_scan_io_queue == NULL) 3779a3874b8bSToomas Soome vdev->vdev_scan_io_queue = scan_io_queue_create(vdev); 3780a3874b8bSToomas Soome ASSERT(dp->dp_scan != NULL); 3781a3874b8bSToomas Soome scan_io_queue_insert(vdev->vdev_scan_io_queue, bp, 3782a3874b8bSToomas Soome i, zio_flags, zb); 3783a3874b8bSToomas Soome mutex_exit(&vdev->vdev_scan_io_queue_lock); 37843f9d6ad7SLin Ling } 37853f9d6ad7SLin Ling } 37863f9d6ad7SLin Ling 37873f9d6ad7SLin Ling static int 37883f9d6ad7SLin Ling dsl_scan_scrub_cb(dsl_pool_t *dp, 37897802d7bfSMatthew Ahrens const blkptr_t *bp, const zbookmark_phys_t *zb) 37903f9d6ad7SLin Ling { 37913f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 37923f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 37933f9d6ad7SLin Ling uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp); 3794a3874b8bSToomas Soome size_t psize = BP_GET_PSIZE(bp); 37953f9d6ad7SLin Ling boolean_t needs_io; 379644ecc532SGeorge Wilson int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL; 3797a3874b8bSToomas Soome int d; 3798dec267e7SMatthew Ahrens 37993f9d6ad7SLin Ling if (phys_birth <= scn->scn_phys.scn_min_txg || 3800a3874b8bSToomas Soome phys_birth >= scn->scn_phys.scn_max_txg) { 3801a3874b8bSToomas Soome count_block(scn, dp->dp_blkstats, bp); 38023f9d6ad7SLin Ling return (0); 3803a3874b8bSToomas Soome } 38043f9d6ad7SLin Ling 3805dec267e7SMatthew Ahrens /* Embedded BP's have phys_birth==0, so we reject them above. */ 3806dec267e7SMatthew Ahrens ASSERT(!BP_IS_EMBEDDED(bp)); 38075d7b4d43SMatthew Ahrens 38083f9d6ad7SLin Ling ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn)); 38093f9d6ad7SLin Ling if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) { 38103f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_SCRUB; 38113f9d6ad7SLin Ling needs_io = B_TRUE; 3812d5285caeSGeorge Wilson } else { 3813d5285caeSGeorge Wilson ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER); 38143f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_RESILVER; 38153f9d6ad7SLin Ling needs_io = B_FALSE; 38163f9d6ad7SLin Ling } 38173f9d6ad7SLin Ling 38183f9d6ad7SLin Ling /* If it's an intent log block, failure is expected. */ 38193f9d6ad7SLin Ling if (zb->zb_level == ZB_ZIL_LEVEL) 38203f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_SPECULATIVE; 38213f9d6ad7SLin Ling 3822a3874b8bSToomas Soome for (d = 0; d < BP_GET_NDVAS(bp); d++) { 3823a3874b8bSToomas Soome const dva_t *dva = &bp->blk_dva[d]; 38243f9d6ad7SLin Ling 38253f9d6ad7SLin Ling /* 38263f9d6ad7SLin Ling * Keep track of how much data we've examined so that 38273f9d6ad7SLin Ling * zpool(1M) status can make useful progress reports. 38283f9d6ad7SLin Ling */ 3829a3874b8bSToomas Soome scn->scn_phys.scn_examined += DVA_GET_ASIZE(dva); 3830a3874b8bSToomas Soome spa->spa_scan_pass_exam += DVA_GET_ASIZE(dva); 38313f9d6ad7SLin Ling 38323f9d6ad7SLin Ling /* if it's a resilver, this may not be in the target range */ 3833a3874b8bSToomas Soome if (!needs_io) 3834a3874b8bSToomas Soome needs_io = dsl_scan_need_resilver(spa, dva, psize, 3835a3874b8bSToomas Soome phys_birth); 38363f9d6ad7SLin Ling } 38373f9d6ad7SLin Ling 38383f9d6ad7SLin Ling if (needs_io && !zfs_no_scrub_io) { 3839a3874b8bSToomas Soome dsl_scan_enqueue(dp, bp, zio_flags, zb); 3840a3874b8bSToomas Soome } else { 3841a3874b8bSToomas Soome count_block(scn, dp->dp_blkstats, bp); 3842a3874b8bSToomas Soome } 3843a3874b8bSToomas Soome 3844a3874b8bSToomas Soome /* do not relocate this block */ 3845a3874b8bSToomas Soome return (0); 3846a3874b8bSToomas Soome } 38473f9d6ad7SLin Ling 3848a3874b8bSToomas Soome static void 3849a3874b8bSToomas Soome dsl_scan_scrub_done(zio_t *zio) 3850a3874b8bSToomas Soome { 3851a3874b8bSToomas Soome spa_t *spa = zio->io_spa; 3852a3874b8bSToomas Soome blkptr_t *bp = zio->io_bp; 3853a3874b8bSToomas Soome dsl_scan_io_queue_t *queue = zio->io_private; 3854a3874b8bSToomas Soome 3855a3874b8bSToomas Soome abd_free(zio->io_abd); 3856a3874b8bSToomas Soome 3857a3874b8bSToomas Soome if (queue == NULL) { 38583f9d6ad7SLin Ling mutex_enter(&spa->spa_scrub_lock); 3859a3874b8bSToomas Soome ASSERT3U(spa->spa_scrub_inflight, >=, BP_GET_PSIZE(bp)); 3860a3874b8bSToomas Soome spa->spa_scrub_inflight -= BP_GET_PSIZE(bp); 3861a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 3862a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 3863a3874b8bSToomas Soome } else { 3864a3874b8bSToomas Soome mutex_enter(&queue->q_vd->vdev_scan_io_queue_lock); 3865a3874b8bSToomas Soome ASSERT3U(queue->q_inflight_bytes, >=, BP_GET_PSIZE(bp)); 3866a3874b8bSToomas Soome queue->q_inflight_bytes -= BP_GET_PSIZE(bp); 3867a3874b8bSToomas Soome cv_broadcast(&queue->q_zio_cv); 3868a3874b8bSToomas Soome mutex_exit(&queue->q_vd->vdev_scan_io_queue_lock); 3869a3874b8bSToomas Soome } 3870a3874b8bSToomas Soome 3871a3874b8bSToomas Soome if (zio->io_error && (zio->io_error != ECKSUM || 3872a3874b8bSToomas Soome !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) { 3873a3874b8bSToomas Soome atomic_inc_64(&spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors); 3874a3874b8bSToomas Soome } 3875a3874b8bSToomas Soome } 3876a3874b8bSToomas Soome 3877a3874b8bSToomas Soome /* 3878a3874b8bSToomas Soome * Given a scanning zio's information, executes the zio. The zio need 3879a3874b8bSToomas Soome * not necessarily be only sortable, this function simply executes the 3880a3874b8bSToomas Soome * zio, no matter what it is. The optional queue argument allows the 3881a3874b8bSToomas Soome * caller to specify that they want per top level vdev IO rate limiting 3882a3874b8bSToomas Soome * instead of the legacy global limiting. 3883a3874b8bSToomas Soome */ 3884a3874b8bSToomas Soome static void 3885a3874b8bSToomas Soome scan_exec_io(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags, 3886a3874b8bSToomas Soome const zbookmark_phys_t *zb, dsl_scan_io_queue_t *queue) 3887a3874b8bSToomas Soome { 3888a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 3889a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 3890a3874b8bSToomas Soome size_t size = BP_GET_PSIZE(bp); 3891a3874b8bSToomas Soome abd_t *data = abd_alloc_for_io(size, B_FALSE); 3892a3874b8bSToomas Soome 3893a3874b8bSToomas Soome if (queue == NULL) { 3894a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 3895a3874b8bSToomas Soome while (spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes) 38963f9d6ad7SLin Ling cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 3897a3874b8bSToomas Soome spa->spa_scrub_inflight += BP_GET_PSIZE(bp); 38983f9d6ad7SLin Ling mutex_exit(&spa->spa_scrub_lock); 3899a3874b8bSToomas Soome } else { 3900a3874b8bSToomas Soome kmutex_t *q_lock = &queue->q_vd->vdev_scan_io_queue_lock; 39013f9d6ad7SLin Ling 3902a3874b8bSToomas Soome mutex_enter(q_lock); 3903a3874b8bSToomas Soome while (queue->q_inflight_bytes >= queue->q_maxinflight_bytes) 3904a3874b8bSToomas Soome cv_wait(&queue->q_zio_cv, q_lock); 3905a3874b8bSToomas Soome queue->q_inflight_bytes += BP_GET_PSIZE(bp); 3906a3874b8bSToomas Soome mutex_exit(q_lock); 3907a3874b8bSToomas Soome } 3908a3874b8bSToomas Soome 3909a3874b8bSToomas Soome count_block(dp->dp_scan, dp->dp_blkstats, bp); 3910a3874b8bSToomas Soome zio_nowait(zio_read(dp->dp_scan->scn_zio_root, spa, bp, data, size, 3911a3874b8bSToomas Soome dsl_scan_scrub_done, queue, ZIO_PRIORITY_SCRUB, zio_flags, zb)); 3912a3874b8bSToomas Soome } 391344ecc532SGeorge Wilson 3914a3874b8bSToomas Soome /* 3915a3874b8bSToomas Soome * This is the primary extent sorting algorithm. We balance two parameters: 3916a3874b8bSToomas Soome * 1) how many bytes of I/O are in an extent 3917a3874b8bSToomas Soome * 2) how well the extent is filled with I/O (as a fraction of its total size) 3918a3874b8bSToomas Soome * Since we allow extents to have gaps between their constituent I/Os, it's 3919a3874b8bSToomas Soome * possible to have a fairly large extent that contains the same amount of 3920a3874b8bSToomas Soome * I/O bytes than a much smaller extent, which just packs the I/O more tightly. 3921a3874b8bSToomas Soome * The algorithm sorts based on a score calculated from the extent's size, 3922a3874b8bSToomas Soome * the relative fill volume (in %) and a "fill weight" parameter that controls 3923a3874b8bSToomas Soome * the split between whether we prefer larger extents or more well populated 3924a3874b8bSToomas Soome * extents: 3925a3874b8bSToomas Soome * 3926a3874b8bSToomas Soome * SCORE = FILL_IN_BYTES + (FILL_IN_PERCENT * FILL_IN_BYTES * FILL_WEIGHT) 3927a3874b8bSToomas Soome * 3928a3874b8bSToomas Soome * Example: 3929a3874b8bSToomas Soome * 1) assume extsz = 64 MiB 3930a3874b8bSToomas Soome * 2) assume fill = 32 MiB (extent is half full) 3931a3874b8bSToomas Soome * 3) assume fill_weight = 3 3932a3874b8bSToomas Soome * 4) SCORE = 32M + (((32M * 100) / 64M) * 3 * 32M) / 100 3933a3874b8bSToomas Soome * SCORE = 32M + (50 * 3 * 32M) / 100 3934a3874b8bSToomas Soome * SCORE = 32M + (4800M / 100) 3935a3874b8bSToomas Soome * SCORE = 32M + 48M 3936a3874b8bSToomas Soome * ^ ^ 3937a3874b8bSToomas Soome * | +--- final total relative fill-based score 3938a3874b8bSToomas Soome * +--------- final total fill-based score 3939a3874b8bSToomas Soome * SCORE = 80M 3940a3874b8bSToomas Soome * 3941a3874b8bSToomas Soome * As can be seen, at fill_ratio=3, the algorithm is slightly biased towards 3942a3874b8bSToomas Soome * extents that are more completely filled (in a 3:2 ratio) vs just larger. 3943a3874b8bSToomas Soome * Note that as an optimization, we replace multiplication and division by 3944a3874b8bSToomas Soome * 100 with bitshifting by 7 (which effecitvely multiplies and divides by 128). 3945a3874b8bSToomas Soome */ 3946a3874b8bSToomas Soome static int 3947a3874b8bSToomas Soome ext_size_compare(const void *x, const void *y) 3948a3874b8bSToomas Soome { 3949a3874b8bSToomas Soome const range_seg_t *rsa = x, *rsb = y; 3950a3874b8bSToomas Soome uint64_t sa = rsa->rs_end - rsa->rs_start, 3951a3874b8bSToomas Soome sb = rsb->rs_end - rsb->rs_start; 3952a3874b8bSToomas Soome uint64_t score_a, score_b; 3953a3874b8bSToomas Soome 3954a3874b8bSToomas Soome score_a = rsa->rs_fill + ((((rsa->rs_fill << 7) / sa) * 3955a3874b8bSToomas Soome fill_weight * rsa->rs_fill) >> 7); 3956a3874b8bSToomas Soome score_b = rsb->rs_fill + ((((rsb->rs_fill << 7) / sb) * 3957a3874b8bSToomas Soome fill_weight * rsb->rs_fill) >> 7); 3958a3874b8bSToomas Soome 3959a3874b8bSToomas Soome if (score_a > score_b) 3960a3874b8bSToomas Soome return (-1); 3961a3874b8bSToomas Soome if (score_a == score_b) { 3962a3874b8bSToomas Soome if (rsa->rs_start < rsb->rs_start) 3963a3874b8bSToomas Soome return (-1); 3964a3874b8bSToomas Soome if (rsa->rs_start == rsb->rs_start) 3965a3874b8bSToomas Soome return (0); 3966a3874b8bSToomas Soome return (1); 39673f9d6ad7SLin Ling } 3968a3874b8bSToomas Soome return (1); 3969a3874b8bSToomas Soome } 39703f9d6ad7SLin Ling 3971a3874b8bSToomas Soome /* 3972a3874b8bSToomas Soome * Comparator for the q_sios_by_addr tree. Sorting is simply performed 3973a3874b8bSToomas Soome * based on LBA-order (from lowest to highest). 3974a3874b8bSToomas Soome */ 3975a3874b8bSToomas Soome static int 397612a8814cSTom Caputi sio_addr_compare(const void *x, const void *y) 3977a3874b8bSToomas Soome { 3978a3874b8bSToomas Soome const scan_io_t *a = x, *b = y; 3979a3874b8bSToomas Soome 398012a8814cSTom Caputi return (AVL_CMP(SIO_GET_OFFSET(a), SIO_GET_OFFSET(b))); 3981a3874b8bSToomas Soome } 3982a3874b8bSToomas Soome 3983a3874b8bSToomas Soome /* IO queues are created on demand when they are needed. */ 3984a3874b8bSToomas Soome static dsl_scan_io_queue_t * 3985a3874b8bSToomas Soome scan_io_queue_create(vdev_t *vd) 3986a3874b8bSToomas Soome { 3987a3874b8bSToomas Soome dsl_scan_t *scn = vd->vdev_spa->spa_dsl_pool->dp_scan; 3988a3874b8bSToomas Soome dsl_scan_io_queue_t *q = kmem_zalloc(sizeof (*q), KM_SLEEP); 3989a3874b8bSToomas Soome 3990a3874b8bSToomas Soome q->q_scn = scn; 3991a3874b8bSToomas Soome q->q_vd = vd; 399212a8814cSTom Caputi q->q_sio_memused = 0; 3993a3874b8bSToomas Soome cv_init(&q->q_zio_cv, NULL, CV_DEFAULT, NULL); 3994a3874b8bSToomas Soome q->q_exts_by_addr = range_tree_create_impl(&rt_avl_ops, 3995a3874b8bSToomas Soome &q->q_exts_by_size, ext_size_compare, zfs_scan_max_ext_gap); 399612a8814cSTom Caputi avl_create(&q->q_sios_by_addr, sio_addr_compare, 3997a3874b8bSToomas Soome sizeof (scan_io_t), offsetof(scan_io_t, sio_nodes.sio_addr_node)); 3998a3874b8bSToomas Soome 3999a3874b8bSToomas Soome return (q); 40003f9d6ad7SLin Ling } 40013f9d6ad7SLin Ling 40021702cce7SAlek Pinchuk /* 4003a3874b8bSToomas Soome * Destroys a scan queue and all segments and scan_io_t's contained in it. 4004a3874b8bSToomas Soome * No further execution of I/O occurs, anything pending in the queue is 4005a3874b8bSToomas Soome * simply freed without being executed. 40061702cce7SAlek Pinchuk */ 4007a3874b8bSToomas Soome void 4008a3874b8bSToomas Soome dsl_scan_io_queue_destroy(dsl_scan_io_queue_t *queue) 40093f9d6ad7SLin Ling { 4010a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 4011a3874b8bSToomas Soome scan_io_t *sio; 4012a3874b8bSToomas Soome void *cookie = NULL; 4013a3874b8bSToomas Soome int64_t bytes_dequeued = 0; 4014a3874b8bSToomas Soome 4015a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 4016a3874b8bSToomas Soome 4017a3874b8bSToomas Soome while ((sio = avl_destroy_nodes(&queue->q_sios_by_addr, &cookie)) != 4018a3874b8bSToomas Soome NULL) { 4019a3874b8bSToomas Soome ASSERT(range_tree_contains(queue->q_exts_by_addr, 402012a8814cSTom Caputi SIO_GET_OFFSET(sio), SIO_GET_ASIZE(sio))); 402112a8814cSTom Caputi bytes_dequeued += SIO_GET_ASIZE(sio); 402212a8814cSTom Caputi queue->q_sio_memused -= SIO_GET_MUSED(sio); 402312a8814cSTom Caputi sio_free(sio); 4024a3874b8bSToomas Soome } 4025a3874b8bSToomas Soome 402612a8814cSTom Caputi ASSERT0(queue->q_sio_memused); 4027a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -bytes_dequeued); 4028a3874b8bSToomas Soome range_tree_vacate(queue->q_exts_by_addr, NULL, queue); 4029a3874b8bSToomas Soome range_tree_destroy(queue->q_exts_by_addr); 4030a3874b8bSToomas Soome avl_destroy(&queue->q_sios_by_addr); 4031a3874b8bSToomas Soome cv_destroy(&queue->q_zio_cv); 4032a3874b8bSToomas Soome 4033a3874b8bSToomas Soome kmem_free(queue, sizeof (*queue)); 4034a3874b8bSToomas Soome } 4035a3874b8bSToomas Soome 4036a3874b8bSToomas Soome /* 4037a3874b8bSToomas Soome * Properly transfers a dsl_scan_queue_t from `svd' to `tvd'. This is 4038a3874b8bSToomas Soome * called on behalf of vdev_top_transfer when creating or destroying 4039a3874b8bSToomas Soome * a mirror vdev due to zpool attach/detach. 4040a3874b8bSToomas Soome */ 4041a3874b8bSToomas Soome void 4042a3874b8bSToomas Soome dsl_scan_io_queue_vdev_xfer(vdev_t *svd, vdev_t *tvd) 4043a3874b8bSToomas Soome { 4044a3874b8bSToomas Soome mutex_enter(&svd->vdev_scan_io_queue_lock); 4045a3874b8bSToomas Soome mutex_enter(&tvd->vdev_scan_io_queue_lock); 4046a3874b8bSToomas Soome 4047a3874b8bSToomas Soome VERIFY3P(tvd->vdev_scan_io_queue, ==, NULL); 4048a3874b8bSToomas Soome tvd->vdev_scan_io_queue = svd->vdev_scan_io_queue; 4049a3874b8bSToomas Soome svd->vdev_scan_io_queue = NULL; 4050a3874b8bSToomas Soome if (tvd->vdev_scan_io_queue != NULL) 4051a3874b8bSToomas Soome tvd->vdev_scan_io_queue->q_vd = tvd; 4052a3874b8bSToomas Soome 4053a3874b8bSToomas Soome mutex_exit(&tvd->vdev_scan_io_queue_lock); 4054a3874b8bSToomas Soome mutex_exit(&svd->vdev_scan_io_queue_lock); 4055a3874b8bSToomas Soome } 4056a3874b8bSToomas Soome 4057a3874b8bSToomas Soome static void 4058a3874b8bSToomas Soome scan_io_queues_destroy(dsl_scan_t *scn) 4059a3874b8bSToomas Soome { 4060a3874b8bSToomas Soome vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; 4061a3874b8bSToomas Soome 4062a3874b8bSToomas Soome for (uint64_t i = 0; i < rvd->vdev_children; i++) { 4063a3874b8bSToomas Soome vdev_t *tvd = rvd->vdev_child[i]; 4064a3874b8bSToomas Soome 4065a3874b8bSToomas Soome mutex_enter(&tvd->vdev_scan_io_queue_lock); 4066a3874b8bSToomas Soome if (tvd->vdev_scan_io_queue != NULL) 4067a3874b8bSToomas Soome dsl_scan_io_queue_destroy(tvd->vdev_scan_io_queue); 4068a3874b8bSToomas Soome tvd->vdev_scan_io_queue = NULL; 4069a3874b8bSToomas Soome mutex_exit(&tvd->vdev_scan_io_queue_lock); 4070a3874b8bSToomas Soome } 4071a3874b8bSToomas Soome } 4072a3874b8bSToomas Soome 4073a3874b8bSToomas Soome static void 4074a3874b8bSToomas Soome dsl_scan_freed_dva(spa_t *spa, const blkptr_t *bp, int dva_i) 4075a3874b8bSToomas Soome { 4076a3874b8bSToomas Soome dsl_pool_t *dp = spa->spa_dsl_pool; 40771702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 4078a3874b8bSToomas Soome vdev_t *vdev; 4079a3874b8bSToomas Soome kmutex_t *q_lock; 4080a3874b8bSToomas Soome dsl_scan_io_queue_t *queue; 408112a8814cSTom Caputi scan_io_t *srch_sio, *sio; 4082a3874b8bSToomas Soome avl_index_t idx; 4083a3874b8bSToomas Soome uint64_t start, size; 4084a3874b8bSToomas Soome 4085a3874b8bSToomas Soome vdev = vdev_lookup_top(spa, DVA_GET_VDEV(&bp->blk_dva[dva_i])); 4086a3874b8bSToomas Soome ASSERT(vdev != NULL); 4087a3874b8bSToomas Soome q_lock = &vdev->vdev_scan_io_queue_lock; 4088a3874b8bSToomas Soome queue = vdev->vdev_scan_io_queue; 4089a3874b8bSToomas Soome 4090a3874b8bSToomas Soome mutex_enter(q_lock); 4091a3874b8bSToomas Soome if (queue == NULL) { 4092a3874b8bSToomas Soome mutex_exit(q_lock); 4093a3874b8bSToomas Soome return; 4094a3874b8bSToomas Soome } 4095a3874b8bSToomas Soome 409612a8814cSTom Caputi srch_sio = sio_alloc(BP_GET_NDVAS(bp)); 409712a8814cSTom Caputi bp2sio(bp, srch_sio, dva_i); 409812a8814cSTom Caputi start = SIO_GET_OFFSET(srch_sio); 409912a8814cSTom Caputi size = SIO_GET_ASIZE(srch_sio); 41003f9d6ad7SLin Ling 41013f9d6ad7SLin Ling /* 4102a3874b8bSToomas Soome * We can find the zio in two states: 4103a3874b8bSToomas Soome * 1) Cold, just sitting in the queue of zio's to be issued at 4104a3874b8bSToomas Soome * some point in the future. In this case, all we do is 4105a3874b8bSToomas Soome * remove the zio from the q_sios_by_addr tree, decrement 4106a3874b8bSToomas Soome * its data volume from the containing range_seg_t and 4107a3874b8bSToomas Soome * resort the q_exts_by_size tree to reflect that the 4108a3874b8bSToomas Soome * range_seg_t has lost some of its 'fill'. We don't shorten 4109a3874b8bSToomas Soome * the range_seg_t - this is usually rare enough not to be 4110a3874b8bSToomas Soome * worth the extra hassle of trying keep track of precise 4111a3874b8bSToomas Soome * extent boundaries. 4112a3874b8bSToomas Soome * 2) Hot, where the zio is currently in-flight in 4113a3874b8bSToomas Soome * dsl_scan_issue_ios. In this case, we can't simply 4114a3874b8bSToomas Soome * reach in and stop the in-flight zio's, so we instead 4115a3874b8bSToomas Soome * block the caller. Eventually, dsl_scan_issue_ios will 4116a3874b8bSToomas Soome * be done with issuing the zio's it gathered and will 4117a3874b8bSToomas Soome * signal us. 41183f9d6ad7SLin Ling */ 411912a8814cSTom Caputi sio = avl_find(&queue->q_sios_by_addr, srch_sio, &idx); 412012a8814cSTom Caputi sio_free(srch_sio); 412112a8814cSTom Caputi 4122a3874b8bSToomas Soome if (sio != NULL) { 412312a8814cSTom Caputi int64_t asize = SIO_GET_ASIZE(sio); 4124a3874b8bSToomas Soome blkptr_t tmpbp; 41253f9d6ad7SLin Ling 4126a3874b8bSToomas Soome /* Got it while it was cold in the queue */ 412712a8814cSTom Caputi ASSERT3U(start, ==, SIO_GET_OFFSET(sio)); 4128a3874b8bSToomas Soome ASSERT3U(size, ==, asize); 4129a3874b8bSToomas Soome avl_remove(&queue->q_sios_by_addr, sio); 413012a8814cSTom Caputi queue->q_sio_memused -= SIO_GET_MUSED(sio); 41311702cce7SAlek Pinchuk 4132a3874b8bSToomas Soome ASSERT(range_tree_contains(queue->q_exts_by_addr, start, size)); 4133a3874b8bSToomas Soome range_tree_remove_fill(queue->q_exts_by_addr, start, size); 41341702cce7SAlek Pinchuk 4135a3874b8bSToomas Soome /* 4136a3874b8bSToomas Soome * We only update scn_bytes_pending in the cold path, 4137a3874b8bSToomas Soome * otherwise it will already have been accounted for as 4138a3874b8bSToomas Soome * part of the zio's execution. 4139a3874b8bSToomas Soome */ 4140a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -asize); 4141a3874b8bSToomas Soome 4142a3874b8bSToomas Soome /* count the block as though we issued it */ 414312a8814cSTom Caputi sio2bp(sio, &tmpbp); 4144a3874b8bSToomas Soome count_block(scn, dp->dp_blkstats, &tmpbp); 4145a3874b8bSToomas Soome 414612a8814cSTom Caputi sio_free(sio); 4147a3874b8bSToomas Soome } 4148a3874b8bSToomas Soome mutex_exit(q_lock); 41493f9d6ad7SLin Ling } 41501825bc56SNav Ravindranath 4151a3874b8bSToomas Soome /* 4152a3874b8bSToomas Soome * Callback invoked when a zio_free() zio is executing. This needs to be 4153a3874b8bSToomas Soome * intercepted to prevent the zio from deallocating a particular portion 4154a3874b8bSToomas Soome * of disk space and it then getting reallocated and written to, while we 4155a3874b8bSToomas Soome * still have it queued up for processing. 4156a3874b8bSToomas Soome */ 4157a3874b8bSToomas Soome void 4158a3874b8bSToomas Soome dsl_scan_freed(spa_t *spa, const blkptr_t *bp) 41591825bc56SNav Ravindranath { 4160a3874b8bSToomas Soome dsl_pool_t *dp = spa->spa_dsl_pool; 4161a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 4162a3874b8bSToomas Soome 4163a3874b8bSToomas Soome ASSERT(!BP_IS_EMBEDDED(bp)); 4164a3874b8bSToomas Soome ASSERT(scn != NULL); 4165a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 4166a3874b8bSToomas Soome return; 4167a3874b8bSToomas Soome 4168a3874b8bSToomas Soome for (int i = 0; i < BP_GET_NDVAS(bp); i++) 4169a3874b8bSToomas Soome dsl_scan_freed_dva(spa, bp, i); 41701825bc56SNav Ravindranath } 4171