13f9d6ad7SLin Ling /* 23f9d6ad7SLin Ling * CDDL HEADER START 33f9d6ad7SLin Ling * 43f9d6ad7SLin Ling * The contents of this file are subject to the terms of the 53f9d6ad7SLin Ling * Common Development and Distribution License (the "License"). 63f9d6ad7SLin Ling * You may not use this file except in compliance with the License. 73f9d6ad7SLin Ling * 83f9d6ad7SLin Ling * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 93f9d6ad7SLin Ling * or http://www.opensolaris.org/os/licensing. 103f9d6ad7SLin Ling * See the License for the specific language governing permissions 113f9d6ad7SLin Ling * and limitations under the License. 123f9d6ad7SLin Ling * 133f9d6ad7SLin Ling * When distributing Covered Code, include this CDDL HEADER in each 143f9d6ad7SLin Ling * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 153f9d6ad7SLin Ling * If applicable, add the following below this CDDL HEADER, with the 163f9d6ad7SLin Ling * fields enclosed by brackets "[]" replaced with your own identifying 173f9d6ad7SLin Ling * information: Portions Copyright [yyyy] [name of copyright owner] 183f9d6ad7SLin Ling * 193f9d6ad7SLin Ling * CDDL HEADER END 203f9d6ad7SLin Ling */ 213f9d6ad7SLin Ling /* 223f9d6ad7SLin Ling * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23bb1f4245SMatthew Ahrens * Copyright (c) 2011, 2018 by Delphix. All rights reserved. 248c04a1faSGary Mills * Copyright 2016 Gary Mills 255cabbc6bSPrashanth Sreenivasa * Copyright (c) 2011, 2017 by Delphix. All rights reserved. 26233f6c49SKody Kantor * Copyright 2019 Joyent, Inc. 271702cce7SAlek Pinchuk * Copyright (c) 2017 Datto Inc. 283f9d6ad7SLin Ling */ 293f9d6ad7SLin Ling 303f9d6ad7SLin Ling #include <sys/dsl_scan.h> 313f9d6ad7SLin Ling #include <sys/dsl_pool.h> 323f9d6ad7SLin Ling #include <sys/dsl_dataset.h> 333f9d6ad7SLin Ling #include <sys/dsl_prop.h> 343f9d6ad7SLin Ling #include <sys/dsl_dir.h> 353f9d6ad7SLin Ling #include <sys/dsl_synctask.h> 363f9d6ad7SLin Ling #include <sys/dnode.h> 373f9d6ad7SLin Ling #include <sys/dmu_tx.h> 383f9d6ad7SLin Ling #include <sys/dmu_objset.h> 393f9d6ad7SLin Ling #include <sys/arc.h> 403f9d6ad7SLin Ling #include <sys/zap.h> 413f9d6ad7SLin Ling #include <sys/zio.h> 423f9d6ad7SLin Ling #include <sys/zfs_context.h> 433f9d6ad7SLin Ling #include <sys/fs/zfs.h> 443f9d6ad7SLin Ling #include <sys/zfs_znode.h> 453f9d6ad7SLin Ling #include <sys/spa_impl.h> 463f9d6ad7SLin Ling #include <sys/vdev_impl.h> 473f9d6ad7SLin Ling #include <sys/zil_impl.h> 483f9d6ad7SLin Ling #include <sys/zio_checksum.h> 493f9d6ad7SLin Ling #include <sys/ddt.h> 503f9d6ad7SLin Ling #include <sys/sa.h> 513f9d6ad7SLin Ling #include <sys/sa_impl.h> 52ad135b5dSChristopher Siden #include <sys/zfeature.h> 53770499e1SDan Kimmel #include <sys/abd.h> 54a3874b8bSToomas Soome #include <sys/range_tree.h> 553f9d6ad7SLin Ling #ifdef _KERNEL 563f9d6ad7SLin Ling #include <sys/zfs_vfsops.h> 573f9d6ad7SLin Ling #endif 583f9d6ad7SLin Ling 59a3874b8bSToomas Soome /* 60a3874b8bSToomas Soome * Grand theory statement on scan queue sorting 61a3874b8bSToomas Soome * 62a3874b8bSToomas Soome * Scanning is implemented by recursively traversing all indirection levels 63a3874b8bSToomas Soome * in an object and reading all blocks referenced from said objects. This 64a3874b8bSToomas Soome * results in us approximately traversing the object from lowest logical 65a3874b8bSToomas Soome * offset to the highest. For best performance, we would want the logical 66a3874b8bSToomas Soome * blocks to be physically contiguous. However, this is frequently not the 67a3874b8bSToomas Soome * case with pools given the allocation patterns of copy-on-write filesystems. 68a3874b8bSToomas Soome * So instead, we put the I/Os into a reordering queue and issue them in a 69a3874b8bSToomas Soome * way that will most benefit physical disks (LBA-order). 70a3874b8bSToomas Soome * 71a3874b8bSToomas Soome * Queue management: 72a3874b8bSToomas Soome * 73a3874b8bSToomas Soome * Ideally, we would want to scan all metadata and queue up all block I/O 74a3874b8bSToomas Soome * prior to starting to issue it, because that allows us to do an optimal 75a3874b8bSToomas Soome * sorting job. This can however consume large amounts of memory. Therefore 76a3874b8bSToomas Soome * we continuously monitor the size of the queues and constrain them to 5% 77a3874b8bSToomas Soome * (zfs_scan_mem_lim_fact) of physmem. If the queues grow larger than this 78a3874b8bSToomas Soome * limit, we clear out a few of the largest extents at the head of the queues 79a3874b8bSToomas Soome * to make room for more scanning. Hopefully, these extents will be fairly 80a3874b8bSToomas Soome * large and contiguous, allowing us to approach sequential I/O throughput 81a3874b8bSToomas Soome * even without a fully sorted tree. 82a3874b8bSToomas Soome * 83a3874b8bSToomas Soome * Metadata scanning takes place in dsl_scan_visit(), which is called from 84a3874b8bSToomas Soome * dsl_scan_sync() every spa_sync(). If we have either fully scanned all 85a3874b8bSToomas Soome * metadata on the pool, or we need to make room in memory because our 86a3874b8bSToomas Soome * queues are too large, dsl_scan_visit() is postponed and 87a3874b8bSToomas Soome * scan_io_queues_run() is called from dsl_scan_sync() instead. This implies 88a3874b8bSToomas Soome * that metadata scanning and queued I/O issuing are mutually exclusive. This 89a3874b8bSToomas Soome * allows us to provide maximum sequential I/O throughput for the majority of 90a3874b8bSToomas Soome * I/O's issued since sequential I/O performance is significantly negatively 91a3874b8bSToomas Soome * impacted if it is interleaved with random I/O. 92a3874b8bSToomas Soome * 93a3874b8bSToomas Soome * Implementation Notes 94a3874b8bSToomas Soome * 95a3874b8bSToomas Soome * One side effect of the queued scanning algorithm is that the scanning code 96a3874b8bSToomas Soome * needs to be notified whenever a block is freed. This is needed to allow 97a3874b8bSToomas Soome * the scanning code to remove these I/Os from the issuing queue. Additionally, 98a3874b8bSToomas Soome * we do not attempt to queue gang blocks to be issued sequentially since this 99a3874b8bSToomas Soome * is very hard to do and would have an extremely limited performance benefit. 100a3874b8bSToomas Soome * Instead, we simply issue gang I/Os as soon as we find them using the legacy 101a3874b8bSToomas Soome * algorithm. 102a3874b8bSToomas Soome * 103a3874b8bSToomas Soome * Backwards compatibility 104a3874b8bSToomas Soome * 105a3874b8bSToomas Soome * This new algorithm is backwards compatible with the legacy on-disk data 106a3874b8bSToomas Soome * structures (and therefore does not require a new feature flag). 107a3874b8bSToomas Soome * Periodically during scanning (see zfs_scan_checkpoint_intval), the scan 108a3874b8bSToomas Soome * will stop scanning metadata (in logical order) and wait for all outstanding 109a3874b8bSToomas Soome * sorted I/O to complete. Once this is done, we write out a checkpoint 110a3874b8bSToomas Soome * bookmark, indicating that we have scanned everything logically before it. 111a3874b8bSToomas Soome * If the pool is imported on a machine without the new sorting algorithm, 112a3874b8bSToomas Soome * the scan simply resumes from the last checkpoint using the legacy algorithm. 113a3874b8bSToomas Soome */ 114a3874b8bSToomas Soome 1157802d7bfSMatthew Ahrens typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, 1167802d7bfSMatthew Ahrens const zbookmark_phys_t *); 1173f9d6ad7SLin Ling 1183f9d6ad7SLin Ling static scan_cb_t dsl_scan_scrub_cb; 1193f9d6ad7SLin Ling 120a3874b8bSToomas Soome static int scan_ds_queue_compare(const void *a, const void *b); 121a3874b8bSToomas Soome static int scan_prefetch_queue_compare(const void *a, const void *b); 122a3874b8bSToomas Soome static void scan_ds_queue_clear(dsl_scan_t *scn); 123a3874b8bSToomas Soome static boolean_t scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj, 124a3874b8bSToomas Soome uint64_t *txg); 125a3874b8bSToomas Soome static void scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg); 126a3874b8bSToomas Soome static void scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj); 127a3874b8bSToomas Soome static void scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx); 128a3874b8bSToomas Soome 129a3874b8bSToomas Soome extern int zfs_vdev_async_write_active_min_dirty_percent; 130a3874b8bSToomas Soome 131a3874b8bSToomas Soome /* 132a3874b8bSToomas Soome * By default zfs will check to ensure it is not over the hard memory 133a3874b8bSToomas Soome * limit before each txg. If finer-grained control of this is needed 134a3874b8bSToomas Soome * this value can be set to 1 to enable checking before scanning each 135a3874b8bSToomas Soome * block. 136a3874b8bSToomas Soome */ 137a3874b8bSToomas Soome int zfs_scan_strict_mem_lim = B_FALSE; 138a3874b8bSToomas Soome 139a3874b8bSToomas Soome /* 140a3874b8bSToomas Soome * Maximum number of parallelly executing I/Os per top-level vdev. 141a3874b8bSToomas Soome * Tune with care. Very high settings (hundreds) are known to trigger 142a3874b8bSToomas Soome * some firmware bugs and resets on certain SSDs. 143a3874b8bSToomas Soome */ 14444ecc532SGeorge Wilson int zfs_top_maxinflight = 32; /* maximum I/Os per top-level */ 145a3874b8bSToomas Soome unsigned int zfs_resilver_delay = 2; /* number of ticks to delay resilver */ 146a3874b8bSToomas Soome unsigned int zfs_scrub_delay = 4; /* number of ticks to delay scrub */ 147a3874b8bSToomas Soome unsigned int zfs_scan_idle = 50; /* idle window in clock ticks */ 148a3874b8bSToomas Soome 149a3874b8bSToomas Soome /* 150a3874b8bSToomas Soome * Maximum number of parallelly executed bytes per leaf vdev. We attempt 151a3874b8bSToomas Soome * to strike a balance here between keeping the vdev queues full of I/Os 152a3874b8bSToomas Soome * at all times and not overflowing the queues to cause long latency, 153a3874b8bSToomas Soome * which would cause long txg sync times. No matter what, we will not 154a3874b8bSToomas Soome * overload the drives with I/O, since that is protected by 155a3874b8bSToomas Soome * zfs_vdev_scrub_max_active. 156a3874b8bSToomas Soome */ 157a3874b8bSToomas Soome unsigned long zfs_scan_vdev_limit = 4 << 20; 158a3874b8bSToomas Soome 159a3874b8bSToomas Soome int zfs_scan_issue_strategy = 0; 160a3874b8bSToomas Soome int zfs_scan_legacy = B_FALSE; /* don't queue & sort zios, go direct */ 161a3874b8bSToomas Soome uint64_t zfs_scan_max_ext_gap = 2 << 20; /* in bytes */ 162a3874b8bSToomas Soome 163a3874b8bSToomas Soome unsigned int zfs_scan_checkpoint_intval = 7200; /* seconds */ 164a3874b8bSToomas Soome #define ZFS_SCAN_CHECKPOINT_INTVAL SEC_TO_TICK(zfs_scan_checkpoint_intval) 165a3874b8bSToomas Soome 166a3874b8bSToomas Soome /* 167a3874b8bSToomas Soome * fill_weight is non-tunable at runtime, so we copy it at module init from 168a3874b8bSToomas Soome * zfs_scan_fill_weight. Runtime adjustments to zfs_scan_fill_weight would 169a3874b8bSToomas Soome * break queue sorting. 170a3874b8bSToomas Soome */ 171a3874b8bSToomas Soome uint64_t zfs_scan_fill_weight = 3; 172a3874b8bSToomas Soome static uint64_t fill_weight; 173a3874b8bSToomas Soome 174a3874b8bSToomas Soome /* See dsl_scan_should_clear() for details on the memory limit tunables */ 175a3874b8bSToomas Soome uint64_t zfs_scan_mem_lim_min = 16 << 20; /* bytes */ 176a3874b8bSToomas Soome uint64_t zfs_scan_mem_lim_soft_max = 128 << 20; /* bytes */ 177a3874b8bSToomas Soome int zfs_scan_mem_lim_fact = 20; /* fraction of physmem */ 178a3874b8bSToomas Soome int zfs_scan_mem_lim_soft_fact = 20; /* fraction of mem lim above */ 179a3874b8bSToomas Soome 180a3874b8bSToomas Soome unsigned int zfs_scrub_min_time_ms = 1000; /* min millisecs to scrub per txg */ 181a3874b8bSToomas Soome unsigned int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */ 182a3874b8bSToomas Soome /* min millisecs to obsolete per txg */ 183a3874b8bSToomas Soome unsigned int zfs_obsolete_min_time_ms = 500; 184a3874b8bSToomas Soome /* min millisecs to resilver per txg */ 185a3874b8bSToomas Soome unsigned int zfs_resilver_min_time_ms = 3000; 186e4c795beSTom Caputi int zfs_scan_suspend_progress = 0; /* set to prevent scans from progressing */ 1873f9d6ad7SLin Ling boolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */ 1887fd05ac4SMatthew Ahrens boolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */ 1893f9d6ad7SLin Ling enum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE; 190af3465daSMax Grossman /* max number of blocks to free in a single TXG */ 1915cabbc6bSPrashanth Sreenivasa uint64_t zfs_async_block_max_blocks = UINT64_MAX; 1923f9d6ad7SLin Ling 193e4c795beSTom Caputi int zfs_resilver_disable_defer = 0; /* set to disable resilver deferring */ 194e4c795beSTom Caputi 195a3874b8bSToomas Soome /* 196a3874b8bSToomas Soome * We wait a few txgs after importing a pool to begin scanning so that 197a3874b8bSToomas Soome * the import / mounting code isn't held up by scrub / resilver IO. 198a3874b8bSToomas Soome * Unfortunately, it is a bit difficult to determine exactly how long 199a3874b8bSToomas Soome * this will take since userspace will trigger fs mounts asynchronously 200a3874b8bSToomas Soome * and the kernel will create zvol minors asynchronously. As a result, 201a3874b8bSToomas Soome * the value provided here is a bit arbitrary, but represents a 202a3874b8bSToomas Soome * reasonable estimate of how many txgs it will take to finish fully 203a3874b8bSToomas Soome * importing a pool 204a3874b8bSToomas Soome */ 205a3874b8bSToomas Soome #define SCAN_IMPORT_WAIT_TXGS 5 206a3874b8bSToomas Soome 207a3874b8bSToomas Soome 2083f9d6ad7SLin Ling #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \ 2093f9d6ad7SLin Ling ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \ 2103f9d6ad7SLin Ling (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER) 2113f9d6ad7SLin Ling 2123f9d6ad7SLin Ling extern int zfs_txg_timeout; 2133f9d6ad7SLin Ling 214139510fbSGeorge Wilson /* 215139510fbSGeorge Wilson * Enable/disable the processing of the free_bpobj object. 216139510fbSGeorge Wilson */ 217139510fbSGeorge Wilson boolean_t zfs_free_bpobj_enabled = B_TRUE; 218139510fbSGeorge Wilson 2193f9d6ad7SLin Ling /* the order has to match pool_scan_type */ 2203f9d6ad7SLin Ling static scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = { 2213f9d6ad7SLin Ling NULL, 2223f9d6ad7SLin Ling dsl_scan_scrub_cb, /* POOL_SCAN_SCRUB */ 2233f9d6ad7SLin Ling dsl_scan_scrub_cb, /* POOL_SCAN_RESILVER */ 2243f9d6ad7SLin Ling }; 2253f9d6ad7SLin Ling 226a3874b8bSToomas Soome /* In core node for the scn->scn_queue. Represents a dataset to be scanned */ 227a3874b8bSToomas Soome typedef struct { 228a3874b8bSToomas Soome uint64_t sds_dsobj; 229a3874b8bSToomas Soome uint64_t sds_txg; 230a3874b8bSToomas Soome avl_node_t sds_node; 231a3874b8bSToomas Soome } scan_ds_t; 232a3874b8bSToomas Soome 233a3874b8bSToomas Soome /* 234a3874b8bSToomas Soome * This controls what conditions are placed on dsl_scan_sync_state(): 235a3874b8bSToomas Soome * SYNC_OPTIONAL) write out scn_phys iff scn_bytes_pending == 0 236a3874b8bSToomas Soome * SYNC_MANDATORY) write out scn_phys always. scn_bytes_pending must be 0. 237a3874b8bSToomas Soome * SYNC_CACHED) if scn_bytes_pending == 0, write out scn_phys. Otherwise 238a3874b8bSToomas Soome * write out the scn_phys_cached version. 239a3874b8bSToomas Soome * See dsl_scan_sync_state for details. 240a3874b8bSToomas Soome */ 241a3874b8bSToomas Soome typedef enum { 242a3874b8bSToomas Soome SYNC_OPTIONAL, 243a3874b8bSToomas Soome SYNC_MANDATORY, 244a3874b8bSToomas Soome SYNC_CACHED 245a3874b8bSToomas Soome } state_sync_type_t; 246a3874b8bSToomas Soome 247a3874b8bSToomas Soome /* 248a3874b8bSToomas Soome * This struct represents the minimum information needed to reconstruct a 249a3874b8bSToomas Soome * zio for sequential scanning. This is useful because many of these will 250a3874b8bSToomas Soome * accumulate in the sequential IO queues before being issued, so saving 251a3874b8bSToomas Soome * memory matters here. 252a3874b8bSToomas Soome */ 253a3874b8bSToomas Soome typedef struct scan_io { 254a3874b8bSToomas Soome /* fields from blkptr_t */ 255a3874b8bSToomas Soome uint64_t sio_blk_prop; 256a3874b8bSToomas Soome uint64_t sio_phys_birth; 257a3874b8bSToomas Soome uint64_t sio_birth; 258a3874b8bSToomas Soome zio_cksum_t sio_cksum; 25912a8814cSTom Caputi uint32_t sio_nr_dvas; 260a3874b8bSToomas Soome 261a3874b8bSToomas Soome /* fields from zio_t */ 26212a8814cSTom Caputi uint32_t sio_flags; 263a3874b8bSToomas Soome zbookmark_phys_t sio_zb; 264a3874b8bSToomas Soome 265a3874b8bSToomas Soome /* members for queue sorting */ 266a3874b8bSToomas Soome union { 26712a8814cSTom Caputi avl_node_t sio_addr_node; /* link into issuing queue */ 268a3874b8bSToomas Soome list_node_t sio_list_node; /* link for issuing to disk */ 269a3874b8bSToomas Soome } sio_nodes; 27012a8814cSTom Caputi 27112a8814cSTom Caputi /* 27212a8814cSTom Caputi * There may be up to SPA_DVAS_PER_BP DVAs here from the bp, 27312a8814cSTom Caputi * depending on how many were in the original bp. Only the 27412a8814cSTom Caputi * first DVA is really used for sorting and issuing purposes. 27512a8814cSTom Caputi * The other DVAs (if provided) simply exist so that the zio 27612a8814cSTom Caputi * layer can find additional copies to repair from in the 27712a8814cSTom Caputi * event of an error. This array must go at the end of the 27812a8814cSTom Caputi * struct to allow this for the variable number of elements. 27912a8814cSTom Caputi */ 28012a8814cSTom Caputi dva_t sio_dva[0]; 281a3874b8bSToomas Soome } scan_io_t; 282a3874b8bSToomas Soome 28312a8814cSTom Caputi #define SIO_SET_OFFSET(sio, x) DVA_SET_OFFSET(&(sio)->sio_dva[0], x) 28412a8814cSTom Caputi #define SIO_SET_ASIZE(sio, x) DVA_SET_ASIZE(&(sio)->sio_dva[0], x) 28512a8814cSTom Caputi #define SIO_GET_OFFSET(sio) DVA_GET_OFFSET(&(sio)->sio_dva[0]) 28612a8814cSTom Caputi #define SIO_GET_ASIZE(sio) DVA_GET_ASIZE(&(sio)->sio_dva[0]) 28712a8814cSTom Caputi #define SIO_GET_END_OFFSET(sio) \ 28812a8814cSTom Caputi (SIO_GET_OFFSET(sio) + SIO_GET_ASIZE(sio)) 28912a8814cSTom Caputi #define SIO_GET_MUSED(sio) \ 29012a8814cSTom Caputi (sizeof (scan_io_t) + ((sio)->sio_nr_dvas * sizeof (dva_t))) 29112a8814cSTom Caputi 292a3874b8bSToomas Soome struct dsl_scan_io_queue { 293a3874b8bSToomas Soome dsl_scan_t *q_scn; /* associated dsl_scan_t */ 294a3874b8bSToomas Soome vdev_t *q_vd; /* top-level vdev that this queue represents */ 295a3874b8bSToomas Soome 296a3874b8bSToomas Soome /* trees used for sorting I/Os and extents of I/Os */ 297a3874b8bSToomas Soome range_tree_t *q_exts_by_addr; 298*4d7988d6SPaul Dagnelie zfs_btree_t q_exts_by_size; 299a3874b8bSToomas Soome avl_tree_t q_sios_by_addr; 30012a8814cSTom Caputi uint64_t q_sio_memused; 301a3874b8bSToomas Soome 302a3874b8bSToomas Soome /* members for zio rate limiting */ 303a3874b8bSToomas Soome uint64_t q_maxinflight_bytes; 304a3874b8bSToomas Soome uint64_t q_inflight_bytes; 305a3874b8bSToomas Soome kcondvar_t q_zio_cv; /* used under vd->vdev_scan_io_queue_lock */ 306a3874b8bSToomas Soome 307a3874b8bSToomas Soome /* per txg statistics */ 308a3874b8bSToomas Soome uint64_t q_total_seg_size_this_txg; 309a3874b8bSToomas Soome uint64_t q_segs_this_txg; 310a3874b8bSToomas Soome uint64_t q_total_zio_size_this_txg; 311a3874b8bSToomas Soome uint64_t q_zios_this_txg; 312a3874b8bSToomas Soome }; 313a3874b8bSToomas Soome 314a3874b8bSToomas Soome /* private data for dsl_scan_prefetch_cb() */ 315a3874b8bSToomas Soome typedef struct scan_prefetch_ctx { 316a3874b8bSToomas Soome zfs_refcount_t spc_refcnt; /* refcount for memory management */ 317a3874b8bSToomas Soome dsl_scan_t *spc_scn; /* dsl_scan_t for the pool */ 318a3874b8bSToomas Soome boolean_t spc_root; /* is this prefetch for an objset? */ 319a3874b8bSToomas Soome uint8_t spc_indblkshift; /* dn_indblkshift of current dnode */ 320a3874b8bSToomas Soome uint16_t spc_datablkszsec; /* dn_idatablkszsec of current dnode */ 321a3874b8bSToomas Soome } scan_prefetch_ctx_t; 322a3874b8bSToomas Soome 323a3874b8bSToomas Soome /* private data for dsl_scan_prefetch() */ 324a3874b8bSToomas Soome typedef struct scan_prefetch_issue_ctx { 325a3874b8bSToomas Soome avl_node_t spic_avl_node; /* link into scn->scn_prefetch_queue */ 326a3874b8bSToomas Soome scan_prefetch_ctx_t *spic_spc; /* spc for the callback */ 327a3874b8bSToomas Soome blkptr_t spic_bp; /* bp to prefetch */ 328a3874b8bSToomas Soome zbookmark_phys_t spic_zb; /* bookmark to prefetch */ 329a3874b8bSToomas Soome } scan_prefetch_issue_ctx_t; 330a3874b8bSToomas Soome 331a3874b8bSToomas Soome static void scan_exec_io(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags, 332a3874b8bSToomas Soome const zbookmark_phys_t *zb, dsl_scan_io_queue_t *queue); 333a3874b8bSToomas Soome static void scan_io_queue_insert_impl(dsl_scan_io_queue_t *queue, 334a3874b8bSToomas Soome scan_io_t *sio); 335a3874b8bSToomas Soome 336a3874b8bSToomas Soome static dsl_scan_io_queue_t *scan_io_queue_create(vdev_t *vd); 337a3874b8bSToomas Soome static void scan_io_queues_destroy(dsl_scan_t *scn); 338a3874b8bSToomas Soome 33912a8814cSTom Caputi static kmem_cache_t *sio_cache[SPA_DVAS_PER_BP]; 34012a8814cSTom Caputi 34112a8814cSTom Caputi /* sio->sio_nr_dvas must be set so we know which cache to free from */ 34212a8814cSTom Caputi static void 34312a8814cSTom Caputi sio_free(scan_io_t *sio) 34412a8814cSTom Caputi { 34512a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, >, 0); 34612a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, <=, SPA_DVAS_PER_BP); 34712a8814cSTom Caputi 34812a8814cSTom Caputi kmem_cache_free(sio_cache[sio->sio_nr_dvas - 1], sio); 34912a8814cSTom Caputi } 35012a8814cSTom Caputi 35112a8814cSTom Caputi /* It is up to the caller to set sio->sio_nr_dvas for freeing */ 35212a8814cSTom Caputi static scan_io_t * 35312a8814cSTom Caputi sio_alloc(unsigned short nr_dvas) 35412a8814cSTom Caputi { 35512a8814cSTom Caputi ASSERT3U(nr_dvas, >, 0); 35612a8814cSTom Caputi ASSERT3U(nr_dvas, <=, SPA_DVAS_PER_BP); 35712a8814cSTom Caputi 35812a8814cSTom Caputi return (kmem_cache_alloc(sio_cache[nr_dvas - 1], KM_SLEEP)); 35912a8814cSTom Caputi } 360a3874b8bSToomas Soome 361a3874b8bSToomas Soome void 362a3874b8bSToomas Soome scan_init(void) 363a3874b8bSToomas Soome { 364a3874b8bSToomas Soome /* 365a3874b8bSToomas Soome * This is used in ext_size_compare() to weight segments 366a3874b8bSToomas Soome * based on how sparse they are. This cannot be changed 367a3874b8bSToomas Soome * mid-scan and the tree comparison functions don't currently 368a3874b8bSToomas Soome * have a mechansim for passing additional context to the 369a3874b8bSToomas Soome * compare functions. Thus we store this value globally and 370a3874b8bSToomas Soome * we only allow it to be set at module intiailization time 371a3874b8bSToomas Soome */ 372a3874b8bSToomas Soome fill_weight = zfs_scan_fill_weight; 373a3874b8bSToomas Soome 37412a8814cSTom Caputi for (int i = 0; i < SPA_DVAS_PER_BP; i++) { 37512a8814cSTom Caputi char name[36]; 37612a8814cSTom Caputi 37712a8814cSTom Caputi (void) sprintf(name, "sio_cache_%d", i); 37812a8814cSTom Caputi sio_cache[i] = kmem_cache_create(name, 37912a8814cSTom Caputi (sizeof (scan_io_t) + ((i + 1) * sizeof (dva_t))), 38012a8814cSTom Caputi 0, NULL, NULL, NULL, NULL, NULL, 0); 38112a8814cSTom Caputi } 382a3874b8bSToomas Soome } 383a3874b8bSToomas Soome 384a3874b8bSToomas Soome void 385a3874b8bSToomas Soome scan_fini(void) 386a3874b8bSToomas Soome { 38712a8814cSTom Caputi for (int i = 0; i < SPA_DVAS_PER_BP; i++) { 38812a8814cSTom Caputi kmem_cache_destroy(sio_cache[i]); 38912a8814cSTom Caputi } 390a3874b8bSToomas Soome } 391a3874b8bSToomas Soome 392a3874b8bSToomas Soome static inline boolean_t 393a3874b8bSToomas Soome dsl_scan_is_running(const dsl_scan_t *scn) 394a3874b8bSToomas Soome { 395a3874b8bSToomas Soome return (scn->scn_phys.scn_state == DSS_SCANNING); 396a3874b8bSToomas Soome } 397a3874b8bSToomas Soome 398a3874b8bSToomas Soome boolean_t 399a3874b8bSToomas Soome dsl_scan_resilvering(dsl_pool_t *dp) 400a3874b8bSToomas Soome { 401a3874b8bSToomas Soome return (dsl_scan_is_running(dp->dp_scan) && 402a3874b8bSToomas Soome dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER); 403a3874b8bSToomas Soome } 404a3874b8bSToomas Soome 405a3874b8bSToomas Soome static inline void 40612a8814cSTom Caputi sio2bp(const scan_io_t *sio, blkptr_t *bp) 407a3874b8bSToomas Soome { 408a3874b8bSToomas Soome bzero(bp, sizeof (*bp)); 409a3874b8bSToomas Soome bp->blk_prop = sio->sio_blk_prop; 410a3874b8bSToomas Soome bp->blk_phys_birth = sio->sio_phys_birth; 411a3874b8bSToomas Soome bp->blk_birth = sio->sio_birth; 412a3874b8bSToomas Soome bp->blk_fill = 1; /* we always only work with data pointers */ 413a3874b8bSToomas Soome bp->blk_cksum = sio->sio_cksum; 41412a8814cSTom Caputi 41512a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, >, 0); 41612a8814cSTom Caputi ASSERT3U(sio->sio_nr_dvas, <=, SPA_DVAS_PER_BP); 41712a8814cSTom Caputi 41812a8814cSTom Caputi bcopy(sio->sio_dva, bp->blk_dva, sio->sio_nr_dvas * sizeof (dva_t)); 419a3874b8bSToomas Soome } 420a3874b8bSToomas Soome 421a3874b8bSToomas Soome static inline void 422a3874b8bSToomas Soome bp2sio(const blkptr_t *bp, scan_io_t *sio, int dva_i) 423a3874b8bSToomas Soome { 424a3874b8bSToomas Soome sio->sio_blk_prop = bp->blk_prop; 425a3874b8bSToomas Soome sio->sio_phys_birth = bp->blk_phys_birth; 426a3874b8bSToomas Soome sio->sio_birth = bp->blk_birth; 427a3874b8bSToomas Soome sio->sio_cksum = bp->blk_cksum; 42812a8814cSTom Caputi sio->sio_nr_dvas = BP_GET_NDVAS(bp); 42912a8814cSTom Caputi 43012a8814cSTom Caputi /* 43112a8814cSTom Caputi * Copy the DVAs to the sio. We need all copies of the block so 43212a8814cSTom Caputi * that the self healing code can use the alternate copies if the 43312a8814cSTom Caputi * first is corrupted. We want the DVA at index dva_i to be first 43412a8814cSTom Caputi * in the sio since this is the primary one that we want to issue. 43512a8814cSTom Caputi */ 43612a8814cSTom Caputi for (int i = 0, j = dva_i; i < sio->sio_nr_dvas; i++, j++) { 43712a8814cSTom Caputi sio->sio_dva[i] = bp->blk_dva[j % sio->sio_nr_dvas]; 43812a8814cSTom Caputi } 439a3874b8bSToomas Soome } 440a3874b8bSToomas Soome 4413f9d6ad7SLin Ling int 4423f9d6ad7SLin Ling dsl_scan_init(dsl_pool_t *dp, uint64_t txg) 4433f9d6ad7SLin Ling { 4443f9d6ad7SLin Ling int err; 4453f9d6ad7SLin Ling dsl_scan_t *scn; 4463f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 4473f9d6ad7SLin Ling uint64_t f; 4483f9d6ad7SLin Ling 4493f9d6ad7SLin Ling scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP); 4503f9d6ad7SLin Ling scn->scn_dp = dp; 4513f9d6ad7SLin Ling 4524a923759SGeorge Wilson /* 4534a923759SGeorge Wilson * It's possible that we're resuming a scan after a reboot so 4544a923759SGeorge Wilson * make sure that the scan_async_destroying flag is initialized 4554a923759SGeorge Wilson * appropriately. 4564a923759SGeorge Wilson */ 4574a923759SGeorge Wilson ASSERT(!scn->scn_async_destroying); 4584a923759SGeorge Wilson scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, 4592acef22dSMatthew Ahrens SPA_FEATURE_ASYNC_DESTROY); 4604a923759SGeorge Wilson 461a3874b8bSToomas Soome avl_create(&scn->scn_queue, scan_ds_queue_compare, sizeof (scan_ds_t), 462a3874b8bSToomas Soome offsetof(scan_ds_t, sds_node)); 463a3874b8bSToomas Soome avl_create(&scn->scn_prefetch_queue, scan_prefetch_queue_compare, 464a3874b8bSToomas Soome sizeof (scan_prefetch_issue_ctx_t), 465a3874b8bSToomas Soome offsetof(scan_prefetch_issue_ctx_t, spic_avl_node)); 466a3874b8bSToomas Soome 4673f9d6ad7SLin Ling err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4683f9d6ad7SLin Ling "scrub_func", sizeof (uint64_t), 1, &f); 4693f9d6ad7SLin Ling if (err == 0) { 4703f9d6ad7SLin Ling /* 4713f9d6ad7SLin Ling * There was an old-style scrub in progress. Restart a 4723f9d6ad7SLin Ling * new-style scrub from the beginning. 4733f9d6ad7SLin Ling */ 4743f9d6ad7SLin Ling scn->scn_restart_txg = txg; 4753f9d6ad7SLin Ling zfs_dbgmsg("old-style scrub was in progress; " 4763f9d6ad7SLin Ling "restarting new-style scrub in txg %llu", 477a3874b8bSToomas Soome (longlong_t)scn->scn_restart_txg); 4783f9d6ad7SLin Ling 4793f9d6ad7SLin Ling /* 4803f9d6ad7SLin Ling * Load the queue obj from the old location so that it 4813f9d6ad7SLin Ling * can be freed by dsl_scan_done(). 4823f9d6ad7SLin Ling */ 4833f9d6ad7SLin Ling (void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4843f9d6ad7SLin Ling "scrub_queue", sizeof (uint64_t), 1, 4853f9d6ad7SLin Ling &scn->scn_phys.scn_queue_obj); 4863f9d6ad7SLin Ling } else { 4873f9d6ad7SLin Ling err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4883f9d6ad7SLin Ling DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 4893f9d6ad7SLin Ling &scn->scn_phys); 490eb633035STom Caputi 491eb633035STom Caputi /* 492eb633035STom Caputi * Detect if the pool contains the signature of #2094. If it 493eb633035STom Caputi * does properly update the scn->scn_phys structure and notify 494eb633035STom Caputi * the administrator by setting an errata for the pool. 495eb633035STom Caputi */ 496eb633035STom Caputi if (err == EOVERFLOW) { 497eb633035STom Caputi uint64_t zaptmp[SCAN_PHYS_NUMINTS + 1]; 498eb633035STom Caputi VERIFY3S(SCAN_PHYS_NUMINTS, ==, 24); 499eb633035STom Caputi VERIFY3S(offsetof(dsl_scan_phys_t, scn_flags), ==, 500eb633035STom Caputi (23 * sizeof (uint64_t))); 501eb633035STom Caputi 502eb633035STom Caputi err = zap_lookup(dp->dp_meta_objset, 503eb633035STom Caputi DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SCAN, 504eb633035STom Caputi sizeof (uint64_t), SCAN_PHYS_NUMINTS + 1, &zaptmp); 505eb633035STom Caputi if (err == 0) { 506eb633035STom Caputi uint64_t overflow = zaptmp[SCAN_PHYS_NUMINTS]; 507eb633035STom Caputi 508eb633035STom Caputi if (overflow & ~DSF_VISIT_DS_AGAIN || 509eb633035STom Caputi scn->scn_async_destroying) { 510eb633035STom Caputi spa->spa_errata = 511eb633035STom Caputi ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY; 512eb633035STom Caputi return (EOVERFLOW); 513eb633035STom Caputi } 514eb633035STom Caputi 515eb633035STom Caputi bcopy(zaptmp, &scn->scn_phys, 516eb633035STom Caputi SCAN_PHYS_NUMINTS * sizeof (uint64_t)); 517eb633035STom Caputi scn->scn_phys.scn_flags = overflow; 518eb633035STom Caputi 519eb633035STom Caputi /* Required scrub already in progress. */ 520eb633035STom Caputi if (scn->scn_phys.scn_state == DSS_FINISHED || 521eb633035STom Caputi scn->scn_phys.scn_state == DSS_CANCELED) 522eb633035STom Caputi spa->spa_errata = 523eb633035STom Caputi ZPOOL_ERRATA_ZOL_2094_SCRUB; 524eb633035STom Caputi } 525eb633035STom Caputi } 526eb633035STom Caputi 5273f9d6ad7SLin Ling if (err == ENOENT) 5283f9d6ad7SLin Ling return (0); 5293f9d6ad7SLin Ling else if (err) 5303f9d6ad7SLin Ling return (err); 5313f9d6ad7SLin Ling 532a3874b8bSToomas Soome /* 533a3874b8bSToomas Soome * We might be restarting after a reboot, so jump the issued 534a3874b8bSToomas Soome * counter to how far we've scanned. We know we're consistent 535a3874b8bSToomas Soome * up to here. 536a3874b8bSToomas Soome */ 537a3874b8bSToomas Soome scn->scn_issued_before_pass = scn->scn_phys.scn_examined; 538a3874b8bSToomas Soome 539a3874b8bSToomas Soome if (dsl_scan_is_running(scn) && 5403f9d6ad7SLin Ling spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) { 5413f9d6ad7SLin Ling /* 5423f9d6ad7SLin Ling * A new-type scrub was in progress on an old 5433f9d6ad7SLin Ling * pool, and the pool was accessed by old 5443f9d6ad7SLin Ling * software. Restart from the beginning, since 5453f9d6ad7SLin Ling * the old software may have changed the pool in 5463f9d6ad7SLin Ling * the meantime. 5473f9d6ad7SLin Ling */ 5483f9d6ad7SLin Ling scn->scn_restart_txg = txg; 5493f9d6ad7SLin Ling zfs_dbgmsg("new-style scrub was modified " 5503f9d6ad7SLin Ling "by old software; restarting in txg %llu", 551a3874b8bSToomas Soome (longlong_t)scn->scn_restart_txg); 552a3874b8bSToomas Soome } 553a3874b8bSToomas Soome } 554a3874b8bSToomas Soome 555e4c795beSTom Caputi bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); 556e4c795beSTom Caputi 557a3874b8bSToomas Soome /* reload the queue into the in-core state */ 558a3874b8bSToomas Soome if (scn->scn_phys.scn_queue_obj != 0) { 559a3874b8bSToomas Soome zap_cursor_t zc; 560a3874b8bSToomas Soome zap_attribute_t za; 561a3874b8bSToomas Soome 562a3874b8bSToomas Soome for (zap_cursor_init(&zc, dp->dp_meta_objset, 563a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj); 564a3874b8bSToomas Soome zap_cursor_retrieve(&zc, &za) == 0; 565a3874b8bSToomas Soome (void) zap_cursor_advance(&zc)) { 566a3874b8bSToomas Soome scan_ds_queue_insert(scn, 567a3874b8bSToomas Soome zfs_strtonum(za.za_name, NULL), 568a3874b8bSToomas Soome za.za_first_integer); 5693f9d6ad7SLin Ling } 570a3874b8bSToomas Soome zap_cursor_fini(&zc); 5713f9d6ad7SLin Ling } 5723f9d6ad7SLin Ling 5733f9d6ad7SLin Ling spa_scan_stat_init(spa); 5743f9d6ad7SLin Ling return (0); 5753f9d6ad7SLin Ling } 5763f9d6ad7SLin Ling 5773f9d6ad7SLin Ling void 5783f9d6ad7SLin Ling dsl_scan_fini(dsl_pool_t *dp) 5793f9d6ad7SLin Ling { 580a3874b8bSToomas Soome if (dp->dp_scan != NULL) { 581a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 582a3874b8bSToomas Soome 583a3874b8bSToomas Soome if (scn->scn_taskq != NULL) 584a3874b8bSToomas Soome taskq_destroy(scn->scn_taskq); 585a3874b8bSToomas Soome scan_ds_queue_clear(scn); 586a3874b8bSToomas Soome avl_destroy(&scn->scn_queue); 587a3874b8bSToomas Soome avl_destroy(&scn->scn_prefetch_queue); 588a3874b8bSToomas Soome 5893f9d6ad7SLin Ling kmem_free(dp->dp_scan, sizeof (dsl_scan_t)); 5903f9d6ad7SLin Ling dp->dp_scan = NULL; 5913f9d6ad7SLin Ling } 5923f9d6ad7SLin Ling } 5933f9d6ad7SLin Ling 594a3874b8bSToomas Soome static boolean_t 595a3874b8bSToomas Soome dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx) 596a3874b8bSToomas Soome { 597a3874b8bSToomas Soome return (scn->scn_restart_txg != 0 && 598a3874b8bSToomas Soome scn->scn_restart_txg <= tx->tx_txg); 599a3874b8bSToomas Soome } 600a3874b8bSToomas Soome 601a3874b8bSToomas Soome boolean_t 602a3874b8bSToomas Soome dsl_scan_scrubbing(const dsl_pool_t *dp) 603a3874b8bSToomas Soome { 604a3874b8bSToomas Soome dsl_scan_phys_t *scn_phys = &dp->dp_scan->scn_phys; 605a3874b8bSToomas Soome 606a3874b8bSToomas Soome return (scn_phys->scn_state == DSS_SCANNING && 607a3874b8bSToomas Soome scn_phys->scn_func == POOL_SCAN_SCRUB); 608a3874b8bSToomas Soome } 609a3874b8bSToomas Soome 610a3874b8bSToomas Soome boolean_t 611a3874b8bSToomas Soome dsl_scan_is_paused_scrub(const dsl_scan_t *scn) 612a3874b8bSToomas Soome { 613a3874b8bSToomas Soome return (dsl_scan_scrubbing(scn->scn_dp) && 614a3874b8bSToomas Soome scn->scn_phys.scn_flags & DSF_SCRUB_PAUSED); 615a3874b8bSToomas Soome } 616a3874b8bSToomas Soome 617a3874b8bSToomas Soome /* 618a3874b8bSToomas Soome * Writes out a persistent dsl_scan_phys_t record to the pool directory. 619a3874b8bSToomas Soome * Because we can be running in the block sorting algorithm, we do not always 620a3874b8bSToomas Soome * want to write out the record, only when it is "safe" to do so. This safety 621a3874b8bSToomas Soome * condition is achieved by making sure that the sorting queues are empty 622a3874b8bSToomas Soome * (scn_bytes_pending == 0). When this condition is not true, the sync'd state 623a3874b8bSToomas Soome * is inconsistent with how much actual scanning progress has been made. The 624a3874b8bSToomas Soome * kind of sync to be performed is specified by the sync_type argument. If the 625a3874b8bSToomas Soome * sync is optional, we only sync if the queues are empty. If the sync is 626a3874b8bSToomas Soome * mandatory, we do a hard ASSERT to make sure that the queues are empty. The 627a3874b8bSToomas Soome * third possible state is a "cached" sync. This is done in response to: 628a3874b8bSToomas Soome * 1) The dataset that was in the last sync'd dsl_scan_phys_t having been 629a3874b8bSToomas Soome * destroyed, so we wouldn't be able to restart scanning from it. 630a3874b8bSToomas Soome * 2) The snapshot that was in the last sync'd dsl_scan_phys_t having been 631a3874b8bSToomas Soome * superseded by a newer snapshot. 632a3874b8bSToomas Soome * 3) The dataset that was in the last sync'd dsl_scan_phys_t having been 633a3874b8bSToomas Soome * swapped with its clone. 634a3874b8bSToomas Soome * In all cases, a cached sync simply rewrites the last record we've written, 635a3874b8bSToomas Soome * just slightly modified. For the modifications that are performed to the 636a3874b8bSToomas Soome * last written dsl_scan_phys_t, see dsl_scan_ds_destroyed, 637a3874b8bSToomas Soome * dsl_scan_ds_snapshotted and dsl_scan_ds_clone_swapped. 638a3874b8bSToomas Soome */ 639a3874b8bSToomas Soome static void 640a3874b8bSToomas Soome dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx, state_sync_type_t sync_type) 641a3874b8bSToomas Soome { 642a3874b8bSToomas Soome int i; 643a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 644a3874b8bSToomas Soome 645a3874b8bSToomas Soome ASSERT(sync_type != SYNC_MANDATORY || scn->scn_bytes_pending == 0); 646a3874b8bSToomas Soome if (scn->scn_bytes_pending == 0) { 647a3874b8bSToomas Soome for (i = 0; i < spa->spa_root_vdev->vdev_children; i++) { 648a3874b8bSToomas Soome vdev_t *vd = spa->spa_root_vdev->vdev_child[i]; 649a3874b8bSToomas Soome dsl_scan_io_queue_t *q = vd->vdev_scan_io_queue; 650a3874b8bSToomas Soome 651a3874b8bSToomas Soome if (q == NULL) 652a3874b8bSToomas Soome continue; 653a3874b8bSToomas Soome 654a3874b8bSToomas Soome mutex_enter(&vd->vdev_scan_io_queue_lock); 655a3874b8bSToomas Soome ASSERT3P(avl_first(&q->q_sios_by_addr), ==, NULL); 656*4d7988d6SPaul Dagnelie ASSERT3P(zfs_btree_first(&q->q_exts_by_size, NULL), ==, 657*4d7988d6SPaul Dagnelie NULL); 658a3874b8bSToomas Soome ASSERT3P(range_tree_first(q->q_exts_by_addr), ==, NULL); 659a3874b8bSToomas Soome mutex_exit(&vd->vdev_scan_io_queue_lock); 660a3874b8bSToomas Soome } 661a3874b8bSToomas Soome 662a3874b8bSToomas Soome if (scn->scn_phys.scn_queue_obj != 0) 663a3874b8bSToomas Soome scan_ds_queue_sync(scn, tx); 664a3874b8bSToomas Soome VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 665a3874b8bSToomas Soome DMU_POOL_DIRECTORY_OBJECT, 666a3874b8bSToomas Soome DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 667a3874b8bSToomas Soome &scn->scn_phys, tx)); 668a3874b8bSToomas Soome bcopy(&scn->scn_phys, &scn->scn_phys_cached, 669a3874b8bSToomas Soome sizeof (scn->scn_phys)); 670a3874b8bSToomas Soome 671a3874b8bSToomas Soome if (scn->scn_checkpointing) 672a3874b8bSToomas Soome zfs_dbgmsg("finish scan checkpoint"); 673a3874b8bSToomas Soome 674a3874b8bSToomas Soome scn->scn_checkpointing = B_FALSE; 675a3874b8bSToomas Soome scn->scn_last_checkpoint = ddi_get_lbolt(); 676a3874b8bSToomas Soome } else if (sync_type == SYNC_CACHED) { 677a3874b8bSToomas Soome VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 678a3874b8bSToomas Soome DMU_POOL_DIRECTORY_OBJECT, 679a3874b8bSToomas Soome DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 680a3874b8bSToomas Soome &scn->scn_phys_cached, tx)); 681a3874b8bSToomas Soome } 682a3874b8bSToomas Soome } 683a3874b8bSToomas Soome 6843f9d6ad7SLin Ling /* ARGSUSED */ 6853f9d6ad7SLin Ling static int 6863b2aab18SMatthew Ahrens dsl_scan_setup_check(void *arg, dmu_tx_t *tx) 6873f9d6ad7SLin Ling { 6883b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 6893f9d6ad7SLin Ling 690a3874b8bSToomas Soome if (dsl_scan_is_running(scn)) 691be6fd75aSMatthew Ahrens return (SET_ERROR(EBUSY)); 6923f9d6ad7SLin Ling 6933f9d6ad7SLin Ling return (0); 6943f9d6ad7SLin Ling } 6953f9d6ad7SLin Ling 6963f9d6ad7SLin Ling static void 6973b2aab18SMatthew Ahrens dsl_scan_setup_sync(void *arg, dmu_tx_t *tx) 6983f9d6ad7SLin Ling { 6993b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 7003b2aab18SMatthew Ahrens pool_scan_func_t *funcp = arg; 7013f9d6ad7SLin Ling dmu_object_type_t ot = 0; 7023f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 7033f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 7043f9d6ad7SLin Ling 705a3874b8bSToomas Soome ASSERT(!dsl_scan_is_running(scn)); 7063f9d6ad7SLin Ling ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS); 7073f9d6ad7SLin Ling bzero(&scn->scn_phys, sizeof (scn->scn_phys)); 7083f9d6ad7SLin Ling scn->scn_phys.scn_func = *funcp; 7093f9d6ad7SLin Ling scn->scn_phys.scn_state = DSS_SCANNING; 7103f9d6ad7SLin Ling scn->scn_phys.scn_min_txg = 0; 7113f9d6ad7SLin Ling scn->scn_phys.scn_max_txg = tx->tx_txg; 7123f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */ 7133f9d6ad7SLin Ling scn->scn_phys.scn_start_time = gethrestime_sec(); 7143f9d6ad7SLin Ling scn->scn_phys.scn_errors = 0; 7153f9d6ad7SLin Ling scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc; 716a3874b8bSToomas Soome scn->scn_issued_before_pass = 0; 7173f9d6ad7SLin Ling scn->scn_restart_txg = 0; 718b4952e17SGeorge Wilson scn->scn_done_txg = 0; 719a3874b8bSToomas Soome scn->scn_last_checkpoint = 0; 720a3874b8bSToomas Soome scn->scn_checkpointing = B_FALSE; 7213f9d6ad7SLin Ling spa_scan_stat_init(spa); 7223f9d6ad7SLin Ling 7233f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 7243f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max; 7253f9d6ad7SLin Ling 7263f9d6ad7SLin Ling /* rewrite all disk labels */ 7273f9d6ad7SLin Ling vdev_config_dirty(spa->spa_root_vdev); 7283f9d6ad7SLin Ling 7293f9d6ad7SLin Ling if (vdev_resilver_needed(spa->spa_root_vdev, 7303f9d6ad7SLin Ling &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) { 731ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, 732ce1577b0SDave Eddy ESC_ZFS_RESILVER_START); 7333f9d6ad7SLin Ling } else { 734ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_START); 7353f9d6ad7SLin Ling } 7363f9d6ad7SLin Ling 7373f9d6ad7SLin Ling spa->spa_scrub_started = B_TRUE; 7383f9d6ad7SLin Ling /* 7393f9d6ad7SLin Ling * If this is an incremental scrub, limit the DDT scrub phase 7403f9d6ad7SLin Ling * to just the auto-ditto class (for correctness); the rest 7413f9d6ad7SLin Ling * of the scrub should go faster using top-down pruning. 7423f9d6ad7SLin Ling */ 7433f9d6ad7SLin Ling if (scn->scn_phys.scn_min_txg > TXG_INITIAL) 7443f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO; 7453f9d6ad7SLin Ling 7463f9d6ad7SLin Ling } 7473f9d6ad7SLin Ling 7483f9d6ad7SLin Ling /* back to the generic stuff */ 7493f9d6ad7SLin Ling 7503f9d6ad7SLin Ling if (dp->dp_blkstats == NULL) { 7513f9d6ad7SLin Ling dp->dp_blkstats = 7523f9d6ad7SLin Ling kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP); 753a3874b8bSToomas Soome mutex_init(&dp->dp_blkstats->zab_lock, NULL, 754a3874b8bSToomas Soome MUTEX_DEFAULT, NULL); 7553f9d6ad7SLin Ling } 756a3874b8bSToomas Soome bzero(&dp->dp_blkstats->zab_type, sizeof (dp->dp_blkstats->zab_type)); 7573f9d6ad7SLin Ling 7583f9d6ad7SLin Ling if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) 7593f9d6ad7SLin Ling ot = DMU_OT_ZAP_OTHER; 7603f9d6ad7SLin Ling 7613f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, 7623f9d6ad7SLin Ling ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx); 7633f9d6ad7SLin Ling 764a3874b8bSToomas Soome bcopy(&scn->scn_phys, &scn->scn_phys_cached, sizeof (scn->scn_phys)); 765a3874b8bSToomas Soome 766a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_MANDATORY); 7673f9d6ad7SLin Ling 7684445fffbSMatthew Ahrens spa_history_log_internal(spa, "scan setup", tx, 7693f9d6ad7SLin Ling "func=%u mintxg=%llu maxtxg=%llu", 7703f9d6ad7SLin Ling *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg); 7713f9d6ad7SLin Ling } 7723f9d6ad7SLin Ling 773a3874b8bSToomas Soome /* 774a3874b8bSToomas Soome * Called by the ZFS_IOC_POOL_SCAN ioctl to start a scrub or resilver. 775a3874b8bSToomas Soome * Can also be called to resume a paused scrub. 776a3874b8bSToomas Soome */ 777a3874b8bSToomas Soome int 778a3874b8bSToomas Soome dsl_scan(dsl_pool_t *dp, pool_scan_func_t func) 779a3874b8bSToomas Soome { 780a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 781a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 782a3874b8bSToomas Soome 783a3874b8bSToomas Soome /* 784a3874b8bSToomas Soome * Purge all vdev caches and probe all devices. We do this here 785a3874b8bSToomas Soome * rather than in sync context because this requires a writer lock 786a3874b8bSToomas Soome * on the spa_config lock, which we can't do from sync context. The 787a3874b8bSToomas Soome * spa_scrub_reopen flag indicates that vdev_open() should not 788a3874b8bSToomas Soome * attempt to start another scrub. 789a3874b8bSToomas Soome */ 790a3874b8bSToomas Soome spa_vdev_state_enter(spa, SCL_NONE); 791a3874b8bSToomas Soome spa->spa_scrub_reopen = B_TRUE; 792a3874b8bSToomas Soome vdev_reopen(spa->spa_root_vdev); 793a3874b8bSToomas Soome spa->spa_scrub_reopen = B_FALSE; 794a3874b8bSToomas Soome (void) spa_vdev_state_exit(spa, NULL, 0); 795a3874b8bSToomas Soome 796e4c795beSTom Caputi if (func == POOL_SCAN_RESILVER) { 797e4c795beSTom Caputi dsl_resilver_restart(spa->spa_dsl_pool, 0); 798e4c795beSTom Caputi return (0); 799e4c795beSTom Caputi } 800e4c795beSTom Caputi 801a3874b8bSToomas Soome if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) { 802a3874b8bSToomas Soome /* got scrub start cmd, resume paused scrub */ 803a3874b8bSToomas Soome int err = dsl_scrub_set_pause_resume(scn->scn_dp, 804a3874b8bSToomas Soome POOL_SCRUB_NORMAL); 805a3874b8bSToomas Soome if (err == 0) { 806a3874b8bSToomas Soome spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_RESUME); 807a3874b8bSToomas Soome return (ECANCELED); 808a3874b8bSToomas Soome } 809a3874b8bSToomas Soome return (SET_ERROR(err)); 810a3874b8bSToomas Soome } 811a3874b8bSToomas Soome 812a3874b8bSToomas Soome return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check, 813a3874b8bSToomas Soome dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_EXTRA_RESERVED)); 814a3874b8bSToomas Soome } 815a3874b8bSToomas Soome 816e4c795beSTom Caputi /* 817e4c795beSTom Caputi * Sets the resilver defer flag to B_FALSE on all leaf devs under vd. Returns 818e4c795beSTom Caputi * B_TRUE if we have devices that need to be resilvered and are available to 819e4c795beSTom Caputi * accept resilver I/Os. 820e4c795beSTom Caputi */ 821e4c795beSTom Caputi static boolean_t 822e4c795beSTom Caputi dsl_scan_clear_deferred(vdev_t *vd, dmu_tx_t *tx) 823e4c795beSTom Caputi { 824e4c795beSTom Caputi boolean_t resilver_needed = B_FALSE; 825e4c795beSTom Caputi spa_t *spa = vd->vdev_spa; 826e4c795beSTom Caputi 827e4c795beSTom Caputi for (int c = 0; c < vd->vdev_children; c++) { 828e4c795beSTom Caputi resilver_needed |= 829e4c795beSTom Caputi dsl_scan_clear_deferred(vd->vdev_child[c], tx); 830e4c795beSTom Caputi } 831e4c795beSTom Caputi 832e4c795beSTom Caputi if (vd == spa->spa_root_vdev && 833e4c795beSTom Caputi spa_feature_is_active(spa, SPA_FEATURE_RESILVER_DEFER)) { 834e4c795beSTom Caputi spa_feature_decr(spa, SPA_FEATURE_RESILVER_DEFER, tx); 835e4c795beSTom Caputi vdev_config_dirty(vd); 836e4c795beSTom Caputi spa->spa_resilver_deferred = B_FALSE; 837e4c795beSTom Caputi return (resilver_needed); 838e4c795beSTom Caputi } 839e4c795beSTom Caputi 840e4c795beSTom Caputi if (!vdev_is_concrete(vd) || vd->vdev_aux || 841e4c795beSTom Caputi !vd->vdev_ops->vdev_op_leaf) 842e4c795beSTom Caputi return (resilver_needed); 843e4c795beSTom Caputi 844e4c795beSTom Caputi if (vd->vdev_resilver_deferred) 845e4c795beSTom Caputi vd->vdev_resilver_deferred = B_FALSE; 846e4c795beSTom Caputi 847e4c795beSTom Caputi return (!vdev_is_dead(vd) && !vd->vdev_offline && 848e4c795beSTom Caputi vdev_resilver_needed(vd, NULL, NULL)); 849e4c795beSTom Caputi } 850e4c795beSTom Caputi 8513f9d6ad7SLin Ling /* ARGSUSED */ 8523f9d6ad7SLin Ling static void 8533f9d6ad7SLin Ling dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) 8543f9d6ad7SLin Ling { 8553f9d6ad7SLin Ling static const char *old_names[] = { 8563f9d6ad7SLin Ling "scrub_bookmark", 8573f9d6ad7SLin Ling "scrub_ddt_bookmark", 8583f9d6ad7SLin Ling "scrub_ddt_class_max", 8593f9d6ad7SLin Ling "scrub_queue", 8603f9d6ad7SLin Ling "scrub_min_txg", 8613f9d6ad7SLin Ling "scrub_max_txg", 8623f9d6ad7SLin Ling "scrub_func", 8633f9d6ad7SLin Ling "scrub_errors", 8643f9d6ad7SLin Ling NULL 8653f9d6ad7SLin Ling }; 8663f9d6ad7SLin Ling 8673f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 8683f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 8693f9d6ad7SLin Ling int i; 8703f9d6ad7SLin Ling 8713f9d6ad7SLin Ling /* Remove any remnants of an old-style scrub. */ 8723f9d6ad7SLin Ling for (i = 0; old_names[i]; i++) { 8733f9d6ad7SLin Ling (void) zap_remove(dp->dp_meta_objset, 8743f9d6ad7SLin Ling DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx); 8753f9d6ad7SLin Ling } 8763f9d6ad7SLin Ling 8773f9d6ad7SLin Ling if (scn->scn_phys.scn_queue_obj != 0) { 878a3874b8bSToomas Soome VERIFY0(dmu_object_free(dp->dp_meta_objset, 8793f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, tx)); 8803f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj = 0; 8813f9d6ad7SLin Ling } 882a3874b8bSToomas Soome scan_ds_queue_clear(scn); 8833f9d6ad7SLin Ling 8841702cce7SAlek Pinchuk scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; 8851702cce7SAlek Pinchuk 8863f9d6ad7SLin Ling /* 8873f9d6ad7SLin Ling * If we were "restarted" from a stopped state, don't bother 8883f9d6ad7SLin Ling * with anything else. 8893f9d6ad7SLin Ling */ 890a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) { 891a3874b8bSToomas Soome ASSERT(!scn->scn_is_sorted); 8923f9d6ad7SLin Ling return; 893a3874b8bSToomas Soome } 8943f9d6ad7SLin Ling 895a3874b8bSToomas Soome if (scn->scn_is_sorted) { 896a3874b8bSToomas Soome scan_io_queues_destroy(scn); 897a3874b8bSToomas Soome scn->scn_is_sorted = B_FALSE; 898a3874b8bSToomas Soome 899a3874b8bSToomas Soome if (scn->scn_taskq != NULL) { 900a3874b8bSToomas Soome taskq_destroy(scn->scn_taskq); 901a3874b8bSToomas Soome scn->scn_taskq = NULL; 902a3874b8bSToomas Soome } 903a3874b8bSToomas Soome } 904a3874b8bSToomas Soome 905a3874b8bSToomas Soome scn->scn_phys.scn_state = complete ? DSS_FINISHED : DSS_CANCELED; 9063f9d6ad7SLin Ling 9071825bc56SNav Ravindranath if (dsl_scan_restarting(scn, tx)) 9081825bc56SNav Ravindranath spa_history_log_internal(spa, "scan aborted, restarting", tx, 9091825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 9101825bc56SNav Ravindranath else if (!complete) 9111825bc56SNav Ravindranath spa_history_log_internal(spa, "scan cancelled", tx, 9121825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 9131825bc56SNav Ravindranath else 9141825bc56SNav Ravindranath spa_history_log_internal(spa, "scan done", tx, 9151825bc56SNav Ravindranath "errors=%llu", spa_get_errlog_size(spa)); 9163f9d6ad7SLin Ling 9173f9d6ad7SLin Ling if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 9183f9d6ad7SLin Ling spa->spa_scrub_started = B_FALSE; 9193f9d6ad7SLin Ling spa->spa_scrub_active = B_FALSE; 9203f9d6ad7SLin Ling 9213f9d6ad7SLin Ling /* 9223f9d6ad7SLin Ling * If the scrub/resilver completed, update all DTLs to 9233f9d6ad7SLin Ling * reflect this. Whether it succeeded or not, vacate 9243f9d6ad7SLin Ling * all temporary scrub DTLs. 92586714001SSerapheim Dimitropoulos * 92686714001SSerapheim Dimitropoulos * As the scrub does not currently support traversing 92786714001SSerapheim Dimitropoulos * data that have been freed but are part of a checkpoint, 92886714001SSerapheim Dimitropoulos * we don't mark the scrub as done in the DTLs as faults 92986714001SSerapheim Dimitropoulos * may still exist in those vdevs. 9303f9d6ad7SLin Ling */ 93186714001SSerapheim Dimitropoulos if (complete && 93286714001SSerapheim Dimitropoulos !spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 93386714001SSerapheim Dimitropoulos vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 93486714001SSerapheim Dimitropoulos scn->scn_phys.scn_max_txg, B_TRUE); 93586714001SSerapheim Dimitropoulos 936ce1577b0SDave Eddy spa_event_notify(spa, NULL, NULL, 937ce1577b0SDave Eddy scn->scn_phys.scn_min_txg ? 9383f9d6ad7SLin Ling ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH); 93986714001SSerapheim Dimitropoulos } else { 94086714001SSerapheim Dimitropoulos vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 94186714001SSerapheim Dimitropoulos 0, B_TRUE); 9423f9d6ad7SLin Ling } 9433f9d6ad7SLin Ling spa_errlog_rotate(spa); 9443f9d6ad7SLin Ling 9453f9d6ad7SLin Ling /* 9463f9d6ad7SLin Ling * We may have finished replacing a device. 9473f9d6ad7SLin Ling * Let the async thread assess this and handle the detach. 9483f9d6ad7SLin Ling */ 9493f9d6ad7SLin Ling spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 950e4c795beSTom Caputi 951e4c795beSTom Caputi /* 952e4c795beSTom Caputi * Clear any deferred_resilver flags in the config. 953e4c795beSTom Caputi * If there are drives that need resilvering, kick 954e4c795beSTom Caputi * off an asynchronous request to start resilver. 955e4c795beSTom Caputi * dsl_scan_clear_deferred() may update the config 956e4c795beSTom Caputi * before the resilver can restart. In the event of 957e4c795beSTom Caputi * a crash during this period, the spa loading code 958e4c795beSTom Caputi * will find the drives that need to be resilvered 959e4c795beSTom Caputi * when the machine reboots and start the resilver then. 960e4c795beSTom Caputi */ 961233f6c49SKody Kantor if (spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 962233f6c49SKody Kantor boolean_t resilver_needed = 963233f6c49SKody Kantor dsl_scan_clear_deferred(spa->spa_root_vdev, tx); 964233f6c49SKody Kantor if (resilver_needed) { 965233f6c49SKody Kantor spa_history_log_internal(spa, 966233f6c49SKody Kantor "starting deferred resilver", tx, 967233f6c49SKody Kantor "errors=%llu", spa_get_errlog_size(spa)); 968233f6c49SKody Kantor spa_async_request(spa, SPA_ASYNC_RESILVER); 969233f6c49SKody Kantor } 970e4c795beSTom Caputi } 9713f9d6ad7SLin Ling } 9723f9d6ad7SLin Ling 9733f9d6ad7SLin Ling scn->scn_phys.scn_end_time = gethrestime_sec(); 974a3874b8bSToomas Soome 975a3874b8bSToomas Soome ASSERT(!dsl_scan_is_running(scn)); 9763f9d6ad7SLin Ling } 9773f9d6ad7SLin Ling 9783f9d6ad7SLin Ling /* ARGSUSED */ 9793f9d6ad7SLin Ling static int 9803b2aab18SMatthew Ahrens dsl_scan_cancel_check(void *arg, dmu_tx_t *tx) 9813f9d6ad7SLin Ling { 9823b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 9833f9d6ad7SLin Ling 984a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 985be6fd75aSMatthew Ahrens return (SET_ERROR(ENOENT)); 9863f9d6ad7SLin Ling return (0); 9873f9d6ad7SLin Ling } 9883f9d6ad7SLin Ling 9893f9d6ad7SLin Ling /* ARGSUSED */ 9903f9d6ad7SLin Ling static void 9913b2aab18SMatthew Ahrens dsl_scan_cancel_sync(void *arg, dmu_tx_t *tx) 9923f9d6ad7SLin Ling { 9933b2aab18SMatthew Ahrens dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 9943f9d6ad7SLin Ling 9953f9d6ad7SLin Ling dsl_scan_done(scn, B_FALSE, tx); 996a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_MANDATORY); 997301fd1d6SSean Eric Fagan spa_event_notify(scn->scn_dp->dp_spa, NULL, NULL, ESC_ZFS_SCRUB_ABORT); 9983f9d6ad7SLin Ling } 9993f9d6ad7SLin Ling 10003f9d6ad7SLin Ling int 10013f9d6ad7SLin Ling dsl_scan_cancel(dsl_pool_t *dp) 10023f9d6ad7SLin Ling { 10033b2aab18SMatthew Ahrens return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check, 10047d46dc6cSMatthew Ahrens dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED)); 10053f9d6ad7SLin Ling } 10063f9d6ad7SLin Ling 10071702cce7SAlek Pinchuk static int 10081702cce7SAlek Pinchuk dsl_scrub_pause_resume_check(void *arg, dmu_tx_t *tx) 10091702cce7SAlek Pinchuk { 10101702cce7SAlek Pinchuk pool_scrub_cmd_t *cmd = arg; 10111702cce7SAlek Pinchuk dsl_pool_t *dp = dmu_tx_pool(tx); 10121702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 10131702cce7SAlek Pinchuk 10141702cce7SAlek Pinchuk if (*cmd == POOL_SCRUB_PAUSE) { 10151702cce7SAlek Pinchuk /* can't pause a scrub when there is no in-progress scrub */ 10161702cce7SAlek Pinchuk if (!dsl_scan_scrubbing(dp)) 10171702cce7SAlek Pinchuk return (SET_ERROR(ENOENT)); 10181702cce7SAlek Pinchuk 10191702cce7SAlek Pinchuk /* can't pause a paused scrub */ 10201702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) 10211702cce7SAlek Pinchuk return (SET_ERROR(EBUSY)); 10221702cce7SAlek Pinchuk } else if (*cmd != POOL_SCRUB_NORMAL) { 10231702cce7SAlek Pinchuk return (SET_ERROR(ENOTSUP)); 10241702cce7SAlek Pinchuk } 10251702cce7SAlek Pinchuk 10261702cce7SAlek Pinchuk return (0); 10271702cce7SAlek Pinchuk } 10281702cce7SAlek Pinchuk 10291702cce7SAlek Pinchuk static void 10301702cce7SAlek Pinchuk dsl_scrub_pause_resume_sync(void *arg, dmu_tx_t *tx) 10311702cce7SAlek Pinchuk { 10321702cce7SAlek Pinchuk pool_scrub_cmd_t *cmd = arg; 10331702cce7SAlek Pinchuk dsl_pool_t *dp = dmu_tx_pool(tx); 10341702cce7SAlek Pinchuk spa_t *spa = dp->dp_spa; 10351702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 10361702cce7SAlek Pinchuk 10371702cce7SAlek Pinchuk if (*cmd == POOL_SCRUB_PAUSE) { 10381702cce7SAlek Pinchuk /* can't pause a scrub when there is no in-progress scrub */ 10391702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_pause = gethrestime_sec(); 10401702cce7SAlek Pinchuk scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED; 1041e4c795beSTom Caputi scn->scn_phys_cached.scn_flags |= DSF_SCRUB_PAUSED; 1042a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 1043301fd1d6SSean Eric Fagan spa_event_notify(spa, NULL, NULL, ESC_ZFS_SCRUB_PAUSED); 10441702cce7SAlek Pinchuk } else { 10451702cce7SAlek Pinchuk ASSERT3U(*cmd, ==, POOL_SCRUB_NORMAL); 10461702cce7SAlek Pinchuk if (dsl_scan_is_paused_scrub(scn)) { 10471702cce7SAlek Pinchuk /* 10481702cce7SAlek Pinchuk * We need to keep track of how much time we spend 10491702cce7SAlek Pinchuk * paused per pass so that we can adjust the scrub rate 10501702cce7SAlek Pinchuk * shown in the output of 'zpool status' 10511702cce7SAlek Pinchuk */ 10521702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_spent_paused += 10531702cce7SAlek Pinchuk gethrestime_sec() - spa->spa_scan_pass_scrub_pause; 10541702cce7SAlek Pinchuk spa->spa_scan_pass_scrub_pause = 0; 10551702cce7SAlek Pinchuk scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED; 1056e4c795beSTom Caputi scn->scn_phys_cached.scn_flags &= ~DSF_SCRUB_PAUSED; 1057a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 10581702cce7SAlek Pinchuk } 10591702cce7SAlek Pinchuk } 10601702cce7SAlek Pinchuk } 10611702cce7SAlek Pinchuk 10621702cce7SAlek Pinchuk /* 10631702cce7SAlek Pinchuk * Set scrub pause/resume state if it makes sense to do so 10641702cce7SAlek Pinchuk */ 10651702cce7SAlek Pinchuk int 10661702cce7SAlek Pinchuk dsl_scrub_set_pause_resume(const dsl_pool_t *dp, pool_scrub_cmd_t cmd) 10671702cce7SAlek Pinchuk { 10681702cce7SAlek Pinchuk return (dsl_sync_task(spa_name(dp->dp_spa), 10691702cce7SAlek Pinchuk dsl_scrub_pause_resume_check, dsl_scrub_pause_resume_sync, &cmd, 3, 10701702cce7SAlek Pinchuk ZFS_SPACE_CHECK_RESERVED)); 10711702cce7SAlek Pinchuk } 10721702cce7SAlek Pinchuk 10731702cce7SAlek Pinchuk 1074a3874b8bSToomas Soome /* start a new scan, or restart an existing one. */ 1075a3874b8bSToomas Soome void 1076a3874b8bSToomas Soome dsl_resilver_restart(dsl_pool_t *dp, uint64_t txg) 1077a3874b8bSToomas Soome { 1078a3874b8bSToomas Soome if (txg == 0) { 1079a3874b8bSToomas Soome dmu_tx_t *tx; 1080a3874b8bSToomas Soome tx = dmu_tx_create_dd(dp->dp_mos_dir); 1081a3874b8bSToomas Soome VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT)); 10821702cce7SAlek Pinchuk 1083a3874b8bSToomas Soome txg = dmu_tx_get_txg(tx); 1084a3874b8bSToomas Soome dp->dp_scan->scn_restart_txg = txg; 1085a3874b8bSToomas Soome dmu_tx_commit(tx); 1086a3874b8bSToomas Soome } else { 1087a3874b8bSToomas Soome dp->dp_scan->scn_restart_txg = txg; 1088a3874b8bSToomas Soome } 1089a3874b8bSToomas Soome zfs_dbgmsg("restarting resilver txg=%llu", txg); 10901702cce7SAlek Pinchuk } 10911702cce7SAlek Pinchuk 10923f9d6ad7SLin Ling void 10933f9d6ad7SLin Ling dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp) 10943f9d6ad7SLin Ling { 10953f9d6ad7SLin Ling zio_free(dp->dp_spa, txg, bp); 10963f9d6ad7SLin Ling } 10973f9d6ad7SLin Ling 10983f9d6ad7SLin Ling void 10993f9d6ad7SLin Ling dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp) 11003f9d6ad7SLin Ling { 11013f9d6ad7SLin Ling ASSERT(dsl_pool_sync_context(dp)); 11023f9d6ad7SLin Ling zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, pio->io_flags)); 11033f9d6ad7SLin Ling } 11043f9d6ad7SLin Ling 1105a3874b8bSToomas Soome static int 1106a3874b8bSToomas Soome scan_ds_queue_compare(const void *a, const void *b) 11073f9d6ad7SLin Ling { 1108a3874b8bSToomas Soome const scan_ds_t *sds_a = a, *sds_b = b; 1109a3874b8bSToomas Soome 1110a3874b8bSToomas Soome if (sds_a->sds_dsobj < sds_b->sds_dsobj) 1111a3874b8bSToomas Soome return (-1); 1112a3874b8bSToomas Soome if (sds_a->sds_dsobj == sds_b->sds_dsobj) 1113a3874b8bSToomas Soome return (0); 1114a3874b8bSToomas Soome return (1); 11153f9d6ad7SLin Ling } 11163f9d6ad7SLin Ling 11173f9d6ad7SLin Ling static void 1118a3874b8bSToomas Soome scan_ds_queue_clear(dsl_scan_t *scn) 11193f9d6ad7SLin Ling { 1120a3874b8bSToomas Soome void *cookie = NULL; 1121a3874b8bSToomas Soome scan_ds_t *sds; 1122a3874b8bSToomas Soome while ((sds = avl_destroy_nodes(&scn->scn_queue, &cookie)) != NULL) { 1123a3874b8bSToomas Soome kmem_free(sds, sizeof (*sds)); 1124a3874b8bSToomas Soome } 11253f9d6ad7SLin Ling } 11263f9d6ad7SLin Ling 1127a3874b8bSToomas Soome static boolean_t 1128a3874b8bSToomas Soome scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj, uint64_t *txg) 1129a3874b8bSToomas Soome { 1130a3874b8bSToomas Soome scan_ds_t srch, *sds; 1131a3874b8bSToomas Soome 1132a3874b8bSToomas Soome srch.sds_dsobj = dsobj; 1133a3874b8bSToomas Soome sds = avl_find(&scn->scn_queue, &srch, NULL); 1134a3874b8bSToomas Soome if (sds != NULL && txg != NULL) 1135a3874b8bSToomas Soome *txg = sds->sds_txg; 1136a3874b8bSToomas Soome return (sds != NULL); 1137a3874b8bSToomas Soome } 1138a3874b8bSToomas Soome 1139a3874b8bSToomas Soome static void 1140a3874b8bSToomas Soome scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg) 1141a3874b8bSToomas Soome { 1142a3874b8bSToomas Soome scan_ds_t *sds; 1143a3874b8bSToomas Soome avl_index_t where; 1144a3874b8bSToomas Soome 1145a3874b8bSToomas Soome sds = kmem_zalloc(sizeof (*sds), KM_SLEEP); 1146a3874b8bSToomas Soome sds->sds_dsobj = dsobj; 1147a3874b8bSToomas Soome sds->sds_txg = txg; 1148a3874b8bSToomas Soome 1149a3874b8bSToomas Soome VERIFY3P(avl_find(&scn->scn_queue, sds, &where), ==, NULL); 1150a3874b8bSToomas Soome avl_insert(&scn->scn_queue, sds, where); 1151a3874b8bSToomas Soome } 1152a3874b8bSToomas Soome 1153a3874b8bSToomas Soome static void 1154a3874b8bSToomas Soome scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj) 1155a3874b8bSToomas Soome { 1156a3874b8bSToomas Soome scan_ds_t srch, *sds; 1157a3874b8bSToomas Soome 1158a3874b8bSToomas Soome srch.sds_dsobj = dsobj; 1159a3874b8bSToomas Soome 1160a3874b8bSToomas Soome sds = avl_find(&scn->scn_queue, &srch, NULL); 1161a3874b8bSToomas Soome VERIFY(sds != NULL); 1162a3874b8bSToomas Soome avl_remove(&scn->scn_queue, sds); 1163a3874b8bSToomas Soome kmem_free(sds, sizeof (*sds)); 1164a3874b8bSToomas Soome } 1165a3874b8bSToomas Soome 1166a3874b8bSToomas Soome static void 1167a3874b8bSToomas Soome scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx) 1168a3874b8bSToomas Soome { 1169a3874b8bSToomas Soome dsl_pool_t *dp = scn->scn_dp; 1170a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 1171a3874b8bSToomas Soome dmu_object_type_t ot = (spa_version(spa) >= SPA_VERSION_DSL_SCRUB) ? 1172a3874b8bSToomas Soome DMU_OT_SCAN_QUEUE : DMU_OT_ZAP_OTHER; 1173a3874b8bSToomas Soome 1174a3874b8bSToomas Soome ASSERT0(scn->scn_bytes_pending); 1175a3874b8bSToomas Soome ASSERT(scn->scn_phys.scn_queue_obj != 0); 1176a3874b8bSToomas Soome 1177a3874b8bSToomas Soome VERIFY0(dmu_object_free(dp->dp_meta_objset, 1178a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj, tx)); 1179a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, ot, 1180a3874b8bSToomas Soome DMU_OT_NONE, 0, tx); 1181a3874b8bSToomas Soome for (scan_ds_t *sds = avl_first(&scn->scn_queue); 1182a3874b8bSToomas Soome sds != NULL; sds = AVL_NEXT(&scn->scn_queue, sds)) { 1183a3874b8bSToomas Soome VERIFY0(zap_add_int_key(dp->dp_meta_objset, 1184a3874b8bSToomas Soome scn->scn_phys.scn_queue_obj, sds->sds_dsobj, 1185a3874b8bSToomas Soome sds->sds_txg, tx)); 1186a3874b8bSToomas Soome } 1187a3874b8bSToomas Soome } 1188a3874b8bSToomas Soome 1189a3874b8bSToomas Soome /* 1190a3874b8bSToomas Soome * Computes the memory limit state that we're currently in. A sorted scan 1191a3874b8bSToomas Soome * needs quite a bit of memory to hold the sorting queue, so we need to 1192a3874b8bSToomas Soome * reasonably constrain the size so it doesn't impact overall system 1193a3874b8bSToomas Soome * performance. We compute two limits: 1194a3874b8bSToomas Soome * 1) Hard memory limit: if the amount of memory used by the sorting 1195a3874b8bSToomas Soome * queues on a pool gets above this value, we stop the metadata 1196a3874b8bSToomas Soome * scanning portion and start issuing the queued up and sorted 1197a3874b8bSToomas Soome * I/Os to reduce memory usage. 1198a3874b8bSToomas Soome * This limit is calculated as a fraction of physmem (by default 5%). 1199a3874b8bSToomas Soome * We constrain the lower bound of the hard limit to an absolute 1200a3874b8bSToomas Soome * minimum of zfs_scan_mem_lim_min (default: 16 MiB). We also constrain 1201a3874b8bSToomas Soome * the upper bound to 5% of the total pool size - no chance we'll 1202a3874b8bSToomas Soome * ever need that much memory, but just to keep the value in check. 1203a3874b8bSToomas Soome * 2) Soft memory limit: once we hit the hard memory limit, we start 1204a3874b8bSToomas Soome * issuing I/O to reduce queue memory usage, but we don't want to 1205a3874b8bSToomas Soome * completely empty out the queues, since we might be able to find I/Os 1206a3874b8bSToomas Soome * that will fill in the gaps of our non-sequential IOs at some point 1207a3874b8bSToomas Soome * in the future. So we stop the issuing of I/Os once the amount of 1208a3874b8bSToomas Soome * memory used drops below the soft limit (at which point we stop issuing 1209a3874b8bSToomas Soome * I/O and start scanning metadata again). 1210a3874b8bSToomas Soome * 1211a3874b8bSToomas Soome * This limit is calculated by subtracting a fraction of the hard 1212a3874b8bSToomas Soome * limit from the hard limit. By default this fraction is 5%, so 1213a3874b8bSToomas Soome * the soft limit is 95% of the hard limit. We cap the size of the 1214a3874b8bSToomas Soome * difference between the hard and soft limits at an absolute 1215a3874b8bSToomas Soome * maximum of zfs_scan_mem_lim_soft_max (default: 128 MiB) - this is 1216a3874b8bSToomas Soome * sufficient to not cause too frequent switching between the 1217a3874b8bSToomas Soome * metadata scan and I/O issue (even at 2k recordsize, 128 MiB's 1218a3874b8bSToomas Soome * worth of queues is about 1.2 GiB of on-pool data, so scanning 1219a3874b8bSToomas Soome * that should take at least a decent fraction of a second). 1220a3874b8bSToomas Soome */ 1221a3874b8bSToomas Soome static boolean_t 1222a3874b8bSToomas Soome dsl_scan_should_clear(dsl_scan_t *scn) 1223a3874b8bSToomas Soome { 1224a3874b8bSToomas Soome vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; 1225a3874b8bSToomas Soome uint64_t mlim_hard, mlim_soft, mused; 1226a3874b8bSToomas Soome uint64_t alloc = metaslab_class_get_alloc(spa_normal_class( 1227a3874b8bSToomas Soome scn->scn_dp->dp_spa)); 1228a3874b8bSToomas Soome 1229a3874b8bSToomas Soome mlim_hard = MAX((physmem / zfs_scan_mem_lim_fact) * PAGESIZE, 1230a3874b8bSToomas Soome zfs_scan_mem_lim_min); 1231a3874b8bSToomas Soome mlim_hard = MIN(mlim_hard, alloc / 20); 1232a3874b8bSToomas Soome mlim_soft = mlim_hard - MIN(mlim_hard / zfs_scan_mem_lim_soft_fact, 1233a3874b8bSToomas Soome zfs_scan_mem_lim_soft_max); 1234a3874b8bSToomas Soome mused = 0; 1235a3874b8bSToomas Soome for (uint64_t i = 0; i < rvd->vdev_children; i++) { 1236a3874b8bSToomas Soome vdev_t *tvd = rvd->vdev_child[i]; 1237a3874b8bSToomas Soome dsl_scan_io_queue_t *queue; 1238a3874b8bSToomas Soome 1239a3874b8bSToomas Soome mutex_enter(&tvd->vdev_scan_io_queue_lock); 1240a3874b8bSToomas Soome queue = tvd->vdev_scan_io_queue; 1241a3874b8bSToomas Soome if (queue != NULL) { 124212a8814cSTom Caputi /* # extents in exts_by_size = # in exts_by_addr */ 1243*4d7988d6SPaul Dagnelie mused += zfs_btree_numnodes(&queue->q_exts_by_size) * 124412a8814cSTom Caputi sizeof (range_seg_t) + queue->q_sio_memused; 1245a3874b8bSToomas Soome } 1246a3874b8bSToomas Soome mutex_exit(&tvd->vdev_scan_io_queue_lock); 1247a3874b8bSToomas Soome } 1248a3874b8bSToomas Soome 1249a3874b8bSToomas Soome dprintf("current scan memory usage: %llu bytes\n", (longlong_t)mused); 1250a3874b8bSToomas Soome 1251a3874b8bSToomas Soome if (mused == 0) 1252a3874b8bSToomas Soome ASSERT0(scn->scn_bytes_pending); 1253a3874b8bSToomas Soome 1254a3874b8bSToomas Soome /* 1255a3874b8bSToomas Soome * If we are above our hard limit, we need to clear out memory. 1256a3874b8bSToomas Soome * If we are below our soft limit, we need to accumulate sequential IOs. 1257a3874b8bSToomas Soome * Otherwise, we should keep doing whatever we are currently doing. 1258a3874b8bSToomas Soome */ 1259a3874b8bSToomas Soome if (mused >= mlim_hard) 1260a3874b8bSToomas Soome return (B_TRUE); 1261a3874b8bSToomas Soome else if (mused < mlim_soft) 1262a3874b8bSToomas Soome return (B_FALSE); 1263a3874b8bSToomas Soome else 1264a3874b8bSToomas Soome return (scn->scn_clearing); 1265a3874b8bSToomas Soome } 12666f6a76adSMatthew Ahrens 12673f9d6ad7SLin Ling static boolean_t 12681702cce7SAlek Pinchuk dsl_scan_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb) 12693f9d6ad7SLin Ling { 12703f9d6ad7SLin Ling /* we never skip user/group accounting objects */ 12713f9d6ad7SLin Ling if (zb && (int64_t)zb->zb_object < 0) 12723f9d6ad7SLin Ling return (B_FALSE); 12733f9d6ad7SLin Ling 12741702cce7SAlek Pinchuk if (scn->scn_suspending) 12751702cce7SAlek Pinchuk return (B_TRUE); /* we're already suspending */ 12763f9d6ad7SLin Ling 1277ad135b5dSChristopher Siden if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) 12783f9d6ad7SLin Ling return (B_FALSE); /* we're resuming */ 12793f9d6ad7SLin Ling 12803f9d6ad7SLin Ling /* We only know how to resume from level-0 blocks. */ 12813f9d6ad7SLin Ling if (zb && zb->zb_level != 0) 12823f9d6ad7SLin Ling return (B_FALSE); 12833f9d6ad7SLin Ling 12846f6a76adSMatthew Ahrens /* 12851702cce7SAlek Pinchuk * We suspend if: 12866f6a76adSMatthew Ahrens * - we have scanned for at least the minimum time (default 1 sec 12876f6a76adSMatthew Ahrens * for scrub, 3 sec for resilver), and either we have sufficient 12886f6a76adSMatthew Ahrens * dirty data that we are starting to write more quickly 12896f6a76adSMatthew Ahrens * (default 30%), or someone is explicitly waiting for this txg 12906f6a76adSMatthew Ahrens * to complete. 12916f6a76adSMatthew Ahrens * or 12926f6a76adSMatthew Ahrens * - the spa is shutting down because this pool is being exported 12936f6a76adSMatthew Ahrens * or the machine is rebooting. 1294a3874b8bSToomas Soome * or 1295a3874b8bSToomas Soome * - the scan queue has reached its memory use limit 12966f6a76adSMatthew Ahrens */ 1297a3874b8bSToomas Soome hrtime_t curr_time_ns = gethrtime(); 1298a3874b8bSToomas Soome uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time; 1299a3874b8bSToomas Soome uint64_t sync_time_ns = curr_time_ns - 1300a3874b8bSToomas Soome scn->scn_dp->dp_spa->spa_sync_starttime; 1301a3874b8bSToomas Soome 13026f6a76adSMatthew Ahrens int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; 1303a3874b8bSToomas Soome int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 1304a3874b8bSToomas Soome zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; 1305a3874b8bSToomas Soome 1306a3874b8bSToomas Soome if ((NSEC2MSEC(scan_time_ns) > mintime && 1307a3874b8bSToomas Soome (dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent || 1308a3874b8bSToomas Soome txg_sync_waiting(scn->scn_dp) || 1309a3874b8bSToomas Soome NSEC2SEC(sync_time_ns) >= zfs_txg_timeout)) || 1310a3874b8bSToomas Soome spa_shutting_down(scn->scn_dp->dp_spa) || 1311a3874b8bSToomas Soome (zfs_scan_strict_mem_lim && dsl_scan_should_clear(scn))) { 13123f9d6ad7SLin Ling if (zb) { 13131702cce7SAlek Pinchuk dprintf("suspending at bookmark %llx/%llx/%llx/%llx\n", 13143f9d6ad7SLin Ling (longlong_t)zb->zb_objset, 13153f9d6ad7SLin Ling (longlong_t)zb->zb_object, 13163f9d6ad7SLin Ling (longlong_t)zb->zb_level, 13173f9d6ad7SLin Ling (longlong_t)zb->zb_blkid); 13183f9d6ad7SLin Ling scn->scn_phys.scn_bookmark = *zb; 1319a3874b8bSToomas Soome } else { 1320a3874b8bSToomas Soome dsl_scan_phys_t *scnp = &scn->scn_phys; 1321a3874b8bSToomas Soome 1322a3874b8bSToomas Soome dprintf("suspending at DDT bookmark " 1323a3874b8bSToomas Soome "%llx/%llx/%llx/%llx\n", 1324a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_class, 1325a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_type, 1326a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_checksum, 1327a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_cursor); 13283f9d6ad7SLin Ling } 13291702cce7SAlek Pinchuk scn->scn_suspending = B_TRUE; 13303f9d6ad7SLin Ling return (B_TRUE); 13313f9d6ad7SLin Ling } 13323f9d6ad7SLin Ling return (B_FALSE); 13333f9d6ad7SLin Ling } 13343f9d6ad7SLin Ling 13353f9d6ad7SLin Ling typedef struct zil_scan_arg { 13363f9d6ad7SLin Ling dsl_pool_t *zsa_dp; 13373f9d6ad7SLin Ling zil_header_t *zsa_zh; 13383f9d6ad7SLin Ling } zil_scan_arg_t; 13393f9d6ad7SLin Ling 13403f9d6ad7SLin Ling /* ARGSUSED */ 13413f9d6ad7SLin Ling static int 13423f9d6ad7SLin Ling dsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 13433f9d6ad7SLin Ling { 13443f9d6ad7SLin Ling zil_scan_arg_t *zsa = arg; 13453f9d6ad7SLin Ling dsl_pool_t *dp = zsa->zsa_dp; 13463f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 13473f9d6ad7SLin Ling zil_header_t *zh = zsa->zsa_zh; 13487802d7bfSMatthew Ahrens zbookmark_phys_t zb; 13493f9d6ad7SLin Ling 135043466aaeSMax Grossman if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 13513f9d6ad7SLin Ling return (0); 13523f9d6ad7SLin Ling 13533f9d6ad7SLin Ling /* 13543f9d6ad7SLin Ling * One block ("stubby") can be allocated a long time ago; we 13553f9d6ad7SLin Ling * want to visit that one because it has been allocated 13563f9d6ad7SLin Ling * (on-disk) even if it hasn't been claimed (even though for 13573f9d6ad7SLin Ling * scrub there's nothing to do to it). 13583f9d6ad7SLin Ling */ 135986714001SSerapheim Dimitropoulos if (claim_txg == 0 && bp->blk_birth >= spa_min_claim_txg(dp->dp_spa)) 13603f9d6ad7SLin Ling return (0); 13613f9d6ad7SLin Ling 13623f9d6ad7SLin Ling SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 13633f9d6ad7SLin Ling ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 13643f9d6ad7SLin Ling 13653f9d6ad7SLin Ling VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 13663f9d6ad7SLin Ling return (0); 13673f9d6ad7SLin Ling } 13683f9d6ad7SLin Ling 13693f9d6ad7SLin Ling /* ARGSUSED */ 13703f9d6ad7SLin Ling static int 13713f9d6ad7SLin Ling dsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg) 13723f9d6ad7SLin Ling { 13733f9d6ad7SLin Ling if (lrc->lrc_txtype == TX_WRITE) { 13743f9d6ad7SLin Ling zil_scan_arg_t *zsa = arg; 13753f9d6ad7SLin Ling dsl_pool_t *dp = zsa->zsa_dp; 13763f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 13773f9d6ad7SLin Ling zil_header_t *zh = zsa->zsa_zh; 13783f9d6ad7SLin Ling lr_write_t *lr = (lr_write_t *)lrc; 13793f9d6ad7SLin Ling blkptr_t *bp = &lr->lr_blkptr; 13807802d7bfSMatthew Ahrens zbookmark_phys_t zb; 13813f9d6ad7SLin Ling 138243466aaeSMax Grossman if (BP_IS_HOLE(bp) || 138343466aaeSMax Grossman bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 13843f9d6ad7SLin Ling return (0); 13853f9d6ad7SLin Ling 13863f9d6ad7SLin Ling /* 13873f9d6ad7SLin Ling * birth can be < claim_txg if this record's txg is 13883f9d6ad7SLin Ling * already txg sync'ed (but this log block contains 13893f9d6ad7SLin Ling * other records that are not synced) 13903f9d6ad7SLin Ling */ 13913f9d6ad7SLin Ling if (claim_txg == 0 || bp->blk_birth < claim_txg) 13923f9d6ad7SLin Ling return (0); 13933f9d6ad7SLin Ling 13943f9d6ad7SLin Ling SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 13953f9d6ad7SLin Ling lr->lr_foid, ZB_ZIL_LEVEL, 13963f9d6ad7SLin Ling lr->lr_offset / BP_GET_LSIZE(bp)); 13973f9d6ad7SLin Ling 13983f9d6ad7SLin Ling VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 13993f9d6ad7SLin Ling } 14003f9d6ad7SLin Ling return (0); 14013f9d6ad7SLin Ling } 14023f9d6ad7SLin Ling 14033f9d6ad7SLin Ling static void 14043f9d6ad7SLin Ling dsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh) 14053f9d6ad7SLin Ling { 14063f9d6ad7SLin Ling uint64_t claim_txg = zh->zh_claim_txg; 14073f9d6ad7SLin Ling zil_scan_arg_t zsa = { dp, zh }; 14083f9d6ad7SLin Ling zilog_t *zilog; 14093f9d6ad7SLin Ling 141086714001SSerapheim Dimitropoulos ASSERT(spa_writeable(dp->dp_spa)); 141186714001SSerapheim Dimitropoulos 14123f9d6ad7SLin Ling /* 141386714001SSerapheim Dimitropoulos * We only want to visit blocks that have been claimed 141486714001SSerapheim Dimitropoulos * but not yet replayed. 14153f9d6ad7SLin Ling */ 141686714001SSerapheim Dimitropoulos if (claim_txg == 0) 14173f9d6ad7SLin Ling return; 14183f9d6ad7SLin Ling 14193f9d6ad7SLin Ling zilog = zil_alloc(dp->dp_meta_objset, zh); 14203f9d6ad7SLin Ling 14213f9d6ad7SLin Ling (void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa, 1422eb633035STom Caputi claim_txg, B_FALSE); 14233f9d6ad7SLin Ling 14243f9d6ad7SLin Ling zil_free(zilog); 14253f9d6ad7SLin Ling } 14263f9d6ad7SLin Ling 1427a3874b8bSToomas Soome /* 1428a3874b8bSToomas Soome * We compare scan_prefetch_issue_ctx_t's based on their bookmarks. The idea 1429a3874b8bSToomas Soome * here is to sort the AVL tree by the order each block will be needed. 1430a3874b8bSToomas Soome */ 1431a3874b8bSToomas Soome static int 1432a3874b8bSToomas Soome scan_prefetch_queue_compare(const void *a, const void *b) 1433a3874b8bSToomas Soome { 1434a3874b8bSToomas Soome const scan_prefetch_issue_ctx_t *spic_a = a, *spic_b = b; 1435a3874b8bSToomas Soome const scan_prefetch_ctx_t *spc_a = spic_a->spic_spc; 1436a3874b8bSToomas Soome const scan_prefetch_ctx_t *spc_b = spic_b->spic_spc; 1437a3874b8bSToomas Soome 1438a3874b8bSToomas Soome return (zbookmark_compare(spc_a->spc_datablkszsec, 1439a3874b8bSToomas Soome spc_a->spc_indblkshift, spc_b->spc_datablkszsec, 1440a3874b8bSToomas Soome spc_b->spc_indblkshift, &spic_a->spic_zb, &spic_b->spic_zb)); 1441a3874b8bSToomas Soome } 1442a3874b8bSToomas Soome 1443a3874b8bSToomas Soome static void 1444a3874b8bSToomas Soome scan_prefetch_ctx_rele(scan_prefetch_ctx_t *spc, void *tag) 1445a3874b8bSToomas Soome { 1446a3874b8bSToomas Soome if (zfs_refcount_remove(&spc->spc_refcnt, tag) == 0) { 1447a3874b8bSToomas Soome zfs_refcount_destroy(&spc->spc_refcnt); 1448a3874b8bSToomas Soome kmem_free(spc, sizeof (scan_prefetch_ctx_t)); 1449a3874b8bSToomas Soome } 1450a3874b8bSToomas Soome } 1451a3874b8bSToomas Soome 1452a3874b8bSToomas Soome static scan_prefetch_ctx_t * 1453a3874b8bSToomas Soome scan_prefetch_ctx_create(dsl_scan_t *scn, dnode_phys_t *dnp, void *tag) 1454a3874b8bSToomas Soome { 1455a3874b8bSToomas Soome scan_prefetch_ctx_t *spc; 1456a3874b8bSToomas Soome 1457a3874b8bSToomas Soome spc = kmem_alloc(sizeof (scan_prefetch_ctx_t), KM_SLEEP); 1458a3874b8bSToomas Soome zfs_refcount_create(&spc->spc_refcnt); 1459a3874b8bSToomas Soome zfs_refcount_add(&spc->spc_refcnt, tag); 1460a3874b8bSToomas Soome spc->spc_scn = scn; 1461a3874b8bSToomas Soome if (dnp != NULL) { 1462a3874b8bSToomas Soome spc->spc_datablkszsec = dnp->dn_datablkszsec; 1463a3874b8bSToomas Soome spc->spc_indblkshift = dnp->dn_indblkshift; 1464a3874b8bSToomas Soome spc->spc_root = B_FALSE; 1465a3874b8bSToomas Soome } else { 1466a3874b8bSToomas Soome spc->spc_datablkszsec = 0; 1467a3874b8bSToomas Soome spc->spc_indblkshift = 0; 1468a3874b8bSToomas Soome spc->spc_root = B_TRUE; 1469a3874b8bSToomas Soome } 1470a3874b8bSToomas Soome 1471a3874b8bSToomas Soome return (spc); 1472a3874b8bSToomas Soome } 1473a3874b8bSToomas Soome 1474a3874b8bSToomas Soome static void 1475a3874b8bSToomas Soome scan_prefetch_ctx_add_ref(scan_prefetch_ctx_t *spc, void *tag) 1476a3874b8bSToomas Soome { 1477a3874b8bSToomas Soome zfs_refcount_add(&spc->spc_refcnt, tag); 1478a3874b8bSToomas Soome } 1479a3874b8bSToomas Soome 1480a3874b8bSToomas Soome static boolean_t 1481a3874b8bSToomas Soome dsl_scan_check_prefetch_resume(scan_prefetch_ctx_t *spc, 1482a3874b8bSToomas Soome const zbookmark_phys_t *zb) 1483a3874b8bSToomas Soome { 1484a3874b8bSToomas Soome zbookmark_phys_t *last_zb = &spc->spc_scn->scn_prefetch_bookmark; 1485a3874b8bSToomas Soome dnode_phys_t tmp_dnp; 1486a3874b8bSToomas Soome dnode_phys_t *dnp = (spc->spc_root) ? NULL : &tmp_dnp; 1487a3874b8bSToomas Soome 1488a3874b8bSToomas Soome if (zb->zb_objset != last_zb->zb_objset) 1489a3874b8bSToomas Soome return (B_TRUE); 1490a3874b8bSToomas Soome if ((int64_t)zb->zb_object < 0) 1491a3874b8bSToomas Soome return (B_FALSE); 1492a3874b8bSToomas Soome 1493a3874b8bSToomas Soome tmp_dnp.dn_datablkszsec = spc->spc_datablkszsec; 1494a3874b8bSToomas Soome tmp_dnp.dn_indblkshift = spc->spc_indblkshift; 1495a3874b8bSToomas Soome 1496a3874b8bSToomas Soome if (zbookmark_subtree_completed(dnp, zb, last_zb)) 1497a3874b8bSToomas Soome return (B_TRUE); 1498a3874b8bSToomas Soome 1499a3874b8bSToomas Soome return (B_FALSE); 1500a3874b8bSToomas Soome } 1501a3874b8bSToomas Soome 1502a3874b8bSToomas Soome static void 1503a3874b8bSToomas Soome dsl_scan_prefetch(scan_prefetch_ctx_t *spc, blkptr_t *bp, zbookmark_phys_t *zb) 15043f9d6ad7SLin Ling { 1505a3874b8bSToomas Soome avl_index_t idx; 1506a3874b8bSToomas Soome dsl_scan_t *scn = spc->spc_scn; 1507a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 1508a3874b8bSToomas Soome scan_prefetch_issue_ctx_t *spic; 15093f9d6ad7SLin Ling 15103f9d6ad7SLin Ling if (zfs_no_scrub_prefetch) 15113f9d6ad7SLin Ling return; 15123f9d6ad7SLin Ling 1513a3874b8bSToomas Soome if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg || 1514a3874b8bSToomas Soome (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE && 1515a3874b8bSToomas Soome BP_GET_TYPE(bp) != DMU_OT_OBJSET)) 1516a3874b8bSToomas Soome return; 1517a3874b8bSToomas Soome 1518a3874b8bSToomas Soome if (dsl_scan_check_prefetch_resume(spc, zb)) 1519a3874b8bSToomas Soome return; 1520a3874b8bSToomas Soome 1521a3874b8bSToomas Soome scan_prefetch_ctx_add_ref(spc, scn); 1522a3874b8bSToomas Soome spic = kmem_alloc(sizeof (scan_prefetch_issue_ctx_t), KM_SLEEP); 1523a3874b8bSToomas Soome spic->spic_spc = spc; 1524a3874b8bSToomas Soome spic->spic_bp = *bp; 1525a3874b8bSToomas Soome spic->spic_zb = *zb; 1526a3874b8bSToomas Soome 1527a3874b8bSToomas Soome /* 1528a3874b8bSToomas Soome * Add the IO to the queue of blocks to prefetch. This allows us to 1529a3874b8bSToomas Soome * prioritize blocks that we will need first for the main traversal 1530a3874b8bSToomas Soome * thread. 1531a3874b8bSToomas Soome */ 1532a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1533a3874b8bSToomas Soome if (avl_find(&scn->scn_prefetch_queue, spic, &idx) != NULL) { 1534a3874b8bSToomas Soome /* this block is already queued for prefetch */ 1535a3874b8bSToomas Soome kmem_free(spic, sizeof (scan_prefetch_issue_ctx_t)); 1536a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, scn); 1537a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1538a3874b8bSToomas Soome return; 1539a3874b8bSToomas Soome } 1540a3874b8bSToomas Soome 1541a3874b8bSToomas Soome avl_insert(&scn->scn_prefetch_queue, spic, idx); 1542a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 1543a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1544a3874b8bSToomas Soome } 1545a3874b8bSToomas Soome 1546a3874b8bSToomas Soome static void 1547a3874b8bSToomas Soome dsl_scan_prefetch_dnode(dsl_scan_t *scn, dnode_phys_t *dnp, 1548a3874b8bSToomas Soome uint64_t objset, uint64_t object) 1549a3874b8bSToomas Soome { 1550a3874b8bSToomas Soome int i; 1551a3874b8bSToomas Soome zbookmark_phys_t zb; 1552a3874b8bSToomas Soome scan_prefetch_ctx_t *spc; 1553a3874b8bSToomas Soome 1554a3874b8bSToomas Soome if (dnp->dn_nblkptr == 0 && !(dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 15553f9d6ad7SLin Ling return; 15563f9d6ad7SLin Ling 1557a3874b8bSToomas Soome SET_BOOKMARK(&zb, objset, object, 0, 0); 1558a3874b8bSToomas Soome 1559a3874b8bSToomas Soome spc = scan_prefetch_ctx_create(scn, dnp, FTAG); 1560a3874b8bSToomas Soome 1561a3874b8bSToomas Soome for (i = 0; i < dnp->dn_nblkptr; i++) { 1562a3874b8bSToomas Soome zb.zb_level = BP_GET_LEVEL(&dnp->dn_blkptr[i]); 1563a3874b8bSToomas Soome zb.zb_blkid = i; 1564a3874b8bSToomas Soome dsl_scan_prefetch(spc, &dnp->dn_blkptr[i], &zb); 1565a3874b8bSToomas Soome } 1566a3874b8bSToomas Soome 1567a3874b8bSToomas Soome if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 1568a3874b8bSToomas Soome zb.zb_level = 0; 1569a3874b8bSToomas Soome zb.zb_blkid = DMU_SPILL_BLKID; 1570a3874b8bSToomas Soome dsl_scan_prefetch(spc, &dnp->dn_spill, &zb); 1571a3874b8bSToomas Soome } 1572a3874b8bSToomas Soome 1573a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, FTAG); 1574a3874b8bSToomas Soome } 1575a3874b8bSToomas Soome 1576a3874b8bSToomas Soome void 1577a3874b8bSToomas Soome dsl_scan_prefetch_cb(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, 1578a3874b8bSToomas Soome arc_buf_t *buf, void *private) 1579a3874b8bSToomas Soome { 1580a3874b8bSToomas Soome scan_prefetch_ctx_t *spc = private; 1581a3874b8bSToomas Soome dsl_scan_t *scn = spc->spc_scn; 1582a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 1583a3874b8bSToomas Soome 1584a3874b8bSToomas Soome /* broadcast that the IO has completed for rate limitting purposes */ 1585a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1586a3874b8bSToomas Soome ASSERT3U(spa->spa_scrub_inflight, >=, BP_GET_PSIZE(bp)); 1587a3874b8bSToomas Soome spa->spa_scrub_inflight -= BP_GET_PSIZE(bp); 1588a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 1589a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1590a3874b8bSToomas Soome 1591a3874b8bSToomas Soome /* if there was an error or we are done prefetching, just cleanup */ 1592a3874b8bSToomas Soome if (buf == NULL || scn->scn_suspending) 1593a3874b8bSToomas Soome goto out; 1594a3874b8bSToomas Soome 1595a3874b8bSToomas Soome if (BP_GET_LEVEL(bp) > 0) { 1596a3874b8bSToomas Soome int i; 1597a3874b8bSToomas Soome blkptr_t *cbp; 1598a3874b8bSToomas Soome int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 1599a3874b8bSToomas Soome zbookmark_phys_t czb; 1600a3874b8bSToomas Soome 1601a3874b8bSToomas Soome for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) { 1602a3874b8bSToomas Soome SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 1603a3874b8bSToomas Soome zb->zb_level - 1, zb->zb_blkid * epb + i); 1604a3874b8bSToomas Soome dsl_scan_prefetch(spc, cbp, &czb); 1605a3874b8bSToomas Soome } 1606a3874b8bSToomas Soome } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 1607a3874b8bSToomas Soome dnode_phys_t *cdnp = buf->b_data; 1608a3874b8bSToomas Soome int i; 1609a3874b8bSToomas Soome int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 1610a3874b8bSToomas Soome 1611a3874b8bSToomas Soome for (i = 0, cdnp = buf->b_data; i < epb; 1612a3874b8bSToomas Soome i += cdnp->dn_extra_slots + 1, 1613a3874b8bSToomas Soome cdnp += cdnp->dn_extra_slots + 1) { 1614a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, cdnp, 1615a3874b8bSToomas Soome zb->zb_objset, zb->zb_blkid * epb + i); 1616a3874b8bSToomas Soome } 1617a3874b8bSToomas Soome } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 1618a3874b8bSToomas Soome objset_phys_t *osp = buf->b_data; 1619a3874b8bSToomas Soome 1620a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, &osp->os_meta_dnode, 1621a3874b8bSToomas Soome zb->zb_objset, DMU_META_DNODE_OBJECT); 16223f9d6ad7SLin Ling 1623a3874b8bSToomas Soome if (OBJSET_BUF_HAS_USERUSED(buf)) { 1624a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, 1625a3874b8bSToomas Soome &osp->os_groupused_dnode, zb->zb_objset, 1626a3874b8bSToomas Soome DMU_GROUPUSED_OBJECT); 1627a3874b8bSToomas Soome dsl_scan_prefetch_dnode(scn, 1628a3874b8bSToomas Soome &osp->os_userused_dnode, zb->zb_objset, 1629a3874b8bSToomas Soome DMU_USERUSED_OBJECT); 1630a3874b8bSToomas Soome } 1631a3874b8bSToomas Soome } 1632a3874b8bSToomas Soome 1633a3874b8bSToomas Soome out: 1634a3874b8bSToomas Soome if (buf != NULL) 1635a3874b8bSToomas Soome arc_buf_destroy(buf, private); 1636a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, scn); 1637a3874b8bSToomas Soome } 1638a3874b8bSToomas Soome 1639a3874b8bSToomas Soome /* ARGSUSED */ 1640a3874b8bSToomas Soome static void 1641a3874b8bSToomas Soome dsl_scan_prefetch_thread(void *arg) 1642a3874b8bSToomas Soome { 1643a3874b8bSToomas Soome dsl_scan_t *scn = arg; 1644a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 1645a3874b8bSToomas Soome vdev_t *rvd = spa->spa_root_vdev; 1646a3874b8bSToomas Soome uint64_t maxinflight = rvd->vdev_children * zfs_top_maxinflight; 1647a3874b8bSToomas Soome scan_prefetch_issue_ctx_t *spic; 1648a3874b8bSToomas Soome 1649a3874b8bSToomas Soome /* loop until we are told to stop */ 1650a3874b8bSToomas Soome while (!scn->scn_prefetch_stop) { 1651a3874b8bSToomas Soome arc_flags_t flags = ARC_FLAG_NOWAIT | 1652a3874b8bSToomas Soome ARC_FLAG_PRESCIENT_PREFETCH | ARC_FLAG_PREFETCH; 1653a3874b8bSToomas Soome int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 1654a3874b8bSToomas Soome 1655a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1656a3874b8bSToomas Soome 1657a3874b8bSToomas Soome /* 1658a3874b8bSToomas Soome * Wait until we have an IO to issue and are not above our 1659a3874b8bSToomas Soome * maximum in flight limit. 1660a3874b8bSToomas Soome */ 1661a3874b8bSToomas Soome while (!scn->scn_prefetch_stop && 1662a3874b8bSToomas Soome (avl_numnodes(&scn->scn_prefetch_queue) == 0 || 1663a3874b8bSToomas Soome spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes)) { 1664a3874b8bSToomas Soome cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 1665a3874b8bSToomas Soome } 1666a3874b8bSToomas Soome 1667a3874b8bSToomas Soome /* recheck if we should stop since we waited for the cv */ 1668a3874b8bSToomas Soome if (scn->scn_prefetch_stop) { 1669a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1670a3874b8bSToomas Soome break; 1671a3874b8bSToomas Soome } 1672a3874b8bSToomas Soome 1673a3874b8bSToomas Soome /* remove the prefetch IO from the tree */ 1674a3874b8bSToomas Soome spic = avl_first(&scn->scn_prefetch_queue); 1675a3874b8bSToomas Soome spa->spa_scrub_inflight += BP_GET_PSIZE(&spic->spic_bp); 1676a3874b8bSToomas Soome avl_remove(&scn->scn_prefetch_queue, spic); 1677a3874b8bSToomas Soome 1678a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 1679a3874b8bSToomas Soome 1680eb633035STom Caputi if (BP_IS_PROTECTED(&spic->spic_bp)) { 1681eb633035STom Caputi ASSERT(BP_GET_TYPE(&spic->spic_bp) == DMU_OT_DNODE || 1682eb633035STom Caputi BP_GET_TYPE(&spic->spic_bp) == DMU_OT_OBJSET); 1683eb633035STom Caputi ASSERT3U(BP_GET_LEVEL(&spic->spic_bp), ==, 0); 1684eb633035STom Caputi zio_flags |= ZIO_FLAG_RAW; 1685eb633035STom Caputi } 1686eb633035STom Caputi 1687a3874b8bSToomas Soome /* issue the prefetch asynchronously */ 1688a3874b8bSToomas Soome (void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, 1689a3874b8bSToomas Soome &spic->spic_bp, dsl_scan_prefetch_cb, spic->spic_spc, 1690a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, &spic->spic_zb); 1691a3874b8bSToomas Soome 1692a3874b8bSToomas Soome kmem_free(spic, sizeof (scan_prefetch_issue_ctx_t)); 1693a3874b8bSToomas Soome } 1694a3874b8bSToomas Soome 1695a3874b8bSToomas Soome ASSERT(scn->scn_prefetch_stop); 1696a3874b8bSToomas Soome 1697a3874b8bSToomas Soome /* free any prefetches we didn't get to complete */ 1698a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 1699a3874b8bSToomas Soome while ((spic = avl_first(&scn->scn_prefetch_queue)) != NULL) { 1700a3874b8bSToomas Soome avl_remove(&scn->scn_prefetch_queue, spic); 1701a3874b8bSToomas Soome scan_prefetch_ctx_rele(spic->spic_spc, scn); 1702a3874b8bSToomas Soome kmem_free(spic, sizeof (scan_prefetch_issue_ctx_t)); 1703a3874b8bSToomas Soome } 1704a3874b8bSToomas Soome ASSERT0(avl_numnodes(&scn->scn_prefetch_queue)); 1705a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 17063f9d6ad7SLin Ling } 17073f9d6ad7SLin Ling 17083f9d6ad7SLin Ling static boolean_t 17093f9d6ad7SLin Ling dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp, 17107802d7bfSMatthew Ahrens const zbookmark_phys_t *zb) 17113f9d6ad7SLin Ling { 17123f9d6ad7SLin Ling /* 17133f9d6ad7SLin Ling * We never skip over user/group accounting objects (obj<0) 17143f9d6ad7SLin Ling */ 1715ad135b5dSChristopher Siden if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) && 17163f9d6ad7SLin Ling (int64_t)zb->zb_object >= 0) { 17173f9d6ad7SLin Ling /* 17183f9d6ad7SLin Ling * If we already visited this bp & everything below (in 17193f9d6ad7SLin Ling * a prior txg sync), don't bother doing it again. 17203f9d6ad7SLin Ling */ 1721a2cdcdd2SPaul Dagnelie if (zbookmark_subtree_completed(dnp, zb, 1722a2cdcdd2SPaul Dagnelie &scn->scn_phys.scn_bookmark)) 17233f9d6ad7SLin Ling return (B_TRUE); 17243f9d6ad7SLin Ling 17253f9d6ad7SLin Ling /* 17263f9d6ad7SLin Ling * If we found the block we're trying to resume from, or 17273f9d6ad7SLin Ling * we went past it to a different object, zero it out to 17281702cce7SAlek Pinchuk * indicate that it's OK to start checking for suspending 17293f9d6ad7SLin Ling * again. 17303f9d6ad7SLin Ling */ 17313f9d6ad7SLin Ling if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 || 17323f9d6ad7SLin Ling zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) { 17333f9d6ad7SLin Ling dprintf("resuming at %llx/%llx/%llx/%llx\n", 17343f9d6ad7SLin Ling (longlong_t)zb->zb_objset, 17353f9d6ad7SLin Ling (longlong_t)zb->zb_object, 17363f9d6ad7SLin Ling (longlong_t)zb->zb_level, 17373f9d6ad7SLin Ling (longlong_t)zb->zb_blkid); 17383f9d6ad7SLin Ling bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb)); 17393f9d6ad7SLin Ling } 17403f9d6ad7SLin Ling } 17413f9d6ad7SLin Ling return (B_FALSE); 17423f9d6ad7SLin Ling } 17433f9d6ad7SLin Ling 1744a3874b8bSToomas Soome static void dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 1745a3874b8bSToomas Soome dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, 1746a3874b8bSToomas Soome dmu_objset_type_t ostype, dmu_tx_t *tx); 1747a3874b8bSToomas Soome static void dsl_scan_visitdnode( 1748a3874b8bSToomas Soome dsl_scan_t *, dsl_dataset_t *ds, dmu_objset_type_t ostype, 1749a3874b8bSToomas Soome dnode_phys_t *dnp, uint64_t object, dmu_tx_t *tx); 1750a3874b8bSToomas Soome 17513f9d6ad7SLin Ling /* 17523f9d6ad7SLin Ling * Return nonzero on i/o error. 17533f9d6ad7SLin Ling * Return new buf to write out in *bufp. 17543f9d6ad7SLin Ling */ 17553f9d6ad7SLin Ling static int 17563f9d6ad7SLin Ling dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, 17573f9d6ad7SLin Ling dnode_phys_t *dnp, const blkptr_t *bp, 17585f37736aSMatthew Ahrens const zbookmark_phys_t *zb, dmu_tx_t *tx) 17593f9d6ad7SLin Ling { 17603f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 176144ecc532SGeorge Wilson int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 17623f9d6ad7SLin Ling int err; 17633f9d6ad7SLin Ling 17643f9d6ad7SLin Ling if (BP_GET_LEVEL(bp) > 0) { 17657adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 17663f9d6ad7SLin Ling int i; 17673f9d6ad7SLin Ling blkptr_t *cbp; 17683f9d6ad7SLin Ling int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 17695f37736aSMatthew Ahrens arc_buf_t *buf; 17703f9d6ad7SLin Ling 17715f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 1772a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, zb); 17733f9d6ad7SLin Ling if (err) { 17743f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 17753f9d6ad7SLin Ling return (err); 17763f9d6ad7SLin Ling } 17775f37736aSMatthew Ahrens for (i = 0, cbp = buf->b_data; i < epb; i++, cbp++) { 17787802d7bfSMatthew Ahrens zbookmark_phys_t czb; 17793f9d6ad7SLin Ling 17803f9d6ad7SLin Ling SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 17813f9d6ad7SLin Ling zb->zb_level - 1, 17823f9d6ad7SLin Ling zb->zb_blkid * epb + i); 17833f9d6ad7SLin Ling dsl_scan_visitbp(cbp, &czb, dnp, 17845f37736aSMatthew Ahrens ds, scn, ostype, tx); 17853f9d6ad7SLin Ling } 1786dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 17873f9d6ad7SLin Ling } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 17887adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 17893f9d6ad7SLin Ling dnode_phys_t *cdnp; 1790a3874b8bSToomas Soome int i; 17913f9d6ad7SLin Ling int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 17925f37736aSMatthew Ahrens arc_buf_t *buf; 17933f9d6ad7SLin Ling 1794eb633035STom Caputi if (BP_IS_PROTECTED(bp)) { 1795eb633035STom Caputi ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF); 1796eb633035STom Caputi zio_flags |= ZIO_FLAG_RAW; 1797eb633035STom Caputi } 1798eb633035STom Caputi 17995f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 1800a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, zb); 18013f9d6ad7SLin Ling if (err) { 18023f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 18033f9d6ad7SLin Ling return (err); 18043f9d6ad7SLin Ling } 180554811da5SToomas Soome for (i = 0, cdnp = buf->b_data; i < epb; 180654811da5SToomas Soome i += cdnp->dn_extra_slots + 1, 180754811da5SToomas Soome cdnp += cdnp->dn_extra_slots + 1) { 18083f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, ostype, 18095f37736aSMatthew Ahrens cdnp, zb->zb_blkid * epb + i, tx); 18103f9d6ad7SLin Ling } 18113f9d6ad7SLin Ling 1812dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 18133f9d6ad7SLin Ling } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 18147adb730bSGeorge Wilson arc_flags_t flags = ARC_FLAG_WAIT; 18153f9d6ad7SLin Ling objset_phys_t *osp; 18165f37736aSMatthew Ahrens arc_buf_t *buf; 18173f9d6ad7SLin Ling 18185f37736aSMatthew Ahrens err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, &buf, 1819a3874b8bSToomas Soome ZIO_PRIORITY_SCRUB, zio_flags, &flags, zb); 18203f9d6ad7SLin Ling if (err) { 18213f9d6ad7SLin Ling scn->scn_phys.scn_errors++; 18223f9d6ad7SLin Ling return (err); 18233f9d6ad7SLin Ling } 18243f9d6ad7SLin Ling 18255f37736aSMatthew Ahrens osp = buf->b_data; 18263f9d6ad7SLin Ling 18273f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 18285f37736aSMatthew Ahrens &osp->os_meta_dnode, DMU_META_DNODE_OBJECT, tx); 18293f9d6ad7SLin Ling 18305f37736aSMatthew Ahrens if (OBJSET_BUF_HAS_USERUSED(buf)) { 18313f9d6ad7SLin Ling /* 1832f67950b2SNasf-Fan * We also always visit user/group/project accounting 18333f9d6ad7SLin Ling * objects, and never skip them, even if we are 18341702cce7SAlek Pinchuk * suspending. This is necessary so that the space 18353f9d6ad7SLin Ling * deltas from this txg get integrated. 18363f9d6ad7SLin Ling */ 1837f67950b2SNasf-Fan if (OBJSET_BUF_HAS_PROJECTUSED(buf)) 1838f67950b2SNasf-Fan dsl_scan_visitdnode(scn, ds, osp->os_type, 1839f67950b2SNasf-Fan &osp->os_projectused_dnode, 1840f67950b2SNasf-Fan DMU_PROJECTUSED_OBJECT, tx); 18413f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 18425f37736aSMatthew Ahrens &osp->os_groupused_dnode, 18433f9d6ad7SLin Ling DMU_GROUPUSED_OBJECT, tx); 18443f9d6ad7SLin Ling dsl_scan_visitdnode(scn, ds, osp->os_type, 18455f37736aSMatthew Ahrens &osp->os_userused_dnode, 18463f9d6ad7SLin Ling DMU_USERUSED_OBJECT, tx); 18473f9d6ad7SLin Ling } 1848dcbf3bd6SGeorge Wilson arc_buf_destroy(buf, &buf); 18493f9d6ad7SLin Ling } 18503f9d6ad7SLin Ling 18513f9d6ad7SLin Ling return (0); 18523f9d6ad7SLin Ling } 18533f9d6ad7SLin Ling 18543f9d6ad7SLin Ling static void 18553f9d6ad7SLin Ling dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, 18565f37736aSMatthew Ahrens dmu_objset_type_t ostype, dnode_phys_t *dnp, 18573f9d6ad7SLin Ling uint64_t object, dmu_tx_t *tx) 18583f9d6ad7SLin Ling { 18593f9d6ad7SLin Ling int j; 18603f9d6ad7SLin Ling 18613f9d6ad7SLin Ling for (j = 0; j < dnp->dn_nblkptr; j++) { 18627802d7bfSMatthew Ahrens zbookmark_phys_t czb; 18633f9d6ad7SLin Ling 18643f9d6ad7SLin Ling SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 18653f9d6ad7SLin Ling dnp->dn_nlevels - 1, j); 18663f9d6ad7SLin Ling dsl_scan_visitbp(&dnp->dn_blkptr[j], 18675f37736aSMatthew Ahrens &czb, dnp, ds, scn, ostype, tx); 18683f9d6ad7SLin Ling } 18693f9d6ad7SLin Ling 18703f9d6ad7SLin Ling if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 18717802d7bfSMatthew Ahrens zbookmark_phys_t czb; 18723f9d6ad7SLin Ling SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 18733f9d6ad7SLin Ling 0, DMU_SPILL_BLKID); 187454811da5SToomas Soome dsl_scan_visitbp(DN_SPILL_BLKPTR(dnp), 18755f37736aSMatthew Ahrens &czb, dnp, ds, scn, ostype, tx); 18763f9d6ad7SLin Ling } 18773f9d6ad7SLin Ling } 18783f9d6ad7SLin Ling 18793f9d6ad7SLin Ling /* 18803f9d6ad7SLin Ling * The arguments are in this order because mdb can only print the 18813f9d6ad7SLin Ling * first 5; we want them to be useful. 18823f9d6ad7SLin Ling */ 18833f9d6ad7SLin Ling static void 18847802d7bfSMatthew Ahrens dsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 18855f37736aSMatthew Ahrens dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn, 18865f37736aSMatthew Ahrens dmu_objset_type_t ostype, dmu_tx_t *tx) 18873f9d6ad7SLin Ling { 18883f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 1889a3874b8bSToomas Soome blkptr_t *bp_toread = NULL; 18903f9d6ad7SLin Ling 18911702cce7SAlek Pinchuk if (dsl_scan_check_suspend(scn, zb)) 18923f9d6ad7SLin Ling return; 18933f9d6ad7SLin Ling 18943f9d6ad7SLin Ling if (dsl_scan_check_resume(scn, dnp, zb)) 18953f9d6ad7SLin Ling return; 18963f9d6ad7SLin Ling 18973f9d6ad7SLin Ling scn->scn_visited_this_txg++; 18983f9d6ad7SLin Ling 1899a3874b8bSToomas Soome /* 1900a3874b8bSToomas Soome * This debugging is commented out to conserve stack space. This 1901a3874b8bSToomas Soome * function is called recursively and the debugging addes several 1902a3874b8bSToomas Soome * bytes to the stack for each call. It can be commented back in 1903a3874b8bSToomas Soome * if required to debug an issue in dsl_scan_visitbp(). 1904a3874b8bSToomas Soome * 1905a3874b8bSToomas Soome * dprintf_bp(bp, 1906a3874b8bSToomas Soome * "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx bp=%p", 1907a3874b8bSToomas Soome * ds, ds ? ds->ds_object : 0, 1908a3874b8bSToomas Soome * zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid, 1909a3874b8bSToomas Soome * bp); 1910a3874b8bSToomas Soome */ 19113f9d6ad7SLin Ling 1912a3874b8bSToomas Soome if (BP_IS_HOLE(bp)) { 1913a3874b8bSToomas Soome scn->scn_holes_this_txg++; 19143f9d6ad7SLin Ling return; 1915a3874b8bSToomas Soome } 19163f9d6ad7SLin Ling 1917a3874b8bSToomas Soome if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) { 1918a3874b8bSToomas Soome scn->scn_lt_min_this_txg++; 19193f9d6ad7SLin Ling return; 1920a3874b8bSToomas Soome } 1921a3874b8bSToomas Soome 1922a3874b8bSToomas Soome bp_toread = kmem_alloc(sizeof (blkptr_t), KM_SLEEP); 1923a3874b8bSToomas Soome *bp_toread = *bp; 1924a3874b8bSToomas Soome 1925a3874b8bSToomas Soome if (dsl_scan_recurse(scn, ds, ostype, dnp, bp_toread, zb, tx) != 0) 1926a3874b8bSToomas Soome goto out; 19273f9d6ad7SLin Ling 19283f9d6ad7SLin Ling /* 192940713f2bSAlan Somers * If dsl_scan_ddt() has already visited this block, it will have 19303f9d6ad7SLin Ling * already done any translations or scrubbing, so don't call the 19313f9d6ad7SLin Ling * callback again. 19323f9d6ad7SLin Ling */ 19333f9d6ad7SLin Ling if (ddt_class_contains(dp->dp_spa, 19343f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max, bp)) { 1935a3874b8bSToomas Soome scn->scn_ddt_contained_this_txg++; 1936a3874b8bSToomas Soome goto out; 19373f9d6ad7SLin Ling } 19383f9d6ad7SLin Ling 19393f9d6ad7SLin Ling /* 19403f9d6ad7SLin Ling * If this block is from the future (after cur_max_txg), then we 19413f9d6ad7SLin Ling * are doing this on behalf of a deleted snapshot, and we will 19423f9d6ad7SLin Ling * revisit the future block on the next pass of this dataset. 19433f9d6ad7SLin Ling * Don't scan it now unless we need to because something 19443f9d6ad7SLin Ling * under it was modified. 19453f9d6ad7SLin Ling */ 1946a3874b8bSToomas Soome if (BP_PHYSICAL_BIRTH(bp) > scn->scn_phys.scn_cur_max_txg) { 1947a3874b8bSToomas Soome scn->scn_gt_max_this_txg++; 1948a3874b8bSToomas Soome goto out; 19493f9d6ad7SLin Ling } 1950a3874b8bSToomas Soome 1951a3874b8bSToomas Soome scan_funcs[scn->scn_phys.scn_func](dp, bp, zb); 1952a3874b8bSToomas Soome 1953a3874b8bSToomas Soome out: 1954a3874b8bSToomas Soome kmem_free(bp_toread, sizeof (blkptr_t)); 19553f9d6ad7SLin Ling } 19563f9d6ad7SLin Ling 19573f9d6ad7SLin Ling static void 19583f9d6ad7SLin Ling dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp, 19593f9d6ad7SLin Ling dmu_tx_t *tx) 19603f9d6ad7SLin Ling { 19617802d7bfSMatthew Ahrens zbookmark_phys_t zb; 1962a3874b8bSToomas Soome scan_prefetch_ctx_t *spc; 19633f9d6ad7SLin Ling 19643f9d6ad7SLin Ling SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 19653f9d6ad7SLin Ling ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID); 1966a3874b8bSToomas Soome 1967a3874b8bSToomas Soome if (ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) { 1968a3874b8bSToomas Soome SET_BOOKMARK(&scn->scn_prefetch_bookmark, 1969a3874b8bSToomas Soome zb.zb_objset, 0, 0, 0); 1970a3874b8bSToomas Soome } else { 1971a3874b8bSToomas Soome scn->scn_prefetch_bookmark = scn->scn_phys.scn_bookmark; 1972a3874b8bSToomas Soome } 1973a3874b8bSToomas Soome 1974a3874b8bSToomas Soome scn->scn_objsets_visited_this_txg++; 1975a3874b8bSToomas Soome 1976a3874b8bSToomas Soome spc = scan_prefetch_ctx_create(scn, NULL, FTAG); 1977a3874b8bSToomas Soome dsl_scan_prefetch(spc, bp, &zb); 1978a3874b8bSToomas Soome scan_prefetch_ctx_rele(spc, FTAG); 1979a3874b8bSToomas Soome 1980a3874b8bSToomas Soome dsl_scan_visitbp(bp, &zb, NULL, ds, scn, DMU_OST_NONE, tx); 19813f9d6ad7SLin Ling 19823f9d6ad7SLin Ling dprintf_ds(ds, "finished scan%s", ""); 19833f9d6ad7SLin Ling } 19843f9d6ad7SLin Ling 1985a3874b8bSToomas Soome static void 1986a3874b8bSToomas Soome ds_destroyed_scn_phys(dsl_dataset_t *ds, dsl_scan_phys_t *scn_phys) 19873f9d6ad7SLin Ling { 1988a3874b8bSToomas Soome if (scn_phys->scn_bookmark.zb_objset == ds->ds_object) { 1989bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) { 199038d61036SMatthew Ahrens /* 199138d61036SMatthew Ahrens * Note: 199238d61036SMatthew Ahrens * - scn_cur_{min,max}_txg stays the same. 199338d61036SMatthew Ahrens * - Setting the flag is not really necessary if 199438d61036SMatthew Ahrens * scn_cur_max_txg == scn_max_txg, because there 199538d61036SMatthew Ahrens * is nothing after this snapshot that we care 199638d61036SMatthew Ahrens * about. However, we set it anyway and then 199738d61036SMatthew Ahrens * ignore it when we retraverse it in 199838d61036SMatthew Ahrens * dsl_scan_visitds(). 199938d61036SMatthew Ahrens */ 2000a3874b8bSToomas Soome scn_phys->scn_bookmark.zb_objset = 2001c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj; 20023f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; currently traversing; " 20033f9d6ad7SLin Ling "reset zb_objset to %llu", 20043f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 2005c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)-> 2006c1379625SJustin T. Gibbs ds_next_snap_obj); 2007a3874b8bSToomas Soome scn_phys->scn_flags |= DSF_VISIT_DS_AGAIN; 20083f9d6ad7SLin Ling } else { 2009a3874b8bSToomas Soome SET_BOOKMARK(&scn_phys->scn_bookmark, 20103f9d6ad7SLin Ling ZB_DESTROYED_OBJSET, 0, 0, 0); 20113f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; currently traversing; " 20123f9d6ad7SLin Ling "reset bookmark to -1,0,0,0", 20133f9d6ad7SLin Ling (u_longlong_t)ds->ds_object); 20143f9d6ad7SLin Ling } 2015a3874b8bSToomas Soome } 2016a3874b8bSToomas Soome } 2017a3874b8bSToomas Soome 2018a3874b8bSToomas Soome /* 2019a3874b8bSToomas Soome * Invoked when a dataset is destroyed. We need to make sure that: 2020a3874b8bSToomas Soome * 2021a3874b8bSToomas Soome * 1) If it is the dataset that was currently being scanned, we write 2022a3874b8bSToomas Soome * a new dsl_scan_phys_t and marking the objset reference in it 2023a3874b8bSToomas Soome * as destroyed. 2024a3874b8bSToomas Soome * 2) Remove it from the work queue, if it was present. 2025a3874b8bSToomas Soome * 2026a3874b8bSToomas Soome * If the dataset was actually a snapshot, instead of marking the dataset 2027a3874b8bSToomas Soome * as destroyed, we instead substitute the next snapshot in line. 2028a3874b8bSToomas Soome */ 2029a3874b8bSToomas Soome void 2030a3874b8bSToomas Soome dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx) 2031a3874b8bSToomas Soome { 2032a3874b8bSToomas Soome dsl_pool_t *dp = ds->ds_dir->dd_pool; 2033a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 2034a3874b8bSToomas Soome uint64_t mintxg; 2035a3874b8bSToomas Soome 2036a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 2037a3874b8bSToomas Soome return; 2038a3874b8bSToomas Soome 2039a3874b8bSToomas Soome ds_destroyed_scn_phys(ds, &scn->scn_phys); 2040a3874b8bSToomas Soome ds_destroyed_scn_phys(ds, &scn->scn_phys_cached); 2041a3874b8bSToomas Soome 2042a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) { 2043a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds->ds_object); 2044a3874b8bSToomas Soome if (ds->ds_is_snapshot) 2045a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2046a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_next_snap_obj, mintxg); 2047a3874b8bSToomas Soome } 2048a3874b8bSToomas Soome 2049a3874b8bSToomas Soome if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 2050a3874b8bSToomas Soome ds->ds_object, &mintxg) == 0) { 2051c1379625SJustin T. Gibbs ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1); 2052b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 20533f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 2054bc9014e6SJustin Gibbs if (ds->ds_is_snapshot) { 20553f9d6ad7SLin Ling /* 20563f9d6ad7SLin Ling * We keep the same mintxg; it could be > 20573f9d6ad7SLin Ling * ds_creation_txg if the previous snapshot was 20583f9d6ad7SLin Ling * deleted too. 20593f9d6ad7SLin Ling */ 20603f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 20613f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 2062c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj, 2063c1379625SJustin T. Gibbs mintxg, tx) == 0); 20643f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; in queue; " 20653f9d6ad7SLin Ling "replacing with %llu", 20663f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 2067c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)-> 2068c1379625SJustin T. Gibbs ds_next_snap_obj); 20693f9d6ad7SLin Ling } else { 20703f9d6ad7SLin Ling zfs_dbgmsg("destroying ds %llu; in queue; removing", 20713f9d6ad7SLin Ling (u_longlong_t)ds->ds_object); 20723f9d6ad7SLin Ling } 20733f9d6ad7SLin Ling } 20743f9d6ad7SLin Ling 20753f9d6ad7SLin Ling /* 20763f9d6ad7SLin Ling * dsl_scan_sync() should be called after this, and should sync 20773f9d6ad7SLin Ling * out our changed state, but just to be safe, do it here. 20783f9d6ad7SLin Ling */ 2079a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 2080a3874b8bSToomas Soome } 2081a3874b8bSToomas Soome 2082a3874b8bSToomas Soome static void 2083a3874b8bSToomas Soome ds_snapshotted_bookmark(dsl_dataset_t *ds, zbookmark_phys_t *scn_bookmark) 2084a3874b8bSToomas Soome { 2085a3874b8bSToomas Soome if (scn_bookmark->zb_objset == ds->ds_object) { 2086a3874b8bSToomas Soome scn_bookmark->zb_objset = 2087a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_obj; 2088a3874b8bSToomas Soome zfs_dbgmsg("snapshotting ds %llu; currently traversing; " 2089a3874b8bSToomas Soome "reset zb_objset to %llu", 2090a3874b8bSToomas Soome (u_longlong_t)ds->ds_object, 2091a3874b8bSToomas Soome (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj); 2092a3874b8bSToomas Soome } 20933f9d6ad7SLin Ling } 20943f9d6ad7SLin Ling 2095a3874b8bSToomas Soome /* 2096a3874b8bSToomas Soome * Called when a dataset is snapshotted. If we were currently traversing 2097a3874b8bSToomas Soome * this snapshot, we reset our bookmark to point at the newly created 2098a3874b8bSToomas Soome * snapshot. We also modify our work queue to remove the old snapshot and 2099a3874b8bSToomas Soome * replace with the new one. 2100a3874b8bSToomas Soome */ 21013f9d6ad7SLin Ling void 21023f9d6ad7SLin Ling dsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx) 21033f9d6ad7SLin Ling { 21043f9d6ad7SLin Ling dsl_pool_t *dp = ds->ds_dir->dd_pool; 21053f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 21063f9d6ad7SLin Ling uint64_t mintxg; 21073f9d6ad7SLin Ling 2108a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 21093f9d6ad7SLin Ling return; 21103f9d6ad7SLin Ling 2111c1379625SJustin T. Gibbs ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0); 21123f9d6ad7SLin Ling 2113a3874b8bSToomas Soome ds_snapshotted_bookmark(ds, &scn->scn_phys.scn_bookmark); 2114a3874b8bSToomas Soome ds_snapshotted_bookmark(ds, &scn->scn_phys_cached.scn_bookmark); 2115a3874b8bSToomas Soome 2116a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) { 2117a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds->ds_object); 2118a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2119a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg); 2120a3874b8bSToomas Soome } 2121a3874b8bSToomas Soome 2122a3874b8bSToomas Soome if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 2123a3874b8bSToomas Soome ds->ds_object, &mintxg) == 0) { 2124b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 21253f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 21263f9d6ad7SLin Ling VERIFY(zap_add_int_key(dp->dp_meta_objset, 21273f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 2128c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, mintxg, tx) == 0); 21293f9d6ad7SLin Ling zfs_dbgmsg("snapshotting ds %llu; in queue; " 21303f9d6ad7SLin Ling "replacing with %llu", 21313f9d6ad7SLin Ling (u_longlong_t)ds->ds_object, 2132c1379625SJustin T. Gibbs (u_longlong_t)dsl_dataset_phys(ds)->ds_prev_snap_obj); 21333f9d6ad7SLin Ling } 2134a3874b8bSToomas Soome 2135a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 21363f9d6ad7SLin Ling } 21373f9d6ad7SLin Ling 2138a3874b8bSToomas Soome static void 2139a3874b8bSToomas Soome ds_clone_swapped_bookmark(dsl_dataset_t *ds1, dsl_dataset_t *ds2, 2140a3874b8bSToomas Soome zbookmark_phys_t *scn_bookmark) 21413f9d6ad7SLin Ling { 2142a3874b8bSToomas Soome if (scn_bookmark->zb_objset == ds1->ds_object) { 2143a3874b8bSToomas Soome scn_bookmark->zb_objset = ds2->ds_object; 21443f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 21453f9d6ad7SLin Ling "reset zb_objset to %llu", 21463f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object, 21473f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object); 2148a3874b8bSToomas Soome } else if (scn_bookmark->zb_objset == ds2->ds_object) { 2149a3874b8bSToomas Soome scn_bookmark->zb_objset = ds1->ds_object; 21503f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 21513f9d6ad7SLin Ling "reset zb_objset to %llu", 21523f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object, 21533f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object); 21543f9d6ad7SLin Ling } 2155a3874b8bSToomas Soome } 2156a3874b8bSToomas Soome 2157a3874b8bSToomas Soome /* 2158a3874b8bSToomas Soome * Called when a parent dataset and its clone are swapped. If we were 2159a3874b8bSToomas Soome * currently traversing the dataset, we need to switch to traversing the 2160a3874b8bSToomas Soome * newly promoted parent. 2161a3874b8bSToomas Soome */ 2162a3874b8bSToomas Soome void 2163a3874b8bSToomas Soome dsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx) 2164a3874b8bSToomas Soome { 2165a3874b8bSToomas Soome dsl_pool_t *dp = ds1->ds_dir->dd_pool; 2166a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 2167a3874b8bSToomas Soome uint64_t mintxg; 2168a3874b8bSToomas Soome 2169a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 2170a3874b8bSToomas Soome return; 2171a3874b8bSToomas Soome 2172a3874b8bSToomas Soome ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys.scn_bookmark); 2173a3874b8bSToomas Soome ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys_cached.scn_bookmark); 2174a3874b8bSToomas Soome 2175a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds1->ds_object, &mintxg)) { 2176a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds1->ds_object); 2177a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds2->ds_object, mintxg); 2178a3874b8bSToomas Soome } 2179a3874b8bSToomas Soome if (scan_ds_queue_contains(scn, ds2->ds_object, &mintxg)) { 2180a3874b8bSToomas Soome scan_ds_queue_remove(scn, ds2->ds_object); 2181a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds1->ds_object, mintxg); 2182a3874b8bSToomas Soome } 21833f9d6ad7SLin Ling 21843f9d6ad7SLin Ling if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 21853f9d6ad7SLin Ling ds1->ds_object, &mintxg) == 0) { 21863f9d6ad7SLin Ling int err; 2187c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg); 2188c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg); 2189b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 21903f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds1->ds_object, tx)); 21913f9d6ad7SLin Ling err = zap_add_int_key(dp->dp_meta_objset, 21923f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx); 21933f9d6ad7SLin Ling VERIFY(err == 0 || err == EEXIST); 21943f9d6ad7SLin Ling if (err == EEXIST) { 21953f9d6ad7SLin Ling /* Both were there to begin with */ 21963f9d6ad7SLin Ling VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 21973f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, 21983f9d6ad7SLin Ling ds1->ds_object, mintxg, tx)); 21993f9d6ad7SLin Ling } 22003f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; in queue; " 22013f9d6ad7SLin Ling "replacing with %llu", 22023f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object, 22033f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object); 2204a3874b8bSToomas Soome } 2205a3874b8bSToomas Soome if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 2206a3874b8bSToomas Soome ds2->ds_object, &mintxg) == 0) { 2207c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds1)->ds_prev_snap_txg); 2208c1379625SJustin T. Gibbs ASSERT3U(mintxg, ==, dsl_dataset_phys(ds2)->ds_prev_snap_txg); 2209b420f3adSRichard Lowe VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 22103f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds2->ds_object, tx)); 22113f9d6ad7SLin Ling VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 22123f9d6ad7SLin Ling scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx)); 22133f9d6ad7SLin Ling zfs_dbgmsg("clone_swap ds %llu; in queue; " 22143f9d6ad7SLin Ling "replacing with %llu", 22153f9d6ad7SLin Ling (u_longlong_t)ds2->ds_object, 22163f9d6ad7SLin Ling (u_longlong_t)ds1->ds_object); 22173f9d6ad7SLin Ling } 22183f9d6ad7SLin Ling 2219a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, SYNC_CACHED); 22203f9d6ad7SLin Ling } 22213f9d6ad7SLin Ling 22223f9d6ad7SLin Ling /* ARGSUSED */ 22233f9d6ad7SLin Ling static int 22243b2aab18SMatthew Ahrens enqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 22253f9d6ad7SLin Ling { 2226a3874b8bSToomas Soome uint64_t originobj = *(uint64_t *)arg; 22273f9d6ad7SLin Ling dsl_dataset_t *ds; 22283f9d6ad7SLin Ling int err; 22293f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 22303f9d6ad7SLin Ling 2231a3874b8bSToomas Soome if (dsl_dir_phys(hds->ds_dir)->dd_origin_obj != originobj) 22323b2aab18SMatthew Ahrens return (0); 22333b2aab18SMatthew Ahrens 22343b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 22353f9d6ad7SLin Ling if (err) 22363f9d6ad7SLin Ling return (err); 22373f9d6ad7SLin Ling 2238a3874b8bSToomas Soome while (dsl_dataset_phys(ds)->ds_prev_snap_obj != originobj) { 22393b2aab18SMatthew Ahrens dsl_dataset_t *prev; 22403b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, 2241c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev); 22423f9d6ad7SLin Ling 22433b2aab18SMatthew Ahrens dsl_dataset_rele(ds, FTAG); 22443b2aab18SMatthew Ahrens if (err) 22453b2aab18SMatthew Ahrens return (err); 22463b2aab18SMatthew Ahrens ds = prev; 22473f9d6ad7SLin Ling } 2248a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds->ds_object, 2249a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_txg); 22503f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 22513f9d6ad7SLin Ling return (0); 22523f9d6ad7SLin Ling } 22533f9d6ad7SLin Ling 22543f9d6ad7SLin Ling static void 22553f9d6ad7SLin Ling dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) 22563f9d6ad7SLin Ling { 22573f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 22583f9d6ad7SLin Ling dsl_dataset_t *ds; 22593f9d6ad7SLin Ling 2260b420f3adSRichard Lowe VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 22613f9d6ad7SLin Ling 226238d61036SMatthew Ahrens if (scn->scn_phys.scn_cur_min_txg >= 226338d61036SMatthew Ahrens scn->scn_phys.scn_max_txg) { 226438d61036SMatthew Ahrens /* 226538d61036SMatthew Ahrens * This can happen if this snapshot was created after the 226638d61036SMatthew Ahrens * scan started, and we already completed a previous snapshot 226738d61036SMatthew Ahrens * that was created after the scan started. This snapshot 226838d61036SMatthew Ahrens * only references blocks with: 226938d61036SMatthew Ahrens * 227038d61036SMatthew Ahrens * birth < our ds_creation_txg 227138d61036SMatthew Ahrens * cur_min_txg is no less than ds_creation_txg. 227238d61036SMatthew Ahrens * We have already visited these blocks. 227338d61036SMatthew Ahrens * or 227438d61036SMatthew Ahrens * birth > scn_max_txg 227538d61036SMatthew Ahrens * The scan requested not to visit these blocks. 227638d61036SMatthew Ahrens * 227738d61036SMatthew Ahrens * Subsequent snapshots (and clones) can reference our 227838d61036SMatthew Ahrens * blocks, or blocks with even higher birth times. 227938d61036SMatthew Ahrens * Therefore we do not need to visit them either, 228038d61036SMatthew Ahrens * so we do not add them to the work queue. 228138d61036SMatthew Ahrens * 228238d61036SMatthew Ahrens * Note that checking for cur_min_txg >= cur_max_txg 228338d61036SMatthew Ahrens * is not sufficient, because in that case we may need to 228438d61036SMatthew Ahrens * visit subsequent snapshots. This happens when min_txg > 0, 228538d61036SMatthew Ahrens * which raises cur_min_txg. In this case we will visit 228638d61036SMatthew Ahrens * this dataset but skip all of its blocks, because the 228738d61036SMatthew Ahrens * rootbp's birth time is < cur_min_txg. Then we will 228838d61036SMatthew Ahrens * add the next snapshots/clones to the work queue. 228938d61036SMatthew Ahrens */ 229038d61036SMatthew Ahrens char *dsname = kmem_alloc(MAXNAMELEN, KM_SLEEP); 229138d61036SMatthew Ahrens dsl_dataset_name(ds, dsname); 229238d61036SMatthew Ahrens zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because " 229338d61036SMatthew Ahrens "cur_min_txg (%llu) >= max_txg (%llu)", 2294a3874b8bSToomas Soome (longlong_t)dsobj, dsname, 2295a3874b8bSToomas Soome (longlong_t)scn->scn_phys.scn_cur_min_txg, 2296a3874b8bSToomas Soome (longlong_t)scn->scn_phys.scn_max_txg); 229738d61036SMatthew Ahrens kmem_free(dsname, MAXNAMELEN); 229838d61036SMatthew Ahrens 229938d61036SMatthew Ahrens goto out; 230038d61036SMatthew Ahrens } 230138d61036SMatthew Ahrens 23026e0cbcaaSMatthew Ahrens /* 23035cabbc6bSPrashanth Sreenivasa * Only the ZIL in the head (non-snapshot) is valid. Even though 23046e0cbcaaSMatthew Ahrens * snapshots can have ZIL block pointers (which may be the same 23055cabbc6bSPrashanth Sreenivasa * BP as in the head), they must be ignored. In addition, $ORIGIN 23065cabbc6bSPrashanth Sreenivasa * doesn't have a objset (i.e. its ds_bp is a hole) so we don't 23075cabbc6bSPrashanth Sreenivasa * need to look for a ZIL in it either. So we traverse the ZIL here, 23085cabbc6bSPrashanth Sreenivasa * rather than in scan_recurse(), because the regular snapshot 23095cabbc6bSPrashanth Sreenivasa * block-sharing rules don't apply to it. 23106e0cbcaaSMatthew Ahrens */ 23115cabbc6bSPrashanth Sreenivasa if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds) && 2312bb1f4245SMatthew Ahrens (dp->dp_origin_snap == NULL || 2313bb1f4245SMatthew Ahrens ds->ds_dir != dp->dp_origin_snap->ds_dir)) { 23145cabbc6bSPrashanth Sreenivasa objset_t *os; 23155cabbc6bSPrashanth Sreenivasa if (dmu_objset_from_ds(ds, &os) != 0) { 23165cabbc6bSPrashanth Sreenivasa goto out; 23175cabbc6bSPrashanth Sreenivasa } 23186e0cbcaaSMatthew Ahrens dsl_scan_zil(dp, &os->os_zil_header); 23195cabbc6bSPrashanth Sreenivasa } 23206e0cbcaaSMatthew Ahrens 23213f9d6ad7SLin Ling /* 23223f9d6ad7SLin Ling * Iterate over the bps in this ds. 23233f9d6ad7SLin Ling */ 23243f9d6ad7SLin Ling dmu_buf_will_dirty(ds->ds_dbuf, tx); 2325c166b69dSPaul Dagnelie rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG); 2326c1379625SJustin T. Gibbs dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx); 2327c166b69dSPaul Dagnelie rrw_exit(&ds->ds_bp_rwlock, FTAG); 23283f9d6ad7SLin Ling 23299adfa60dSMatthew Ahrens char *dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); 23303f9d6ad7SLin Ling dsl_dataset_name(ds, dsname); 23313f9d6ad7SLin Ling zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; " 23321702cce7SAlek Pinchuk "suspending=%u", 23333f9d6ad7SLin Ling (longlong_t)dsobj, dsname, 23343f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_cur_min_txg, 23353f9d6ad7SLin Ling (longlong_t)scn->scn_phys.scn_cur_max_txg, 23361702cce7SAlek Pinchuk (int)scn->scn_suspending); 23379adfa60dSMatthew Ahrens kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN); 23383f9d6ad7SLin Ling 23391702cce7SAlek Pinchuk if (scn->scn_suspending) 23403f9d6ad7SLin Ling goto out; 23413f9d6ad7SLin Ling 23423f9d6ad7SLin Ling /* 23433f9d6ad7SLin Ling * We've finished this pass over this dataset. 23443f9d6ad7SLin Ling */ 23453f9d6ad7SLin Ling 23463f9d6ad7SLin Ling /* 23473f9d6ad7SLin Ling * If we did not completely visit this dataset, do another pass. 23483f9d6ad7SLin Ling */ 23493f9d6ad7SLin Ling if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) { 23503f9d6ad7SLin Ling zfs_dbgmsg("incomplete pass; visiting again"); 23513f9d6ad7SLin Ling scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN; 2352a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds->ds_object, 2353a3874b8bSToomas Soome scn->scn_phys.scn_cur_max_txg); 23543f9d6ad7SLin Ling goto out; 23553f9d6ad7SLin Ling } 23563f9d6ad7SLin Ling 23573f9d6ad7SLin Ling /* 23583f9d6ad7SLin Ling * Add descendent datasets to work queue. 23593f9d6ad7SLin Ling */ 2360c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) { 2361a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2362c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_snap_obj, 2363a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_creation_txg); 23643f9d6ad7SLin Ling } 2365c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_num_children > 1) { 23663f9d6ad7SLin Ling boolean_t usenext = B_FALSE; 2367c1379625SJustin T. Gibbs if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) { 23683f9d6ad7SLin Ling uint64_t count; 23693f9d6ad7SLin Ling /* 23703f9d6ad7SLin Ling * A bug in a previous version of the code could 23713f9d6ad7SLin Ling * cause upgrade_clones_cb() to not set 23723f9d6ad7SLin Ling * ds_next_snap_obj when it should, leading to a 23733f9d6ad7SLin Ling * missing entry. Therefore we can only use the 23743f9d6ad7SLin Ling * next_clones_obj when its count is correct. 23753f9d6ad7SLin Ling */ 23763f9d6ad7SLin Ling int err = zap_count(dp->dp_meta_objset, 2377c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_next_clones_obj, &count); 23783f9d6ad7SLin Ling if (err == 0 && 2379c1379625SJustin T. Gibbs count == dsl_dataset_phys(ds)->ds_num_children - 1) 23803f9d6ad7SLin Ling usenext = B_TRUE; 23813f9d6ad7SLin Ling } 23823f9d6ad7SLin Ling 23833f9d6ad7SLin Ling if (usenext) { 2384a3874b8bSToomas Soome zap_cursor_t zc; 2385a3874b8bSToomas Soome zap_attribute_t za; 2386a3874b8bSToomas Soome for (zap_cursor_init(&zc, dp->dp_meta_objset, 2387a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_next_clones_obj); 2388a3874b8bSToomas Soome zap_cursor_retrieve(&zc, &za) == 0; 2389a3874b8bSToomas Soome (void) zap_cursor_advance(&zc)) { 2390a3874b8bSToomas Soome scan_ds_queue_insert(scn, 2391a3874b8bSToomas Soome zfs_strtonum(za.za_name, NULL), 2392a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_creation_txg); 2393a3874b8bSToomas Soome } 2394a3874b8bSToomas Soome zap_cursor_fini(&zc); 23953f9d6ad7SLin Ling } else { 23963b2aab18SMatthew Ahrens VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2397a3874b8bSToomas Soome enqueue_clones_cb, &ds->ds_object, 2398a3874b8bSToomas Soome DS_FIND_CHILDREN)); 23993f9d6ad7SLin Ling } 24003f9d6ad7SLin Ling } 24013f9d6ad7SLin Ling 24023f9d6ad7SLin Ling out: 24033f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24043f9d6ad7SLin Ling } 24053f9d6ad7SLin Ling 24063f9d6ad7SLin Ling /* ARGSUSED */ 24073f9d6ad7SLin Ling static int 24083b2aab18SMatthew Ahrens enqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 24093f9d6ad7SLin Ling { 24103f9d6ad7SLin Ling dsl_dataset_t *ds; 24113f9d6ad7SLin Ling int err; 24123f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 24133f9d6ad7SLin Ling 24143b2aab18SMatthew Ahrens err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 24153f9d6ad7SLin Ling if (err) 24163f9d6ad7SLin Ling return (err); 24173f9d6ad7SLin Ling 2418c1379625SJustin T. Gibbs while (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) { 24193f9d6ad7SLin Ling dsl_dataset_t *prev; 2420c1379625SJustin T. Gibbs err = dsl_dataset_hold_obj(dp, 2421c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev); 24223f9d6ad7SLin Ling if (err) { 24233f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24243f9d6ad7SLin Ling return (err); 24253f9d6ad7SLin Ling } 24263f9d6ad7SLin Ling 24273f9d6ad7SLin Ling /* 24283f9d6ad7SLin Ling * If this is a clone, we don't need to worry about it for now. 24293f9d6ad7SLin Ling */ 2430c1379625SJustin T. Gibbs if (dsl_dataset_phys(prev)->ds_next_snap_obj != ds->ds_object) { 24313f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24323f9d6ad7SLin Ling dsl_dataset_rele(prev, FTAG); 24333f9d6ad7SLin Ling return (0); 24343f9d6ad7SLin Ling } 24353f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24363f9d6ad7SLin Ling ds = prev; 24373f9d6ad7SLin Ling } 24383f9d6ad7SLin Ling 2439a3874b8bSToomas Soome scan_ds_queue_insert(scn, ds->ds_object, 2440a3874b8bSToomas Soome dsl_dataset_phys(ds)->ds_prev_snap_txg); 24413f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 24423f9d6ad7SLin Ling return (0); 24433f9d6ad7SLin Ling } 24443f9d6ad7SLin Ling 2445a3874b8bSToomas Soome /* ARGSUSED */ 2446a3874b8bSToomas Soome void 2447a3874b8bSToomas Soome dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, 2448a3874b8bSToomas Soome ddt_entry_t *dde, dmu_tx_t *tx) 2449a3874b8bSToomas Soome { 2450a3874b8bSToomas Soome const ddt_key_t *ddk = &dde->dde_key; 2451a3874b8bSToomas Soome ddt_phys_t *ddp = dde->dde_phys; 2452a3874b8bSToomas Soome blkptr_t bp; 2453a3874b8bSToomas Soome zbookmark_phys_t zb = { 0 }; 2454a3874b8bSToomas Soome int p; 2455a3874b8bSToomas Soome 2456a3874b8bSToomas Soome if (scn->scn_phys.scn_state != DSS_SCANNING) 2457a3874b8bSToomas Soome return; 2458a3874b8bSToomas Soome 2459e4c795beSTom Caputi /* 2460e4c795beSTom Caputi * This function is special because it is the only thing 2461e4c795beSTom Caputi * that can add scan_io_t's to the vdev scan queues from 2462e4c795beSTom Caputi * outside dsl_scan_sync(). For the most part this is ok 2463e4c795beSTom Caputi * as long as it is called from within syncing context. 2464e4c795beSTom Caputi * However, dsl_scan_sync() expects that no new sio's will 2465e4c795beSTom Caputi * be added between when all the work for a scan is done 2466e4c795beSTom Caputi * and the next txg when the scan is actually marked as 2467e4c795beSTom Caputi * completed. This check ensures we do not issue new sio's 2468e4c795beSTom Caputi * during this period. 2469e4c795beSTom Caputi */ 2470e4c795beSTom Caputi if (scn->scn_done_txg != 0) 2471e4c795beSTom Caputi return; 2472e4c795beSTom Caputi 2473a3874b8bSToomas Soome for (p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 2474a3874b8bSToomas Soome if (ddp->ddp_phys_birth == 0 || 2475a3874b8bSToomas Soome ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) 2476a3874b8bSToomas Soome continue; 2477a3874b8bSToomas Soome ddt_bp_create(checksum, ddk, ddp, &bp); 2478a3874b8bSToomas Soome 2479a3874b8bSToomas Soome scn->scn_visited_this_txg++; 2480a3874b8bSToomas Soome scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb); 2481a3874b8bSToomas Soome } 2482a3874b8bSToomas Soome } 2483a3874b8bSToomas Soome 2484a3874b8bSToomas Soome /* 24853f9d6ad7SLin Ling * Scrub/dedup interaction. 24863f9d6ad7SLin Ling * 24873f9d6ad7SLin Ling * If there are N references to a deduped block, we don't want to scrub it 24883f9d6ad7SLin Ling * N times -- ideally, we should scrub it exactly once. 24893f9d6ad7SLin Ling * 24903f9d6ad7SLin Ling * We leverage the fact that the dde's replication class (enum ddt_class) 24913f9d6ad7SLin Ling * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest 24923f9d6ad7SLin Ling * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order. 24933f9d6ad7SLin Ling * 24943f9d6ad7SLin Ling * To prevent excess scrubbing, the scrub begins by walking the DDT 24953f9d6ad7SLin Ling * to find all blocks with refcnt > 1, and scrubs each of these once. 24963f9d6ad7SLin Ling * Since there are two replication classes which contain blocks with 24973f9d6ad7SLin Ling * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first. 24983f9d6ad7SLin Ling * Finally the top-down scrub begins, only visiting blocks with refcnt == 1. 24993f9d6ad7SLin Ling * 25003f9d6ad7SLin Ling * There would be nothing more to say if a block's refcnt couldn't change 25013f9d6ad7SLin Ling * during a scrub, but of course it can so we must account for changes 25023f9d6ad7SLin Ling * in a block's replication class. 25033f9d6ad7SLin Ling * 25043f9d6ad7SLin Ling * Here's an example of what can occur: 25053f9d6ad7SLin Ling * 25063f9d6ad7SLin Ling * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1 25073f9d6ad7SLin Ling * when visited during the top-down scrub phase, it will be scrubbed twice. 25083f9d6ad7SLin Ling * This negates our scrub optimization, but is otherwise harmless. 25093f9d6ad7SLin Ling * 25103f9d6ad7SLin Ling * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1 25113f9d6ad7SLin Ling * on each visit during the top-down scrub phase, it will never be scrubbed. 25123f9d6ad7SLin Ling * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's 25133f9d6ad7SLin Ling * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to 25143f9d6ad7SLin Ling * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1 25153f9d6ad7SLin Ling * while a scrub is in progress, it scrubs the block right then. 25163f9d6ad7SLin Ling */ 25173f9d6ad7SLin Ling static void 25183f9d6ad7SLin Ling dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx) 25193f9d6ad7SLin Ling { 25203f9d6ad7SLin Ling ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark; 25213f9d6ad7SLin Ling ddt_entry_t dde = { 0 }; 25223f9d6ad7SLin Ling int error; 25233f9d6ad7SLin Ling uint64_t n = 0; 25243f9d6ad7SLin Ling 25253f9d6ad7SLin Ling while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) { 25263f9d6ad7SLin Ling ddt_t *ddt; 25273f9d6ad7SLin Ling 25283f9d6ad7SLin Ling if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max) 25293f9d6ad7SLin Ling break; 25303f9d6ad7SLin Ling dprintf("visiting ddb=%llu/%llu/%llu/%llx\n", 25313f9d6ad7SLin Ling (longlong_t)ddb->ddb_class, 25323f9d6ad7SLin Ling (longlong_t)ddb->ddb_type, 25333f9d6ad7SLin Ling (longlong_t)ddb->ddb_checksum, 25343f9d6ad7SLin Ling (longlong_t)ddb->ddb_cursor); 25353f9d6ad7SLin Ling 25363f9d6ad7SLin Ling /* There should be no pending changes to the dedup table */ 25373f9d6ad7SLin Ling ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum]; 25383f9d6ad7SLin Ling ASSERT(avl_first(&ddt->ddt_tree) == NULL); 25393f9d6ad7SLin Ling 25403f9d6ad7SLin Ling dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx); 25413f9d6ad7SLin Ling n++; 25423f9d6ad7SLin Ling 25431702cce7SAlek Pinchuk if (dsl_scan_check_suspend(scn, NULL)) 25443f9d6ad7SLin Ling break; 25453f9d6ad7SLin Ling } 25463f9d6ad7SLin Ling 25471702cce7SAlek Pinchuk zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; " 25481702cce7SAlek Pinchuk "suspending=%u", (longlong_t)n, 25491702cce7SAlek Pinchuk (int)scn->scn_phys.scn_ddt_class_max, (int)scn->scn_suspending); 25503f9d6ad7SLin Ling 25513f9d6ad7SLin Ling ASSERT(error == 0 || error == ENOENT); 25523f9d6ad7SLin Ling ASSERT(error != ENOENT || 25533f9d6ad7SLin Ling ddb->ddb_class > scn->scn_phys.scn_ddt_class_max); 25543f9d6ad7SLin Ling } 25553f9d6ad7SLin Ling 2556a3874b8bSToomas Soome static uint64_t 2557a3874b8bSToomas Soome dsl_scan_ds_maxtxg(dsl_dataset_t *ds) 25583f9d6ad7SLin Ling { 2559a3874b8bSToomas Soome uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg; 2560a3874b8bSToomas Soome if (ds->ds_is_snapshot) 2561a3874b8bSToomas Soome return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg)); 2562a3874b8bSToomas Soome return (smt); 25633f9d6ad7SLin Ling } 25643f9d6ad7SLin Ling 25653f9d6ad7SLin Ling static void 25663f9d6ad7SLin Ling dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx) 25673f9d6ad7SLin Ling { 2568a3874b8bSToomas Soome scan_ds_t *sds; 25693f9d6ad7SLin Ling dsl_pool_t *dp = scn->scn_dp; 25703f9d6ad7SLin Ling 25713f9d6ad7SLin Ling if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 25723f9d6ad7SLin Ling scn->scn_phys.scn_ddt_class_max) { 25733f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 25743f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 25753f9d6ad7SLin Ling dsl_scan_ddt(scn, tx); 25761702cce7SAlek Pinchuk if (scn->scn_suspending) 25773f9d6ad7SLin Ling return; 25783f9d6ad7SLin Ling } 25793f9d6ad7SLin Ling 25803f9d6ad7SLin Ling if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) { 25813f9d6ad7SLin Ling /* First do the MOS & ORIGIN */ 25823f9d6ad7SLin Ling 25833f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 25843f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 25853f9d6ad7SLin Ling dsl_scan_visit_rootbp(scn, NULL, 25863f9d6ad7SLin Ling &dp->dp_meta_rootbp, tx); 25873f9d6ad7SLin Ling spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp); 25881702cce7SAlek Pinchuk if (scn->scn_suspending) 25893f9d6ad7SLin Ling return; 25903f9d6ad7SLin Ling 25913f9d6ad7SLin Ling if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) { 25923b2aab18SMatthew Ahrens VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2593a3874b8bSToomas Soome enqueue_cb, NULL, DS_FIND_CHILDREN)); 25943f9d6ad7SLin Ling } else { 25953f9d6ad7SLin Ling dsl_scan_visitds(scn, 25963f9d6ad7SLin Ling dp->dp_origin_snap->ds_object, tx); 25973f9d6ad7SLin Ling } 25981702cce7SAlek Pinchuk ASSERT(!scn->scn_suspending); 25993f9d6ad7SLin Ling } else if (scn->scn_phys.scn_bookmark.zb_objset != 26003f9d6ad7SLin Ling ZB_DESTROYED_OBJSET) { 2601a3874b8bSToomas Soome uint64_t dsobj = scn->scn_phys.scn_bookmark.zb_objset; 26023f9d6ad7SLin Ling /* 2603a3874b8bSToomas Soome * If we were suspended, continue from here. Note if the 26041702cce7SAlek Pinchuk * ds we were suspended on was deleted, the zb_objset may 26053f9d6ad7SLin Ling * be -1, so we will skip this and find a new objset 26063f9d6ad7SLin Ling * below. 26073f9d6ad7SLin Ling */ 2608a3874b8bSToomas Soome dsl_scan_visitds(scn, dsobj, tx); 26091702cce7SAlek Pinchuk if (scn->scn_suspending) 26103f9d6ad7SLin Ling return; 26113f9d6ad7SLin Ling } 26123f9d6ad7SLin Ling 26133f9d6ad7SLin Ling /* 2614a3874b8bSToomas Soome * In case we suspended right at the end of the ds, zero the 26153f9d6ad7SLin Ling * bookmark so we don't think that we're still trying to resume. 26163f9d6ad7SLin Ling */ 26177802d7bfSMatthew Ahrens bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t)); 26183f9d6ad7SLin Ling 2619a3874b8bSToomas Soome /* 2620a3874b8bSToomas Soome * Keep pulling things out of the dataset avl queue. Updates to the 2621a3874b8bSToomas Soome * persistent zap-object-as-queue happen only at checkpoints. 2622a3874b8bSToomas Soome */ 2623a3874b8bSToomas Soome while ((sds = avl_first(&scn->scn_queue)) != NULL) { 26243f9d6ad7SLin Ling dsl_dataset_t *ds; 2625a3874b8bSToomas Soome uint64_t dsobj = sds->sds_dsobj; 2626a3874b8bSToomas Soome uint64_t txg = sds->sds_txg; 26273f9d6ad7SLin Ling 2628a3874b8bSToomas Soome /* dequeue and free the ds from the queue */ 2629a3874b8bSToomas Soome scan_ds_queue_remove(scn, dsobj); 2630a3874b8bSToomas Soome sds = NULL; /* must not be touched after removal */ 26313f9d6ad7SLin Ling 2632a3874b8bSToomas Soome /* Set up min / max txg */ 2633b420f3adSRichard Lowe VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 2634a3874b8bSToomas Soome if (txg != 0) { 26353f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = 2636a3874b8bSToomas Soome MAX(scn->scn_phys.scn_min_txg, txg); 26373f9d6ad7SLin Ling } else { 26383f9d6ad7SLin Ling scn->scn_phys.scn_cur_min_txg = 26393f9d6ad7SLin Ling MAX(scn->scn_phys.scn_min_txg, 2640c1379625SJustin T. Gibbs dsl_dataset_phys(ds)->ds_prev_snap_txg); 26413f9d6ad7SLin Ling } 26423f9d6ad7SLin Ling scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds); 26433f9d6ad7SLin Ling dsl_dataset_rele(ds, FTAG); 26443f9d6ad7SLin Ling 26453f9d6ad7SLin Ling dsl_scan_visitds(scn, dsobj, tx); 26461702cce7SAlek Pinchuk if (scn->scn_suspending) 26473f9d6ad7SLin Ling return; 26483f9d6ad7SLin Ling } 2649a3874b8bSToomas Soome /* No more objsets to fetch, we're done */ 2650a3874b8bSToomas Soome scn->scn_phys.scn_bookmark.zb_objset = ZB_DESTROYED_OBJSET; 2651a3874b8bSToomas Soome ASSERT0(scn->scn_suspending); 2652a3874b8bSToomas Soome } 2653a3874b8bSToomas Soome 2654a3874b8bSToomas Soome static uint64_t 2655a3874b8bSToomas Soome dsl_scan_count_leaves(vdev_t *vd) 2656a3874b8bSToomas Soome { 2657a3874b8bSToomas Soome uint64_t i, leaves = 0; 2658a3874b8bSToomas Soome 2659a3874b8bSToomas Soome /* we only count leaves that belong to the main pool and are readable */ 2660a3874b8bSToomas Soome if (vd->vdev_islog || vd->vdev_isspare || 2661a3874b8bSToomas Soome vd->vdev_isl2cache || !vdev_readable(vd)) 2662a3874b8bSToomas Soome return (0); 2663a3874b8bSToomas Soome 2664a3874b8bSToomas Soome if (vd->vdev_ops->vdev_op_leaf) 2665a3874b8bSToomas Soome return (1); 2666a3874b8bSToomas Soome 2667a3874b8bSToomas Soome for (i = 0; i < vd->vdev_children; i++) { 2668a3874b8bSToomas Soome leaves += dsl_scan_count_leaves(vd->vdev_child[i]); 2669a3874b8bSToomas Soome } 2670a3874b8bSToomas Soome 2671a3874b8bSToomas Soome return (leaves); 2672a3874b8bSToomas Soome } 2673a3874b8bSToomas Soome 2674a3874b8bSToomas Soome 2675a3874b8bSToomas Soome static void 2676a3874b8bSToomas Soome scan_io_queues_update_zio_stats(dsl_scan_io_queue_t *q, const blkptr_t *bp) 2677a3874b8bSToomas Soome { 2678a3874b8bSToomas Soome int i; 2679a3874b8bSToomas Soome uint64_t cur_size = 0; 2680a3874b8bSToomas Soome 2681a3874b8bSToomas Soome for (i = 0; i < BP_GET_NDVAS(bp); i++) { 2682a3874b8bSToomas Soome cur_size += DVA_GET_ASIZE(&bp->blk_dva[i]); 2683a3874b8bSToomas Soome } 2684a3874b8bSToomas Soome 2685a3874b8bSToomas Soome q->q_total_zio_size_this_txg += cur_size; 2686a3874b8bSToomas Soome q->q_zios_this_txg++; 2687a3874b8bSToomas Soome } 2688a3874b8bSToomas Soome 2689a3874b8bSToomas Soome static void 2690a3874b8bSToomas Soome scan_io_queues_update_seg_stats(dsl_scan_io_queue_t *q, uint64_t start, 2691a3874b8bSToomas Soome uint64_t end) 2692a3874b8bSToomas Soome { 2693a3874b8bSToomas Soome q->q_total_seg_size_this_txg += end - start; 2694a3874b8bSToomas Soome q->q_segs_this_txg++; 2695a3874b8bSToomas Soome } 2696a3874b8bSToomas Soome 2697a3874b8bSToomas Soome static boolean_t 2698a3874b8bSToomas Soome scan_io_queue_check_suspend(dsl_scan_t *scn) 2699a3874b8bSToomas Soome { 2700a3874b8bSToomas Soome /* See comment in dsl_scan_check_suspend() */ 2701a3874b8bSToomas Soome uint64_t curr_time_ns = gethrtime(); 2702a3874b8bSToomas Soome uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time; 2703a3874b8bSToomas Soome uint64_t sync_time_ns = curr_time_ns - 2704a3874b8bSToomas Soome scn->scn_dp->dp_spa->spa_sync_starttime; 2705a3874b8bSToomas Soome int dirty_pct = scn->scn_dp->dp_dirty_total * 100 / zfs_dirty_data_max; 2706a3874b8bSToomas Soome int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 2707a3874b8bSToomas Soome zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; 2708a3874b8bSToomas Soome 2709a3874b8bSToomas Soome return ((NSEC2MSEC(scan_time_ns) > mintime && 2710a3874b8bSToomas Soome (dirty_pct >= zfs_vdev_async_write_active_min_dirty_percent || 2711a3874b8bSToomas Soome txg_sync_waiting(scn->scn_dp) || 2712a3874b8bSToomas Soome NSEC2SEC(sync_time_ns) >= zfs_txg_timeout)) || 2713a3874b8bSToomas Soome spa_shutting_down(scn->scn_dp->dp_spa)); 2714a3874b8bSToomas Soome } 2715a3874b8bSToomas Soome 2716a3874b8bSToomas Soome /* 2717a3874b8bSToomas Soome * Given a list of scan_io_t's in io_list, this issues the io's out to 2718a3874b8bSToomas Soome * disk. This consumes the io_list and frees the scan_io_t's. This is 2719a3874b8bSToomas Soome * called when emptying queues, either when we're up against the memory 2720a3874b8bSToomas Soome * limit or when we have finished scanning. Returns B_TRUE if we stopped 2721a3874b8bSToomas Soome * processing the list before we finished. Any zios that were not issued 2722a3874b8bSToomas Soome * will remain in the io_list. 2723a3874b8bSToomas Soome */ 2724a3874b8bSToomas Soome static boolean_t 2725a3874b8bSToomas Soome scan_io_queue_issue(dsl_scan_io_queue_t *queue, list_t *io_list) 2726a3874b8bSToomas Soome { 2727a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 2728a3874b8bSToomas Soome scan_io_t *sio; 2729a3874b8bSToomas Soome int64_t bytes_issued = 0; 2730a3874b8bSToomas Soome boolean_t suspended = B_FALSE; 2731a3874b8bSToomas Soome 2732a3874b8bSToomas Soome while ((sio = list_head(io_list)) != NULL) { 2733a3874b8bSToomas Soome blkptr_t bp; 2734a3874b8bSToomas Soome 2735a3874b8bSToomas Soome if (scan_io_queue_check_suspend(scn)) { 2736a3874b8bSToomas Soome suspended = B_TRUE; 2737a3874b8bSToomas Soome break; 2738a3874b8bSToomas Soome } 2739a3874b8bSToomas Soome 274012a8814cSTom Caputi sio2bp(sio, &bp); 274112a8814cSTom Caputi bytes_issued += SIO_GET_ASIZE(sio); 2742a3874b8bSToomas Soome scan_exec_io(scn->scn_dp, &bp, sio->sio_flags, 2743a3874b8bSToomas Soome &sio->sio_zb, queue); 2744a3874b8bSToomas Soome (void) list_remove_head(io_list); 2745a3874b8bSToomas Soome scan_io_queues_update_zio_stats(queue, &bp); 274612a8814cSTom Caputi sio_free(sio); 2747a3874b8bSToomas Soome } 2748a3874b8bSToomas Soome 2749a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -bytes_issued); 2750a3874b8bSToomas Soome 2751a3874b8bSToomas Soome return (suspended); 2752a3874b8bSToomas Soome } 2753a3874b8bSToomas Soome 2754a3874b8bSToomas Soome /* 2755a3874b8bSToomas Soome * Given a range_seg_t (extent) and a list, this function passes over a 2756a3874b8bSToomas Soome * scan queue and gathers up the appropriate ios which fit into that 2757a3874b8bSToomas Soome * scan seg (starting from lowest LBA). At the end, we remove the segment 2758a3874b8bSToomas Soome * from the q_exts_by_addr range tree. 2759a3874b8bSToomas Soome */ 2760a3874b8bSToomas Soome static boolean_t 2761a3874b8bSToomas Soome scan_io_queue_gather(dsl_scan_io_queue_t *queue, range_seg_t *rs, list_t *list) 2762a3874b8bSToomas Soome { 276312a8814cSTom Caputi scan_io_t *srch_sio, *sio, *next_sio; 2764a3874b8bSToomas Soome avl_index_t idx; 2765a3874b8bSToomas Soome uint_t num_sios = 0; 2766a3874b8bSToomas Soome int64_t bytes_issued = 0; 2767a3874b8bSToomas Soome 2768a3874b8bSToomas Soome ASSERT(rs != NULL); 2769a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 2770a3874b8bSToomas Soome 277112a8814cSTom Caputi srch_sio = sio_alloc(1); 277212a8814cSTom Caputi srch_sio->sio_nr_dvas = 1; 2773*4d7988d6SPaul Dagnelie SIO_SET_OFFSET(srch_sio, rs_get_start(rs, queue->q_exts_by_addr)); 2774a3874b8bSToomas Soome 2775a3874b8bSToomas Soome /* 2776a3874b8bSToomas Soome * The exact start of the extent might not contain any matching zios, 2777a3874b8bSToomas Soome * so if that's the case, examine the next one in the tree. 2778a3874b8bSToomas Soome */ 277912a8814cSTom Caputi sio = avl_find(&queue->q_sios_by_addr, srch_sio, &idx); 278012a8814cSTom Caputi sio_free(srch_sio); 278112a8814cSTom Caputi 2782a3874b8bSToomas Soome if (sio == NULL) 2783a3874b8bSToomas Soome sio = avl_nearest(&queue->q_sios_by_addr, idx, AVL_AFTER); 2784a3874b8bSToomas Soome 2785*4d7988d6SPaul Dagnelie while (sio != NULL && SIO_GET_OFFSET(sio) < rs_get_end(rs, 2786*4d7988d6SPaul Dagnelie queue->q_exts_by_addr) && num_sios <= 32) { 2787*4d7988d6SPaul Dagnelie ASSERT3U(SIO_GET_OFFSET(sio), >=, rs_get_start(rs, 2788*4d7988d6SPaul Dagnelie queue->q_exts_by_addr)); 2789*4d7988d6SPaul Dagnelie ASSERT3U(SIO_GET_END_OFFSET(sio), <=, rs_get_end(rs, 2790*4d7988d6SPaul Dagnelie queue->q_exts_by_addr)); 2791a3874b8bSToomas Soome 2792a3874b8bSToomas Soome next_sio = AVL_NEXT(&queue->q_sios_by_addr, sio); 2793a3874b8bSToomas Soome avl_remove(&queue->q_sios_by_addr, sio); 279412a8814cSTom Caputi queue->q_sio_memused -= SIO_GET_MUSED(sio); 2795a3874b8bSToomas Soome 279612a8814cSTom Caputi bytes_issued += SIO_GET_ASIZE(sio); 2797a3874b8bSToomas Soome num_sios++; 2798a3874b8bSToomas Soome list_insert_tail(list, sio); 2799a3874b8bSToomas Soome sio = next_sio; 2800a3874b8bSToomas Soome } 2801a3874b8bSToomas Soome 2802a3874b8bSToomas Soome /* 2803a3874b8bSToomas Soome * We limit the number of sios we process at once to 32 to avoid 2804a3874b8bSToomas Soome * biting off more than we can chew. If we didn't take everything 2805a3874b8bSToomas Soome * in the segment we update it to reflect the work we were able to 2806a3874b8bSToomas Soome * complete. Otherwise, we remove it from the range tree entirely. 2807a3874b8bSToomas Soome */ 2808*4d7988d6SPaul Dagnelie if (sio != NULL && SIO_GET_OFFSET(sio) < rs_get_end(rs, 2809*4d7988d6SPaul Dagnelie queue->q_exts_by_addr)) { 2810a3874b8bSToomas Soome range_tree_adjust_fill(queue->q_exts_by_addr, rs, 2811a3874b8bSToomas Soome -bytes_issued); 2812a3874b8bSToomas Soome range_tree_resize_segment(queue->q_exts_by_addr, rs, 2813*4d7988d6SPaul Dagnelie SIO_GET_OFFSET(sio), rs_get_end(rs, 2814*4d7988d6SPaul Dagnelie queue->q_exts_by_addr) - SIO_GET_OFFSET(sio)); 2815a3874b8bSToomas Soome 2816a3874b8bSToomas Soome return (B_TRUE); 2817a3874b8bSToomas Soome } else { 2818*4d7988d6SPaul Dagnelie uint64_t rstart = rs_get_start(rs, queue->q_exts_by_addr); 2819*4d7988d6SPaul Dagnelie uint64_t rend = rs_get_end(rs, queue->q_exts_by_addr); 2820*4d7988d6SPaul Dagnelie range_tree_remove(queue->q_exts_by_addr, rstart, rend - rstart); 2821a3874b8bSToomas Soome return (B_FALSE); 2822a3874b8bSToomas Soome } 2823a3874b8bSToomas Soome } 2824a3874b8bSToomas Soome 2825a3874b8bSToomas Soome 2826a3874b8bSToomas Soome /* 2827a3874b8bSToomas Soome * This is called from the queue emptying thread and selects the next 2828a3874b8bSToomas Soome * extent from which we are to issue io's. The behavior of this function 2829a3874b8bSToomas Soome * depends on the state of the scan, the current memory consumption and 2830a3874b8bSToomas Soome * whether or not we are performing a scan shutdown. 2831a3874b8bSToomas Soome * 1) We select extents in an elevator algorithm (LBA-order) if the scan 2832a3874b8bSToomas Soome * needs to perform a checkpoint 2833a3874b8bSToomas Soome * 2) We select the largest available extent if we are up against the 2834a3874b8bSToomas Soome * memory limit. 2835a3874b8bSToomas Soome * 3) Otherwise we don't select any extents. 2836a3874b8bSToomas Soome */ 2837a3874b8bSToomas Soome static const range_seg_t * 2838a3874b8bSToomas Soome scan_io_queue_fetch_ext(dsl_scan_io_queue_t *queue) 2839a3874b8bSToomas Soome { 2840a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 2841*4d7988d6SPaul Dagnelie range_tree_t *rt = queue->q_exts_by_addr; 2842a3874b8bSToomas Soome 2843a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 2844a3874b8bSToomas Soome ASSERT(scn->scn_is_sorted); 2845a3874b8bSToomas Soome 2846a3874b8bSToomas Soome /* handle tunable overrides */ 2847a3874b8bSToomas Soome if (scn->scn_checkpointing || scn->scn_clearing) { 2848a3874b8bSToomas Soome if (zfs_scan_issue_strategy == 1) { 2849*4d7988d6SPaul Dagnelie return (range_tree_first(rt)); 2850a3874b8bSToomas Soome } else if (zfs_scan_issue_strategy == 2) { 2851*4d7988d6SPaul Dagnelie range_seg_t *size_rs = 2852*4d7988d6SPaul Dagnelie zfs_btree_first(&queue->q_exts_by_size, NULL); 2853*4d7988d6SPaul Dagnelie uint64_t start = rs_get_start(size_rs, rt); 2854*4d7988d6SPaul Dagnelie uint64_t size = rs_get_end(size_rs, rt) - start; 2855*4d7988d6SPaul Dagnelie range_seg_t *addr_rs = range_tree_find(rt, start, 2856*4d7988d6SPaul Dagnelie size); 2857*4d7988d6SPaul Dagnelie ASSERT3P(addr_rs, !=, NULL); 2858*4d7988d6SPaul Dagnelie return (addr_rs); 2859a3874b8bSToomas Soome } 2860a3874b8bSToomas Soome } 2861a3874b8bSToomas Soome 2862a3874b8bSToomas Soome /* 2863a3874b8bSToomas Soome * During normal clearing, we want to issue our largest segments 2864a3874b8bSToomas Soome * first, keeping IO as sequential as possible, and leaving the 2865a3874b8bSToomas Soome * smaller extents for later with the hope that they might eventually 2866a3874b8bSToomas Soome * grow to larger sequential segments. However, when the scan is 2867a3874b8bSToomas Soome * checkpointing, no new extents will be added to the sorting queue, 2868a3874b8bSToomas Soome * so the way we are sorted now is as good as it will ever get. 2869a3874b8bSToomas Soome * In this case, we instead switch to issuing extents in LBA order. 2870a3874b8bSToomas Soome */ 2871a3874b8bSToomas Soome if (scn->scn_checkpointing) { 2872*4d7988d6SPaul Dagnelie return (range_tree_first(rt)); 2873a3874b8bSToomas Soome } else if (scn->scn_clearing) { 2874*4d7988d6SPaul Dagnelie range_seg_t *size_rs = zfs_btree_first(&queue->q_exts_by_size, 2875*4d7988d6SPaul Dagnelie NULL); 2876*4d7988d6SPaul Dagnelie uint64_t start = rs_get_start(size_rs, rt); 2877*4d7988d6SPaul Dagnelie uint64_t size = rs_get_end(size_rs, rt) - start; 2878*4d7988d6SPaul Dagnelie range_seg_t *addr_rs = range_tree_find(rt, start, size); 2879*4d7988d6SPaul Dagnelie ASSERT3P(addr_rs, !=, NULL); 2880*4d7988d6SPaul Dagnelie return (addr_rs); 2881a3874b8bSToomas Soome } else { 2882a3874b8bSToomas Soome return (NULL); 2883a3874b8bSToomas Soome } 2884a3874b8bSToomas Soome } 2885a3874b8bSToomas Soome 2886a3874b8bSToomas Soome static void 2887a3874b8bSToomas Soome scan_io_queues_run_one(void *arg) 2888a3874b8bSToomas Soome { 2889a3874b8bSToomas Soome dsl_scan_io_queue_t *queue = arg; 2890a3874b8bSToomas Soome kmutex_t *q_lock = &queue->q_vd->vdev_scan_io_queue_lock; 2891a3874b8bSToomas Soome boolean_t suspended = B_FALSE; 2892a3874b8bSToomas Soome range_seg_t *rs = NULL; 2893a3874b8bSToomas Soome scan_io_t *sio = NULL; 2894a3874b8bSToomas Soome list_t sio_list; 2895a3874b8bSToomas Soome uint64_t bytes_per_leaf = zfs_scan_vdev_limit; 2896a3874b8bSToomas Soome uint64_t nr_leaves = dsl_scan_count_leaves(queue->q_vd); 2897a3874b8bSToomas Soome 2898a3874b8bSToomas Soome ASSERT(queue->q_scn->scn_is_sorted); 2899a3874b8bSToomas Soome 2900a3874b8bSToomas Soome list_create(&sio_list, sizeof (scan_io_t), 2901a3874b8bSToomas Soome offsetof(scan_io_t, sio_nodes.sio_list_node)); 2902a3874b8bSToomas Soome mutex_enter(q_lock); 2903a3874b8bSToomas Soome 2904a3874b8bSToomas Soome /* calculate maximum in-flight bytes for this txg (min 1MB) */ 2905a3874b8bSToomas Soome queue->q_maxinflight_bytes = 2906a3874b8bSToomas Soome MAX(nr_leaves * bytes_per_leaf, 1ULL << 20); 2907a3874b8bSToomas Soome 2908a3874b8bSToomas Soome /* reset per-queue scan statistics for this txg */ 2909a3874b8bSToomas Soome queue->q_total_seg_size_this_txg = 0; 2910a3874b8bSToomas Soome queue->q_segs_this_txg = 0; 2911a3874b8bSToomas Soome queue->q_total_zio_size_this_txg = 0; 2912a3874b8bSToomas Soome queue->q_zios_this_txg = 0; 2913a3874b8bSToomas Soome 2914a3874b8bSToomas Soome /* loop until we have run out of time or sios */ 2915a3874b8bSToomas Soome while ((rs = (range_seg_t *)scan_io_queue_fetch_ext(queue)) != NULL) { 2916a3874b8bSToomas Soome uint64_t seg_start = 0, seg_end = 0; 2917a3874b8bSToomas Soome boolean_t more_left = B_TRUE; 2918a3874b8bSToomas Soome 2919a3874b8bSToomas Soome ASSERT(list_is_empty(&sio_list)); 2920a3874b8bSToomas Soome 2921a3874b8bSToomas Soome /* loop while we still have sios left to process in this rs */ 2922a3874b8bSToomas Soome while (more_left) { 2923a3874b8bSToomas Soome scan_io_t *first_sio, *last_sio; 2924a3874b8bSToomas Soome 2925a3874b8bSToomas Soome /* 2926a3874b8bSToomas Soome * We have selected which extent needs to be 2927a3874b8bSToomas Soome * processed next. Gather up the corresponding sios. 2928a3874b8bSToomas Soome */ 2929a3874b8bSToomas Soome more_left = scan_io_queue_gather(queue, rs, &sio_list); 2930a3874b8bSToomas Soome ASSERT(!list_is_empty(&sio_list)); 2931a3874b8bSToomas Soome first_sio = list_head(&sio_list); 2932a3874b8bSToomas Soome last_sio = list_tail(&sio_list); 2933a3874b8bSToomas Soome 293412a8814cSTom Caputi seg_end = SIO_GET_END_OFFSET(last_sio); 2935a3874b8bSToomas Soome if (seg_start == 0) 293612a8814cSTom Caputi seg_start = SIO_GET_OFFSET(first_sio); 2937a3874b8bSToomas Soome 2938a3874b8bSToomas Soome /* 2939a3874b8bSToomas Soome * Issuing sios can take a long time so drop the 2940a3874b8bSToomas Soome * queue lock. The sio queue won't be updated by 2941a3874b8bSToomas Soome * other threads since we're in syncing context so 2942a3874b8bSToomas Soome * we can be sure that our trees will remain exactly 2943a3874b8bSToomas Soome * as we left them. 2944a3874b8bSToomas Soome */ 2945a3874b8bSToomas Soome mutex_exit(q_lock); 2946a3874b8bSToomas Soome suspended = scan_io_queue_issue(queue, &sio_list); 2947a3874b8bSToomas Soome mutex_enter(q_lock); 2948a3874b8bSToomas Soome 2949a3874b8bSToomas Soome if (suspended) 2950a3874b8bSToomas Soome break; 2951a3874b8bSToomas Soome } 2952a3874b8bSToomas Soome /* update statistics for debugging purposes */ 2953a3874b8bSToomas Soome scan_io_queues_update_seg_stats(queue, seg_start, seg_end); 2954a3874b8bSToomas Soome 2955a3874b8bSToomas Soome if (suspended) 2956a3874b8bSToomas Soome break; 2957a3874b8bSToomas Soome } 2958a3874b8bSToomas Soome 2959a3874b8bSToomas Soome 2960a3874b8bSToomas Soome /* 2961a3874b8bSToomas Soome * If we were suspended in the middle of processing, 2962a3874b8bSToomas Soome * requeue any unfinished sios and exit. 2963a3874b8bSToomas Soome */ 2964a3874b8bSToomas Soome while ((sio = list_head(&sio_list)) != NULL) { 2965a3874b8bSToomas Soome list_remove(&sio_list, sio); 2966a3874b8bSToomas Soome scan_io_queue_insert_impl(queue, sio); 2967a3874b8bSToomas Soome } 2968a3874b8bSToomas Soome 2969a3874b8bSToomas Soome mutex_exit(q_lock); 2970a3874b8bSToomas Soome list_destroy(&sio_list); 2971a3874b8bSToomas Soome } 2972a3874b8bSToomas Soome 2973a3874b8bSToomas Soome /* 2974a3874b8bSToomas Soome * Performs an emptying run on all scan queues in the pool. This just 2975a3874b8bSToomas Soome * punches out one thread per top-level vdev, each of which processes 2976a3874b8bSToomas Soome * only that vdev's scan queue. We can parallelize the I/O here because 2977a3874b8bSToomas Soome * we know that each queue's io's only affect its own top-level vdev. 2978a3874b8bSToomas Soome * 2979a3874b8bSToomas Soome * This function waits for the queue runs to complete, and must be 2980a3874b8bSToomas Soome * called from dsl_scan_sync (or in general, syncing context). 2981a3874b8bSToomas Soome */ 2982a3874b8bSToomas Soome static void 2983a3874b8bSToomas Soome scan_io_queues_run(dsl_scan_t *scn) 2984a3874b8bSToomas Soome { 2985a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 2986a3874b8bSToomas Soome 2987a3874b8bSToomas Soome ASSERT(scn->scn_is_sorted); 2988a3874b8bSToomas Soome ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 2989a3874b8bSToomas Soome 2990a3874b8bSToomas Soome if (scn->scn_bytes_pending == 0) 2991a3874b8bSToomas Soome return; 2992a3874b8bSToomas Soome 2993a3874b8bSToomas Soome if (scn->scn_taskq == NULL) { 2994a3874b8bSToomas Soome char *tq_name = kmem_zalloc(ZFS_MAX_DATASET_NAME_LEN + 16, 2995a3874b8bSToomas Soome KM_SLEEP); 2996a3874b8bSToomas Soome int nthreads = spa->spa_root_vdev->vdev_children; 2997a3874b8bSToomas Soome 2998a3874b8bSToomas Soome /* 2999a3874b8bSToomas Soome * We need to make this taskq *always* execute as many 3000a3874b8bSToomas Soome * threads in parallel as we have top-level vdevs and no 3001a3874b8bSToomas Soome * less, otherwise strange serialization of the calls to 3002a3874b8bSToomas Soome * scan_io_queues_run_one can occur during spa_sync runs 3003a3874b8bSToomas Soome * and that significantly impacts performance. 3004a3874b8bSToomas Soome */ 3005a3874b8bSToomas Soome (void) snprintf(tq_name, ZFS_MAX_DATASET_NAME_LEN + 16, 3006a3874b8bSToomas Soome "dsl_scan_tq_%s", spa->spa_name); 3007a3874b8bSToomas Soome scn->scn_taskq = taskq_create(tq_name, nthreads, minclsyspri, 3008a3874b8bSToomas Soome nthreads, nthreads, TASKQ_PREPOPULATE); 3009a3874b8bSToomas Soome kmem_free(tq_name, ZFS_MAX_DATASET_NAME_LEN + 16); 3010a3874b8bSToomas Soome } 3011a3874b8bSToomas Soome 3012a3874b8bSToomas Soome for (uint64_t i = 0; i < spa->spa_root_vdev->vdev_children; i++) { 3013a3874b8bSToomas Soome vdev_t *vd = spa->spa_root_vdev->vdev_child[i]; 3014a3874b8bSToomas Soome 3015a3874b8bSToomas Soome mutex_enter(&vd->vdev_scan_io_queue_lock); 3016a3874b8bSToomas Soome if (vd->vdev_scan_io_queue != NULL) { 3017a3874b8bSToomas Soome VERIFY(taskq_dispatch(scn->scn_taskq, 3018a3874b8bSToomas Soome scan_io_queues_run_one, vd->vdev_scan_io_queue, 3019a3874b8bSToomas Soome TQ_SLEEP) != TASKQID_INVALID); 3020a3874b8bSToomas Soome } 3021a3874b8bSToomas Soome mutex_exit(&vd->vdev_scan_io_queue_lock); 3022a3874b8bSToomas Soome } 3023a3874b8bSToomas Soome 3024a3874b8bSToomas Soome /* 3025a3874b8bSToomas Soome * Wait for the queues to finish issuing thir IOs for this run 3026a3874b8bSToomas Soome * before we return. There may still be IOs in flight at this 3027a3874b8bSToomas Soome * point. 3028a3874b8bSToomas Soome */ 3029a3874b8bSToomas Soome taskq_wait(scn->scn_taskq); 30303f9d6ad7SLin Ling } 30313f9d6ad7SLin Ling 3032ad135b5dSChristopher Siden static boolean_t 30335cabbc6bSPrashanth Sreenivasa dsl_scan_async_block_should_pause(dsl_scan_t *scn) 3034cde58dbcSMatthew Ahrens { 3035cde58dbcSMatthew Ahrens uint64_t elapsed_nanosecs; 3036cde58dbcSMatthew Ahrens 30378b36997aSMatthew Ahrens if (zfs_recover) 30388b36997aSMatthew Ahrens return (B_FALSE); 30398b36997aSMatthew Ahrens 30405cabbc6bSPrashanth Sreenivasa if (scn->scn_visited_this_txg >= zfs_async_block_max_blocks) 3041af3465daSMax Grossman return (B_TRUE); 3042af3465daSMax Grossman 3043cde58dbcSMatthew Ahrens elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 3044ad135b5dSChristopher Siden return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 30455cabbc6bSPrashanth Sreenivasa (NSEC2MSEC(elapsed_nanosecs) > scn->scn_async_block_min_time_ms && 3046cde58dbcSMatthew Ahrens txg_sync_waiting(scn->scn_dp)) || 3047ad135b5dSChristopher Siden spa_shutting_down(scn->scn_dp->dp_spa)); 3048ad135b5dSChristopher Siden } 3049ad135b5dSChristopher Siden 3050ad135b5dSChristopher Siden static int 3051ad135b5dSChristopher Siden dsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 3052ad135b5dSChristopher Siden { 3053ad135b5dSChristopher Siden dsl_scan_t *scn = arg; 3054ad135b5dSChristopher Siden 3055ad135b5dSChristopher Siden if (!scn->scn_is_bptree || 3056ad135b5dSChristopher Siden (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)) { 30575cabbc6bSPrashanth Sreenivasa if (dsl_scan_async_block_should_pause(scn)) 3058be6fd75aSMatthew Ahrens return (SET_ERROR(ERESTART)); 3059ad135b5dSChristopher Siden } 3060cde58dbcSMatthew Ahrens 3061cde58dbcSMatthew Ahrens zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa, 3062cde58dbcSMatthew Ahrens dmu_tx_get_txg(tx), bp, 0)); 3063cde58dbcSMatthew Ahrens dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 3064cde58dbcSMatthew Ahrens -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp), 3065cde58dbcSMatthew Ahrens -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 3066cde58dbcSMatthew Ahrens scn->scn_visited_this_txg++; 3067cde58dbcSMatthew Ahrens return (0); 3068cde58dbcSMatthew Ahrens } 3069cde58dbcSMatthew Ahrens 3070a3874b8bSToomas Soome static void 3071a3874b8bSToomas Soome dsl_scan_update_stats(dsl_scan_t *scn) 3072a3874b8bSToomas Soome { 3073a3874b8bSToomas Soome spa_t *spa = scn->scn_dp->dp_spa; 3074a3874b8bSToomas Soome uint64_t i; 3075a3874b8bSToomas Soome uint64_t seg_size_total = 0, zio_size_total = 0; 3076a3874b8bSToomas Soome uint64_t seg_count_total = 0, zio_count_total = 0; 3077a3874b8bSToomas Soome 3078a3874b8bSToomas Soome for (i = 0; i < spa->spa_root_vdev->vdev_children; i++) { 3079a3874b8bSToomas Soome vdev_t *vd = spa->spa_root_vdev->vdev_child[i]; 3080a3874b8bSToomas Soome dsl_scan_io_queue_t *queue = vd->vdev_scan_io_queue; 3081a3874b8bSToomas Soome 3082a3874b8bSToomas Soome if (queue == NULL) 3083a3874b8bSToomas Soome continue; 3084a3874b8bSToomas Soome 3085a3874b8bSToomas Soome seg_size_total += queue->q_total_seg_size_this_txg; 3086a3874b8bSToomas Soome zio_size_total += queue->q_total_zio_size_this_txg; 3087a3874b8bSToomas Soome seg_count_total += queue->q_segs_this_txg; 3088a3874b8bSToomas Soome zio_count_total += queue->q_zios_this_txg; 3089a3874b8bSToomas Soome } 3090a3874b8bSToomas Soome 3091a3874b8bSToomas Soome if (seg_count_total == 0 || zio_count_total == 0) { 3092a3874b8bSToomas Soome scn->scn_avg_seg_size_this_txg = 0; 3093a3874b8bSToomas Soome scn->scn_avg_zio_size_this_txg = 0; 3094a3874b8bSToomas Soome scn->scn_segs_this_txg = 0; 3095a3874b8bSToomas Soome scn->scn_zios_this_txg = 0; 3096a3874b8bSToomas Soome return; 3097a3874b8bSToomas Soome } 3098a3874b8bSToomas Soome 3099a3874b8bSToomas Soome scn->scn_avg_seg_size_this_txg = seg_size_total / seg_count_total; 3100a3874b8bSToomas Soome scn->scn_avg_zio_size_this_txg = zio_size_total / zio_count_total; 3101a3874b8bSToomas Soome scn->scn_segs_this_txg = seg_count_total; 3102a3874b8bSToomas Soome scn->scn_zios_this_txg = zio_count_total; 3103a3874b8bSToomas Soome } 3104a3874b8bSToomas Soome 31055cabbc6bSPrashanth Sreenivasa static int 31065cabbc6bSPrashanth Sreenivasa dsl_scan_obsolete_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 31075cabbc6bSPrashanth Sreenivasa { 31085cabbc6bSPrashanth Sreenivasa dsl_scan_t *scn = arg; 31095cabbc6bSPrashanth Sreenivasa const dva_t *dva = &bp->blk_dva[0]; 31105cabbc6bSPrashanth Sreenivasa 31115cabbc6bSPrashanth Sreenivasa if (dsl_scan_async_block_should_pause(scn)) 31125cabbc6bSPrashanth Sreenivasa return (SET_ERROR(ERESTART)); 31135cabbc6bSPrashanth Sreenivasa 31145cabbc6bSPrashanth Sreenivasa spa_vdev_indirect_mark_obsolete(scn->scn_dp->dp_spa, 31155cabbc6bSPrashanth Sreenivasa DVA_GET_VDEV(dva), DVA_GET_OFFSET(dva), 31165cabbc6bSPrashanth Sreenivasa DVA_GET_ASIZE(dva), tx); 31175cabbc6bSPrashanth Sreenivasa scn->scn_visited_this_txg++; 31185cabbc6bSPrashanth Sreenivasa return (0); 31195cabbc6bSPrashanth Sreenivasa } 31205cabbc6bSPrashanth Sreenivasa 3121cde58dbcSMatthew Ahrens boolean_t 3122cde58dbcSMatthew Ahrens dsl_scan_active(dsl_scan_t *scn) 3123cde58dbcSMatthew Ahrens { 3124cde58dbcSMatthew Ahrens spa_t *spa = scn->scn_dp->dp_spa; 3125cde58dbcSMatthew Ahrens uint64_t used = 0, comp, uncomp; 3126cde58dbcSMatthew Ahrens 3127cde58dbcSMatthew Ahrens if (spa->spa_load_state != SPA_LOAD_NONE) 3128cde58dbcSMatthew Ahrens return (B_FALSE); 3129cde58dbcSMatthew Ahrens if (spa_shutting_down(spa)) 3130cde58dbcSMatthew Ahrens return (B_FALSE); 3131a3874b8bSToomas Soome if ((dsl_scan_is_running(scn) && !dsl_scan_is_paused_scrub(scn)) || 31327fd05ac4SMatthew Ahrens (scn->scn_async_destroying && !scn->scn_async_stalled)) 3133cde58dbcSMatthew Ahrens return (B_TRUE); 3134cde58dbcSMatthew Ahrens 3135cde58dbcSMatthew Ahrens if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 3136cde58dbcSMatthew Ahrens (void) bpobj_space(&scn->scn_dp->dp_free_bpobj, 3137cde58dbcSMatthew Ahrens &used, &comp, &uncomp); 3138cde58dbcSMatthew Ahrens } 3139cde58dbcSMatthew Ahrens return (used != 0); 3140cde58dbcSMatthew Ahrens } 3141cde58dbcSMatthew Ahrens 3142e4c795beSTom Caputi static boolean_t 3143e4c795beSTom Caputi dsl_scan_check_deferred(vdev_t *vd) 3144e4c795beSTom Caputi { 3145e4c795beSTom Caputi boolean_t need_resilver = B_FALSE; 3146e4c795beSTom Caputi 3147e4c795beSTom Caputi for (int c = 0; c < vd->vdev_children; c++) { 3148e4c795beSTom Caputi need_resilver |= 3149e4c795beSTom Caputi dsl_scan_check_deferred(vd->vdev_child[c]); 3150e4c795beSTom Caputi } 3151e4c795beSTom Caputi 3152e4c795beSTom Caputi if (!vdev_is_concrete(vd) || vd->vdev_aux || 3153e4c795beSTom Caputi !vd->vdev_ops->vdev_op_leaf) 3154e4c795beSTom Caputi return (need_resilver); 3155e4c795beSTom Caputi 3156e4c795beSTom Caputi if (!vd->vdev_resilver_deferred) 3157e4c795beSTom Caputi need_resilver = B_TRUE; 3158e4c795beSTom Caputi 3159e4c795beSTom Caputi return (need_resilver); 3160e4c795beSTom Caputi } 3161e4c795beSTom Caputi 3162a3874b8bSToomas Soome static boolean_t 3163a3874b8bSToomas Soome dsl_scan_need_resilver(spa_t *spa, const dva_t *dva, size_t psize, 3164a3874b8bSToomas Soome uint64_t phys_birth) 3165a3874b8bSToomas Soome { 3166a3874b8bSToomas Soome vdev_t *vd; 3167a3874b8bSToomas Soome 3168a3874b8bSToomas Soome vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva)); 3169a3874b8bSToomas Soome 3170a3874b8bSToomas Soome if (vd->vdev_ops == &vdev_indirect_ops) { 3171a3874b8bSToomas Soome /* 3172a3874b8bSToomas Soome * The indirect vdev can point to multiple 3173a3874b8bSToomas Soome * vdevs. For simplicity, always create 3174a3874b8bSToomas Soome * the resilver zio_t. zio_vdev_io_start() 3175a3874b8bSToomas Soome * will bypass the child resilver i/o's if 3176a3874b8bSToomas Soome * they are on vdevs that don't have DTL's. 3177a3874b8bSToomas Soome */ 3178a3874b8bSToomas Soome return (B_TRUE); 3179a3874b8bSToomas Soome } 3180a3874b8bSToomas Soome 3181a3874b8bSToomas Soome if (DVA_GET_GANG(dva)) { 3182a3874b8bSToomas Soome /* 3183a3874b8bSToomas Soome * Gang members may be spread across multiple 3184a3874b8bSToomas Soome * vdevs, so the best estimate we have is the 3185a3874b8bSToomas Soome * scrub range, which has already been checked. 3186a3874b8bSToomas Soome * XXX -- it would be better to change our 3187a3874b8bSToomas Soome * allocation policy to ensure that all 3188a3874b8bSToomas Soome * gang members reside on the same vdev. 3189a3874b8bSToomas Soome */ 3190a3874b8bSToomas Soome return (B_TRUE); 3191a3874b8bSToomas Soome } 3192a3874b8bSToomas Soome 3193a3874b8bSToomas Soome /* 3194a3874b8bSToomas Soome * Check if the txg falls within the range which must be 3195a3874b8bSToomas Soome * resilvered. DVAs outside this range can always be skipped. 3196a3874b8bSToomas Soome */ 3197a3874b8bSToomas Soome if (!vdev_dtl_contains(vd, DTL_PARTIAL, phys_birth, 1)) 3198a3874b8bSToomas Soome return (B_FALSE); 3199a3874b8bSToomas Soome 3200a3874b8bSToomas Soome /* 3201a3874b8bSToomas Soome * Check if the top-level vdev must resilver this offset. 3202a3874b8bSToomas Soome * When the offset does not intersect with a dirty leaf DTL 3203a3874b8bSToomas Soome * then it may be possible to skip the resilver IO. The psize 3204a3874b8bSToomas Soome * is provided instead of asize to simplify the check for RAIDZ. 3205a3874b8bSToomas Soome */ 3206a3874b8bSToomas Soome if (!vdev_dtl_need_resilver(vd, DVA_GET_OFFSET(dva), psize)) 3207a3874b8bSToomas Soome return (B_FALSE); 3208a3874b8bSToomas Soome 3209e4c795beSTom Caputi /* 3210e4c795beSTom Caputi * Check that this top-level vdev has a device under it which 3211e4c795beSTom Caputi * is resilvering and is not deferred. 3212e4c795beSTom Caputi */ 3213e4c795beSTom Caputi if (!dsl_scan_check_deferred(vd)) 3214e4c795beSTom Caputi return (B_FALSE); 3215e4c795beSTom Caputi 3216a3874b8bSToomas Soome return (B_TRUE); 3217a3874b8bSToomas Soome } 3218a3874b8bSToomas Soome 321986714001SSerapheim Dimitropoulos static int 322086714001SSerapheim Dimitropoulos dsl_process_async_destroys(dsl_pool_t *dp, dmu_tx_t *tx) 32213f9d6ad7SLin Ling { 3222a3874b8bSToomas Soome int err = 0; 32233f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 32243f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 32253f9d6ad7SLin Ling 322686714001SSerapheim Dimitropoulos if (spa_suspend_async_destroy(spa)) 322786714001SSerapheim Dimitropoulos return (0); 32283f9d6ad7SLin Ling 3229139510fbSGeorge Wilson if (zfs_free_bpobj_enabled && 3230a3874b8bSToomas Soome spa_version(spa) >= SPA_VERSION_DEADLISTS) { 3231ad135b5dSChristopher Siden scn->scn_is_bptree = B_FALSE; 32325cabbc6bSPrashanth Sreenivasa scn->scn_async_block_min_time_ms = zfs_free_min_time_ms; 3233a3874b8bSToomas Soome scn->scn_zio_root = zio_root(spa, NULL, 3234cde58dbcSMatthew Ahrens NULL, ZIO_FLAG_MUSTSUCCEED); 3235cde58dbcSMatthew Ahrens err = bpobj_iterate(&dp->dp_free_bpobj, 3236ad135b5dSChristopher Siden dsl_scan_free_block_cb, scn, tx); 3237a3874b8bSToomas Soome VERIFY0(zio_wait(scn->scn_zio_root)); 3238a3874b8bSToomas Soome scn->scn_zio_root = NULL; 3239ad135b5dSChristopher Siden 32407fd05ac4SMatthew Ahrens if (err != 0 && err != ERESTART) 32417fd05ac4SMatthew Ahrens zfs_panic_recover("error %u from bpobj_iterate()", err); 32427fd05ac4SMatthew Ahrens } 32437fd05ac4SMatthew Ahrens 32447fd05ac4SMatthew Ahrens if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) { 32457fd05ac4SMatthew Ahrens ASSERT(scn->scn_async_destroying); 32467fd05ac4SMatthew Ahrens scn->scn_is_bptree = B_TRUE; 3247a3874b8bSToomas Soome scn->scn_zio_root = zio_root(spa, NULL, 32487fd05ac4SMatthew Ahrens NULL, ZIO_FLAG_MUSTSUCCEED); 32497fd05ac4SMatthew Ahrens err = bptree_iterate(dp->dp_meta_objset, 32507fd05ac4SMatthew Ahrens dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx); 32517fd05ac4SMatthew Ahrens VERIFY0(zio_wait(scn->scn_zio_root)); 3252a3874b8bSToomas Soome scn->scn_zio_root = NULL; 32537fd05ac4SMatthew Ahrens 32547fd05ac4SMatthew Ahrens if (err == EIO || err == ECKSUM) { 32557fd05ac4SMatthew Ahrens err = 0; 32567fd05ac4SMatthew Ahrens } else if (err != 0 && err != ERESTART) { 32577fd05ac4SMatthew Ahrens zfs_panic_recover("error %u from " 32587fd05ac4SMatthew Ahrens "traverse_dataset_destroyed()", err); 3259ad135b5dSChristopher Siden } 32607fd05ac4SMatthew Ahrens 32617fd05ac4SMatthew Ahrens if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) { 32627fd05ac4SMatthew Ahrens /* finished; deactivate async destroy feature */ 32637fd05ac4SMatthew Ahrens spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx); 32647fd05ac4SMatthew Ahrens ASSERT(!spa_feature_is_active(spa, 32657fd05ac4SMatthew Ahrens SPA_FEATURE_ASYNC_DESTROY)); 32667fd05ac4SMatthew Ahrens VERIFY0(zap_remove(dp->dp_meta_objset, 32677fd05ac4SMatthew Ahrens DMU_POOL_DIRECTORY_OBJECT, 32687fd05ac4SMatthew Ahrens DMU_POOL_BPTREE_OBJ, tx)); 32697fd05ac4SMatthew Ahrens VERIFY0(bptree_free(dp->dp_meta_objset, 32707fd05ac4SMatthew Ahrens dp->dp_bptree_obj, tx)); 32717fd05ac4SMatthew Ahrens dp->dp_bptree_obj = 0; 32727fd05ac4SMatthew Ahrens scn->scn_async_destroying = B_FALSE; 3273231aab85SMatthew Ahrens scn->scn_async_stalled = B_FALSE; 3274231aab85SMatthew Ahrens } else { 3275231aab85SMatthew Ahrens /* 3276231aab85SMatthew Ahrens * If we didn't make progress, mark the async 3277231aab85SMatthew Ahrens * destroy as stalled, so that we will not initiate 3278231aab85SMatthew Ahrens * a spa_sync() on its behalf. Note that we only 3279231aab85SMatthew Ahrens * check this if we are not finished, because if the 3280231aab85SMatthew Ahrens * bptree had no blocks for us to visit, we can 3281231aab85SMatthew Ahrens * finish without "making progress". 3282231aab85SMatthew Ahrens */ 3283231aab85SMatthew Ahrens scn->scn_async_stalled = 3284231aab85SMatthew Ahrens (scn->scn_visited_this_txg == 0); 3285cde58dbcSMatthew Ahrens } 32867fd05ac4SMatthew Ahrens } 32877fd05ac4SMatthew Ahrens if (scn->scn_visited_this_txg) { 32887fd05ac4SMatthew Ahrens zfs_dbgmsg("freed %llu blocks in %llums from " 3289a3874b8bSToomas Soome "free_bpobj/bptree txg %llu; err=%d", 32907fd05ac4SMatthew Ahrens (longlong_t)scn->scn_visited_this_txg, 32917fd05ac4SMatthew Ahrens (longlong_t) 32927fd05ac4SMatthew Ahrens NSEC2MSEC(gethrtime() - scn->scn_sync_start_time), 32937fd05ac4SMatthew Ahrens (longlong_t)tx->tx_txg, err); 32947fd05ac4SMatthew Ahrens scn->scn_visited_this_txg = 0; 32957fd05ac4SMatthew Ahrens 32967fd05ac4SMatthew Ahrens /* 32977fd05ac4SMatthew Ahrens * Write out changes to the DDT that may be required as a 32987fd05ac4SMatthew Ahrens * result of the blocks freed. This ensures that the DDT 32997fd05ac4SMatthew Ahrens * is clean when a scrub/resilver runs. 33007fd05ac4SMatthew Ahrens */ 33017fd05ac4SMatthew Ahrens ddt_sync(spa, tx->tx_txg); 33027fd05ac4SMatthew Ahrens } 33037fd05ac4SMatthew Ahrens if (err != 0) 330486714001SSerapheim Dimitropoulos return (err); 33058c04a1faSGary Mills if (dp->dp_free_dir != NULL && !scn->scn_async_destroying && 33068c04a1faSGary Mills zfs_free_leak_on_eio && 3307c1379625SJustin T. Gibbs (dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes != 0 || 3308c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes != 0 || 3309c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes != 0)) { 33107fd05ac4SMatthew Ahrens /* 33117fd05ac4SMatthew Ahrens * We have finished background destroying, but there is still 33127fd05ac4SMatthew Ahrens * some space left in the dp_free_dir. Transfer this leaked 33137fd05ac4SMatthew Ahrens * space to the dp_leak_dir. 33147fd05ac4SMatthew Ahrens */ 33157fd05ac4SMatthew Ahrens if (dp->dp_leak_dir == NULL) { 33167fd05ac4SMatthew Ahrens rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 33177fd05ac4SMatthew Ahrens (void) dsl_dir_create_sync(dp, dp->dp_root_dir, 33187fd05ac4SMatthew Ahrens LEAK_DIR_NAME, tx); 33197fd05ac4SMatthew Ahrens VERIFY0(dsl_pool_open_special_dir(dp, 33207fd05ac4SMatthew Ahrens LEAK_DIR_NAME, &dp->dp_leak_dir)); 33217fd05ac4SMatthew Ahrens rrw_exit(&dp->dp_config_rwlock, FTAG); 33227fd05ac4SMatthew Ahrens } 33237fd05ac4SMatthew Ahrens dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD, 3324c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes, 3325c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes, 3326c1379625SJustin T. Gibbs dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx); 33277fd05ac4SMatthew Ahrens dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD, 3328c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes, 3329c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes, 3330c1379625SJustin T. Gibbs -dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes, tx); 33317fd05ac4SMatthew Ahrens } 33325cabbc6bSPrashanth Sreenivasa 33338c04a1faSGary Mills if (dp->dp_free_dir != NULL && !scn->scn_async_destroying) { 33345d7b4d43SMatthew Ahrens /* finished; verify that space accounting went to zero */ 3335c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_used_bytes); 3336c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_compressed_bytes); 3337c1379625SJustin T. Gibbs ASSERT0(dsl_dir_phys(dp->dp_free_dir)->dd_uncompressed_bytes); 3338cde58dbcSMatthew Ahrens } 3339cde58dbcSMatthew Ahrens 33405cabbc6bSPrashanth Sreenivasa EQUIV(bpobj_is_open(&dp->dp_obsolete_bpobj), 33415cabbc6bSPrashanth Sreenivasa 0 == zap_contains(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 33425cabbc6bSPrashanth Sreenivasa DMU_POOL_OBSOLETE_BPOBJ)); 33435cabbc6bSPrashanth Sreenivasa if (err == 0 && bpobj_is_open(&dp->dp_obsolete_bpobj)) { 33445cabbc6bSPrashanth Sreenivasa ASSERT(spa_feature_is_active(dp->dp_spa, 33455cabbc6bSPrashanth Sreenivasa SPA_FEATURE_OBSOLETE_COUNTS)); 33465cabbc6bSPrashanth Sreenivasa 33475cabbc6bSPrashanth Sreenivasa scn->scn_is_bptree = B_FALSE; 33485cabbc6bSPrashanth Sreenivasa scn->scn_async_block_min_time_ms = zfs_obsolete_min_time_ms; 33495cabbc6bSPrashanth Sreenivasa err = bpobj_iterate(&dp->dp_obsolete_bpobj, 33505cabbc6bSPrashanth Sreenivasa dsl_scan_obsolete_block_cb, scn, tx); 33515cabbc6bSPrashanth Sreenivasa if (err != 0 && err != ERESTART) 33525cabbc6bSPrashanth Sreenivasa zfs_panic_recover("error %u from bpobj_iterate()", err); 33535cabbc6bSPrashanth Sreenivasa 33545cabbc6bSPrashanth Sreenivasa if (bpobj_is_empty(&dp->dp_obsolete_bpobj)) 33555cabbc6bSPrashanth Sreenivasa dsl_pool_destroy_obsolete_bpobj(dp, tx); 33565cabbc6bSPrashanth Sreenivasa } 33575cabbc6bSPrashanth Sreenivasa 335886714001SSerapheim Dimitropoulos return (0); 335986714001SSerapheim Dimitropoulos } 336086714001SSerapheim Dimitropoulos 3361a3874b8bSToomas Soome /* 3362a3874b8bSToomas Soome * This is the primary entry point for scans that is called from syncing 3363a3874b8bSToomas Soome * context. Scans must happen entirely during syncing context so that we 3364a3874b8bSToomas Soome * cna guarantee that blocks we are currently scanning will not change out 3365a3874b8bSToomas Soome * from under us. While a scan is active, this funciton controls how quickly 3366a3874b8bSToomas Soome * transaction groups proceed, instead of the normal handling provided by 3367a3874b8bSToomas Soome * txg_sync_thread(). 3368a3874b8bSToomas Soome */ 336986714001SSerapheim Dimitropoulos void 337086714001SSerapheim Dimitropoulos dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx) 337186714001SSerapheim Dimitropoulos { 337286714001SSerapheim Dimitropoulos dsl_scan_t *scn = dp->dp_scan; 337386714001SSerapheim Dimitropoulos spa_t *spa = dp->dp_spa; 337486714001SSerapheim Dimitropoulos int err = 0; 3375a3874b8bSToomas Soome state_sync_type_t sync_type = SYNC_OPTIONAL; 337686714001SSerapheim Dimitropoulos 3377e4c795beSTom Caputi if (spa->spa_resilver_deferred && 3378e4c795beSTom Caputi !spa_feature_is_active(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER)) 3379e4c795beSTom Caputi spa_feature_incr(spa, SPA_FEATURE_RESILVER_DEFER, tx); 3380e4c795beSTom Caputi 338186714001SSerapheim Dimitropoulos /* 338286714001SSerapheim Dimitropoulos * Check for scn_restart_txg before checking spa_load_state, so 338386714001SSerapheim Dimitropoulos * that we can restart an old-style scan while the pool is being 3384e4c795beSTom Caputi * imported (see dsl_scan_init). We also restart scans if there 3385e4c795beSTom Caputi * is a deferred resilver and the user has manually disabled 3386e4c795beSTom Caputi * deferred resilvers via the tunable. 338786714001SSerapheim Dimitropoulos */ 3388e4c795beSTom Caputi if (dsl_scan_restarting(scn, tx) || 3389e4c795beSTom Caputi (spa->spa_resilver_deferred && zfs_resilver_disable_defer)) { 339086714001SSerapheim Dimitropoulos pool_scan_func_t func = POOL_SCAN_SCRUB; 339186714001SSerapheim Dimitropoulos dsl_scan_done(scn, B_FALSE, tx); 339286714001SSerapheim Dimitropoulos if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) 339386714001SSerapheim Dimitropoulos func = POOL_SCAN_RESILVER; 339486714001SSerapheim Dimitropoulos zfs_dbgmsg("restarting scan func=%u txg=%llu", 3395a3874b8bSToomas Soome func, (longlong_t)tx->tx_txg); 339686714001SSerapheim Dimitropoulos dsl_scan_setup_sync(&func, tx); 339786714001SSerapheim Dimitropoulos } 339886714001SSerapheim Dimitropoulos 339986714001SSerapheim Dimitropoulos /* 340086714001SSerapheim Dimitropoulos * Only process scans in sync pass 1. 340186714001SSerapheim Dimitropoulos */ 340286714001SSerapheim Dimitropoulos if (spa_sync_pass(dp->dp_spa) > 1) 340386714001SSerapheim Dimitropoulos return; 340486714001SSerapheim Dimitropoulos 340586714001SSerapheim Dimitropoulos /* 340686714001SSerapheim Dimitropoulos * If the spa is shutting down, then stop scanning. This will 340786714001SSerapheim Dimitropoulos * ensure that the scan does not dirty any new data during the 340886714001SSerapheim Dimitropoulos * shutdown phase. 340986714001SSerapheim Dimitropoulos */ 341086714001SSerapheim Dimitropoulos if (spa_shutting_down(spa)) 341186714001SSerapheim Dimitropoulos return; 341286714001SSerapheim Dimitropoulos 341386714001SSerapheim Dimitropoulos /* 341486714001SSerapheim Dimitropoulos * If the scan is inactive due to a stalled async destroy, try again. 341586714001SSerapheim Dimitropoulos */ 341686714001SSerapheim Dimitropoulos if (!scn->scn_async_stalled && !dsl_scan_active(scn)) 341786714001SSerapheim Dimitropoulos return; 341886714001SSerapheim Dimitropoulos 3419a3874b8bSToomas Soome /* reset scan statistics */ 342086714001SSerapheim Dimitropoulos scn->scn_visited_this_txg = 0; 3421a3874b8bSToomas Soome scn->scn_holes_this_txg = 0; 3422a3874b8bSToomas Soome scn->scn_lt_min_this_txg = 0; 3423a3874b8bSToomas Soome scn->scn_gt_max_this_txg = 0; 3424a3874b8bSToomas Soome scn->scn_ddt_contained_this_txg = 0; 3425a3874b8bSToomas Soome scn->scn_objsets_visited_this_txg = 0; 3426a3874b8bSToomas Soome scn->scn_avg_seg_size_this_txg = 0; 3427a3874b8bSToomas Soome scn->scn_segs_this_txg = 0; 3428a3874b8bSToomas Soome scn->scn_avg_zio_size_this_txg = 0; 3429a3874b8bSToomas Soome scn->scn_zios_this_txg = 0; 343086714001SSerapheim Dimitropoulos scn->scn_suspending = B_FALSE; 343186714001SSerapheim Dimitropoulos scn->scn_sync_start_time = gethrtime(); 343286714001SSerapheim Dimitropoulos spa->spa_scrub_active = B_TRUE; 343386714001SSerapheim Dimitropoulos 343486714001SSerapheim Dimitropoulos /* 343586714001SSerapheim Dimitropoulos * First process the async destroys. If we pause, don't do 343686714001SSerapheim Dimitropoulos * any scrubbing or resilvering. This ensures that there are no 343786714001SSerapheim Dimitropoulos * async destroys while we are scanning, so the scan code doesn't 343886714001SSerapheim Dimitropoulos * have to worry about traversing it. It is also faster to free the 343986714001SSerapheim Dimitropoulos * blocks than to scrub them. 344086714001SSerapheim Dimitropoulos */ 344186714001SSerapheim Dimitropoulos err = dsl_process_async_destroys(dp, tx); 344286714001SSerapheim Dimitropoulos if (err != 0) 344386714001SSerapheim Dimitropoulos return; 344486714001SSerapheim Dimitropoulos 3445a3874b8bSToomas Soome if (!dsl_scan_is_running(scn) || dsl_scan_is_paused_scrub(scn)) 3446cde58dbcSMatthew Ahrens return; 3447cde58dbcSMatthew Ahrens 3448a3874b8bSToomas Soome /* 3449a3874b8bSToomas Soome * Wait a few txgs after importing to begin scanning so that 3450a3874b8bSToomas Soome * we can get the pool imported quickly. 3451a3874b8bSToomas Soome */ 3452a3874b8bSToomas Soome if (spa->spa_syncing_txg < spa->spa_first_txg + SCAN_IMPORT_WAIT_TXGS) 3453b4952e17SGeorge Wilson return; 3454b4952e17SGeorge Wilson 3455e4c795beSTom Caputi /* 3456e4c795beSTom Caputi * zfs_scan_suspend_progress can be set to disable scan progress. 3457e4c795beSTom Caputi * We don't want to spin the txg_sync thread, so we add a delay 3458e4c795beSTom Caputi * here to simulate the time spent doing a scan. This is mostly 3459e4c795beSTom Caputi * useful for testing and debugging. 3460e4c795beSTom Caputi */ 3461e4c795beSTom Caputi if (zfs_scan_suspend_progress) { 3462e4c795beSTom Caputi uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time; 3463e4c795beSTom Caputi int mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 3464e4c795beSTom Caputi zfs_resilver_min_time_ms : zfs_scrub_min_time_ms; 3465e4c795beSTom Caputi 3466e4c795beSTom Caputi while (zfs_scan_suspend_progress && 3467e4c795beSTom Caputi !txg_sync_waiting(scn->scn_dp) && 3468e4c795beSTom Caputi !spa_shutting_down(scn->scn_dp->dp_spa) && 3469e4c795beSTom Caputi NSEC2MSEC(scan_time_ns) < mintime) { 3470e4c795beSTom Caputi delay(hz); 3471e4c795beSTom Caputi scan_time_ns = gethrtime() - scn->scn_sync_start_time; 3472e4c795beSTom Caputi } 3473e4c795beSTom Caputi return; 3474e4c795beSTom Caputi } 3475e4c795beSTom Caputi 3476a3874b8bSToomas Soome /* 3477a3874b8bSToomas Soome * It is possible to switch from unsorted to sorted at any time, 3478a3874b8bSToomas Soome * but afterwards the scan will remain sorted unless reloaded from 3479a3874b8bSToomas Soome * a checkpoint after a reboot. 3480a3874b8bSToomas Soome */ 3481a3874b8bSToomas Soome if (!zfs_scan_legacy) { 3482a3874b8bSToomas Soome scn->scn_is_sorted = B_TRUE; 3483a3874b8bSToomas Soome if (scn->scn_last_checkpoint == 0) 3484a3874b8bSToomas Soome scn->scn_last_checkpoint = ddi_get_lbolt(); 3485a3874b8bSToomas Soome } 34861702cce7SAlek Pinchuk 3487a3874b8bSToomas Soome /* 3488a3874b8bSToomas Soome * For sorted scans, determine what kind of work we will be doing 3489a3874b8bSToomas Soome * this txg based on our memory limitations and whether or not we 3490a3874b8bSToomas Soome * need to perform a checkpoint. 3491a3874b8bSToomas Soome */ 3492a3874b8bSToomas Soome if (scn->scn_is_sorted) { 3493a3874b8bSToomas Soome /* 3494a3874b8bSToomas Soome * If we are over our checkpoint interval, set scn_clearing 3495a3874b8bSToomas Soome * so that we can begin checkpointing immediately. The 3496a3874b8bSToomas Soome * checkpoint allows us to save a consisent bookmark 3497a3874b8bSToomas Soome * representing how much data we have scrubbed so far. 3498a3874b8bSToomas Soome * Otherwise, use the memory limit to determine if we should 3499a3874b8bSToomas Soome * scan for metadata or start issue scrub IOs. We accumulate 3500a3874b8bSToomas Soome * metadata until we hit our hard memory limit at which point 3501a3874b8bSToomas Soome * we issue scrub IOs until we are at our soft memory limit. 3502a3874b8bSToomas Soome */ 3503a3874b8bSToomas Soome if (scn->scn_checkpointing || 3504a3874b8bSToomas Soome ddi_get_lbolt() - scn->scn_last_checkpoint > 3505a3874b8bSToomas Soome SEC_TO_TICK(zfs_scan_checkpoint_intval)) { 3506a3874b8bSToomas Soome if (!scn->scn_checkpointing) 3507a3874b8bSToomas Soome zfs_dbgmsg("begin scan checkpoint"); 3508a3874b8bSToomas Soome 3509a3874b8bSToomas Soome scn->scn_checkpointing = B_TRUE; 3510a3874b8bSToomas Soome scn->scn_clearing = B_TRUE; 3511a3874b8bSToomas Soome } else { 3512a3874b8bSToomas Soome boolean_t should_clear = dsl_scan_should_clear(scn); 3513a3874b8bSToomas Soome if (should_clear && !scn->scn_clearing) { 3514a3874b8bSToomas Soome zfs_dbgmsg("begin scan clearing"); 3515a3874b8bSToomas Soome scn->scn_clearing = B_TRUE; 3516a3874b8bSToomas Soome } else if (!should_clear && scn->scn_clearing) { 3517a3874b8bSToomas Soome zfs_dbgmsg("finish scan clearing"); 3518a3874b8bSToomas Soome scn->scn_clearing = B_FALSE; 3519a3874b8bSToomas Soome } 3520a3874b8bSToomas Soome } 35213f9d6ad7SLin Ling } else { 3522a3874b8bSToomas Soome ASSERT0(scn->scn_checkpointing); 3523a3874b8bSToomas Soome ASSERT0(scn->scn_clearing); 35243f9d6ad7SLin Ling } 35253f9d6ad7SLin Ling 3526a3874b8bSToomas Soome if (!scn->scn_clearing && scn->scn_done_txg == 0) { 3527a3874b8bSToomas Soome /* Need to scan metadata for more blocks to scrub */ 3528a3874b8bSToomas Soome dsl_scan_phys_t *scnp = &scn->scn_phys; 3529a3874b8bSToomas Soome taskqid_t prefetch_tqid; 3530a3874b8bSToomas Soome uint64_t bytes_per_leaf = zfs_scan_vdev_limit; 3531a3874b8bSToomas Soome uint64_t nr_leaves = dsl_scan_count_leaves(spa->spa_root_vdev); 35323f9d6ad7SLin Ling 3533a3874b8bSToomas Soome /* 3534a3874b8bSToomas Soome * Calculate the max number of in-flight bytes for pool-wide 3535a3874b8bSToomas Soome * scanning operations (minimum 1MB). Limits for the issuing 3536a3874b8bSToomas Soome * phase are done per top-level vdev and are handled separately. 3537a3874b8bSToomas Soome */ 3538a3874b8bSToomas Soome scn->scn_maxinflight_bytes = 3539a3874b8bSToomas Soome MAX(nr_leaves * bytes_per_leaf, 1ULL << 20); 3540a3874b8bSToomas Soome 3541a3874b8bSToomas Soome if (scnp->scn_ddt_bookmark.ddb_class <= 3542a3874b8bSToomas Soome scnp->scn_ddt_class_max) { 3543a3874b8bSToomas Soome ASSERT(ZB_IS_ZERO(&scnp->scn_bookmark)); 3544a3874b8bSToomas Soome zfs_dbgmsg("doing scan sync txg %llu; " 3545a3874b8bSToomas Soome "ddt bm=%llu/%llu/%llu/%llx", 3546a3874b8bSToomas Soome (longlong_t)tx->tx_txg, 3547a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_class, 3548a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_type, 3549a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_checksum, 3550a3874b8bSToomas Soome (longlong_t)scnp->scn_ddt_bookmark.ddb_cursor); 3551a3874b8bSToomas Soome } else { 3552a3874b8bSToomas Soome zfs_dbgmsg("doing scan sync txg %llu; " 3553a3874b8bSToomas Soome "bm=%llu/%llu/%llu/%llu", 3554a3874b8bSToomas Soome (longlong_t)tx->tx_txg, 3555a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_objset, 3556a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_object, 3557a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_level, 3558a3874b8bSToomas Soome (longlong_t)scnp->scn_bookmark.zb_blkid); 3559a3874b8bSToomas Soome } 35603f9d6ad7SLin Ling 3561a3874b8bSToomas Soome scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 3562a3874b8bSToomas Soome NULL, ZIO_FLAG_CANFAIL); 35633f9d6ad7SLin Ling 3564a3874b8bSToomas Soome scn->scn_prefetch_stop = B_FALSE; 3565a3874b8bSToomas Soome prefetch_tqid = taskq_dispatch(dp->dp_sync_taskq, 3566a3874b8bSToomas Soome dsl_scan_prefetch_thread, scn, TQ_SLEEP); 3567a3874b8bSToomas Soome ASSERT(prefetch_tqid != TASKQID_INVALID); 35683f9d6ad7SLin Ling 3569a3874b8bSToomas Soome dsl_pool_config_enter(dp, FTAG); 3570a3874b8bSToomas Soome dsl_scan_visit(scn, tx); 3571a3874b8bSToomas Soome dsl_pool_config_exit(dp, FTAG); 35723f9d6ad7SLin Ling 3573a3874b8bSToomas Soome mutex_enter(&dp->dp_spa->spa_scrub_lock); 3574a3874b8bSToomas Soome scn->scn_prefetch_stop = B_TRUE; 3575a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 3576a3874b8bSToomas Soome mutex_exit(&dp->dp_spa->spa_scrub_lock); 35773f9d6ad7SLin Ling 3578a3874b8bSToomas Soome taskq_wait_id(dp->dp_sync_taskq, prefetch_tqid); 3579a3874b8bSToomas Soome (void) zio_wait(scn->scn_zio_root); 3580a3874b8bSToomas Soome scn->scn_zio_root = NULL; 3581a3874b8bSToomas Soome 3582a3874b8bSToomas Soome zfs_dbgmsg("scan visited %llu blocks in %llums " 3583a3874b8bSToomas Soome "(%llu os's, %llu holes, %llu < mintxg, " 3584a3874b8bSToomas Soome "%llu in ddt, %llu > maxtxg)", 3585a3874b8bSToomas Soome (longlong_t)scn->scn_visited_this_txg, 3586a3874b8bSToomas Soome (longlong_t)NSEC2MSEC(gethrtime() - 3587a3874b8bSToomas Soome scn->scn_sync_start_time), 3588a3874b8bSToomas Soome (longlong_t)scn->scn_objsets_visited_this_txg, 3589a3874b8bSToomas Soome (longlong_t)scn->scn_holes_this_txg, 3590a3874b8bSToomas Soome (longlong_t)scn->scn_lt_min_this_txg, 3591a3874b8bSToomas Soome (longlong_t)scn->scn_ddt_contained_this_txg, 3592a3874b8bSToomas Soome (longlong_t)scn->scn_gt_max_this_txg); 3593a3874b8bSToomas Soome 3594a3874b8bSToomas Soome if (!scn->scn_suspending) { 3595a3874b8bSToomas Soome ASSERT0(avl_numnodes(&scn->scn_queue)); 3596a3874b8bSToomas Soome scn->scn_done_txg = tx->tx_txg + 1; 3597a3874b8bSToomas Soome if (scn->scn_is_sorted) { 3598a3874b8bSToomas Soome scn->scn_checkpointing = B_TRUE; 3599a3874b8bSToomas Soome scn->scn_clearing = B_TRUE; 3600a3874b8bSToomas Soome } 3601a3874b8bSToomas Soome zfs_dbgmsg("scan complete txg %llu", 3602a3874b8bSToomas Soome (longlong_t)tx->tx_txg); 3603a3874b8bSToomas Soome } 3604a3874b8bSToomas Soome } else if (scn->scn_is_sorted && scn->scn_bytes_pending != 0) { 3605e4c795beSTom Caputi ASSERT(scn->scn_clearing); 3606e4c795beSTom Caputi 3607a3874b8bSToomas Soome /* need to issue scrubbing IOs from per-vdev queues */ 3608a3874b8bSToomas Soome scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 3609a3874b8bSToomas Soome NULL, ZIO_FLAG_CANFAIL); 3610a3874b8bSToomas Soome scan_io_queues_run(scn); 3611a3874b8bSToomas Soome (void) zio_wait(scn->scn_zio_root); 3612a3874b8bSToomas Soome scn->scn_zio_root = NULL; 3613a3874b8bSToomas Soome 3614a3874b8bSToomas Soome /* calculate and dprintf the current memory usage */ 3615a3874b8bSToomas Soome (void) dsl_scan_should_clear(scn); 3616a3874b8bSToomas Soome dsl_scan_update_stats(scn); 3617a3874b8bSToomas Soome 3618a3874b8bSToomas Soome zfs_dbgmsg("scrubbed %llu blocks (%llu segs) in %llums " 3619a3874b8bSToomas Soome "(avg_block_size = %llu, avg_seg_size = %llu)", 3620a3874b8bSToomas Soome (longlong_t)scn->scn_zios_this_txg, 3621a3874b8bSToomas Soome (longlong_t)scn->scn_segs_this_txg, 3622a3874b8bSToomas Soome (longlong_t)NSEC2MSEC(gethrtime() - 3623a3874b8bSToomas Soome scn->scn_sync_start_time), 3624a3874b8bSToomas Soome (longlong_t)scn->scn_avg_zio_size_this_txg, 3625a3874b8bSToomas Soome (longlong_t)scn->scn_avg_seg_size_this_txg); 3626a3874b8bSToomas Soome } else if (scn->scn_done_txg != 0 && scn->scn_done_txg <= tx->tx_txg) { 3627a3874b8bSToomas Soome /* Finished with everything. Mark the scrub as complete */ 3628a3874b8bSToomas Soome zfs_dbgmsg("scan issuing complete txg %llu", 3629a3874b8bSToomas Soome (longlong_t)tx->tx_txg); 3630a3874b8bSToomas Soome ASSERT3U(scn->scn_done_txg, !=, 0); 3631a3874b8bSToomas Soome ASSERT0(spa->spa_scrub_inflight); 3632a3874b8bSToomas Soome ASSERT0(scn->scn_bytes_pending); 3633a3874b8bSToomas Soome dsl_scan_done(scn, B_TRUE, tx); 3634a3874b8bSToomas Soome sync_type = SYNC_MANDATORY; 36353f9d6ad7SLin Ling } 36363f9d6ad7SLin Ling 3637a3874b8bSToomas Soome dsl_scan_sync_state(scn, tx, sync_type); 36383f9d6ad7SLin Ling } 36393f9d6ad7SLin Ling 36403f9d6ad7SLin Ling static void 3641a3874b8bSToomas Soome count_block(dsl_scan_t *scn, zfs_all_blkstats_t *zab, const blkptr_t *bp) 36423f9d6ad7SLin Ling { 36433f9d6ad7SLin Ling int i; 36443f9d6ad7SLin Ling 3645ee2f9ca4SBill Sommerfeld /* 3646ee2f9ca4SBill Sommerfeld * Don't count embedded bp's, since we already did the work of 3647ee2f9ca4SBill Sommerfeld * scanning these when we scanned the containing block. 3648ee2f9ca4SBill Sommerfeld */ 3649ee2f9ca4SBill Sommerfeld if (BP_IS_EMBEDDED(bp)) 3650ee2f9ca4SBill Sommerfeld return; 3651ee2f9ca4SBill Sommerfeld 365212a8814cSTom Caputi /* 365312a8814cSTom Caputi * Update the spa's stats on how many bytes we have issued. 365412a8814cSTom Caputi * Sequential scrubs create a zio for each DVA of the bp. Each 365512a8814cSTom Caputi * of these will include all DVAs for repair purposes, but the 365612a8814cSTom Caputi * zio code will only try the first one unless there is an issue. 365712a8814cSTom Caputi * Therefore, we should only count the first DVA for these IOs. 365812a8814cSTom Caputi */ 365912a8814cSTom Caputi if (scn->scn_is_sorted) { 3660a3874b8bSToomas Soome atomic_add_64(&scn->scn_dp->dp_spa->spa_scan_pass_issued, 366112a8814cSTom Caputi DVA_GET_ASIZE(&bp->blk_dva[0])); 366212a8814cSTom Caputi } else { 366312a8814cSTom Caputi spa_t *spa = scn->scn_dp->dp_spa; 366412a8814cSTom Caputi 366512a8814cSTom Caputi for (i = 0; i < BP_GET_NDVAS(bp); i++) { 366612a8814cSTom Caputi atomic_add_64(&spa->spa_scan_pass_issued, 366712a8814cSTom Caputi DVA_GET_ASIZE(&bp->blk_dva[i])); 366812a8814cSTom Caputi } 3669a3874b8bSToomas Soome } 3670a3874b8bSToomas Soome 36713f9d6ad7SLin Ling /* 36723f9d6ad7SLin Ling * If we resume after a reboot, zab will be NULL; don't record 36733f9d6ad7SLin Ling * incomplete stats in that case. 36743f9d6ad7SLin Ling */ 36753f9d6ad7SLin Ling if (zab == NULL) 36763f9d6ad7SLin Ling return; 36773f9d6ad7SLin Ling 3678a3874b8bSToomas Soome mutex_enter(&zab->zab_lock); 3679a3874b8bSToomas Soome 36803f9d6ad7SLin Ling for (i = 0; i < 4; i++) { 36813f9d6ad7SLin Ling int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS; 36823f9d6ad7SLin Ling int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL; 3683ad135b5dSChristopher Siden if (t & DMU_OT_NEWTYPE) 3684ad135b5dSChristopher Siden t = DMU_OT_OTHER; 36853f9d6ad7SLin Ling zfs_blkstat_t *zb = &zab->zab_type[l][t]; 36863f9d6ad7SLin Ling int equal; 36873f9d6ad7SLin Ling 36883f9d6ad7SLin Ling zb->zb_count++; 36893f9d6ad7SLin Ling zb->zb_asize += BP_GET_ASIZE(bp); 36903f9d6ad7SLin Ling zb->zb_lsize += BP_GET_LSIZE(bp); 36913f9d6ad7SLin Ling zb->zb_psize += BP_GET_PSIZE(bp); 36923f9d6ad7SLin Ling zb->zb_gangs += BP_COUNT_GANG(bp); 36933f9d6ad7SLin Ling 36943f9d6ad7SLin Ling switch (BP_GET_NDVAS(bp)) { 36953f9d6ad7SLin Ling case 2: 36963f9d6ad7SLin Ling if (DVA_GET_VDEV(&bp->blk_dva[0]) == 36973f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[1])) 36983f9d6ad7SLin Ling zb->zb_ditto_2_of_2_samevdev++; 36993f9d6ad7SLin Ling break; 37003f9d6ad7SLin Ling case 3: 37013f9d6ad7SLin Ling equal = (DVA_GET_VDEV(&bp->blk_dva[0]) == 37023f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[1])) + 37033f9d6ad7SLin Ling (DVA_GET_VDEV(&bp->blk_dva[0]) == 37043f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[2])) + 37053f9d6ad7SLin Ling (DVA_GET_VDEV(&bp->blk_dva[1]) == 37063f9d6ad7SLin Ling DVA_GET_VDEV(&bp->blk_dva[2])); 37073f9d6ad7SLin Ling if (equal == 1) 37083f9d6ad7SLin Ling zb->zb_ditto_2_of_3_samevdev++; 37093f9d6ad7SLin Ling else if (equal == 3) 37103f9d6ad7SLin Ling zb->zb_ditto_3_of_3_samevdev++; 37113f9d6ad7SLin Ling break; 37123f9d6ad7SLin Ling } 37133f9d6ad7SLin Ling } 3714a3874b8bSToomas Soome 3715a3874b8bSToomas Soome mutex_exit(&zab->zab_lock); 37163f9d6ad7SLin Ling } 37173f9d6ad7SLin Ling 37183f9d6ad7SLin Ling static void 3719a3874b8bSToomas Soome scan_io_queue_insert_impl(dsl_scan_io_queue_t *queue, scan_io_t *sio) 37203f9d6ad7SLin Ling { 3721a3874b8bSToomas Soome avl_index_t idx; 372212a8814cSTom Caputi int64_t asize = SIO_GET_ASIZE(sio); 3723a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 37243f9d6ad7SLin Ling 3725a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 37263f9d6ad7SLin Ling 3727a3874b8bSToomas Soome if (avl_find(&queue->q_sios_by_addr, sio, &idx) != NULL) { 3728a3874b8bSToomas Soome /* block is already scheduled for reading */ 3729a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -asize); 373012a8814cSTom Caputi sio_free(sio); 3731a3874b8bSToomas Soome return; 3732a3874b8bSToomas Soome } 3733a3874b8bSToomas Soome avl_insert(&queue->q_sios_by_addr, sio, idx); 373412a8814cSTom Caputi queue->q_sio_memused += SIO_GET_MUSED(sio); 373512a8814cSTom Caputi range_tree_add(queue->q_exts_by_addr, SIO_GET_OFFSET(sio), asize); 3736a3874b8bSToomas Soome } 37373f9d6ad7SLin Ling 3738a3874b8bSToomas Soome /* 3739a3874b8bSToomas Soome * Given all the info we got from our metadata scanning process, we 3740a3874b8bSToomas Soome * construct a scan_io_t and insert it into the scan sorting queue. The 3741a3874b8bSToomas Soome * I/O must already be suitable for us to process. This is controlled 3742a3874b8bSToomas Soome * by dsl_scan_enqueue(). 3743a3874b8bSToomas Soome */ 3744a3874b8bSToomas Soome static void 3745a3874b8bSToomas Soome scan_io_queue_insert(dsl_scan_io_queue_t *queue, const blkptr_t *bp, int dva_i, 3746a3874b8bSToomas Soome int zio_flags, const zbookmark_phys_t *zb) 3747a3874b8bSToomas Soome { 3748a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 374912a8814cSTom Caputi scan_io_t *sio = sio_alloc(BP_GET_NDVAS(bp)); 3750a3874b8bSToomas Soome 3751a3874b8bSToomas Soome ASSERT0(BP_IS_GANG(bp)); 3752a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 3753a3874b8bSToomas Soome 3754a3874b8bSToomas Soome bp2sio(bp, sio, dva_i); 3755a3874b8bSToomas Soome sio->sio_flags = zio_flags; 3756a3874b8bSToomas Soome sio->sio_zb = *zb; 3757a3874b8bSToomas Soome 3758a3874b8bSToomas Soome /* 3759a3874b8bSToomas Soome * Increment the bytes pending counter now so that we can't 3760a3874b8bSToomas Soome * get an integer underflow in case the worker processes the 3761a3874b8bSToomas Soome * zio before we get to incrementing this counter. 3762a3874b8bSToomas Soome */ 376312a8814cSTom Caputi atomic_add_64(&scn->scn_bytes_pending, SIO_GET_ASIZE(sio)); 3764a3874b8bSToomas Soome 3765a3874b8bSToomas Soome scan_io_queue_insert_impl(queue, sio); 3766a3874b8bSToomas Soome } 3767a3874b8bSToomas Soome 3768a3874b8bSToomas Soome /* 3769a3874b8bSToomas Soome * Given a set of I/O parameters as discovered by the metadata traversal 3770a3874b8bSToomas Soome * process, attempts to place the I/O into the sorted queues (if allowed), 3771a3874b8bSToomas Soome * or immediately executes the I/O. 3772a3874b8bSToomas Soome */ 3773a3874b8bSToomas Soome static void 3774a3874b8bSToomas Soome dsl_scan_enqueue(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags, 3775a3874b8bSToomas Soome const zbookmark_phys_t *zb) 3776a3874b8bSToomas Soome { 3777a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 3778a3874b8bSToomas Soome 3779a3874b8bSToomas Soome ASSERT(!BP_IS_EMBEDDED(bp)); 3780a3874b8bSToomas Soome 3781a3874b8bSToomas Soome /* 3782a3874b8bSToomas Soome * Gang blocks are hard to issue sequentially, so we just issue them 3783a3874b8bSToomas Soome * here immediately instead of queuing them. 3784a3874b8bSToomas Soome */ 3785a3874b8bSToomas Soome if (!dp->dp_scan->scn_is_sorted || BP_IS_GANG(bp)) { 3786a3874b8bSToomas Soome scan_exec_io(dp, bp, zio_flags, zb, NULL); 3787a3874b8bSToomas Soome return; 3788a3874b8bSToomas Soome } 3789a3874b8bSToomas Soome for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 3790a3874b8bSToomas Soome dva_t dva; 3791a3874b8bSToomas Soome vdev_t *vdev; 3792a3874b8bSToomas Soome 3793a3874b8bSToomas Soome dva = bp->blk_dva[i]; 3794a3874b8bSToomas Soome vdev = vdev_lookup_top(spa, DVA_GET_VDEV(&dva)); 3795a3874b8bSToomas Soome ASSERT(vdev != NULL); 3796a3874b8bSToomas Soome 3797a3874b8bSToomas Soome mutex_enter(&vdev->vdev_scan_io_queue_lock); 3798a3874b8bSToomas Soome if (vdev->vdev_scan_io_queue == NULL) 3799a3874b8bSToomas Soome vdev->vdev_scan_io_queue = scan_io_queue_create(vdev); 3800a3874b8bSToomas Soome ASSERT(dp->dp_scan != NULL); 3801a3874b8bSToomas Soome scan_io_queue_insert(vdev->vdev_scan_io_queue, bp, 3802a3874b8bSToomas Soome i, zio_flags, zb); 3803a3874b8bSToomas Soome mutex_exit(&vdev->vdev_scan_io_queue_lock); 38043f9d6ad7SLin Ling } 38053f9d6ad7SLin Ling } 38063f9d6ad7SLin Ling 38073f9d6ad7SLin Ling static int 38083f9d6ad7SLin Ling dsl_scan_scrub_cb(dsl_pool_t *dp, 38097802d7bfSMatthew Ahrens const blkptr_t *bp, const zbookmark_phys_t *zb) 38103f9d6ad7SLin Ling { 38113f9d6ad7SLin Ling dsl_scan_t *scn = dp->dp_scan; 38123f9d6ad7SLin Ling spa_t *spa = dp->dp_spa; 38133f9d6ad7SLin Ling uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp); 3814a3874b8bSToomas Soome size_t psize = BP_GET_PSIZE(bp); 38153f9d6ad7SLin Ling boolean_t needs_io; 381644ecc532SGeorge Wilson int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL; 3817a3874b8bSToomas Soome int d; 3818dec267e7SMatthew Ahrens 38193f9d6ad7SLin Ling if (phys_birth <= scn->scn_phys.scn_min_txg || 3820a3874b8bSToomas Soome phys_birth >= scn->scn_phys.scn_max_txg) { 3821a3874b8bSToomas Soome count_block(scn, dp->dp_blkstats, bp); 38223f9d6ad7SLin Ling return (0); 3823a3874b8bSToomas Soome } 38243f9d6ad7SLin Ling 3825dec267e7SMatthew Ahrens /* Embedded BP's have phys_birth==0, so we reject them above. */ 3826dec267e7SMatthew Ahrens ASSERT(!BP_IS_EMBEDDED(bp)); 38275d7b4d43SMatthew Ahrens 38283f9d6ad7SLin Ling ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn)); 38293f9d6ad7SLin Ling if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) { 38303f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_SCRUB; 38313f9d6ad7SLin Ling needs_io = B_TRUE; 3832d5285caeSGeorge Wilson } else { 3833d5285caeSGeorge Wilson ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER); 38343f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_RESILVER; 38353f9d6ad7SLin Ling needs_io = B_FALSE; 38363f9d6ad7SLin Ling } 38373f9d6ad7SLin Ling 38383f9d6ad7SLin Ling /* If it's an intent log block, failure is expected. */ 38393f9d6ad7SLin Ling if (zb->zb_level == ZB_ZIL_LEVEL) 38403f9d6ad7SLin Ling zio_flags |= ZIO_FLAG_SPECULATIVE; 38413f9d6ad7SLin Ling 3842a3874b8bSToomas Soome for (d = 0; d < BP_GET_NDVAS(bp); d++) { 3843a3874b8bSToomas Soome const dva_t *dva = &bp->blk_dva[d]; 38443f9d6ad7SLin Ling 38453f9d6ad7SLin Ling /* 38463f9d6ad7SLin Ling * Keep track of how much data we've examined so that 38473f9d6ad7SLin Ling * zpool(1M) status can make useful progress reports. 38483f9d6ad7SLin Ling */ 3849a3874b8bSToomas Soome scn->scn_phys.scn_examined += DVA_GET_ASIZE(dva); 3850a3874b8bSToomas Soome spa->spa_scan_pass_exam += DVA_GET_ASIZE(dva); 38513f9d6ad7SLin Ling 38523f9d6ad7SLin Ling /* if it's a resilver, this may not be in the target range */ 3853a3874b8bSToomas Soome if (!needs_io) 3854a3874b8bSToomas Soome needs_io = dsl_scan_need_resilver(spa, dva, psize, 3855a3874b8bSToomas Soome phys_birth); 38563f9d6ad7SLin Ling } 38573f9d6ad7SLin Ling 38583f9d6ad7SLin Ling if (needs_io && !zfs_no_scrub_io) { 3859a3874b8bSToomas Soome dsl_scan_enqueue(dp, bp, zio_flags, zb); 3860a3874b8bSToomas Soome } else { 3861a3874b8bSToomas Soome count_block(scn, dp->dp_blkstats, bp); 3862a3874b8bSToomas Soome } 3863a3874b8bSToomas Soome 3864a3874b8bSToomas Soome /* do not relocate this block */ 3865a3874b8bSToomas Soome return (0); 3866a3874b8bSToomas Soome } 38673f9d6ad7SLin Ling 3868a3874b8bSToomas Soome static void 3869a3874b8bSToomas Soome dsl_scan_scrub_done(zio_t *zio) 3870a3874b8bSToomas Soome { 3871a3874b8bSToomas Soome spa_t *spa = zio->io_spa; 3872a3874b8bSToomas Soome blkptr_t *bp = zio->io_bp; 3873a3874b8bSToomas Soome dsl_scan_io_queue_t *queue = zio->io_private; 3874a3874b8bSToomas Soome 3875a3874b8bSToomas Soome abd_free(zio->io_abd); 3876a3874b8bSToomas Soome 3877a3874b8bSToomas Soome if (queue == NULL) { 38783f9d6ad7SLin Ling mutex_enter(&spa->spa_scrub_lock); 3879a3874b8bSToomas Soome ASSERT3U(spa->spa_scrub_inflight, >=, BP_GET_PSIZE(bp)); 3880a3874b8bSToomas Soome spa->spa_scrub_inflight -= BP_GET_PSIZE(bp); 3881a3874b8bSToomas Soome cv_broadcast(&spa->spa_scrub_io_cv); 3882a3874b8bSToomas Soome mutex_exit(&spa->spa_scrub_lock); 3883a3874b8bSToomas Soome } else { 3884a3874b8bSToomas Soome mutex_enter(&queue->q_vd->vdev_scan_io_queue_lock); 3885a3874b8bSToomas Soome ASSERT3U(queue->q_inflight_bytes, >=, BP_GET_PSIZE(bp)); 3886a3874b8bSToomas Soome queue->q_inflight_bytes -= BP_GET_PSIZE(bp); 3887a3874b8bSToomas Soome cv_broadcast(&queue->q_zio_cv); 3888a3874b8bSToomas Soome mutex_exit(&queue->q_vd->vdev_scan_io_queue_lock); 3889a3874b8bSToomas Soome } 3890a3874b8bSToomas Soome 3891a3874b8bSToomas Soome if (zio->io_error && (zio->io_error != ECKSUM || 3892a3874b8bSToomas Soome !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) { 3893a3874b8bSToomas Soome atomic_inc_64(&spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors); 3894a3874b8bSToomas Soome } 3895a3874b8bSToomas Soome } 3896a3874b8bSToomas Soome 3897a3874b8bSToomas Soome /* 3898a3874b8bSToomas Soome * Given a scanning zio's information, executes the zio. The zio need 3899a3874b8bSToomas Soome * not necessarily be only sortable, this function simply executes the 3900a3874b8bSToomas Soome * zio, no matter what it is. The optional queue argument allows the 3901a3874b8bSToomas Soome * caller to specify that they want per top level vdev IO rate limiting 3902a3874b8bSToomas Soome * instead of the legacy global limiting. 3903a3874b8bSToomas Soome */ 3904a3874b8bSToomas Soome static void 3905a3874b8bSToomas Soome scan_exec_io(dsl_pool_t *dp, const blkptr_t *bp, int zio_flags, 3906a3874b8bSToomas Soome const zbookmark_phys_t *zb, dsl_scan_io_queue_t *queue) 3907a3874b8bSToomas Soome { 3908a3874b8bSToomas Soome spa_t *spa = dp->dp_spa; 3909a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 3910a3874b8bSToomas Soome size_t size = BP_GET_PSIZE(bp); 3911a3874b8bSToomas Soome abd_t *data = abd_alloc_for_io(size, B_FALSE); 3912a3874b8bSToomas Soome 3913a3874b8bSToomas Soome if (queue == NULL) { 3914a3874b8bSToomas Soome mutex_enter(&spa->spa_scrub_lock); 3915a3874b8bSToomas Soome while (spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes) 39163f9d6ad7SLin Ling cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 3917a3874b8bSToomas Soome spa->spa_scrub_inflight += BP_GET_PSIZE(bp); 39183f9d6ad7SLin Ling mutex_exit(&spa->spa_scrub_lock); 3919a3874b8bSToomas Soome } else { 3920a3874b8bSToomas Soome kmutex_t *q_lock = &queue->q_vd->vdev_scan_io_queue_lock; 39213f9d6ad7SLin Ling 3922a3874b8bSToomas Soome mutex_enter(q_lock); 3923a3874b8bSToomas Soome while (queue->q_inflight_bytes >= queue->q_maxinflight_bytes) 3924a3874b8bSToomas Soome cv_wait(&queue->q_zio_cv, q_lock); 3925a3874b8bSToomas Soome queue->q_inflight_bytes += BP_GET_PSIZE(bp); 3926a3874b8bSToomas Soome mutex_exit(q_lock); 3927a3874b8bSToomas Soome } 3928a3874b8bSToomas Soome 3929a3874b8bSToomas Soome count_block(dp->dp_scan, dp->dp_blkstats, bp); 3930a3874b8bSToomas Soome zio_nowait(zio_read(dp->dp_scan->scn_zio_root, spa, bp, data, size, 3931a3874b8bSToomas Soome dsl_scan_scrub_done, queue, ZIO_PRIORITY_SCRUB, zio_flags, zb)); 3932a3874b8bSToomas Soome } 393344ecc532SGeorge Wilson 3934a3874b8bSToomas Soome /* 3935a3874b8bSToomas Soome * This is the primary extent sorting algorithm. We balance two parameters: 3936a3874b8bSToomas Soome * 1) how many bytes of I/O are in an extent 3937a3874b8bSToomas Soome * 2) how well the extent is filled with I/O (as a fraction of its total size) 3938a3874b8bSToomas Soome * Since we allow extents to have gaps between their constituent I/Os, it's 3939a3874b8bSToomas Soome * possible to have a fairly large extent that contains the same amount of 3940a3874b8bSToomas Soome * I/O bytes than a much smaller extent, which just packs the I/O more tightly. 3941a3874b8bSToomas Soome * The algorithm sorts based on a score calculated from the extent's size, 3942a3874b8bSToomas Soome * the relative fill volume (in %) and a "fill weight" parameter that controls 3943a3874b8bSToomas Soome * the split between whether we prefer larger extents or more well populated 3944a3874b8bSToomas Soome * extents: 3945a3874b8bSToomas Soome * 3946a3874b8bSToomas Soome * SCORE = FILL_IN_BYTES + (FILL_IN_PERCENT * FILL_IN_BYTES * FILL_WEIGHT) 3947a3874b8bSToomas Soome * 3948a3874b8bSToomas Soome * Example: 3949a3874b8bSToomas Soome * 1) assume extsz = 64 MiB 3950a3874b8bSToomas Soome * 2) assume fill = 32 MiB (extent is half full) 3951a3874b8bSToomas Soome * 3) assume fill_weight = 3 3952a3874b8bSToomas Soome * 4) SCORE = 32M + (((32M * 100) / 64M) * 3 * 32M) / 100 3953a3874b8bSToomas Soome * SCORE = 32M + (50 * 3 * 32M) / 100 3954a3874b8bSToomas Soome * SCORE = 32M + (4800M / 100) 3955a3874b8bSToomas Soome * SCORE = 32M + 48M 3956a3874b8bSToomas Soome * ^ ^ 3957a3874b8bSToomas Soome * | +--- final total relative fill-based score 3958a3874b8bSToomas Soome * +--------- final total fill-based score 3959a3874b8bSToomas Soome * SCORE = 80M 3960a3874b8bSToomas Soome * 3961a3874b8bSToomas Soome * As can be seen, at fill_ratio=3, the algorithm is slightly biased towards 3962a3874b8bSToomas Soome * extents that are more completely filled (in a 3:2 ratio) vs just larger. 3963a3874b8bSToomas Soome * Note that as an optimization, we replace multiplication and division by 3964a3874b8bSToomas Soome * 100 with bitshifting by 7 (which effecitvely multiplies and divides by 128). 3965a3874b8bSToomas Soome */ 3966a3874b8bSToomas Soome static int 3967a3874b8bSToomas Soome ext_size_compare(const void *x, const void *y) 3968a3874b8bSToomas Soome { 3969*4d7988d6SPaul Dagnelie const range_seg_gap_t *rsa = x, *rsb = y; 3970*4d7988d6SPaul Dagnelie 3971*4d7988d6SPaul Dagnelie uint64_t sa = rsa->rs_end - rsa->rs_start; 3972*4d7988d6SPaul Dagnelie uint64_t sb = rsb->rs_end - rsb->rs_start; 3973a3874b8bSToomas Soome uint64_t score_a, score_b; 3974a3874b8bSToomas Soome 3975a3874b8bSToomas Soome score_a = rsa->rs_fill + ((((rsa->rs_fill << 7) / sa) * 3976a3874b8bSToomas Soome fill_weight * rsa->rs_fill) >> 7); 3977a3874b8bSToomas Soome score_b = rsb->rs_fill + ((((rsb->rs_fill << 7) / sb) * 3978a3874b8bSToomas Soome fill_weight * rsb->rs_fill) >> 7); 3979a3874b8bSToomas Soome 3980a3874b8bSToomas Soome if (score_a > score_b) 3981a3874b8bSToomas Soome return (-1); 3982a3874b8bSToomas Soome if (score_a == score_b) { 3983a3874b8bSToomas Soome if (rsa->rs_start < rsb->rs_start) 3984a3874b8bSToomas Soome return (-1); 3985a3874b8bSToomas Soome if (rsa->rs_start == rsb->rs_start) 3986a3874b8bSToomas Soome return (0); 3987a3874b8bSToomas Soome return (1); 39883f9d6ad7SLin Ling } 3989a3874b8bSToomas Soome return (1); 3990a3874b8bSToomas Soome } 39913f9d6ad7SLin Ling 3992a3874b8bSToomas Soome /* 3993a3874b8bSToomas Soome * Comparator for the q_sios_by_addr tree. Sorting is simply performed 3994a3874b8bSToomas Soome * based on LBA-order (from lowest to highest). 3995a3874b8bSToomas Soome */ 3996a3874b8bSToomas Soome static int 399712a8814cSTom Caputi sio_addr_compare(const void *x, const void *y) 3998a3874b8bSToomas Soome { 3999a3874b8bSToomas Soome const scan_io_t *a = x, *b = y; 4000a3874b8bSToomas Soome 4001*4d7988d6SPaul Dagnelie return (TREE_CMP(SIO_GET_OFFSET(a), SIO_GET_OFFSET(b))); 4002a3874b8bSToomas Soome } 4003a3874b8bSToomas Soome 4004a3874b8bSToomas Soome /* IO queues are created on demand when they are needed. */ 4005a3874b8bSToomas Soome static dsl_scan_io_queue_t * 4006a3874b8bSToomas Soome scan_io_queue_create(vdev_t *vd) 4007a3874b8bSToomas Soome { 4008a3874b8bSToomas Soome dsl_scan_t *scn = vd->vdev_spa->spa_dsl_pool->dp_scan; 4009a3874b8bSToomas Soome dsl_scan_io_queue_t *q = kmem_zalloc(sizeof (*q), KM_SLEEP); 4010a3874b8bSToomas Soome 4011a3874b8bSToomas Soome q->q_scn = scn; 4012a3874b8bSToomas Soome q->q_vd = vd; 401312a8814cSTom Caputi q->q_sio_memused = 0; 4014a3874b8bSToomas Soome cv_init(&q->q_zio_cv, NULL, CV_DEFAULT, NULL); 4015*4d7988d6SPaul Dagnelie q->q_exts_by_addr = range_tree_create_impl(&rt_btree_ops, RANGE_SEG_GAP, 4016*4d7988d6SPaul Dagnelie &q->q_exts_by_size, 0, 0, ext_size_compare, zfs_scan_max_ext_gap); 401712a8814cSTom Caputi avl_create(&q->q_sios_by_addr, sio_addr_compare, 4018a3874b8bSToomas Soome sizeof (scan_io_t), offsetof(scan_io_t, sio_nodes.sio_addr_node)); 4019a3874b8bSToomas Soome 4020a3874b8bSToomas Soome return (q); 40213f9d6ad7SLin Ling } 40223f9d6ad7SLin Ling 40231702cce7SAlek Pinchuk /* 4024a3874b8bSToomas Soome * Destroys a scan queue and all segments and scan_io_t's contained in it. 4025a3874b8bSToomas Soome * No further execution of I/O occurs, anything pending in the queue is 4026a3874b8bSToomas Soome * simply freed without being executed. 40271702cce7SAlek Pinchuk */ 4028a3874b8bSToomas Soome void 4029a3874b8bSToomas Soome dsl_scan_io_queue_destroy(dsl_scan_io_queue_t *queue) 40303f9d6ad7SLin Ling { 4031a3874b8bSToomas Soome dsl_scan_t *scn = queue->q_scn; 4032a3874b8bSToomas Soome scan_io_t *sio; 4033a3874b8bSToomas Soome void *cookie = NULL; 4034a3874b8bSToomas Soome int64_t bytes_dequeued = 0; 4035a3874b8bSToomas Soome 4036a3874b8bSToomas Soome ASSERT(MUTEX_HELD(&queue->q_vd->vdev_scan_io_queue_lock)); 4037a3874b8bSToomas Soome 4038a3874b8bSToomas Soome while ((sio = avl_destroy_nodes(&queue->q_sios_by_addr, &cookie)) != 4039a3874b8bSToomas Soome NULL) { 4040a3874b8bSToomas Soome ASSERT(range_tree_contains(queue->q_exts_by_addr, 404112a8814cSTom Caputi SIO_GET_OFFSET(sio), SIO_GET_ASIZE(sio))); 404212a8814cSTom Caputi bytes_dequeued += SIO_GET_ASIZE(sio); 404312a8814cSTom Caputi queue->q_sio_memused -= SIO_GET_MUSED(sio); 404412a8814cSTom Caputi sio_free(sio); 4045a3874b8bSToomas Soome } 4046a3874b8bSToomas Soome 404712a8814cSTom Caputi ASSERT0(queue->q_sio_memused); 4048a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -bytes_dequeued); 4049a3874b8bSToomas Soome range_tree_vacate(queue->q_exts_by_addr, NULL, queue); 4050a3874b8bSToomas Soome range_tree_destroy(queue->q_exts_by_addr); 4051a3874b8bSToomas Soome avl_destroy(&queue->q_sios_by_addr); 4052a3874b8bSToomas Soome cv_destroy(&queue->q_zio_cv); 4053a3874b8bSToomas Soome 4054a3874b8bSToomas Soome kmem_free(queue, sizeof (*queue)); 4055a3874b8bSToomas Soome } 4056a3874b8bSToomas Soome 4057a3874b8bSToomas Soome /* 4058a3874b8bSToomas Soome * Properly transfers a dsl_scan_queue_t from `svd' to `tvd'. This is 4059a3874b8bSToomas Soome * called on behalf of vdev_top_transfer when creating or destroying 4060a3874b8bSToomas Soome * a mirror vdev due to zpool attach/detach. 4061a3874b8bSToomas Soome */ 4062a3874b8bSToomas Soome void 4063a3874b8bSToomas Soome dsl_scan_io_queue_vdev_xfer(vdev_t *svd, vdev_t *tvd) 4064a3874b8bSToomas Soome { 4065a3874b8bSToomas Soome mutex_enter(&svd->vdev_scan_io_queue_lock); 4066a3874b8bSToomas Soome mutex_enter(&tvd->vdev_scan_io_queue_lock); 4067a3874b8bSToomas Soome 4068a3874b8bSToomas Soome VERIFY3P(tvd->vdev_scan_io_queue, ==, NULL); 4069a3874b8bSToomas Soome tvd->vdev_scan_io_queue = svd->vdev_scan_io_queue; 4070a3874b8bSToomas Soome svd->vdev_scan_io_queue = NULL; 4071a3874b8bSToomas Soome if (tvd->vdev_scan_io_queue != NULL) 4072a3874b8bSToomas Soome tvd->vdev_scan_io_queue->q_vd = tvd; 4073a3874b8bSToomas Soome 4074a3874b8bSToomas Soome mutex_exit(&tvd->vdev_scan_io_queue_lock); 4075a3874b8bSToomas Soome mutex_exit(&svd->vdev_scan_io_queue_lock); 4076a3874b8bSToomas Soome } 4077a3874b8bSToomas Soome 4078a3874b8bSToomas Soome static void 4079a3874b8bSToomas Soome scan_io_queues_destroy(dsl_scan_t *scn) 4080a3874b8bSToomas Soome { 4081a3874b8bSToomas Soome vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev; 4082a3874b8bSToomas Soome 4083a3874b8bSToomas Soome for (uint64_t i = 0; i < rvd->vdev_children; i++) { 4084a3874b8bSToomas Soome vdev_t *tvd = rvd->vdev_child[i]; 4085a3874b8bSToomas Soome 4086a3874b8bSToomas Soome mutex_enter(&tvd->vdev_scan_io_queue_lock); 4087a3874b8bSToomas Soome if (tvd->vdev_scan_io_queue != NULL) 4088a3874b8bSToomas Soome dsl_scan_io_queue_destroy(tvd->vdev_scan_io_queue); 4089a3874b8bSToomas Soome tvd->vdev_scan_io_queue = NULL; 4090a3874b8bSToomas Soome mutex_exit(&tvd->vdev_scan_io_queue_lock); 4091a3874b8bSToomas Soome } 4092a3874b8bSToomas Soome } 4093a3874b8bSToomas Soome 4094a3874b8bSToomas Soome static void 4095a3874b8bSToomas Soome dsl_scan_freed_dva(spa_t *spa, const blkptr_t *bp, int dva_i) 4096a3874b8bSToomas Soome { 4097a3874b8bSToomas Soome dsl_pool_t *dp = spa->spa_dsl_pool; 40981702cce7SAlek Pinchuk dsl_scan_t *scn = dp->dp_scan; 4099a3874b8bSToomas Soome vdev_t *vdev; 4100a3874b8bSToomas Soome kmutex_t *q_lock; 4101a3874b8bSToomas Soome dsl_scan_io_queue_t *queue; 410212a8814cSTom Caputi scan_io_t *srch_sio, *sio; 4103a3874b8bSToomas Soome avl_index_t idx; 4104a3874b8bSToomas Soome uint64_t start, size; 4105a3874b8bSToomas Soome 4106a3874b8bSToomas Soome vdev = vdev_lookup_top(spa, DVA_GET_VDEV(&bp->blk_dva[dva_i])); 4107a3874b8bSToomas Soome ASSERT(vdev != NULL); 4108a3874b8bSToomas Soome q_lock = &vdev->vdev_scan_io_queue_lock; 4109a3874b8bSToomas Soome queue = vdev->vdev_scan_io_queue; 4110a3874b8bSToomas Soome 4111a3874b8bSToomas Soome mutex_enter(q_lock); 4112a3874b8bSToomas Soome if (queue == NULL) { 4113a3874b8bSToomas Soome mutex_exit(q_lock); 4114a3874b8bSToomas Soome return; 4115a3874b8bSToomas Soome } 4116a3874b8bSToomas Soome 411712a8814cSTom Caputi srch_sio = sio_alloc(BP_GET_NDVAS(bp)); 411812a8814cSTom Caputi bp2sio(bp, srch_sio, dva_i); 411912a8814cSTom Caputi start = SIO_GET_OFFSET(srch_sio); 412012a8814cSTom Caputi size = SIO_GET_ASIZE(srch_sio); 41213f9d6ad7SLin Ling 41223f9d6ad7SLin Ling /* 4123a3874b8bSToomas Soome * We can find the zio in two states: 4124a3874b8bSToomas Soome * 1) Cold, just sitting in the queue of zio's to be issued at 4125a3874b8bSToomas Soome * some point in the future. In this case, all we do is 4126a3874b8bSToomas Soome * remove the zio from the q_sios_by_addr tree, decrement 4127a3874b8bSToomas Soome * its data volume from the containing range_seg_t and 4128a3874b8bSToomas Soome * resort the q_exts_by_size tree to reflect that the 4129a3874b8bSToomas Soome * range_seg_t has lost some of its 'fill'. We don't shorten 4130a3874b8bSToomas Soome * the range_seg_t - this is usually rare enough not to be 4131a3874b8bSToomas Soome * worth the extra hassle of trying keep track of precise 4132a3874b8bSToomas Soome * extent boundaries. 4133a3874b8bSToomas Soome * 2) Hot, where the zio is currently in-flight in 4134a3874b8bSToomas Soome * dsl_scan_issue_ios. In this case, we can't simply 4135a3874b8bSToomas Soome * reach in and stop the in-flight zio's, so we instead 4136a3874b8bSToomas Soome * block the caller. Eventually, dsl_scan_issue_ios will 4137a3874b8bSToomas Soome * be done with issuing the zio's it gathered and will 4138a3874b8bSToomas Soome * signal us. 41393f9d6ad7SLin Ling */ 414012a8814cSTom Caputi sio = avl_find(&queue->q_sios_by_addr, srch_sio, &idx); 414112a8814cSTom Caputi sio_free(srch_sio); 414212a8814cSTom Caputi 4143a3874b8bSToomas Soome if (sio != NULL) { 414412a8814cSTom Caputi int64_t asize = SIO_GET_ASIZE(sio); 4145a3874b8bSToomas Soome blkptr_t tmpbp; 41463f9d6ad7SLin Ling 4147a3874b8bSToomas Soome /* Got it while it was cold in the queue */ 414812a8814cSTom Caputi ASSERT3U(start, ==, SIO_GET_OFFSET(sio)); 4149a3874b8bSToomas Soome ASSERT3U(size, ==, asize); 4150a3874b8bSToomas Soome avl_remove(&queue->q_sios_by_addr, sio); 415112a8814cSTom Caputi queue->q_sio_memused -= SIO_GET_MUSED(sio); 41521702cce7SAlek Pinchuk 4153a3874b8bSToomas Soome ASSERT(range_tree_contains(queue->q_exts_by_addr, start, size)); 4154a3874b8bSToomas Soome range_tree_remove_fill(queue->q_exts_by_addr, start, size); 41551702cce7SAlek Pinchuk 4156a3874b8bSToomas Soome /* 4157a3874b8bSToomas Soome * We only update scn_bytes_pending in the cold path, 4158a3874b8bSToomas Soome * otherwise it will already have been accounted for as 4159a3874b8bSToomas Soome * part of the zio's execution. 4160a3874b8bSToomas Soome */ 4161a3874b8bSToomas Soome atomic_add_64(&scn->scn_bytes_pending, -asize); 4162a3874b8bSToomas Soome 4163a3874b8bSToomas Soome /* count the block as though we issued it */ 416412a8814cSTom Caputi sio2bp(sio, &tmpbp); 4165a3874b8bSToomas Soome count_block(scn, dp->dp_blkstats, &tmpbp); 4166a3874b8bSToomas Soome 416712a8814cSTom Caputi sio_free(sio); 4168a3874b8bSToomas Soome } 4169a3874b8bSToomas Soome mutex_exit(q_lock); 41703f9d6ad7SLin Ling } 41711825bc56SNav Ravindranath 4172a3874b8bSToomas Soome /* 4173a3874b8bSToomas Soome * Callback invoked when a zio_free() zio is executing. This needs to be 4174a3874b8bSToomas Soome * intercepted to prevent the zio from deallocating a particular portion 4175a3874b8bSToomas Soome * of disk space and it then getting reallocated and written to, while we 4176a3874b8bSToomas Soome * still have it queued up for processing. 4177a3874b8bSToomas Soome */ 4178a3874b8bSToomas Soome void 4179a3874b8bSToomas Soome dsl_scan_freed(spa_t *spa, const blkptr_t *bp) 41801825bc56SNav Ravindranath { 4181a3874b8bSToomas Soome dsl_pool_t *dp = spa->spa_dsl_pool; 4182a3874b8bSToomas Soome dsl_scan_t *scn = dp->dp_scan; 4183a3874b8bSToomas Soome 4184a3874b8bSToomas Soome ASSERT(!BP_IS_EMBEDDED(bp)); 4185a3874b8bSToomas Soome ASSERT(scn != NULL); 4186a3874b8bSToomas Soome if (!dsl_scan_is_running(scn)) 4187a3874b8bSToomas Soome return; 4188a3874b8bSToomas Soome 4189a3874b8bSToomas Soome for (int i = 0; i < BP_GET_NDVAS(bp); i++) 4190a3874b8bSToomas Soome dsl_scan_freed_dva(spa, bp, i); 41911825bc56SNav Ravindranath } 4192